From 87f6237bb0d87a6cf23879f24b1bb3a488b3ad26 Mon Sep 17 00:00:00 2001 From: vbvg2008 Date: Wed, 26 Sep 2018 17:10:24 -0700 Subject: [PATCH 001/332] fixed calling Estimator '_distribution' method The code is calling a '_distribution' method which no longer exists. It will produce error message when training mirrored strategy with hooks. Changing '_distribution' to '_train_distribution' can fix the issue. --- tensorflow/python/estimator/estimator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index fd62a79c840..f2601ee55ca 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -1303,7 +1303,7 @@ class Estimator(object): # TODO(yuefengz): add a test for unwrapping per_device_hooks. def get_hooks_from_the_first_device(per_device_hooks): return [ - self._distribution.unwrap(per_device_hook)[0] + self._train_distribution.unwrap(per_device_hook)[0] for per_device_hook in per_device_hooks ] From 2a3ef832522bd55dc0b0e02a97f49352da35a01f Mon Sep 17 00:00:00 2001 From: frreiss Date: Tue, 11 Jun 2019 15:21:27 -0700 Subject: [PATCH 002/332] Address review comments --- tensorflow/core/util/tensor_bundle/byte_swap.cc | 6 +++--- tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/util/tensor_bundle/byte_swap.cc b/tensorflow/core/util/tensor_bundle/byte_swap.cc index 2f70aad75ca..f963e527d00 100644 --- a/tensorflow/core/util/tensor_bundle/byte_swap.cc +++ b/tensorflow/core/util/tensor_bundle/byte_swap.cc @@ -23,19 +23,19 @@ Status ByteSwapArray(char* array, size_t bytes_per_elem, int array_len) { // No-op return Status::OK(); } else if (bytes_per_elem == 2) { - auto array_16 = (uint16_t*)array; + auto array_16 = reinterpret_cast(array); for (int i = 0; i < array_len; i++) { array_16[i] = BYTE_SWAP_16(array_16[i]); } return Status::OK(); } else if (bytes_per_elem == 4) { - auto array_32 = (uint32_t*)array; + auto array_32 = reinterpret_cast(array); for (int i = 0; i < array_len; i++) { array_32[i] = BYTE_SWAP_32(array_32[i]); } return Status::OK(); } else if (bytes_per_elem == 8) { - auto array_64 = (uint64_t*)array; + auto array_64 = reinterpret_cast(array); for (int i = 0; i < array_len; i++) { array_64[i] = BYTE_SWAP_64(array_64[i]); } diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc index dc5250a6460..b05e0a3bead 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc @@ -63,7 +63,7 @@ Tensor Constant_2x3(T v) { Tensor ByteSwap(Tensor t) { Tensor ret = tensor::DeepCopy(t); - EXPECT_OK(ByteSwapTensor(&ret)); + TF_EXPECT_OK(ByteSwapTensor(&ret)); return ret; } From 8651888c2201632b47f3ccf3d180bc43f5ca6dd5 Mon Sep 17 00:00:00 2001 From: frreiss Date: Thu, 13 Jun 2019 14:49:38 -0700 Subject: [PATCH 003/332] Fix Windows build issue and correct formatting of comments --- .../core/util/tensor_bundle/byte_swap.h | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/util/tensor_bundle/byte_swap.h b/tensorflow/core/util/tensor_bundle/byte_swap.h index 6404cc37186..948cd2e82bd 100644 --- a/tensorflow/core/util/tensor_bundle/byte_swap.h +++ b/tensorflow/core/util/tensor_bundle/byte_swap.h @@ -16,17 +16,17 @@ limitations under the License. #ifndef BYTE_SWAP_H #define BYTE_SWAP_H -#include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/byte_order.h" // Define basic byte swapping operations. // These operations must be macros to use compiler intrinsics. -// Note that the code here is written for portability, not speed. Byte swapping -// only happens when importing a checkpoint from one hardware architecture onto -// a different architecture. If these operations become part of a fast path, -// then the function ByteSwapArray() below should be rewritten to use +// Note that the code here is written for portability, not speed. Byte swapping +// only happens when importing a checkpoint from one hardware architecture onto +// a different architecture. If these operations become part of a fast path, +// then the function ByteSwapArray() below should be rewritten to use // architecture-appropriate SIMD instructions that swap multiple words at once. #if defined(__linux__) @@ -37,11 +37,20 @@ limitations under the License. #define BYTE_SWAP_32(x) bswap_32 (x) #define BYTE_SWAP_64(x) bswap_64 (x) +#elif defined(PLATFORM_WINDOWS) + +// On windows, byte-swapping is in winsock.h, and there is a version of htonl +// that can byte-swap 64-bit values +#include +#define BYTE_SWAP_16(x) htons (x) +#define BYTE_SWAP_32(x) htonl (x) +#define BYTE_SWAP_64(x) htonll (x) + #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -// On non-Linux, but little-endian, environments, use htonl/s, which byte-swap -// when the host byte order is little-endian. POSIX doesn't define a 64-bit -// version of these library functions, so we roll our own. +// On non-Linux, non-Windows, but little-endian, environments, use htonl/s, +// which byte-swap when the host byte order is little-endian. POSIX doesn't +// define a 64-bit version of these library functions, so we roll our own. #include #define BYTE_SWAP_16(x) htons (x) #define BYTE_SWAP_32(x) htonl (x) @@ -50,7 +59,8 @@ limitations under the License. | (htonl(((x) & 0xffffffff00000000UL) >> 32)) \ ) -#else // not defined(__linux__) and (__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__) +#else // not defined(__linux__) and not defined(PLATFORM_WINDOWS) + // and (__BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__) // Fall back on a non-optimized implementation on other big-endian targets. // This code swaps one byte at a time and is probably an order of magnitude @@ -79,7 +89,7 @@ limitations under the License. | (((x) & 0xff00000000000000UL) >> 56) \ ) -#endif // defined(__linux__) +#endif // defined(__linux__) namespace tensorflow { @@ -106,6 +116,6 @@ Status ByteSwapArray(char *array, size_t bytes_per_elem, int array_len); // TODO(frreiss): Should this be a member of the Tensor class? Status ByteSwapTensor(Tensor *t); -} // namespace tensorflow +} // namespace tensorflow -#endif // BYTE_SWAP_H +#endif // BYTE_SWAP_H From ef6d28be5978cfea218d9830dc3b49d856b3c6f3 Mon Sep 17 00:00:00 2001 From: frreiss Date: Fri, 14 Jun 2019 14:37:39 -0700 Subject: [PATCH 004/332] Restore changes lost during merge --- tensorflow/core/util/tensor_bundle/BUILD | 5 +- .../core/util/tensor_bundle/tensor_bundle.cc | 55 +++-- .../core/util/tensor_bundle/tensor_bundle.h | 4 + .../util/tensor_bundle/tensor_bundle_test.cc | 219 ++++++++++++++++-- 4 files changed, 251 insertions(+), 32 deletions(-) diff --git a/tensorflow/core/util/tensor_bundle/BUILD b/tensorflow/core/util/tensor_bundle/BUILD index 91efede0269..ef767469c6c 100644 --- a/tensorflow/core/util/tensor_bundle/BUILD +++ b/tensorflow/core/util/tensor_bundle/BUILD @@ -22,6 +22,7 @@ filegroup( "naming.h", "tensor_bundle.cc", "tensor_bundle.h", + "byte_swap.h" ], ) @@ -32,8 +33,8 @@ alias( cc_library( name = "tensor_bundle", - srcs = ["tensor_bundle.cc"], - hdrs = ["tensor_bundle.h"], + srcs = ["tensor_bundle.cc", "byte_swap.cc"], + hdrs = ["tensor_bundle.h", "byte_swap.h"], copts = tf_copts() + if_not_windows(["-Wno-sign-compare"]), deps = [ ":naming", diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc index 111ccdc48f4..bd5cce4f1b7 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/util/tensor_bundle/tensor_bundle.h" +#include "tensorflow/core/util/tensor_bundle/byte_swap.h" #include #include @@ -23,8 +24,8 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.pb.h" -#include "tensorflow/core/framework/tensor_shape.pb_text.h" #include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb_text.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb_text.h" #include "tensorflow/core/framework/variant.h" @@ -69,7 +70,7 @@ namespace { // bytes) and string bytes, and stores it into "actual_crc32c". Status ReadStringTensor(io::InputBuffer* buffered_file, size_t num_elements, size_t offset, size_t size, string* destination, - uint32* actual_crc32c) { + uint32* actual_crc32c, bool need_to_swap_bytes) { if (size == 0) return Status::OK(); CHECK_GT(size, 0); @@ -81,14 +82,22 @@ Status ReadStringTensor(io::InputBuffer* buffered_file, size_t num_elements, if (string_lengths[i] <= UINT32_MAX) { // We need to do this because older checkpoints only used uint32s and we // should still support them. - const uint32 elem_size_uint32 = static_cast(string_lengths[i]); + uint32 elem_size_uint32 = static_cast(string_lengths[i]); + if (need_to_swap_bytes) { + // Checksum would have been computed on the source machine's byte order + elem_size_uint32 = BYTE_SWAP_32(elem_size_uint32); + } *actual_crc32c = crc32c::Extend( *actual_crc32c, reinterpret_cast(&elem_size_uint32), sizeof(uint32)); } else { - *actual_crc32c = crc32c::Extend( - *actual_crc32c, reinterpret_cast(&string_lengths[i]), - sizeof(uint64)); + uint64 length = string_lengths[i]; + if (need_to_swap_bytes) { + length = BYTE_SWAP_64(length); + } + *actual_crc32c = + crc32c::Extend(*actual_crc32c, reinterpret_cast(&length), + sizeof(uint64)); } } if (offset + size < buffered_file->Tell()) { @@ -97,20 +106,23 @@ Status ReadStringTensor(io::InputBuffer* buffered_file, size_t num_elements, } // Reads the length-checksum. - uint32 length_checksum = 0; + uint32 raw_length_checksum = 0; // Bytes in file + uint32 length_checksum = 0; // In-memory representation size_t unused_bytes_read = 0; TF_RETURN_IF_ERROR(buffered_file->ReadNBytes( - sizeof(uint32), reinterpret_cast(&length_checksum), + sizeof(uint32), reinterpret_cast(&raw_length_checksum), &unused_bytes_read)); + length_checksum = need_to_swap_bytes ? BYTE_SWAP_32(raw_length_checksum) + : raw_length_checksum; if (crc32c::Unmask(length_checksum) != *actual_crc32c) { return errors::DataLoss( "The length checksum does not match: expected ", strings::Printf("%08u", crc32c::Unmask(length_checksum)), " but actual is ", strings::Printf("%08u", *actual_crc32c)); } - *actual_crc32c = - crc32c::Extend(*actual_crc32c, reinterpret_cast(&length_checksum), - sizeof(uint32)); + *actual_crc32c = crc32c::Extend(*actual_crc32c, + reinterpret_cast(&raw_length_checksum), + sizeof(uint32)); // Reads the actual string bytes. for (size_t i = 0; i < num_elements; ++i) { @@ -719,7 +731,8 @@ BundleReader::BundleReader(Env* env, StringPiece prefix) prefix_(prefix), metadata_(nullptr), table_(nullptr), - iter_(nullptr) { + iter_(nullptr), + need_to_swap_bytes_(false) { const string filename = MetaFilename(prefix_); uint64 file_size; status_ = env_->GetFileSize(filename, &file_size); @@ -751,9 +764,7 @@ BundleReader::BundleReader(Env* env, StringPiece prefix) if ((header.endianness() == BundleHeaderProto::BIG && port::kLittleEndian) || (header.endianness() == BundleHeaderProto::LITTLE && !port::kLittleEndian)) { - status_ = errors::Unimplemented( - "Reading a bundle with different endianness from the reader"); - return; + need_to_swap_bytes_ = true; } status_ = CheckVersions(header.version(), kTensorBundleVersion, kTensorBundleMinProducer, "Checkpoint", "checkpoint"); @@ -852,8 +863,20 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) { TF_RETURN_IF_ERROR(buffered_file->ReadNBytes(entry.size(), backing_buffer, &unused_bytes_read)); } + // Note that we compute the checksum *before* byte-swapping. The checksum + // should be on the bytes in the order they appear in the file. actual_crc32c = crc32c::Value(backing_buffer, entry.size()); + if (need_to_swap_bytes_) { + TF_RETURN_IF_ERROR(ByteSwapTensor(ret)); + } } else if (entry.dtype() == DT_VARIANT) { + if (need_to_swap_bytes_) { + return errors::Unimplemented( + "TensorBundle at ", prefix_, + "is of a different endianness than this machine's hardware, and " + "the bundle contains a variant (arbitrary C++ type) tensor. " + "Byte-swapping of variant tensors is not currently implemented."); + } // Relies on io::InputBuffer's buffering, because we issue many neighboring // reads for a single string tensor. TF_RETURN_IF_ERROR(ReadVariantTensor(buffered_file, ret, entry.offset(), @@ -863,7 +886,7 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) { // reads for a single string tensor. TF_RETURN_IF_ERROR(ReadStringTensor( buffered_file, ret->NumElements(), entry.offset(), entry.size(), - GetStringBackingBuffer(*ret), &actual_crc32c)); + GetStringBackingBuffer(*ret), &actual_crc32c, need_to_swap_bytes_)); } if (crc32c::Unmask(entry.crc32c()) != actual_crc32c) { return errors::DataLoss( diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.h b/tensorflow/core/util/tensor_bundle/tensor_bundle.h index 3a2ffbb4952..0320878df8d 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle.h +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.h @@ -300,6 +300,10 @@ class BundleReader { // the header entry in the metadata table. int num_shards_; + // Flag that this class sets to true when the endianness of the target bundle + // differs from that of the current system's processor architecture. + bool need_to_swap_bytes_; + friend class TensorBundleAlignmentTest; // For testing data alignment. TF_DISALLOW_COPY_AND_ASSIGN(BundleReader); diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc index df465397dec..b05e0a3bead 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc @@ -14,11 +14,13 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/util/tensor_bundle/tensor_bundle.h" +#include "tensorflow/core/util/tensor_bundle/byte_swap.h" #include #include #include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/framework/variant.h" #include "tensorflow/core/framework/variant_op_registry.h" @@ -35,10 +37,13 @@ namespace tensorflow { namespace { +// Prepend the current test case's working temporary directory to string Prefix(const string& prefix) { return strings::StrCat(testing::TmpDir(), "/", prefix); } +// Construct a data input directory by prepending the test data root +// directory to string TestdataPrefix(const string& prefix) { return strings::StrCat(testing::TensorFlowSrcRoot(), "/core/util/tensor_bundle/testdata/", prefix); @@ -267,6 +272,206 @@ void TestBasic() { } } +// Reinterpret the bytes of an rvalue +#define _CAST(x, T) (*reinterpret_cast(&x)) + +// Type-specific subroutine of SwapBytes test below +template +void TestByteSwap(const T* forward, const T* swapped, int array_len) { + auto bytes_per_elem = sizeof(T); + + // Convert the entire array at once + std::unique_ptr forward_copy(new T[array_len]); + std::memcpy(forward_copy.get(), forward, array_len * bytes_per_elem); + TF_EXPECT_OK( + ByteSwapArray((char*)forward_copy.get(), bytes_per_elem, array_len)); + for (int i = 0; i < array_len; i++) { + EXPECT_EQ(forward_copy.get()[i], swapped[i]); + } + + // Then the array wrapped in a tensor + auto shape = TensorShape({array_len}); + auto dtype = DataTypeToEnum::value; + Tensor forward_tensor(dtype, shape); + Tensor swapped_tensor(dtype, shape); + std::memcpy((char*)forward_tensor.tensor_data().data(), forward, + array_len * bytes_per_elem); + std::memcpy((char*)swapped_tensor.tensor_data().data(), swapped, + array_len * bytes_per_elem); + TF_EXPECT_OK(ByteSwapTensor(&forward_tensor)); + test::ExpectTensorEqual(forward_tensor, swapped_tensor); +} + +// Unit test of the byte-swapping operations that TensorBundle uses. +TEST(TensorBundleTest, SwapBytes) { + // A bug in the compiler on MacOS causes ByteSwap() and FlipEndiannessBit() + // to be removed from the executable if they are only called from templated + // functions. As a workaround, we make some dummy calls here. + // TODO(frreiss): Remove this workaround when the compiler bug is fixed. + ByteSwap(Constant_2x3(42)); + EXPECT_NE(Status::OK(), FlipEndiannessBit(Prefix("not_a_valid_prefix"))); + + // Test patterns, manually swapped so that we aren't relying on the + // correctness of our own byte-swapping macros when testing those macros. + // At least one of the entries in each list has the sign bit set when + // interpreted as a signed int. + const int arr_len_16 = 4; + const uint16_t forward_16[] = {0x1de5, 0xd017, 0xf1ea, 0xc0a1}; + const uint16_t swapped_16[] = {0xe51d, 0x17d0, 0xeaf1, 0xa1c0}; + const int arr_len_32 = 2; + const uint32_t forward_32[] = {0x0ddba115, 0xf01dab1e}; + const uint32_t swapped_32[] = {0x15a1db0d, 0x1eab1df0}; + const int arr_len_64 = 2; + const uint64_t forward_64[] = {0xf005ba11caba1000, 0x5ca1ab1ecab005e5}; + const uint64_t swapped_64[] = {0x0010baca11ba05f0, 0xe505b0ca1eaba15c}; + + // 16-bit types + TestByteSwap(forward_16, swapped_16, arr_len_16); + TestByteSwap((int16_t*)forward_16, (int16_t*)swapped_16, arr_len_16); + // TODO(frreiss): Test half-precision float + + // 32-bit types + TestByteSwap(forward_32, swapped_32, arr_len_32); + TestByteSwap((const int32_t*)forward_32, (const int32_t*)swapped_32, + arr_len_32); + TestByteSwap((const float*)forward_32, (const float*)swapped_32, arr_len_32); + + // 64-bit types + // Cast to uint64*/int64* to make DataTypeToEnum happy + TestByteSwap((const uint64*)forward_64, (const uint64*)swapped_64, + arr_len_64); + TestByteSwap((const int64*)forward_64, (const int64*)swapped_64, arr_len_64); + TestByteSwap((const double*)forward_64, (const double*)swapped_64, + arr_len_64); + + // Complex types. + // Logic for complex number handling is only in ByteSwapTensor, so don't test + // ByteSwapArray + const float* forward_float = (const float*)forward_32; + const float* swapped_float = (const float*)swapped_32; + const double* forward_double = (const double*)forward_64; + const double* swapped_double = (const double*)swapped_64; + Tensor forward_complex64 = Constant_2x3( + std::complex(forward_float[0], forward_float[1])); + Tensor swapped_complex64 = Constant_2x3( + std::complex(swapped_float[0], swapped_float[1])); + Tensor forward_complex128 = Constant_2x3( + std::complex(forward_double[0], forward_double[1])); + Tensor swapped_complex128 = Constant_2x3( + std::complex(swapped_double[0], swapped_double[1])); + + TF_EXPECT_OK(ByteSwapTensor(&forward_complex64)); + test::ExpectTensorEqual(forward_complex64, swapped_complex64); + + TF_EXPECT_OK(ByteSwapTensor(&forward_complex128)); + test::ExpectTensorEqual(forward_complex128, swapped_complex128); +} + +// Basic test of alternate-endianness support. Generates a bundle in +// the opposite of the current system's endianness and attempts to +// read the bundle back in. Does not exercise sharding or access to +// nonaligned tensors. Does cover the major access types exercised +// in TestBasic. +template +void TestEndianness() { + { + // Write out a TensorBundle in the opposite of this host's endianness. + BundleWriter writer(Env::Default(), Prefix("foo")); + TF_EXPECT_OK(writer.Add("foo_003", ByteSwap(Constant_2x3(3)))); + TF_EXPECT_OK(writer.Add("foo_000", ByteSwap(Constant_2x3(0)))); + TF_EXPECT_OK(writer.Add("foo_002", ByteSwap(Constant_2x3(2)))); + TF_EXPECT_OK(writer.Add("foo_001", ByteSwap(Constant_2x3(1)))); + TF_ASSERT_OK(writer.Finish()); + TF_ASSERT_OK(FlipEndiannessBit(Prefix("foo"))); + } + { + BundleReader reader(Env::Default(), Prefix("foo")); + TF_ASSERT_OK(reader.status()); + EXPECT_EQ( + AllTensorKeys(&reader), + std::vector({"foo_000", "foo_001", "foo_002", "foo_003"})); + Expect(&reader, "foo_000", Constant_2x3(0)); + Expect(&reader, "foo_001", Constant_2x3(1)); + Expect(&reader, "foo_002", Constant_2x3(2)); + Expect(&reader, "foo_003", Constant_2x3(3)); + } + { + BundleReader reader(Env::Default(), Prefix("foo")); + TF_ASSERT_OK(reader.status()); + ExpectNext(&reader, Constant_2x3(0)); + ExpectNext(&reader, Constant_2x3(1)); + ExpectNext(&reader, Constant_2x3(2)); + ExpectNext(&reader, Constant_2x3(3)); + EXPECT_TRUE(reader.Valid()); + reader.Next(); + EXPECT_FALSE(reader.Valid()); + } + { + BundleWriter writer(Env::Default(), Prefix("bar")); + TF_EXPECT_OK(writer.Add("bar_003", ByteSwap(Constant_2x3(3)))); + TF_EXPECT_OK(writer.Add("bar_000", ByteSwap(Constant_2x3(0)))); + TF_EXPECT_OK(writer.Add("bar_002", ByteSwap(Constant_2x3(2)))); + TF_EXPECT_OK(writer.Add("bar_001", ByteSwap(Constant_2x3(1)))); + TF_ASSERT_OK(writer.Finish()); + TF_ASSERT_OK(FlipEndiannessBit(Prefix("bar"))); + } + { + BundleReader reader(Env::Default(), Prefix("bar")); + TF_ASSERT_OK(reader.status()); + EXPECT_EQ( + AllTensorKeys(&reader), + std::vector({"bar_000", "bar_001", "bar_002", "bar_003"})); + Expect(&reader, "bar_003", Constant_2x3(3)); + Expect(&reader, "bar_002", Constant_2x3(2)); + Expect(&reader, "bar_001", Constant_2x3(1)); + Expect(&reader, "bar_000", Constant_2x3(0)); + } + { + BundleReader reader(Env::Default(), Prefix("bar")); + TF_ASSERT_OK(reader.status()); + ExpectNext(&reader, Constant_2x3(0)); + ExpectNext(&reader, Constant_2x3(1)); + ExpectNext(&reader, Constant_2x3(2)); + ExpectNext(&reader, Constant_2x3(3)); + EXPECT_TRUE(reader.Valid()); + reader.Next(); + EXPECT_FALSE(reader.Valid()); + } + TF_ASSERT_OK(MergeBundles(Env::Default(), {Prefix("foo"), Prefix("bar")}, + Prefix("merged"))); + { + BundleReader reader(Env::Default(), Prefix("merged")); + TF_ASSERT_OK(reader.status()); + EXPECT_EQ( + AllTensorKeys(&reader), + std::vector({"bar_000", "bar_001", "bar_002", "bar_003", + "foo_000", "foo_001", "foo_002", "foo_003"})); + Expect(&reader, "bar_000", Constant_2x3(0)); + Expect(&reader, "bar_001", Constant_2x3(1)); + Expect(&reader, "bar_002", Constant_2x3(2)); + Expect(&reader, "bar_003", Constant_2x3(3)); + Expect(&reader, "foo_000", Constant_2x3(0)); + Expect(&reader, "foo_001", Constant_2x3(1)); + Expect(&reader, "foo_002", Constant_2x3(2)); + Expect(&reader, "foo_003", Constant_2x3(3)); + } + { + BundleReader reader(Env::Default(), Prefix("merged")); + TF_ASSERT_OK(reader.status()); + ExpectNext(&reader, Constant_2x3(0)); + ExpectNext(&reader, Constant_2x3(1)); + ExpectNext(&reader, Constant_2x3(2)); + ExpectNext(&reader, Constant_2x3(3)); + ExpectNext(&reader, Constant_2x3(0)); + ExpectNext(&reader, Constant_2x3(1)); + ExpectNext(&reader, Constant_2x3(2)); + ExpectNext(&reader, Constant_2x3(3)); + EXPECT_TRUE(reader.Valid()); + reader.Next(); + EXPECT_FALSE(reader.Valid()); + } +} + template void TestNonStandardShapes() { { @@ -738,20 +943,6 @@ TEST(TensorBundleTest, Checksum) { } } -TEST(TensorBundleTest, Endianness) { - BundleWriter writer(Env::Default(), Prefix("end")); - TF_EXPECT_OK(writer.Add("key", Constant_2x3(1.0))); - TF_ASSERT_OK(writer.Finish()); - - // Flips the endianness bit. - TF_ASSERT_OK(FlipEndiannessBit(Prefix("end"))); - - BundleReader reader(Env::Default(), Prefix("end")); - EXPECT_TRUE(errors::IsUnimplemented(reader.status())); - EXPECT_TRUE(absl::StrContains(reader.status().ToString(), - "different endianness from the reader")); -} - TEST(TensorBundleTest, TruncatedTensorContents) { Env* env = Env::Default(); BundleWriter writer(env, Prefix("end")); From 63463d853e7a26183d02008d23037da48f7747f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=8C=AF=E5=8D=8E=20=28Zhenhua=20WANG=29?= Date: Wed, 12 Jun 2019 15:27:08 +0800 Subject: [PATCH 005/332] lite: enable int8 for op sparse_to_dense Test: bazel run tensorflow/lite/kernels:sparse_to_dense_test --- tensorflow/lite/kernels/sparse_to_dense.cc | 16 +++++++----- .../lite/kernels/sparse_to_dense_test.cc | 26 +++++++++++++++---- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/tensorflow/lite/kernels/sparse_to_dense.cc b/tensorflow/lite/kernels/sparse_to_dense.cc index 74eef2d698e..0e49d503ad5 100644 --- a/tensorflow/lite/kernels/sparse_to_dense.cc +++ b/tensorflow/lite/kernels/sparse_to_dense.cc @@ -170,10 +170,11 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE( context, indices->type == kTfLiteInt32 || indices->type == kTfLiteInt64); TF_LITE_ENSURE(context, output_shape->type == kTfLiteInt32 || - output_shape->type == kTfLiteInt64); + output_shape->type == kTfLiteInt64); TF_LITE_ENSURE(context, values->type == kTfLiteInt32 || - values->type == kTfLiteInt64 || - values->type == kTfLiteFloat32); + values->type == kTfLiteInt64 || + values->type == kTfLiteInt8 || + values->type == kTfLiteFloat32); TF_LITE_ENSURE_EQ(context, values->type, default_value->type); // Ensure dimensions match. @@ -232,7 +233,8 @@ TfLiteStatus EvalForIndexType(TfLiteContext* context, TfLiteNode* node, } default: context->ReportError( - context, "Type %d is currently not supported by sparse to dense.", + context, + "Indice type %d is currently not supported by sparse to dense.", indices->type); return kTfLiteError; } @@ -242,7 +244,6 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteTensor* indices = GetInput(context, node, kIndicesTensor); const TfLiteTensor* values = GetInput(context, node, kValueInputTensor); - // Currently only supports float32, int32 and int64. switch (values->type) { case kTfLiteFloat32: return EvalForIndexType(context, node, indices); @@ -250,9 +251,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { return EvalForIndexType(context, node, indices); case kTfLiteInt64: return EvalForIndexType(context, node, indices); + case kTfLiteInt8: + return EvalForIndexType(context, node, indices); default: context->ReportError( - context, "Type %d is currently not supported by sparse to dense.", + context, + "Value type %d is currently not supported by sparse to dense.", values->type); return kTfLiteError; } diff --git a/tensorflow/lite/kernels/sparse_to_dense_test.cc b/tensorflow/lite/kernels/sparse_to_dense_test.cc index 4a5ce6a36b5..27d733afd3c 100644 --- a/tensorflow/lite/kernels/sparse_to_dense_test.cc +++ b/tensorflow/lite/kernels/sparse_to_dense_test.cc @@ -100,6 +100,21 @@ TEST(SparseToDenseOpModelTest, TwoDimensionsTest) { EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 3, 3})); } +TEST(SparseToDenseOpModelTest, Int64IndexTest) { + SparseToDenseOpModel m({3, 3}, {3}, {3}, -1, TensorType_INT64, + TensorType_FLOAT32); + m.PopulateTensor(m.indices(), {0, 0, 0, 1, 2, 1, 2, 0, 1}); + m.PopulateTensor(m.output_shape(), {3, 3, 3}); + m.PopulateTensor(m.values(), {2, 4, 6}); + m.Invoke(); + + EXPECT_THAT( + m.GetOutput(), + ElementsAreArray({2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 4, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1})); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 3, 3})); +} + TEST(SparseToDenseOpModelTest, DefaultValueTest) { SparseToDenseOpModel m({3, 3}, {3}, {3}, -1, TensorType_INT32, TensorType_FLOAT32); @@ -145,12 +160,12 @@ TEST(SparseToDenseOpModelTest, Int64ValueTest) { EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 3, 3})); } -TEST(SparseToDenseOpModelTest, Int64IndexTest) { - SparseToDenseOpModel m({3, 3}, {3}, {3}, -1, TensorType_INT64, - TensorType_FLOAT32); - m.PopulateTensor(m.indices(), {0, 0, 0, 1, 2, 1, 2, 0, 1}); +TEST(SparseToDenseOpModelTest, Int8ValueTest) { + SparseToDenseOpModel m({3, 3}, {3}, {3}, -1, TensorType_INT32, + TensorType_INT8); + m.PopulateTensor(m.indices(), {0, 0, 0, 1, 2, 1, 2, 0, 1}); m.PopulateTensor(m.output_shape(), {3, 3, 3}); - m.PopulateTensor(m.values(), {2, 4, 6}); + m.PopulateTensor(m.values(), {2, 4, 6}); m.Invoke(); EXPECT_THAT( @@ -160,5 +175,6 @@ TEST(SparseToDenseOpModelTest, Int64IndexTest) { EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 3, 3})); } + } // namespace } // namespace tflite From f0ad53a42e274489f7743ac86455b45cea3ab025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=8C=AF=E5=8D=8E=20=28Zhenhua=20WANG=29?= Date: Wed, 12 Jun 2019 15:29:56 +0800 Subject: [PATCH 006/332] lite: enable uint8 for op sparse_to_dense Test: bazel run tensorflow/lite/kernels:sparse_to_dense_test --- tensorflow/lite/kernels/sparse_to_dense.cc | 3 +++ tensorflow/lite/kernels/sparse_to_dense_test.cc | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/tensorflow/lite/kernels/sparse_to_dense.cc b/tensorflow/lite/kernels/sparse_to_dense.cc index 0e49d503ad5..ffc60b67249 100644 --- a/tensorflow/lite/kernels/sparse_to_dense.cc +++ b/tensorflow/lite/kernels/sparse_to_dense.cc @@ -174,6 +174,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE(context, values->type == kTfLiteInt32 || values->type == kTfLiteInt64 || values->type == kTfLiteInt8 || + values->type == kTfLiteUInt8 || values->type == kTfLiteFloat32); TF_LITE_ENSURE_EQ(context, values->type, default_value->type); @@ -253,6 +254,8 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { return EvalForIndexType(context, node, indices); case kTfLiteInt8: return EvalForIndexType(context, node, indices); + case kTfLiteUInt8: + return EvalForIndexType(context, node, indices); default: context->ReportError( context, diff --git a/tensorflow/lite/kernels/sparse_to_dense_test.cc b/tensorflow/lite/kernels/sparse_to_dense_test.cc index 27d733afd3c..358a4b9282f 100644 --- a/tensorflow/lite/kernels/sparse_to_dense_test.cc +++ b/tensorflow/lite/kernels/sparse_to_dense_test.cc @@ -176,5 +176,20 @@ TEST(SparseToDenseOpModelTest, Int8ValueTest) { } +TEST(SparseToDenseOpModelTest, UInt8ValueTest) { + SparseToDenseOpModel m({3, 3}, {3}, {3}, 1, TensorType_INT32, + TensorType_UINT8); + m.PopulateTensor(m.indices(), {0, 0, 0, 1, 2, 1, 2, 0, 1}); + m.PopulateTensor(m.output_shape(), {3, 3, 3}); + m.PopulateTensor(m.values(), {2, 4, 6}); + m.Invoke(); + + EXPECT_THAT( + m.GetOutput(), + ElementsAreArray({2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 4, 1, 1, 6, 1, 1, 1, 1, 1, 1, 1})); + EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 3, 3})); +} + } // namespace } // namespace tflite From 3e1342453ffff5a4655c8250a9afba957d4288d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=8C=AF=E5=8D=8E=20=28Zhenhua=20WANG=29?= Date: Wed, 12 Jun 2019 15:30:43 +0800 Subject: [PATCH 007/332] lite: enable op kSparseToDense for quantization in toco --- tensorflow/lite/toco/graph_transformations/quantize.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tensorflow/lite/toco/graph_transformations/quantize.cc b/tensorflow/lite/toco/graph_transformations/quantize.cc index 680b2711488..89468ff1cc2 100644 --- a/tensorflow/lite/toco/graph_transformations/quantize.cc +++ b/tensorflow/lite/toco/graph_transformations/quantize.cc @@ -67,11 +67,14 @@ bool SupportsQuantization(const Operator& op) { type == OperatorType::kUnpack || type == OperatorType::kTopK_V2 || type == OperatorType::kRandomUniform || type == OperatorType::kResizeNearestNeighbor || - type == OperatorType::kPRelu || type == OperatorType::kReduceMax || + type == OperatorType::kPRelu || + type == OperatorType::kReduceMax || type == OperatorType::kReduceMin || type == OperatorType::kTransposeConv || type == OperatorType::kMatrixSetDiag || - type == OperatorType::kMatrixDiag || type == OperatorType::kHardSwish; + type == OperatorType::kMatrixDiag || + type == OperatorType::kSparseToDense || + type == OperatorType::kHardSwish; } // The quantized op allows output arrays of type float using From 45651397446fe127c7964f7137bb997982295500 Mon Sep 17 00:00:00 2001 From: frreiss Date: Mon, 17 Jun 2019 10:08:28 -0700 Subject: [PATCH 008/332] Replace winsock.h with winsock2.h --- tensorflow/core/util/tensor_bundle/byte_swap.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/util/tensor_bundle/byte_swap.h b/tensorflow/core/util/tensor_bundle/byte_swap.h index 948cd2e82bd..ec577c749b1 100644 --- a/tensorflow/core/util/tensor_bundle/byte_swap.h +++ b/tensorflow/core/util/tensor_bundle/byte_swap.h @@ -39,9 +39,9 @@ limitations under the License. #elif defined(PLATFORM_WINDOWS) -// On windows, byte-swapping is in winsock.h, and there is a version of htonl -// that can byte-swap 64-bit values -#include +// On windows, byte-swapping is in winsock.h, and winsock2.h has a version of +// of htonl that can byte-swap 64-bit values +#include #define BYTE_SWAP_16(x) htons (x) #define BYTE_SWAP_32(x) htonl (x) #define BYTE_SWAP_64(x) htonll (x) From bd3d417082083ce3e57a2119e58646214a2cb210 Mon Sep 17 00:00:00 2001 From: frreiss Date: Mon, 17 Jun 2019 13:53:27 -0700 Subject: [PATCH 009/332] Fall back on htonl() on windows due to htonll being disabled in default build config --- tensorflow/core/util/tensor_bundle/byte_swap.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/util/tensor_bundle/byte_swap.h b/tensorflow/core/util/tensor_bundle/byte_swap.h index ec577c749b1..b5987b129a8 100644 --- a/tensorflow/core/util/tensor_bundle/byte_swap.h +++ b/tensorflow/core/util/tensor_bundle/byte_swap.h @@ -40,11 +40,17 @@ limitations under the License. #elif defined(PLATFORM_WINDOWS) // On windows, byte-swapping is in winsock.h, and winsock2.h has a version of -// of htonl that can byte-swap 64-bit values +// of htonl that can byte-swap 64-bit values. #include #define BYTE_SWAP_16(x) htons (x) #define BYTE_SWAP_32(x) htonl (x) -#define BYTE_SWAP_64(x) htonll (x) +// At the moment the 64-bit and 128-bit byte-swapping routines in Winsock2 are +// disabled in TensorFlow's standard Windows build environment, so we use +// htonl() instead of "#define BYTE_SWAP_64(x) htonll (x)". +#define BYTE_SWAP_64(x) ( \ + (uint64_t(htonl((x) & 0x00000000ffffffffUL)) << 32) \ + | (htonl(((x) & 0xffffffff00000000UL) >> 32)) \ +) #elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ From 4285d4956f350ce8eefa5841d8efdce839f46072 Mon Sep 17 00:00:00 2001 From: Vishnuvardhan Janapati <46058173+jvishnuvardhan@users.noreply.github.com> Date: Tue, 18 Jun 2019 15:15:51 -0700 Subject: [PATCH 010/332] Correction in keras.layers.Reshape Here is the GitHub [gist](https://colab.sandbox.google.com/gist/jvishnuvardhan/0609804e9d2f18453b6eae1dba1d8284/reshape_error.ipynb). I tried TF1.13.1 also and has same output. Thanks --- tensorflow/python/keras/layers/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/keras/layers/core.py b/tensorflow/python/keras/layers/core.py index a57c4ab59dc..e9a5d0eaf69 100644 --- a/tensorflow/python/keras/layers/core.py +++ b/tensorflow/python/keras/layers/core.py @@ -400,7 +400,7 @@ class Reshape(Layer): # also supports shape inference using `-1` as dimension model.add(Reshape((-1, 2, 2))) - # now: model.output_shape == (None, 3, 2, 2) + # now: model.output_shape == (None, None, 2, 2) ``` """ From c6d57e6d055a5eaba52f4843f1d0c304c4d8f46c Mon Sep 17 00:00:00 2001 From: frreiss Date: Thu, 20 Jun 2019 17:14:31 -0700 Subject: [PATCH 011/332] Fix swapping of bfloat16 and restore testing code lost in merge --- .../core/util/tensor_bundle/byte_swap.cc | 2 +- .../util/tensor_bundle/tensor_bundle_test.cc | 100 +++++++++++------- 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/tensorflow/core/util/tensor_bundle/byte_swap.cc b/tensorflow/core/util/tensor_bundle/byte_swap.cc index f963e527d00..8d4c9c5531d 100644 --- a/tensorflow/core/util/tensor_bundle/byte_swap.cc +++ b/tensorflow/core/util/tensor_bundle/byte_swap.cc @@ -56,12 +56,12 @@ Status ByteSwapTensor(Tensor* t) { case DT_QINT8: case DT_QUINT8: case DT_BOOL: - case DT_BFLOAT16: case DT_UINT8: case DT_INT8: return Status::OK(); // 16-bit types + case DT_BFLOAT16: case DT_HALF: case DT_QINT16: case DT_QUINT16: diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc index b05e0a3bead..310a71ba321 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc @@ -328,7 +328,7 @@ TEST(TensorBundleTest, SwapBytes) { // 16-bit types TestByteSwap(forward_16, swapped_16, arr_len_16); TestByteSwap((int16_t*)forward_16, (int16_t*)swapped_16, arr_len_16); - // TODO(frreiss): Test half-precision float + TestByteSwap((bfloat16*)forward_16, (bfloat16*)swapped_16, arr_len_16); // 32-bit types TestByteSwap(forward_32, swapped_32, arr_len_32); @@ -377,10 +377,10 @@ void TestEndianness() { { // Write out a TensorBundle in the opposite of this host's endianness. BundleWriter writer(Env::Default(), Prefix("foo")); - TF_EXPECT_OK(writer.Add("foo_003", ByteSwap(Constant_2x3(3)))); - TF_EXPECT_OK(writer.Add("foo_000", ByteSwap(Constant_2x3(0)))); - TF_EXPECT_OK(writer.Add("foo_002", ByteSwap(Constant_2x3(2)))); - TF_EXPECT_OK(writer.Add("foo_001", ByteSwap(Constant_2x3(1)))); + TF_EXPECT_OK(writer.Add("foo_003", ByteSwap(Constant_2x3(T(3))))); + TF_EXPECT_OK(writer.Add("foo_000", ByteSwap(Constant_2x3(T(0))))); + TF_EXPECT_OK(writer.Add("foo_002", ByteSwap(Constant_2x3(T(2))))); + TF_EXPECT_OK(writer.Add("foo_001", ByteSwap(Constant_2x3(T(1))))); TF_ASSERT_OK(writer.Finish()); TF_ASSERT_OK(FlipEndiannessBit(Prefix("foo"))); } @@ -390,28 +390,28 @@ void TestEndianness() { EXPECT_EQ( AllTensorKeys(&reader), std::vector({"foo_000", "foo_001", "foo_002", "foo_003"})); - Expect(&reader, "foo_000", Constant_2x3(0)); - Expect(&reader, "foo_001", Constant_2x3(1)); - Expect(&reader, "foo_002", Constant_2x3(2)); - Expect(&reader, "foo_003", Constant_2x3(3)); + Expect(&reader, "foo_000", Constant_2x3(T(0))); + Expect(&reader, "foo_001", Constant_2x3(T(1))); + Expect(&reader, "foo_002", Constant_2x3(T(2))); + Expect(&reader, "foo_003", Constant_2x3(T(3))); } { BundleReader reader(Env::Default(), Prefix("foo")); TF_ASSERT_OK(reader.status()); - ExpectNext(&reader, Constant_2x3(0)); - ExpectNext(&reader, Constant_2x3(1)); - ExpectNext(&reader, Constant_2x3(2)); - ExpectNext(&reader, Constant_2x3(3)); + ExpectNext(&reader, Constant_2x3(T(0))); + ExpectNext(&reader, Constant_2x3(T(1))); + ExpectNext(&reader, Constant_2x3(T(2))); + ExpectNext(&reader, Constant_2x3(T(3))); EXPECT_TRUE(reader.Valid()); reader.Next(); EXPECT_FALSE(reader.Valid()); } { BundleWriter writer(Env::Default(), Prefix("bar")); - TF_EXPECT_OK(writer.Add("bar_003", ByteSwap(Constant_2x3(3)))); - TF_EXPECT_OK(writer.Add("bar_000", ByteSwap(Constant_2x3(0)))); - TF_EXPECT_OK(writer.Add("bar_002", ByteSwap(Constant_2x3(2)))); - TF_EXPECT_OK(writer.Add("bar_001", ByteSwap(Constant_2x3(1)))); + TF_EXPECT_OK(writer.Add("bar_003", ByteSwap(Constant_2x3(T(3))))); + TF_EXPECT_OK(writer.Add("bar_000", ByteSwap(Constant_2x3(T(0))))); + TF_EXPECT_OK(writer.Add("bar_002", ByteSwap(Constant_2x3(T(2))))); + TF_EXPECT_OK(writer.Add("bar_001", ByteSwap(Constant_2x3(T(1))))); TF_ASSERT_OK(writer.Finish()); TF_ASSERT_OK(FlipEndiannessBit(Prefix("bar"))); } @@ -421,18 +421,18 @@ void TestEndianness() { EXPECT_EQ( AllTensorKeys(&reader), std::vector({"bar_000", "bar_001", "bar_002", "bar_003"})); - Expect(&reader, "bar_003", Constant_2x3(3)); - Expect(&reader, "bar_002", Constant_2x3(2)); - Expect(&reader, "bar_001", Constant_2x3(1)); - Expect(&reader, "bar_000", Constant_2x3(0)); + Expect(&reader, "bar_003", Constant_2x3(T(3))); + Expect(&reader, "bar_002", Constant_2x3(T(2))); + Expect(&reader, "bar_001", Constant_2x3(T(1))); + Expect(&reader, "bar_000", Constant_2x3(T(0))); } { BundleReader reader(Env::Default(), Prefix("bar")); TF_ASSERT_OK(reader.status()); - ExpectNext(&reader, Constant_2x3(0)); - ExpectNext(&reader, Constant_2x3(1)); - ExpectNext(&reader, Constant_2x3(2)); - ExpectNext(&reader, Constant_2x3(3)); + ExpectNext(&reader, Constant_2x3(T(0))); + ExpectNext(&reader, Constant_2x3(T(1))); + ExpectNext(&reader, Constant_2x3(T(2))); + ExpectNext(&reader, Constant_2x3(T(3))); EXPECT_TRUE(reader.Valid()); reader.Next(); EXPECT_FALSE(reader.Valid()); @@ -446,32 +446,33 @@ void TestEndianness() { AllTensorKeys(&reader), std::vector({"bar_000", "bar_001", "bar_002", "bar_003", "foo_000", "foo_001", "foo_002", "foo_003"})); - Expect(&reader, "bar_000", Constant_2x3(0)); - Expect(&reader, "bar_001", Constant_2x3(1)); - Expect(&reader, "bar_002", Constant_2x3(2)); - Expect(&reader, "bar_003", Constant_2x3(3)); - Expect(&reader, "foo_000", Constant_2x3(0)); - Expect(&reader, "foo_001", Constant_2x3(1)); - Expect(&reader, "foo_002", Constant_2x3(2)); - Expect(&reader, "foo_003", Constant_2x3(3)); + Expect(&reader, "bar_000", Constant_2x3(T(0))); + Expect(&reader, "bar_001", Constant_2x3(T(1))); + Expect(&reader, "bar_002", Constant_2x3(T(2))); + Expect(&reader, "bar_003", Constant_2x3(T(3))); + Expect(&reader, "foo_000", Constant_2x3(T(0))); + Expect(&reader, "foo_001", Constant_2x3(T(1))); + Expect(&reader, "foo_002", Constant_2x3(T(2))); + Expect(&reader, "foo_003", Constant_2x3(T(3))); } { BundleReader reader(Env::Default(), Prefix("merged")); TF_ASSERT_OK(reader.status()); - ExpectNext(&reader, Constant_2x3(0)); - ExpectNext(&reader, Constant_2x3(1)); - ExpectNext(&reader, Constant_2x3(2)); - ExpectNext(&reader, Constant_2x3(3)); - ExpectNext(&reader, Constant_2x3(0)); - ExpectNext(&reader, Constant_2x3(1)); - ExpectNext(&reader, Constant_2x3(2)); - ExpectNext(&reader, Constant_2x3(3)); + ExpectNext(&reader, Constant_2x3(T(0))); + ExpectNext(&reader, Constant_2x3(T(1))); + ExpectNext(&reader, Constant_2x3(T(2))); + ExpectNext(&reader, Constant_2x3(T(3))); + ExpectNext(&reader, Constant_2x3(T(0))); + ExpectNext(&reader, Constant_2x3(T(1))); + ExpectNext(&reader, Constant_2x3(T(2))); + ExpectNext(&reader, Constant_2x3(T(3))); EXPECT_TRUE(reader.Valid()); reader.Next(); EXPECT_FALSE(reader.Valid()); } } + template void TestNonStandardShapes() { { @@ -534,6 +535,23 @@ TEST(TensorBundleTest, Basic) { TestBasic(); } +TEST(TensorBundleTest, Endianness) { + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); + TestEndianness(); +} + TEST(TensorBundleTest, PartitionedVariables) { const TensorShape kFullShape({5, 10}); // Adds two slices. From 70fbb2cc023cbe336eb9df1d5d89a12eaade2893 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:40:46 -0700 Subject: [PATCH 012/332] Removed not diff flag for RGB to HSV. --- tensorflow/python/ops/image_ops_impl.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index c6e11edb347..f89739c1f70 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -43,9 +43,6 @@ from tensorflow.python.util.tf_export import tf_export ops.NotDifferentiable('RandomCrop') # TODO(b/31222613): This op may be differentiable, and there may be # latent bugs here. -ops.NotDifferentiable('RGBToHSV') -# TODO(b/31222613): This op may be differentiable, and there may be -# latent bugs here. ops.NotDifferentiable('HSVToRGB') ops.NotDifferentiable('DrawBoundingBoxes') ops.NotDifferentiable('SampleDistortedBoundingBox') From 6520a50a1f088f5b2e461418c1b30976db1d2f32 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:42:36 -0700 Subject: [PATCH 013/332] Set flags for max and min layers. --- tensorflow/python/ops/image_grad.py | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 7d240dc6b63..d2c7db37e5e 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -153,3 +153,40 @@ def _CropAndResizeGrad(op, grad): grad, op.inputs[0], op.inputs[1], op.inputs[2]) return [grad0, grad1, None, None] + +@ops.RegisterGradient("RGBToHSV") +def _rgb_to_hsv_grad(op, grad): + """The gradients for `zero_out`. + + Args: + op: The `rgb_to_hsv` `Operation` that we are differentiating, which we can use + to find the inputs and outputs of the original op. + grad: Gradient with respect to the output of the `rgb_to_hsv` op. + + Returns: + Gradients with respect to the input of `rgb_to_hsv`. + """ + print("******** This is a test implementation ********** \n") + + # Input Channels + reds = op.inputs[0][..., 0] + greens = op.inputs[0][..., 1] + blues = op.inputs[0][..., 2] + # Output Channels + hue = op.outputs[0][..., 0] + saturation = op.outputs[0][..., 1] + value = op.outputs[0][..., 2] + + # Mask/Indicator for max and min values of each pixel. Arbitrary assignment in case + # of tie breakers with R>G>B. + # Max values + red_biggest = cast((reds >= blues) & (reds >= greens), dtypes.float32) + green_biggest = cast((greens > reds) & (greens >= blues), dtypes.float32) + blue_biggest = cast((blues > reds) & (blues > greens), dtypes.float32) + # Min values + red_smallest = cast((reds < blues) & (reds < greens), dtypes.float32) + green_smallest = cast((greens <= reds) & (greens < blues), dtypes.float32) + blue_smallest = cast((blues <= reds) & (blues <= greens), dtypes.float32) + + + return None From b01034f62fddbcefd040ac05b92765282f55db3b Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:43:08 -0700 Subject: [PATCH 014/332] Derivative of rgb wrt value. --- tensorflow/python/ops/image_grad.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index d2c7db37e5e..45f45cb4d89 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -188,5 +188,12 @@ def _rgb_to_hsv_grad(op, grad): green_smallest = cast((greens <= reds) & (greens < blues), dtypes.float32) blue_smallest = cast((blues <= reds) & (blues <= greens), dtypes.float32) + ############################################################## + # Derivatives of R, G, B wrt Value slice + ############################################################## + dv_dr = red_biggest + dv_dg = green_biggest + dv_db = blue_biggest + return None From 3570dd4cf2eb6b5c485686380c1a351c055c4c01 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:43:34 -0700 Subject: [PATCH 015/332] derivative wrt saturation. --- tensorflow/python/ops/image_grad.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 45f45cb4d89..9f3406e0191 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -194,6 +194,21 @@ def _rgb_to_hsv_grad(op, grad): dv_dr = red_biggest dv_dg = green_biggest dv_db = blue_biggest + ############################################################## + # Derivatives of R, G, B wrt Saturation slice + ############################################################## + # The first term in the addition is the case when the corresponding color from (r,g,b) was "MAX" + # -> derivative = MIN/square(MAX), MIN could be one of the other two colors + # The second term is the case when the corresponding color from (r,g,b) was "MIN" + # -> derivative = -1/MAX, MAX could be one of the other two colours. + # Defining a custom replacement for machine epsilon (eps) to avoid NaNs or divide by zeros + my_eps = 0.000000001 + ds_dr = cast(reds > 0, dtypes.float32) * add(red_biggest * divide(add(green_smallest * greens, blue_smallest * blues), square(reds + my_eps)), + red_smallest * -reciprocal(add(green_biggest * greens, blue_biggest * blues + my_eps))) + ds_dg = cast(greens > 0, dtypes.float32) * add(green_biggest * divide(add(red_smallest * reds, blue_smallest * blues), square(greens + my_eps)), + green_smallest * -reciprocal(add(red_biggest * reds, blue_biggest * blues + my_eps))) + ds_db = cast(blues > 0, dtypes.float32) * add(blue_biggest * divide(add(green_smallest * greens, red_smallest * reds), square(blues + my_eps)), + blue_smallest * -reciprocal(add(green_biggest * greens, red_biggest * reds + my_eps))) return None From 66e4d05467db18784e5e413ec74a970a4dcba093 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:44:03 -0700 Subject: [PATCH 016/332] dh_dr --- tensorflow/python/ops/image_grad.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 9f3406e0191..843804ce009 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -209,6 +209,31 @@ def _rgb_to_hsv_grad(op, grad): green_smallest * -reciprocal(add(red_biggest * reds, blue_biggest * blues + my_eps))) ds_db = cast(blues > 0, dtypes.float32) * add(blue_biggest * divide(add(green_smallest * greens, red_smallest * reds), square(blues + my_eps)), blue_smallest * -reciprocal(add(green_biggest * greens, red_biggest * reds + my_eps))) + ############################################################## + # Derivatives of R, G, B wrt Hue slice + ############################################################## + # Need to go case by case for each color. + + # for red, dh_dr -> dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 + # dh_dr_1 -> + # if red was MAX, then derivative = 60 * -1 * (G-B)/square(MAX-MIN) == 60 * -1 * (greens-blues) * reciprocal(square(saturation)) * reciprical(square(value)) + # elif green was MAX, there are two subcases ie when red was MIN and when red was NOT MIN + # dh_dr_2 -> + # if red was MIN (use UV rule) -> 60 * ((1 * -1/(MAX-MIN)) + (B-R)*(-1/square(MAX-MIN) * -1)) == 60 * (blues - greens) * reciprocal(square(reds - greens)) + # dh_dr_3 -> + # if red was NOT MIN -> 60 * -1/MAX-MIN == -60 * reciprocal(greens-blues) + # elif blue was MAX, there are two subcases + # dh_dr_4 -> + # if red was MIN (similarly use the UV rule) -> 60 * (blues - greens) * reciprocal(square(blues - reds)) + # dh_dr_5 -> + # if red was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(blues-greens) + dh_dr_1 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * -1 * (greens - blues) * _custom_reciprocal(square(saturation)) * _custom_reciprocal(square(value))) + dh_dr_2 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * red_smallest * (blues - greens) * _custom_reciprocal(square(reds - greens))) + dh_dr_3 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * blue_smallest * -1 * _custom_reciprocal(greens - blues)) + dh_dr_4 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * red_smallest * (blues - greens) * _custom_reciprocal(square(blues - reds))) + dh_dr_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * green_smallest * _custom_reciprocal(blues - greens)) + + dh_dr = dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 return None From ea037f6633199ac1a7823e8bda630f4614a39cc9 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:44:25 -0700 Subject: [PATCH 017/332] dh_dg --- tensorflow/python/ops/image_grad.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 843804ce009..369c9eed8e0 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -235,5 +235,26 @@ def _rgb_to_hsv_grad(op, grad): dh_dr = dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 + # for green, dh_dg -> dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 + # dh_dg_1 -> + # if green was MAX, then derivative = 60 * -1 * (B-R)/square(MAX-MIN) == 60 * -1 * (blues - reds) * reciprocal(square(saturation)) * reciprocal(square(value)) + # elif red was MAX, there are two subcases ie when green was MIN and when green was NOT MIN + # dh_dg_2 -> + # if green was MIN (use UV rule) -> 60 * ((1 * 1/(MAX-MIN)) + (greens-blues)*(-1/square(MAX-MIN) * -1)) == 60 * ((reciprocal(reds-greens) + (greens-blues)*reciprocal(square(reds-greens)))) + # dh_dg_3 -> + # if green was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(reds - blues) + # elif blue was MAX, there are two subcases + # dh_dg_4 -> + # if green was MIN (similarly use the UV rule) -> 60 * -1 * (reciprocal(blues - greens) + (reds-greens)* -1 * reciprocal(square(blues-greens))) + # dh_dr_5 -> + # if green was NOT MIN -> 60 * -1/MAX-MIN == 60 * -1 * reciprocal(blues - reds) + dh_dg_1 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * -1 * (blues - reds) * _custom_reciprocal(square(saturation)) * _custom_reciprocal(square(value))) + dh_dg_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * green_smallest * (reds - blues) * _custom_reciprocal(square(reds - greens))) + dh_dg_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * blue_smallest * _custom_reciprocal(reds - blues)) + dh_dg_4 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * green_smallest * (reds - blues) * _custom_reciprocal(square(blues - greens))) + dh_dg_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * red_smallest * -1 * _custom_reciprocal(blues - reds)) + + dh_dg = dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 + return None From 7188d116e2661522dbf6f1bd9644a45ecd636702 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:44:47 -0700 Subject: [PATCH 018/332] dh_db --- tensorflow/python/ops/image_grad.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 369c9eed8e0..f017b4f0d7b 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -256,5 +256,26 @@ def _rgb_to_hsv_grad(op, grad): dh_dg = dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 + # for blue, dh_db -> dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 + # dh_db_1 -> + # if blue was MAX, then derivative = 60 * -1 * (R-G)/square(MAX-MIN) == 60 * -1 * reciprocal(square(saturation)) * reciprocal(square(value)) + # elif red was MAX, there are two subcases ie when blue was MIN and when blue was NOT MIN + # dh_dg_2 -> + # if blue was MIN (use UV rule) -> 60 * ((1 * -1/(MAX-MIN)) + (greens-blues)*(-1/square(MAX-MIN) * -1)) == 60 * (greens - reds) * reciprocal(square(reds - blues)) + # dh_dg_3 -> + # if blue was NOT MIN -> 60 * -1/MAX-MIN == 60 * -1 * reciprocal(reds - greens) + # elif green was MAX, there are two subcases + # dh_dg_4 -> + # if blue was MIN (similarly use the UV rule) -> 60 * -1 * (reciprocal(greens - blues) + (blues - reds)* -1 * reciprocal(square(greens - blues))) + # dh_dr_5 -> + # if blue was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(greens - reds) + dh_db_1 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * -1 * (reds - greens) * _custom_reciprocal(square(saturation)) * _custom_reciprocal(square(value))) + dh_db_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * blue_smallest * (greens - reds) * _custom_reciprocal(square(reds - blues))) + dh_db_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * green_smallest * -1 * _custom_reciprocal(reds - greens)) + dh_db_4 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * blue_smallest * (greens - reds) * _custom_reciprocal(square(greebs - blues))) + dh_db_5 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * red_smallest * _custom_reciprocal(greens - reds)) + + dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 + return None From 260dfce3828e1b8555e9978588011053861287d1 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:45:19 -0700 Subject: [PATCH 019/332] adding all gradients. --- tensorflow/python/ops/image_grad.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index f017b4f0d7b..cd9f94bfdd7 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -276,6 +276,11 @@ def _rgb_to_hsv_grad(op, grad): dh_db_5 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * red_smallest * _custom_reciprocal(greens - reds)) dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 + # Gradients wrt to inputs + dv_drgb = stack([grad[...,2] * dv_dr, grad[...,2] * dv_dg, grad[...,2] * dv_db], axis=-1) + ds_drgb = stack([grad[...,1] * ds_dr, grad[...,1] * ds_dg, grad[...,1] * ds_db], axis=-1) + dh_drgb = stack([grad[...,0] * dh_dr, grad[...,0] * dh_dg, grad[...,0] * dh_db], axis=-1) + gradient_input = add(add(dv_drgb, ds_drgb), dh_drgb) - return None + return gradient_input \ No newline at end of file From f002c39fbfbcf13bc272acfb8ad4a2c43c26137a Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 21 Jun 2019 15:46:25 -0700 Subject: [PATCH 020/332] Added custom reciprocal function. --- tensorflow/python/ops/image_grad.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index cd9f94bfdd7..0040b0bc46d 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -154,6 +154,20 @@ def _CropAndResizeGrad(op, grad): return [grad0, grad1, None, None] + +def _custom_reciprocal(x, my_eps=1e-10): + """ + Performs reciprocal with an EPS added to the input. This is to avoid inversion errors + or divide by zeros or NaNs. + Inputs: + x -> input tensor to be reciprocat-ed + my_eps -> custom machine precision epsilon + Returns: + x_reciprocal -> reciprocal of x added with my_eps + """ + return reciprocal(x + my_eps) + + @ops.RegisterGradient("RGBToHSV") def _rgb_to_hsv_grad(op, grad): """The gradients for `zero_out`. @@ -234,7 +248,7 @@ def _rgb_to_hsv_grad(op, grad): dh_dr_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * green_smallest * _custom_reciprocal(blues - greens)) dh_dr = dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 - + # for green, dh_dg -> dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 # dh_dg_1 -> # if green was MAX, then derivative = 60 * -1 * (B-R)/square(MAX-MIN) == 60 * -1 * (blues - reds) * reciprocal(square(saturation)) * reciprocal(square(value)) @@ -276,6 +290,7 @@ def _rgb_to_hsv_grad(op, grad): dh_db_5 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * red_smallest * _custom_reciprocal(greens - reds)) dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 + # Gradients wrt to inputs dv_drgb = stack([grad[...,2] * dv_dr, grad[...,2] * dv_dg, grad[...,2] * dv_db], axis=-1) ds_drgb = stack([grad[...,1] * ds_dr, grad[...,1] * ds_dg, grad[...,1] * ds_db], axis=-1) From 0bacbc090d50cace55c4bd8757f24f87fa2e360b Mon Sep 17 00:00:00 2001 From: frreiss Date: Fri, 21 Jun 2019 17:23:02 -0700 Subject: [PATCH 021/332] Run build file through buildifier again --- tensorflow/core/util/tensor_bundle/BUILD | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/util/tensor_bundle/BUILD b/tensorflow/core/util/tensor_bundle/BUILD index ef767469c6c..9f3d45c23d8 100644 --- a/tensorflow/core/util/tensor_bundle/BUILD +++ b/tensorflow/core/util/tensor_bundle/BUILD @@ -18,11 +18,11 @@ load( filegroup( name = "mobile_srcs", srcs = [ + "byte_swap.h", "naming.cc", "naming.h", "tensor_bundle.cc", "tensor_bundle.h", - "byte_swap.h" ], ) @@ -33,8 +33,14 @@ alias( cc_library( name = "tensor_bundle", - srcs = ["tensor_bundle.cc", "byte_swap.cc"], - hdrs = ["tensor_bundle.h", "byte_swap.h"], + srcs = [ + "byte_swap.cc", + "tensor_bundle.cc", + ], + hdrs = [ + "byte_swap.h", + "tensor_bundle.h", + ], copts = tf_copts() + if_not_windows(["-Wno-sign-compare"]), deps = [ ":naming", From e1a9841e06a0c5b01ad481d25c883136df13eeb6 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Tue, 18 Jun 2019 19:10:04 +0000 Subject: [PATCH 022/332] Adding ROCm support for resource_variable_ops --- .../core/kernels/resource_variable_ops.cc | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index 5948db34c81..967d4a4734e 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -47,7 +47,7 @@ limitations under the License. #define EIGEN_USE_THREADS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU #endif @@ -260,7 +260,7 @@ void VarHandleOp::Compute(OpKernelContext* ctx) { REGISTER_KERNEL_BUILDER(Name("VarHandleOp").Device(DEVICE_CPU), VarHandleOp); -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER_KERNEL_BUILDER( Name("ReadVariableOp").Device(DEVICE_GPU).HostMemory("resource"), ReadVariableOp); @@ -295,7 +295,7 @@ REGISTER_KERNEL_BUILDER(Name("_VarHandlesOp") DT_DOUBLE, DT_BOOL, DT_VARIANT}), ResourceHandlesOp); -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM template class VariableShapeOp : public OpKernel { @@ -324,7 +324,7 @@ REGISTER_KERNEL_BUILDER( Name("VariableShape").Device(DEVICE_CPU).TypeConstraint("out_type"), VariableShapeOp); -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER_KERNEL_BUILDER(Name("VariableShape") .Device(DEVICE_GPU) @@ -339,7 +339,7 @@ REGISTER_KERNEL_BUILDER(Name("VariableShape") .HostMemory("input"), VariableShapeOp); -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM DestroyResourceOp::DestroyResourceOp(OpKernelConstruction* ctx) : OpKernel(ctx) { @@ -515,7 +515,7 @@ TF_CALL_ALL_TYPES(REGISTER_KERNELS); TF_CALL_QUANTIZED_TYPES(REGISTER_KERNELS); #undef REGISTER_KERNELS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GPU_KERNELS(type) \ REGISTER_KERNEL_BUILDER(Name("AssignVariableOp") \ .Device(DEVICE_GPU) \ @@ -527,7 +527,7 @@ TF_CALL_GPU_ALL_TYPES(REGISTER_GPU_KERNELS); TF_CALL_int64(REGISTER_GPU_KERNELS); TF_CALL_variant(REGISTER_GPU_KERNELS); #undef REGISTER_GPU_KERNELS -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM template class AssignUpdateVariableOp : public OpKernel { @@ -575,7 +575,7 @@ class AssignUpdateVariableOp : public OpKernel { TF_CALL_NUMBER_TYPES(REGISTER_KERNELS); #undef REGISTER_KERNELS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GPU_KERNELS(type) \ REGISTER_KERNEL_BUILDER(Name("AssignAddVariableOp") \ .Device(DEVICE_GPU) \ @@ -591,7 +591,7 @@ TF_CALL_NUMBER_TYPES(REGISTER_KERNELS); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_int64(REGISTER_GPU_KERNELS); #undef REGISTER_GPU_KERNELS -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM class VarIsInitializedOp : public OpKernel { public: @@ -616,13 +616,13 @@ class VarIsInitializedOp : public OpKernel { REGISTER_KERNEL_BUILDER(Name("VarIsInitializedOp").Device(DEVICE_CPU), VarIsInitializedOp); -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER_KERNEL_BUILDER(Name("VarIsInitializedOp") .Device(DEVICE_GPU) .HostMemory("resource") .HostMemory("is_initialized"), IsResourceInitialized); -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM template class ResourceGatherOp : public OpKernel { @@ -762,7 +762,7 @@ TF_CALL_ALL_TYPES(REGISTER_GATHER_CPU); TF_CALL_QUANTIZED_TYPES(REGISTER_GATHER_CPU); // Registers GPU kernels. -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GATHER_GPU(type) REGISTER_GATHER_ALL_INDICES(GPU, type) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GATHER_GPU); @@ -784,7 +784,7 @@ REGISTER_KERNEL_BUILDER(Name("ResourceGather") .TypeConstraint("Tindices"), ResourceGatherOp) -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #undef REGISTER_GATHER_CPU #undef REGISTER_GATHER_GPU @@ -834,12 +834,12 @@ class ResourceGatherNdOp : public OpKernel { TF_CALL_ALL_TYPES(REGISTER_GATHER_ND_CPU); // Registers GPU kernels. -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GATHER_ND_GPU(type) REGISTER_GATHER_ND_ALL_INDICES(GPU, type) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GATHER_ND_GPU); -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #undef REGISTER_GATHER_ND_CPU #undef REGISTER_GATHER_ND_GPU @@ -958,7 +958,7 @@ REGISTER_SCATTER_KERNEL(Variant, CPU, "ResourceScatterUpdate", scatter_op::UpdateOp::ASSIGN); // Registers GPU kernels. -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_SCATTER_ARITHMETIC_GPU(type) \ REGISTER_SCATTER_ARITHMETIC(type, GPU); #define REGISTER_SCATTER_MINMAX_GPU(type) REGISTER_SCATTER_MINMAX(type, GPU); @@ -992,7 +992,7 @@ REGISTER_KERNEL_BUILDER(Name("ResourceScatterUpdate") ResourceScatterUpdateOp) -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #undef REGISTER_SCATTER_ARITHMETIC #undef REGISTER_SCATTER_ARITHMETIC_CPU From 4d9f734575e09f2f7440d32cab570f11184f013e Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Mon, 24 Jun 2019 11:50:13 -0700 Subject: [PATCH 023/332] Fixed typo. --- tensorflow/python/ops/image_grad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 0040b0bc46d..af4d63e17e6 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -286,7 +286,7 @@ def _rgb_to_hsv_grad(op, grad): dh_db_1 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * -1 * (reds - greens) * _custom_reciprocal(square(saturation)) * _custom_reciprocal(square(value))) dh_db_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * blue_smallest * (greens - reds) * _custom_reciprocal(square(reds - blues))) dh_db_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * green_smallest * -1 * _custom_reciprocal(reds - greens)) - dh_db_4 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * blue_smallest * (greens - reds) * _custom_reciprocal(square(greebs - blues))) + dh_db_4 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * blue_smallest * (greens - reds) * _custom_reciprocal(square(greens - blues))) dh_db_5 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * red_smallest * _custom_reciprocal(greens - reds)) dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 From 2617f7f8c7a81019da3da8d7088f199a4b8f9029 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Mon, 24 Jun 2019 14:45:47 -0700 Subject: [PATCH 024/332] pylint fixes. --- tensorflow/python/ops/image_grad.py | 170 +++++++++++++++++++--------- 1 file changed, 115 insertions(+), 55 deletions(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index af4d63e17e6..f85f26d114d 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -155,10 +155,10 @@ def _CropAndResizeGrad(op, grad): return [grad0, grad1, None, None] -def _custom_reciprocal(x, my_eps=1e-10): +def _CustomReciprocal(x, my_eps=1e-10): """ - Performs reciprocal with an EPS added to the input. This is to avoid inversion errors - or divide by zeros or NaNs. + Performs reciprocal with an eps added to the input. This is to avoid + inversion errors or divide by zeros or NaNs. Inputs: x -> input tensor to be reciprocat-ed my_eps -> custom machine precision epsilon @@ -169,11 +169,12 @@ def _custom_reciprocal(x, my_eps=1e-10): @ops.RegisterGradient("RGBToHSV") -def _rgb_to_hsv_grad(op, grad): +def _RGBToHSVGrad(op, grad): """The gradients for `zero_out`. Args: - op: The `rgb_to_hsv` `Operation` that we are differentiating, which we can use + op: The `rgb_to_hsv` `Operation` that we are differentiating, + which we can use to find the inputs and outputs of the original op. grad: Gradient with respect to the output of the `rgb_to_hsv` op. @@ -181,7 +182,6 @@ def _rgb_to_hsv_grad(op, grad): Gradients with respect to the input of `rgb_to_hsv`. """ print("******** This is a test implementation ********** \n") - # Input Channels reds = op.inputs[0][..., 0] greens = op.inputs[0][..., 1] @@ -191,8 +191,8 @@ def _rgb_to_hsv_grad(op, grad): saturation = op.outputs[0][..., 1] value = op.outputs[0][..., 2] - # Mask/Indicator for max and min values of each pixel. Arbitrary assignment in case - # of tie breakers with R>G>B. + # Mask/Indicator for max and min values of each pixel. + # Arbitrary assignment in case of tie breakers with R>G>B. # Max values red_biggest = cast((reds >= blues) & (reds >= greens), dtypes.float32) green_biggest = cast((greens > reds) & (greens >= blues), dtypes.float32) @@ -211,18 +211,27 @@ def _rgb_to_hsv_grad(op, grad): ############################################################## # Derivatives of R, G, B wrt Saturation slice ############################################################## - # The first term in the addition is the case when the corresponding color from (r,g,b) was "MAX" + # The first term in the addition is the case when the corresponding color + # from (r,g,b) was "MAX" # -> derivative = MIN/square(MAX), MIN could be one of the other two colors - # The second term is the case when the corresponding color from (r,g,b) was "MIN" + # The second term is the case when the corresponding color from + # (r,g,b) was "MIN" # -> derivative = -1/MAX, MAX could be one of the other two colours. - # Defining a custom replacement for machine epsilon (eps) to avoid NaNs or divide by zeros - my_eps = 0.000000001 - ds_dr = cast(reds > 0, dtypes.float32) * add(red_biggest * divide(add(green_smallest * greens, blue_smallest * blues), square(reds + my_eps)), - red_smallest * -reciprocal(add(green_biggest * greens, blue_biggest * blues + my_eps))) - ds_dg = cast(greens > 0, dtypes.float32) * add(green_biggest * divide(add(red_smallest * reds, blue_smallest * blues), square(greens + my_eps)), - green_smallest * -reciprocal(add(red_biggest * reds, blue_biggest * blues + my_eps))) - ds_db = cast(blues > 0, dtypes.float32) * add(blue_biggest * divide(add(green_smallest * greens, red_smallest * reds), square(blues + my_eps)), - blue_smallest * -reciprocal(add(green_biggest * greens, red_biggest * reds + my_eps))) + ds_dr = cast(reds > 0, dtypes.float32) * add(red_biggest * \ + add(green_smallest * greens, blue_smallest * blues) * \ + _CustomReciprocal(square(reds)),\ + red_smallest * -1 * _CustomReciprocal((green_biggest * \ + greens) + (blue_biggest * blues))) + ds_dg = cast(greens > 0, dtypes.float32) * add(green_biggest * \ + add(red_smallest * reds, blue_smallest * blues) * \ + _CustomReciprocal(square(greens)),\ + green_smallest * -1 * _CustomReciprocal((red_biggest * \ + reds) + (blue_biggest * blues))) + ds_db = cast(blues > 0, dtypes.float32) * add(blue_biggest * \ + add(green_smallest * greens, red_smallest * reds) * \ + _CustomReciprocal(square(blues)),\ + blue_smallest * -1 * _CustomReciprocal((green_biggest * \ + greens) + (red_biggest * reds))) ############################################################## # Derivatives of R, G, B wrt Hue slice ############################################################## @@ -230,72 +239,123 @@ def _rgb_to_hsv_grad(op, grad): # for red, dh_dr -> dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 # dh_dr_1 -> - # if red was MAX, then derivative = 60 * -1 * (G-B)/square(MAX-MIN) == 60 * -1 * (greens-blues) * reciprocal(square(saturation)) * reciprical(square(value)) - # elif green was MAX, there are two subcases ie when red was MIN and when red was NOT MIN + # if red was MAX, then derivative = 60 * -1 * (G-B)/square(MAX-MIN) == 60 *\ + # -1 * (greens-blues) * reciprocal(square(saturation)) * \ + # reciprical(square(value)) + # elif green was MAX, there are two subcases + # ie when red was MIN and when red was NOT MIN # dh_dr_2 -> - # if red was MIN (use UV rule) -> 60 * ((1 * -1/(MAX-MIN)) + (B-R)*(-1/square(MAX-MIN) * -1)) == 60 * (blues - greens) * reciprocal(square(reds - greens)) + # if red was MIN (use UV rule) -> 60 * ((1 * -1/(MAX-MIN)) +\ + # (B-R)*(-1/square(MAX-MIN) * -1)) == 60 * (blues - greens) *\ + # reciprocal(square(reds - greens)) # dh_dr_3 -> # if red was NOT MIN -> 60 * -1/MAX-MIN == -60 * reciprocal(greens-blues) # elif blue was MAX, there are two subcases # dh_dr_4 -> - # if red was MIN (similarly use the UV rule) -> 60 * (blues - greens) * reciprocal(square(blues - reds)) - # dh_dr_5 -> + # if red was MIN (similarly use the UV rule) -> 60 * (blues - greens) *\ + # reciprocal(square(blues - reds)) + # dh_dr_5 -> # if red was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(blues-greens) - dh_dr_1 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * -1 * (greens - blues) * _custom_reciprocal(square(saturation)) * _custom_reciprocal(square(value))) - dh_dr_2 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * red_smallest * (blues - greens) * _custom_reciprocal(square(reds - greens))) - dh_dr_3 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * blue_smallest * -1 * _custom_reciprocal(greens - blues)) - dh_dr_4 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * red_smallest * (blues - greens) * _custom_reciprocal(square(blues - reds))) - dh_dr_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * green_smallest * _custom_reciprocal(blues - greens)) + dh_dr_1 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * -1 * \ + (greens - blues) * _CustomReciprocal(square(saturation)) *\ + _CustomReciprocal(square(value))) + dh_dr_2 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + red_smallest * (blues - greens) * \ + _CustomReciprocal(square(reds - greens))) + dh_dr_3 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + blue_smallest * -1 * _CustomReciprocal(greens - blues)) + dh_dr_4 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * \ + red_smallest * (blues - greens) * \ + _CustomReciprocal(square(blues - reds))) + dh_dr_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * \ + green_smallest * _CustomReciprocal(blues - greens)) dh_dr = dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 - + # for green, dh_dg -> dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 # dh_dg_1 -> - # if green was MAX, then derivative = 60 * -1 * (B-R)/square(MAX-MIN) == 60 * -1 * (blues - reds) * reciprocal(square(saturation)) * reciprocal(square(value)) - # elif red was MAX, there are two subcases ie when green was MIN and when green was NOT MIN + # if green was MAX, then derivative = 60 * -1 * (B-R)/square(MAX-MIN) == 60 *\ + # -1 * (blues - reds) * reciprocal(square(saturation)) * \ + # reciprocal(square(value)) + # elif red was MAX, there are two subcases ie + # when green was MIN and when green was NOT MIN # dh_dg_2 -> - # if green was MIN (use UV rule) -> 60 * ((1 * 1/(MAX-MIN)) + (greens-blues)*(-1/square(MAX-MIN) * -1)) == 60 * ((reciprocal(reds-greens) + (greens-blues)*reciprocal(square(reds-greens)))) + # if green was MIN (use UV rule) -> 60 * ((1 * 1/(MAX-MIN)) + \ + # (greens-blues) * (-1/square(MAX-MIN) * -1)) == 60 * \ + # ((reciprocal(reds-greens) + (greens-blues) * \ + # reciprocal(square(reds-greens)))) # dh_dg_3 -> # if green was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(reds - blues) # elif blue was MAX, there are two subcases # dh_dg_4 -> - # if green was MIN (similarly use the UV rule) -> 60 * -1 * (reciprocal(blues - greens) + (reds-greens)* -1 * reciprocal(square(blues-greens))) - # dh_dr_5 -> - # if green was NOT MIN -> 60 * -1/MAX-MIN == 60 * -1 * reciprocal(blues - reds) - dh_dg_1 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * -1 * (blues - reds) * _custom_reciprocal(square(saturation)) * _custom_reciprocal(square(value))) - dh_dg_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * green_smallest * (reds - blues) * _custom_reciprocal(square(reds - greens))) - dh_dg_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * blue_smallest * _custom_reciprocal(reds - blues)) - dh_dg_4 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * green_smallest * (reds - blues) * _custom_reciprocal(square(blues - greens))) - dh_dg_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * red_smallest * -1 * _custom_reciprocal(blues - reds)) + # if green was MIN (similarly use the UV rule) -> 60 * -1 * \ + # (reciprocal(blues - greens) + (reds-greens)* -1 * \ + # reciprocal(square(blues-greens))) + # dh_dr_5 -> + # if green was NOT MIN -> 60 * -1/MAX-MIN == -60 * reciprocal(blues - reds) + dh_dg_1 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + -1 * (blues - reds) * _CustomReciprocal(square(saturation))\ + * _CustomReciprocal(square(value))) + dh_dg_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * \ + green_smallest * (reds - blues) * \ + _CustomReciprocal(square(reds - greens))) + dh_dg_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * \ + blue_smallest * _CustomReciprocal(reds - blues)) + dh_dg_4 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * \ + green_smallest * (reds - blues) * \ + _CustomReciprocal(square(blues - greens))) + dh_dg_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * \ + red_smallest * -1 * _CustomReciprocal(blues - reds)) dh_dg = dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 # for blue, dh_db -> dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 # dh_db_1 -> - # if blue was MAX, then derivative = 60 * -1 * (R-G)/square(MAX-MIN) == 60 * -1 * reciprocal(square(saturation)) * reciprocal(square(value)) - # elif red was MAX, there are two subcases ie when blue was MIN and when blue was NOT MIN + # if blue was MAX, then derivative = 60 * -1 * (R-G)/square(MAX-MIN) == 60 *\ + # -1 * reciprocal(square(saturation)) * reciprocal(square(value)) + # elif red was MAX, there are two subcases + # ie when blue was MIN and when blue was NOT MIN # dh_dg_2 -> - # if blue was MIN (use UV rule) -> 60 * ((1 * -1/(MAX-MIN)) + (greens-blues)*(-1/square(MAX-MIN) * -1)) == 60 * (greens - reds) * reciprocal(square(reds - blues)) + # if blue was MIN (use UV rule) -> 60 * ((1 * -1/(MAX-MIN)) + \ + # (greens-blues) * (-1/square(MAX-MIN) * -1)) == 60 * (greens - reds) *\ + # reciprocal(square(reds - blues)) # dh_dg_3 -> - # if blue was NOT MIN -> 60 * -1/MAX-MIN == 60 * -1 * reciprocal(reds - greens) + # if blue was NOT MIN -> 60 * -1/MAX-MIN == 60 * -1 * \ + # reciprocal(reds - greens) # elif green was MAX, there are two subcases # dh_dg_4 -> - # if blue was MIN (similarly use the UV rule) -> 60 * -1 * (reciprocal(greens - blues) + (blues - reds)* -1 * reciprocal(square(greens - blues))) - # dh_dr_5 -> + # if blue was MIN (similarly use the UV rule) -> 60 * -1 * \ + # (reciprocal(greens - blues) + (blues - reds) * -1 * \ + # reciprocal(square(greens - blues))) + # dh_dr_5 -> # if blue was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(greens - reds) - dh_db_1 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * -1 * (reds - greens) * _custom_reciprocal(square(saturation)) * _custom_reciprocal(square(value))) - dh_db_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * blue_smallest * (greens - reds) * _custom_reciprocal(square(reds - blues))) - dh_db_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * green_smallest * -1 * _custom_reciprocal(reds - greens)) - dh_db_4 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * blue_smallest * (greens - reds) * _custom_reciprocal(square(greens - blues))) - dh_db_5 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * red_smallest * _custom_reciprocal(greens - reds)) + dh_db_1 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * -1 * \ + (reds - greens) * _CustomReciprocal(square(saturation)) * \ + _CustomReciprocal(square(value))) + dh_db_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest *\ + blue_smallest * (greens - reds) * \ + _CustomReciprocal(square(reds - blues))) + dh_db_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * \ + green_smallest * -1 * _CustomReciprocal(reds - greens)) + dh_db_4 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + blue_smallest * (greens - reds) * \ + _CustomReciprocal(square(greens - blues))) + dh_db_5 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + red_smallest * _CustomReciprocal(greens - reds)) dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 # Gradients wrt to inputs - dv_drgb = stack([grad[...,2] * dv_dr, grad[...,2] * dv_dg, grad[...,2] * dv_db], axis=-1) - ds_drgb = stack([grad[...,1] * ds_dr, grad[...,1] * ds_dg, grad[...,1] * ds_db], axis=-1) - dh_drgb = stack([grad[...,0] * dh_dr, grad[...,0] * dh_dg, grad[...,0] * dh_db], axis=-1) + dv_drgb = stack([grad[..., 2] * dv_dr, + grad[..., 2] * dv_dg, + grad[..., 2] * dv_db], axis=-1) + ds_drgb = stack([grad[..., 1] * ds_dr, + grad[..., 1] * ds_dg, + grad[..., 1] * ds_db], axis=-1) + dh_drgb = stack([grad[..., 0] * dh_dr, + grad[..., 0] * dh_dg, + grad[..., 0] * dh_db], axis=-1) gradient_input = add(add(dv_drgb, ds_drgb), dh_drgb) - return gradient_input \ No newline at end of file + return gradient_input From e2cfaf95fab994e653806ebdea00b920aca61fb4 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Mon, 24 Jun 2019 14:54:47 -0700 Subject: [PATCH 025/332] Updated function description. --- tensorflow/python/ops/image_grad.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index f85f26d114d..d900fc86b60 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -170,18 +170,20 @@ def _CustomReciprocal(x, my_eps=1e-10): @ops.RegisterGradient("RGBToHSV") def _RGBToHSVGrad(op, grad): - """The gradients for `zero_out`. + """The gradients for `rgb_to_hsv` operation. + This function is a piecewise continuous function as defined here: + https://en.wikipedia.org/wiki/HSL_and_HSV#From_RGB + We perform the multi variate derivative and compute all partial derivates + seperately before adding them in the end. Formulas are given before each + partial derivative calculation. Args: - op: The `rgb_to_hsv` `Operation` that we are differentiating, - which we can use - to find the inputs and outputs of the original op. + op: The `rgb_to_hsv` `Operation` that we are differentiating. grad: Gradient with respect to the output of the `rgb_to_hsv` op. Returns: Gradients with respect to the input of `rgb_to_hsv`. """ - print("******** This is a test implementation ********** \n") # Input Channels reds = op.inputs[0][..., 0] greens = op.inputs[0][..., 1] From 688456f32a74b3ce4041a3b37a472b270bba21dd Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Mon, 24 Jun 2019 14:55:34 -0700 Subject: [PATCH 026/332] Linting issue. --- tensorflow/python/ops/image_grad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index d900fc86b60..40592cff9fb 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -174,7 +174,7 @@ def _RGBToHSVGrad(op, grad): This function is a piecewise continuous function as defined here: https://en.wikipedia.org/wiki/HSL_and_HSV#From_RGB We perform the multi variate derivative and compute all partial derivates - seperately before adding them in the end. Formulas are given before each + seperately before adding them in the end. Formulas are given before each partial derivative calculation. Args: From c13767e6093e06208367b3d1f86fd5ff0edd5d43 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Tue, 25 Jun 2019 14:47:42 -0700 Subject: [PATCH 027/332] fixed import and linting --- tensorflow/python/ops/image_grad.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 40592cff9fb..a3345c8fb74 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -22,6 +22,8 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_image_ops +from tensorflow.python.ops.math_ops import cast, add, reciprocal, square +from tensorflow.python.ops.array_ops import stack @ops.RegisterGradient("ResizeNearestNeighbor") @@ -189,7 +191,6 @@ def _RGBToHSVGrad(op, grad): greens = op.inputs[0][..., 1] blues = op.inputs[0][..., 2] # Output Channels - hue = op.outputs[0][..., 0] saturation = op.outputs[0][..., 1] value = op.outputs[0][..., 2] @@ -204,15 +205,13 @@ def _RGBToHSVGrad(op, grad): green_smallest = cast((greens <= reds) & (greens < blues), dtypes.float32) blue_smallest = cast((blues <= reds) & (blues <= greens), dtypes.float32) - ############################################################## - # Derivatives of R, G, B wrt Value slice - ############################################################## + # Derivatives of R, G, B wrt Value slice dv_dr = red_biggest dv_dg = green_biggest dv_db = blue_biggest - ############################################################## + # Derivatives of R, G, B wrt Saturation slice - ############################################################## + # The first term in the addition is the case when the corresponding color # from (r,g,b) was "MAX" # -> derivative = MIN/square(MAX), MIN could be one of the other two colors @@ -234,11 +233,10 @@ def _RGBToHSVGrad(op, grad): _CustomReciprocal(square(blues)),\ blue_smallest * -1 * _CustomReciprocal((green_biggest * \ greens) + (red_biggest * reds))) - ############################################################## - # Derivatives of R, G, B wrt Hue slice - ############################################################## - # Need to go case by case for each color. + # Derivatives of R, G, B wrt Hue slice + + # Need to go case by case for each color. # for red, dh_dr -> dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 # dh_dr_1 -> # if red was MAX, then derivative = 60 * -1 * (G-B)/square(MAX-MIN) == 60 *\ From 0c7561a11f843a4b707272c917feb3996df4a499 Mon Sep 17 00:00:00 2001 From: wenxizhu Date: Wed, 26 Jun 2019 13:31:09 +0800 Subject: [PATCH 028/332] Bug fix: support 3d dilated convolution. --- tensorflow/core/kernels/mkl_conv_ops.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h index c12a4ff0f0c..8222619deca 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.h +++ b/tensorflow/core/kernels/mkl_conv_ops.h @@ -411,14 +411,15 @@ class MklDnnConvUtil { input_cols, filter_cols, dilation_cols, stride_cols, padding_type, &out_cols, &pad_left, &pad_right)); } else { - OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose( - input_planes, filter_planes, stride_planes, - padding_, &out_planes, &pad_D1, &pad_D2)); - OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose( - input_rows, filter_rows, stride_rows, + OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerboseV2( + input_planes, filter_planes, dilation_planes, + stride_planes, padding_, &out_planes, &pad_D1, + &pad_D2)); + OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerboseV2( + input_rows, filter_rows, dilation_rows, stride_rows, padding_, &out_rows, &pad_top, &pad_bottom)); - OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose( - input_cols, filter_cols, stride_cols, + OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerboseV2( + input_cols, filter_cols, dilation_cols, stride_cols, padding_, &out_cols, &pad_left, &pad_right)); } From 48671af095225b8a16c3bb2854bda7e4f1451339 Mon Sep 17 00:00:00 2001 From: Bhavani Subramanian Date: Wed, 26 Jun 2019 13:11:12 -0700 Subject: [PATCH 029/332] Added a config option to enable MKL-DNN v1.0. --- .bazelrc | 1 + tensorflow/tensorflow.bzl | 2 + tensorflow/workspace.bzl | 11 ++++ third_party/mkl/build_defs.bzl | 1 + third_party/mkl_dnn/BUILD | 9 +++ third_party/mkl_dnn/build_defs.bzl | 20 ++++++- third_party/mkl_dnn/mkldnn.BUILD | 90 ++++++++++++++++++++++++++++++ 7 files changed, 131 insertions(+), 3 deletions(-) diff --git a/.bazelrc b/.bazelrc index 95e2b369a89..590a87f5732 100644 --- a/.bazelrc +++ b/.bazelrc @@ -40,6 +40,7 @@ build:mkl -c opt # This config option is used to enable MKL-DNN open source library only, # without depending on MKL binary version. build:mkl_open_source_only --define=build_with_mkl_dnn_only=true +build:mkl_open_source_only --define=build_with_mkl_dnn_v1_only=true build:mkl_open_source_only --define=build_with_mkl=true --define=enable_mkl=true build:mkl_open_source_only --define=tensorflow_mkldnn_contraction_kernel=0 diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 84bd175c4c2..0adb6383d25 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -44,6 +44,7 @@ load( load( "//third_party/mkl_dnn:build_defs.bzl", "if_mkl_open_source_only", + "if_mkl_v1_open_source_only", ) load( "//third_party/ngraph:build_defs.bzl", @@ -292,6 +293,7 @@ def tf_copts(android_optimization_level_override = "-O2", is_external = False): if_tensorrt(["-DGOOGLE_TENSORRT=1"]) + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"]) + if_mkl_open_source_only(["-DINTEL_MKL_DNN_ONLY"]) + + if_mkl_v1_open_source_only(["-DENABLE_MKLDNN_v1"]) + if_enable_mkl(["-DENABLE_MKL"]) + if_ngraph(["-DINTEL_NGRAPH=1"]) + if_mkl_lnx_x64(["-fopenmp"]) + diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 99f5447105b..ac327fa367a 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -139,6 +139,17 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ], ) + tf_http_archive( + name = "mkl_dnn_v1", + build_file = clean_dep("//third_party/mkl_dnn:mkldnn.BUILD"), + sha256 = "fcc2d951f7170eade0cfdd0d8d1d58e3e7785bd326bca6555f3722f8cba71811", + strip_prefix = "mkl-dnn-1.0-pc2", + urls = [ + "http://mirror.tensorflow.org/github.com/intel/mkl-dnn/archive/v1.0-pc2.tar.gz", + "https://github.com/intel/mkl-dnn/archive/v1.0-pc2.tar.gz", + ], + ) + tf_http_archive( name = "com_google_absl", build_file = clean_dep("//third_party:com_google_absl.BUILD"), diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl index 10c2d90c848..00789019a8b 100644 --- a/third_party/mkl/build_defs.bzl +++ b/third_party/mkl/build_defs.bzl @@ -107,6 +107,7 @@ def mkl_deps(): """ return select({ str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_only")): ["@mkl_dnn"], + str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_v1_only")): ["@mkl_dnn_v1"], str(Label("//third_party/mkl:build_with_mkl_ml_only")): ["//third_party/mkl:intel_binary_blob"], str(Label("//third_party/mkl:build_with_mkl")): [ "//third_party/mkl:intel_binary_blob", diff --git a/third_party/mkl_dnn/BUILD b/third_party/mkl_dnn/BUILD index 047d062cdda..5229dd5aa74 100644 --- a/third_party/mkl_dnn/BUILD +++ b/third_party/mkl_dnn/BUILD @@ -10,3 +10,12 @@ config_setting( }, visibility = ["//visibility:public"], ) + +config_setting( + name = "build_with_mkl_dnn_v1_only", + define_values = { + "build_with_mkl": "true", + "build_with_mkl_dnn_v1_only": "true", + }, + visibility = ["//visibility:public"], +) diff --git a/third_party/mkl_dnn/build_defs.bzl b/third_party/mkl_dnn/build_defs.bzl index 6388f31971c..e851d94048a 100644 --- a/third_party/mkl_dnn/build_defs.bzl +++ b/third_party/mkl_dnn/build_defs.bzl @@ -1,13 +1,27 @@ def if_mkl_open_source_only(if_true, if_false = []): """Shorthand for select()'ing on whether we're building with - MKL-DNN open source lib only, without depending on MKL binary form. + MKL-DNN v0.x open source library only, without depending on MKL binary form. Returns a select statement which evaluates to if_true if we're building - with MKL-DNN open source lib only. Otherwise, - the select statement evaluates to if_false. + with MKL-DNN v0.x open source library only. Otherwise, the select statement + evaluates to if_false. """ return select({ str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_only")): if_true, "//conditions:default": if_false, }) + +def if_mkl_v1_open_source_only(if_true, if_false = []): + """Shorthand for select()'ing on whether we're building with MKL-DNN v1.x + (and v0.x) open source library only, without depending on MKL binary form. + + Returns a select statement which evaluates to if_true if we're building + with MKL-DNN v1.x (and v0.x) open source library only. Otherwise, the + select statement evaluates to if_false. + + """ + return select({ + str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_v1_only")): if_true, + "//conditions:default": if_false, + }) diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index 487e24adc11..8bd082aa308 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -17,6 +17,16 @@ config_setting( }, ) +template_rule( + name = "mkldnn_config_h", + src = "include/mkldnn_config.h.in", + out = "include/mkldnn_config.h", + substitutions = { + "#cmakedefine MKLDNN_CPU_BACKEND MKLDNN_BACKEND_${MKLDNN_CPU_BACKEND}": "#define MKLDNN_CPU_BACKEND MKLDNN_BACKEND_NATIVE", + "#cmakedefine MKLDNN_GPU_BACKEND MKLDNN_BACKEND_${MKLDNN_GPU_BACKEND}": "#define MKLDNN_GPU_BACKEND MKLDNN_BACKEND_NONE", + }, +) + # Create the file mkldnn_version.h with MKL-DNN version numbers. # Currently, the version numbers are hard coded here. If MKL-DNN is upgraded then # the version numbers have to be updated manually. The version numbers can be @@ -37,6 +47,20 @@ template_rule( }, ) +# TODO(bhavanis): Once we automatically get version numbers from CMakeLists.txt, +# the following template rule needs to be removed. +template_rule( + name = "mkldnn_v1_version_h", + src = "include/mkldnn_version.h.in", + out = "include/mkldnn_version.h", + substitutions = { + "@MKLDNN_VERSION_MAJOR@": "0", + "@MKLDNN_VERSION_MINOR@": "95", + "@MKLDNN_VERSION_PATCH@": "0", + "@MKLDNN_VERSION_HASH@": "N/A", + }, +) + cc_library( name = "mkl_dnn", srcs = glob([ @@ -98,6 +122,72 @@ cc_library( }), ) +cc_library( + name = "mkl_dnn_v1", + srcs = glob([ + "src/common/*.cpp", + "src/common/*.hpp", + "src/cpu/*.cpp", + "src/cpu/*.hpp", + "src/cpu/gemm/*.cpp", + "src/cpu/gemm/*.hpp", + "src/cpu/gemm/f32/*.cpp", + "src/cpu/gemm/f32/*.hpp", + "src/cpu/gemm/s8x8s32/*.cpp", + "src/cpu/gemm/s8x8s32/*.hpp", + "src/cpu/jit_utils/*.cpp", # newly added + "src/cpu/jit_utils/*.hpp", # newly added + "src/cpu/rnn/*.cpp", + "src/cpu/rnn/*.hpp", + "src/cpu/xbyak/*.h", + ]) + [ + ":mkldnn_v1_version_h", # newly added + ":mkldnn_config_h", # newly added + ], + hdrs = glob(["include/*"]), + copts = [ + "-fexceptions", + "-DUSE_MKL", + "-DUSE_CBLAS", + ] + if_mkl_open_source_only([ + "-UUSE_MKL", + "-UUSE_CBLAS", + ]) + select({ + "@org_tensorflow//tensorflow:linux_x86_64": [ + "-fopenmp", # only works with gcc + ], + # TODO(ibiryukov): enable openmp with clang by including libomp as a + # dependency. + ":clang_linux_x86_64": [], + "//conditions:default": [], + }), + includes = [ + "include", + "src", + "src/common", + "src/cpu", + "src/cpu/gemm", + "src/cpu/xbyak", + ], + nocopts = "-fno-exceptions", + visibility = ["//visibility:public"], + deps = select({ + "@org_tensorflow//tensorflow:linux_x86_64": [ + "@mkl_linux//:mkl_headers", + "@mkl_linux//:mkl_libs_linux", + ], + "@org_tensorflow//tensorflow:macos": [ + "@mkl_darwin//:mkl_headers", + "@mkl_darwin//:mkl_libs_darwin", + ], + "@org_tensorflow//tensorflow:windows": [ + "@mkl_windows//:mkl_headers", + "@mkl_windows//:mkl_libs_windows", + ], + "//conditions:default": [], + }), +) + cc_library( name = "mkldnn_single_threaded", srcs = glob([ From 351c56becdf64ed7d58ab96f317db0e26a733276 Mon Sep 17 00:00:00 2001 From: Bhavani Subramanian Date: Wed, 26 Jun 2019 13:27:56 -0700 Subject: [PATCH 030/332] Using the same version number for MKL-DNN v1.x as v0.x until we figure out how to automatically get version numbers from CMakeLists.txt. --- third_party/mkl_dnn/mkldnn.BUILD | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index 8bd082aa308..cc17dd2b4c0 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -34,6 +34,7 @@ template_rule( # set to "version_major.version_minor.version_patch". The git hash version can # be set to NA. # TODO(agramesh1) Automatically get the version numbers from CMakeLists.txt. +# TODO(bhavanis): MKL-DNN version needs to be updated for MKL-DNN v1.x template_rule( name = "mkldnn_version_h", @@ -47,20 +48,6 @@ template_rule( }, ) -# TODO(bhavanis): Once we automatically get version numbers from CMakeLists.txt, -# the following template rule needs to be removed. -template_rule( - name = "mkldnn_v1_version_h", - src = "include/mkldnn_version.h.in", - out = "include/mkldnn_version.h", - substitutions = { - "@MKLDNN_VERSION_MAJOR@": "0", - "@MKLDNN_VERSION_MINOR@": "95", - "@MKLDNN_VERSION_PATCH@": "0", - "@MKLDNN_VERSION_HASH@": "N/A", - }, -) - cc_library( name = "mkl_dnn", srcs = glob([ @@ -141,7 +128,7 @@ cc_library( "src/cpu/rnn/*.hpp", "src/cpu/xbyak/*.h", ]) + [ - ":mkldnn_v1_version_h", # newly added + ":mkldnn_version_h", ":mkldnn_config_h", # newly added ], hdrs = glob(["include/*"]), From 3639789b8fdac02cac17686e7a1d501c13e1a432 Mon Sep 17 00:00:00 2001 From: Bhavani Subramanian Date: Wed, 26 Jun 2019 16:19:03 -0700 Subject: [PATCH 031/332] Minor change. --- third_party/mkl_dnn/build_defs.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/mkl_dnn/build_defs.bzl b/third_party/mkl_dnn/build_defs.bzl index e851d94048a..ed3f0619623 100644 --- a/third_party/mkl_dnn/build_defs.bzl +++ b/third_party/mkl_dnn/build_defs.bzl @@ -13,11 +13,11 @@ def if_mkl_open_source_only(if_true, if_false = []): }) def if_mkl_v1_open_source_only(if_true, if_false = []): - """Shorthand for select()'ing on whether we're building with MKL-DNN v1.x - (and v0.x) open source library only, without depending on MKL binary form. + """Shorthand for select()'ing on whether we're building with + MKL-DNN v1.x open source library only, without depending on MKL binary form. Returns a select statement which evaluates to if_true if we're building - with MKL-DNN v1.x (and v0.x) open source library only. Otherwise, the + with MKL-DNN v1.x open source library only. Otherwise, the select statement evaluates to if_false. """ From 7e1cced44cb8c92478af0c98827e990552255626 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Wed, 26 Jun 2019 16:32:14 -0700 Subject: [PATCH 032/332] Review comments. --- tensorflow/python/ops/image_grad.py | 104 ++++++++++++++++------------ 1 file changed, 59 insertions(+), 45 deletions(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index a3345c8fb74..52d5d9dcb7c 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -22,7 +22,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_image_ops -from tensorflow.python.ops.math_ops import cast, add, reciprocal, square +from tensorflow.python.ops import math_ops from tensorflow.python.ops.array_ops import stack @@ -167,7 +167,7 @@ def _CustomReciprocal(x, my_eps=1e-10): Returns: x_reciprocal -> reciprocal of x added with my_eps """ - return reciprocal(x + my_eps) + return math_ops.div_no_nan(1.0, x) @ops.RegisterGradient("RGBToHSV") @@ -197,13 +197,19 @@ def _RGBToHSVGrad(op, grad): # Mask/Indicator for max and min values of each pixel. # Arbitrary assignment in case of tie breakers with R>G>B. # Max values - red_biggest = cast((reds >= blues) & (reds >= greens), dtypes.float32) - green_biggest = cast((greens > reds) & (greens >= blues), dtypes.float32) - blue_biggest = cast((blues > reds) & (blues > greens), dtypes.float32) + red_biggest = math_ops.cast((reds >= blues) & \ + (reds >= greens), dtypes.float32) + green_biggest = math_ops.cast((greens > reds) & \ + (greens >= blues), dtypes.float32) + blue_biggest = math_ops.cast((blues > reds) & \ + (blues > greens), dtypes.float32) # Min values - red_smallest = cast((reds < blues) & (reds < greens), dtypes.float32) - green_smallest = cast((greens <= reds) & (greens < blues), dtypes.float32) - blue_smallest = cast((blues <= reds) & (blues <= greens), dtypes.float32) + red_smallest = math_ops.cast((reds < blues) & \ + (reds < greens), dtypes.float32) + green_smallest = math_ops.cast((greens <= reds) & \ + (greens < blues), dtypes.float32) + blue_smallest = math_ops.cast((blues <= reds) & \ + (blues <= greens), dtypes.float32) # Derivatives of R, G, B wrt Value slice dv_dr = red_biggest @@ -218,19 +224,22 @@ def _RGBToHSVGrad(op, grad): # The second term is the case when the corresponding color from # (r,g,b) was "MIN" # -> derivative = -1/MAX, MAX could be one of the other two colours. - ds_dr = cast(reds > 0, dtypes.float32) * add(red_biggest * \ - add(green_smallest * greens, blue_smallest * blues) * \ - _CustomReciprocal(square(reds)),\ + ds_dr = math_ops.cast(reds > 0, dtypes.float32) * \ + math_ops.add(red_biggest * \ + math_ops.add(green_smallest * greens, blue_smallest * blues) * \ + _CustomReciprocal(math_ops.square(reds)),\ red_smallest * -1 * _CustomReciprocal((green_biggest * \ greens) + (blue_biggest * blues))) - ds_dg = cast(greens > 0, dtypes.float32) * add(green_biggest * \ - add(red_smallest * reds, blue_smallest * blues) * \ - _CustomReciprocal(square(greens)),\ + ds_dg = math_ops.cast(greens > 0, dtypes.float32) * \ + math_ops.add(green_biggest * \ + math_ops.add(red_smallest * reds, blue_smallest * blues) * \ + _CustomReciprocal(math_ops.square(greens)),\ green_smallest * -1 * _CustomReciprocal((red_biggest * \ reds) + (blue_biggest * blues))) - ds_db = cast(blues > 0, dtypes.float32) * add(blue_biggest * \ - add(green_smallest * greens, red_smallest * reds) * \ - _CustomReciprocal(square(blues)),\ + ds_db = math_ops.cast(blues > 0, dtypes.float32) * \ + math_ops.add(blue_biggest * \ + math_ops.add(green_smallest * greens, red_smallest * reds) * \ + _CustomReciprocal(math_ops.square(blues)),\ blue_smallest * -1 * _CustomReciprocal((green_biggest * \ greens) + (red_biggest * reds))) @@ -256,18 +265,20 @@ def _RGBToHSVGrad(op, grad): # reciprocal(square(blues - reds)) # dh_dr_5 -> # if red was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(blues-greens) - dh_dr_1 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * -1 * \ - (greens - blues) * _CustomReciprocal(square(saturation)) *\ - _CustomReciprocal(square(value))) - dh_dr_2 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + dh_dr_1 = 60 * (math_ops.cast(reds > 0, dtypes.float32) * red_biggest * \ + -1 * \ + (greens - blues) * \ + _CustomReciprocal(math_ops.square(saturation)) *\ + _CustomReciprocal(math_ops.square(value))) + dh_dr_2 = 60 * (math_ops.cast(greens > 0, dtypes.float32) * green_biggest * \ red_smallest * (blues - greens) * \ - _CustomReciprocal(square(reds - greens))) - dh_dr_3 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + _CustomReciprocal(math_ops.square(reds - greens))) + dh_dr_3 = 60 * (math_ops.cast(greens > 0, dtypes.float32) * green_biggest * \ blue_smallest * -1 * _CustomReciprocal(greens - blues)) - dh_dr_4 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * \ + dh_dr_4 = 60 * (math_ops.cast(blues > 0, dtypes.float32) * blue_biggest * \ red_smallest * (blues - greens) * \ - _CustomReciprocal(square(blues - reds))) - dh_dr_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * \ + _CustomReciprocal(math_ops.square(blues - reds))) + dh_dr_5 = 60 * (math_ops.cast(blues > 0, dtypes.float32) * blue_biggest * \ green_smallest * _CustomReciprocal(blues - greens)) dh_dr = dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 @@ -293,18 +304,19 @@ def _RGBToHSVGrad(op, grad): # reciprocal(square(blues-greens))) # dh_dr_5 -> # if green was NOT MIN -> 60 * -1/MAX-MIN == -60 * reciprocal(blues - reds) - dh_dg_1 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ - -1 * (blues - reds) * _CustomReciprocal(square(saturation))\ - * _CustomReciprocal(square(value))) - dh_dg_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * \ + dh_dg_1 = 60 * (math_ops.cast(greens > 0, dtypes.float32) * green_biggest * \ + -1 * (blues - reds) * \ + _CustomReciprocal(math_ops.square(saturation))\ + * _CustomReciprocal(math_ops.square(value))) + dh_dg_2 = 60 * (math_ops.cast(reds > 0, dtypes.float32) * red_biggest * \ green_smallest * (reds - blues) * \ - _CustomReciprocal(square(reds - greens))) - dh_dg_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * \ + _CustomReciprocal(math_ops.square(reds - greens))) + dh_dg_3 = 60 * (math_ops.cast(reds > 0, dtypes.float32) * red_biggest * \ blue_smallest * _CustomReciprocal(reds - blues)) - dh_dg_4 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * \ + dh_dg_4 = 60 * (math_ops.cast(blues > 0, dtypes.float32) * blue_biggest * \ green_smallest * (reds - blues) * \ - _CustomReciprocal(square(blues - greens))) - dh_dg_5 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * \ + _CustomReciprocal(math_ops.square(blues - greens))) + dh_dg_5 = 60 * (math_ops.cast(blues > 0, dtypes.float32) * blue_biggest * \ red_smallest * -1 * _CustomReciprocal(blues - reds)) dh_dg = dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 @@ -329,18 +341,20 @@ def _RGBToHSVGrad(op, grad): # reciprocal(square(greens - blues))) # dh_dr_5 -> # if blue was NOT MIN -> 60 * 1/MAX-MIN == 60 * reciprocal(greens - reds) - dh_db_1 = 60 * (cast(blues > 0, dtypes.float32) * blue_biggest * -1 * \ - (reds - greens) * _CustomReciprocal(square(saturation)) * \ - _CustomReciprocal(square(value))) - dh_db_2 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest *\ + dh_db_1 = 60 * (math_ops.cast(blues > 0, dtypes.float32) * blue_biggest * \ + -1 * \ + (reds - greens) * \ + _CustomReciprocal(math_ops.square(saturation)) * \ + _CustomReciprocal(math_ops.square(value))) + dh_db_2 = 60 * (math_ops.cast(reds > 0, dtypes.float32) * red_biggest *\ blue_smallest * (greens - reds) * \ - _CustomReciprocal(square(reds - blues))) - dh_db_3 = 60 * (cast(reds > 0, dtypes.float32) * red_biggest * \ + _CustomReciprocal(math_ops.square(reds - blues))) + dh_db_3 = 60 * (math_ops.cast(reds > 0, dtypes.float32) * red_biggest * \ green_smallest * -1 * _CustomReciprocal(reds - greens)) - dh_db_4 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + dh_db_4 = 60 * (math_ops.cast(greens > 0, dtypes.float32) * green_biggest * \ blue_smallest * (greens - reds) * \ - _CustomReciprocal(square(greens - blues))) - dh_db_5 = 60 * (cast(greens > 0, dtypes.float32) * green_biggest * \ + _CustomReciprocal(math_ops.square(greens - blues))) + dh_db_5 = 60 * (math_ops.cast(greens > 0, dtypes.float32) * green_biggest * \ red_smallest * _CustomReciprocal(greens - reds)) dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 @@ -356,6 +370,6 @@ def _RGBToHSVGrad(op, grad): grad[..., 0] * dh_dg, grad[..., 0] * dh_db], axis=-1) - gradient_input = add(add(dv_drgb, ds_drgb), dh_drgb) + gradient_input = math_ops.add(math_ops.add(dv_drgb, ds_drgb), dh_drgb) return gradient_input From c1a8ec77e52afafd41b1671aecc89eaf3e34810f Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Fri, 28 Jun 2019 11:00:50 -0700 Subject: [PATCH 033/332] Add tests for rgb_to_hsv grad and fixes. --- tensorflow/python/ops/image_grad.py | 22 +++++++-------- tensorflow/python/ops/image_grad_test.py | 34 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 52d5d9dcb7c..9eb671d2481 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -23,8 +23,6 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_image_ops from tensorflow.python.ops import math_ops -from tensorflow.python.ops.array_ops import stack - @ops.RegisterGradient("ResizeNearestNeighbor") def _ResizeNearestNeighborGrad(op, grad): @@ -157,7 +155,7 @@ def _CropAndResizeGrad(op, grad): return [grad0, grad1, None, None] -def _CustomReciprocal(x, my_eps=1e-10): +def _CustomReciprocal(x): """ Performs reciprocal with an eps added to the input. This is to avoid inversion errors or divide by zeros or NaNs. @@ -360,15 +358,15 @@ def _RGBToHSVGrad(op, grad): dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 # Gradients wrt to inputs - dv_drgb = stack([grad[..., 2] * dv_dr, - grad[..., 2] * dv_dg, - grad[..., 2] * dv_db], axis=-1) - ds_drgb = stack([grad[..., 1] * ds_dr, - grad[..., 1] * ds_dg, - grad[..., 1] * ds_db], axis=-1) - dh_drgb = stack([grad[..., 0] * dh_dr, - grad[..., 0] * dh_dg, - grad[..., 0] * dh_db], axis=-1) + dv_drgb = array_ops.stack([grad[..., 2] * dv_dr, + grad[..., 2] * dv_dg, + grad[..., 2] * dv_db], axis=-1) + ds_drgb = array_ops.stack([grad[..., 1] * ds_dr, + grad[..., 1] * ds_dg, + grad[..., 1] * ds_db], axis=-1) + dh_drgb = array_ops.stack([grad[..., 0] * dh_dr, + grad[..., 0] * dh_dg, + grad[..., 0] * dh_db], axis=-1) gradient_input = math_ops.add(math_ops.add(dv_drgb, ds_drgb), dh_drgb) diff --git a/tensorflow/python/ops/image_grad_test.py b/tensorflow/python/ops/image_grad_test.py index 43d9699980e..b4326e07eb5 100644 --- a/tensorflow/python/ops/image_grad_test.py +++ b/tensorflow/python/ops/image_grad_test.py @@ -24,8 +24,10 @@ from tensorflow.python.eager import backprop from tensorflow.python.framework import constant_op from tensorflow.python.framework import test_util from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import gradient_checker_v2 from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import image_ops +from tensorflow.python.ops import gen_image_ops from tensorflow.python.platform import test @@ -456,6 +458,38 @@ class CropAndResizeOpTest(test.TestCase): self.assertLess(err, 2e-3) +@test_util.run_all_in_graph_and_eager_modes +class RGBToHSVOpTest(test.TestCase): + + TYPES = [np.float32, np.float64] + + def testShapeIsCorrectAfterOp(self): + in_shape = [2, 20, 30, 3] + out_shape = [2, 20, 30, 3] + + for nptype in self.TYPES: + x = np.random.randint(0, high=255, size=[2, 20, 30, 3]).astype(nptype) + with self.cached_session(use_gpu=True): + rgb_input_tensor = constant_op.constant(x, shape=in_shape) + hsv_out = gen_image_ops.rgb_to_hsv(rgb_input_tensor) + self.assertEqual(out_shape, list(hsv_out.get_shape())) + + hsv_out = self.evaluate(hsv_out) + self.assertEqual(out_shape, list(hsv_out.shape)) + + def testRGBToHSVGrad(self): + in_shape = [2, 20, 30, 3] + def f(x): + return gen_image_ops.rgb_to_hsv(x) + + x = np.random.rand(2, 20, 30, 3).astype(np.float32) + rgb_input_tensor = constant_op.constant(x, shape=in_shape) + analytical, numerical = gradient_checker_v2.compute_gradient( + f, [rgb_input_tensor]) + self.assertLess( + gradient_checker_v2.max_error(analytical, numerical), 1e-2) + + if __name__ == "__main__": test.main() From 52c80f42ff0ffe6de3b27ae0f9b76710ee4dddcf Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Sat, 29 Jun 2019 00:12:59 +0000 Subject: [PATCH 034/332] Use environment markers for enum34 This fix tries to addres the issue raised in 30200 where enum34 caused poetry breaks due to the conditional sys.version_info This fix changes to use environment markers for enum34 (conform to PEP508) This fix fixes 30200. Signed-off-by: Yong Tang --- tensorflow/tools/pip_package/setup.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 5868b763d77..2b70f64c093 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -52,6 +52,8 @@ _VERSION = '1.14.0' REQUIRED_PACKAGES = [ 'absl-py >= 0.7.0', 'astor >= 0.6.0', + 'backports.weakref >= 1.0rc1;python_version<"3.4"', + 'enum34 >= 1.1.6;python_version<"3.4"', 'gast >= 0.2.0', 'google_pasta >= 0.1.6', 'keras_applications >= 1.0.8', @@ -97,11 +99,6 @@ if 'tf_nightly' in project_name: elif 'tensorflow_estimator' in pkg: REQUIRED_PACKAGES[i] = 'tf-estimator-nightly' -# weakref.finalize and enum were introduced in Python 3.4 -if sys.version_info < (3, 4): - REQUIRED_PACKAGES.append('backports.weakref >= 1.0rc1') - REQUIRED_PACKAGES.append('enum34 >= 1.1.6') - # pylint: disable=line-too-long CONSOLE_SCRIPTS = [ 'toco_from_protos = tensorflow.lite.toco.python.toco_from_protos:main', From 950fac03a362d1df71231cd6b515804209062c14 Mon Sep 17 00:00:00 2001 From: wenxizhu Date: Mon, 1 Jul 2019 13:53:23 +0800 Subject: [PATCH 035/332] Add testcases. --- .../python/kernel_tests/conv_ops_3d_test.py | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py index e136d091393..bdb8bc514c9 100644 --- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py @@ -220,13 +220,12 @@ class Conv3DTest(test.TestCase): expected=expected_output) def testConv3D1x1x1Filter2x1x1Dilation(self): - if test.is_gpu_available(cuda_only=True): - self._VerifyDilatedConvValues( - tensor_in_sizes=[1, 3, 6, 1, 1], - filter_in_sizes=[1, 1, 1, 1, 1], - stride=1, - padding="VALID", - dilations=[2, 1, 1]) + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 3, 6, 1, 1], + filter_in_sizes=[1, 1, 1, 1, 1], + stride=1, + padding="VALID", + dilations=[2, 1, 1]) # Expected values computed using scipy's correlate function. def testConv3D2x2x2Filter(self): @@ -245,13 +244,12 @@ class Conv3DTest(test.TestCase): expected=expected_output) def testConv3D2x2x2Filter1x2x1Dilation(self): - if test.is_gpu_available(cuda_only=True): - self._VerifyDilatedConvValues( - tensor_in_sizes=[1, 4, 6, 3, 1], - filter_in_sizes=[2, 2, 2, 1, 1], - stride=1, - padding="VALID", - dilations=[1, 2, 1]) + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 4, 6, 3, 1], + filter_in_sizes=[2, 2, 2, 1, 1], + stride=1, + padding="VALID", + dilations=[1, 2, 1]) def testConv3DStrides(self): expected_output = [ From 80c1ca103f51ab361f76ac140e97f448fa8ed8da Mon Sep 17 00:00:00 2001 From: Rasmus Diederichsen Date: Fri, 28 Jun 2019 09:16:14 +0200 Subject: [PATCH 036/332] Fix macos makefile build - Adding `CoreFoundation` to `HOST_LDFLAGS` is necessary as the library target uses that variable - Adding `CoreFoundation` to `LIBS` is necessary as the benchmark target uses that variable --- tensorflow/contrib/makefile/Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index f90af791aa4..e2e665e0479 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -910,12 +910,18 @@ $(PROTO_TEXT_OBJS) : $(PROTO_TEXT_PB_H_FILES) # Ensures we link CoreFoundation as it is used for time library when building # for Mac. +ifeq ($(TARGET),OSX) + HOST_LDOPTS += -framework CoreFoundation + LIBS += -framework CoreFoundation +endif ifeq ($(TARGET),IOS) ifeq ($(IOS_ARCH),X86_64) HOST_LDOPTS += -framework CoreFoundation endif endif + + # Runs proto_text to generate C++ source files from protos. $(PROTO_TEXT): $(PROTO_TEXT_OBJS) $(PROTO_TEXT_PB_H_FILES) @mkdir -p $(dir $@) From c4608083779f4ae2e5124b41860b6d6735e08fbd Mon Sep 17 00:00:00 2001 From: mbhuiyan Date: Mon, 1 Jul 2019 13:18:04 -0700 Subject: [PATCH 037/332] Initialized private variables in constructor to pass Clockworks and clang format fix --- tensorflow/core/kernels/mkl_concat_op.cc | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc index 6c50ad0ccea..6e87e70f587 100644 --- a/tensorflow/core/kernels/mkl_concat_op.cc +++ b/tensorflow/core/kernels/mkl_concat_op.cc @@ -17,7 +17,6 @@ limitations under the License. #include #include "mkldnn.hpp" -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -30,6 +29,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/mkl_util.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" using mkldnn::concat; using mkldnn::stream; @@ -152,9 +152,8 @@ class EigenConcatBaseOp : public OpKernel { int32 axis = (concat_dim < 0) ? (concat_dim + input_dims) : concat_dim; OP_REQUIRES( - c, - (0 <= axis && axis < input_dims) || - (allow_legacy_scalars() && concat_dim == 0), + c, (0 <= axis && axis < input_dims) || + (allow_legacy_scalars() && concat_dim == 0), errors::InvalidArgument( "ConcatOp : Expected concatenating dimensions in the range [", -input_dims, ", ", input_dims, "), but got ", concat_dim)); @@ -184,13 +183,12 @@ class EigenConcatBaseOp : public OpKernel { const auto in = values[i]; const bool in_is_scalar = IsLegacyScalar(input_shapes[i]); OP_REQUIRES( - c, - (input_shapes[i].dims() == input_dims) || - (input_is_scalar && in_is_scalar), + c, (input_shapes[i].dims() == input_dims) || + (input_is_scalar && in_is_scalar), errors::InvalidArgument( "ConcatOp : Ranks of all input tensors should match: shape[0] = ", - input_shape.DebugString(), " vs. shape[", i, - "] = ", input_shapes[i].DebugString())); + input_shape.DebugString(), " vs. shape[", i, "] = ", + input_shapes[i].DebugString())); if (in.NumElements() > 0) { int64 inputs_flat_dim1 = in.NumElements() / inputs_flat_dim0; inputs_flat.emplace_back(new typename TTypes::ConstMatrix( @@ -247,7 +245,9 @@ class MklConcatOp : public OpKernel { ConstMatrixVector; explicit MklConcatOp(OpKernelConstruction* c) - : OpKernel(c), eigen_concat_op_(c) {} + : OpKernel(c), + eigen_concat_op_(c), + data_format_(TensorFormat::FORMAT_NCHW) {} void Compute(OpKernelContext* context) override { try { @@ -555,9 +555,9 @@ class MklConcatOp : public OpKernel { } } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status: " + std::to_string(e.status) + ", message: " + + string(e.message) + ", in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK( context, errors::Aborted("Operation received an exception:", error_msg)); From 9770870291d7a4107d9c1d6fd49a8434f343fd21 Mon Sep 17 00:00:00 2001 From: "Li, Guizi" Date: Tue, 2 Jul 2019 08:21:08 +0800 Subject: [PATCH 038/332] [Intel MKL] Fix klockworks issue --- tensorflow/core/kernels/mkl_fused_batch_norm_op.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc index 455522baf5b..9c6c2b1ea37 100644 --- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc +++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc @@ -510,6 +510,7 @@ class MklFusedBatchNormOp : public OpKernel { OP_REQUIRES(context, FormatFromString(tensor_format, &tensor_format_), errors::InvalidArgument("Invalid data format")); OP_REQUIRES_OK(context, context->GetAttr("is_training", &is_training_)); + depth_ = 0; } void Compute(OpKernelContext* context) override { @@ -834,6 +835,7 @@ class MklFusedBatchNormGradOp : public OpKernel { OP_REQUIRES(context, FormatFromString(tensor_format, &tensor_format_), errors::InvalidArgument("Invalid data format")); OP_REQUIRES_OK(context, context->GetAttr("is_training", &is_training_)); + depth_ = 0; } void Compute(OpKernelContext* context) override { @@ -1046,7 +1048,7 @@ class MklFusedBatchNormGradOp : public OpKernel { private: float epsilon_; TensorFormat tensor_format_; - int depth_; // batch normalization is done for per channel. + size_t depth_; // batch normalization is done for per channel. bool is_training_; engine cpu_engine = engine(engine::cpu, 0); From 8d00ced88d6dfa369c2387d3745c1451f3f1ae64 Mon Sep 17 00:00:00 2001 From: "Li, Guizi" Date: Tue, 2 Jul 2019 08:21:58 +0800 Subject: [PATCH 039/332] Fix clang format issue --- tensorflow/core/kernels/mkl_fused_batch_norm_op.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc index 9c6c2b1ea37..10b7714dabb 100644 --- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc +++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc @@ -14,13 +14,13 @@ limitations under the License. ==============================================================================*/ #ifdef INTEL_MKL #include "mkldnn.hpp" -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/util/mkl_util.h" #include "tensorflow/core/util/tensor_format.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" using mkldnn::batch_normalization_backward; using mkldnn::batch_normalization_forward; @@ -711,9 +711,9 @@ class MklFusedBatchNormOp : public OpKernel { std::memcpy(batch_variance_data, variance_data, depth_ * sizeof(U)); } } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status: " + std::to_string(e.status) + ", message: " + + string(e.message) + ", in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK( context, errors::Aborted("Operation received an exception:", error_msg)); @@ -1036,9 +1036,9 @@ class MklFusedBatchNormGradOp : public OpKernel { reinterpret_cast(diff_weights_data + depth_), depth_ * sizeof(U)); } catch (mkldnn::error& e) { - string error_msg = "Status: " + std::to_string(e.status) + - ", message: " + string(e.message) + ", in file " + - string(__FILE__) + ":" + std::to_string(__LINE__); + string error_msg = "Status: " + std::to_string(e.status) + ", message: " + + string(e.message) + ", in file " + string(__FILE__) + + ":" + std::to_string(__LINE__); OP_REQUIRES_OK( context, errors::Aborted("Operation received an exception:", error_msg)); From d75510457b1dcda7e84c86acfa38bdec4f3177e5 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Tue, 2 Jul 2019 03:49:46 +0000 Subject: [PATCH 040/332] cosmetic / formating changes --- tensorflow/stream_executor/rocm/rocm_blas.cc | 158 +++++++++--------- tensorflow/stream_executor/rocm/rocm_blas.h | 18 +- .../rocm/rocm_driver_wrapper.h | 6 +- tensorflow/stream_executor/rocm/rocm_fft.cc | 72 ++++---- tensorflow/stream_executor/rocm/rocm_rng.cc | 3 +- 5 files changed, 132 insertions(+), 125 deletions(-) diff --git a/tensorflow/stream_executor/rocm/rocm_blas.cc b/tensorflow/stream_executor/rocm/rocm_blas.cc index 52c654617f8..fd444a4cd6b 100644 --- a/tensorflow/stream_executor/rocm/rocm_blas.cc +++ b/tensorflow/stream_executor/rocm/rocm_blas.cc @@ -53,14 +53,14 @@ namespace wrap { #ifdef PLATFORM_GOOGLE #define STREAM_EXECUTOR_ROCBLAS_WRAP(__name) \ struct WrapperShim__##__name { \ - static const char *kName; \ + static const char* kName; \ template \ - rocblas_status operator()(GpuExecutor *parent, Args... args) { \ + rocblas_status operator()(GpuExecutor* parent, Args... args) { \ gpu::ScopedActivateExecutorContext sac{parent}; \ return ::__name(args...); \ } \ } __name; \ - const char *WrapperShim__##__name::kName = #__name; + const char* WrapperShim__##__name::kName = #__name; #define STREAM_EXECUTOR_ROCBLAS_V2_WRAP(__name) \ STREAM_EXECUTOR_ROCBLAS_WRAP(__name) @@ -69,14 +69,14 @@ namespace wrap { #define STREAM_EXECUTOR_ROCBLAS_WRAP(__name) \ struct DynLoadShim__##__name { \ - static const char *kName; \ + static const char* kName; \ using FuncPtrT = std::add_pointer::type; \ - static void *GetDsoHandle() { \ + static void* GetDsoHandle() { \ auto s = internal::CachedDsoLoader::GetRocblasDsoHandle(); \ return s.ValueOrDie(); \ } \ static FuncPtrT LoadOrDie() { \ - void *f; \ + void* f; \ auto s = port::Env::Default()->GetSymbolFromLibrary(GetDsoHandle(), \ kName, &f); \ CHECK(s.ok()) << "could not find " << kName \ @@ -88,12 +88,12 @@ namespace wrap { return f; \ } \ template \ - rocblas_status operator()(GpuExecutor *parent, Args... args) { \ + rocblas_status operator()(GpuExecutor* parent, Args... args) { \ gpu::ScopedActivateExecutorContext sac{parent}; \ return DynLoad()(args...); \ } \ } __name; \ - const char *DynLoadShim__##__name::kName = #__name; + const char* DynLoadShim__##__name::kName = #__name; #define STREAM_EXECUTOR_ROCBLAS_V2_WRAP(__name) \ STREAM_EXECUTOR_ROCBLAS_WRAP(__name) @@ -322,7 +322,7 @@ bool ROCMBlas::Init() { return true; } -ROCMBlas::ROCMBlas(gpu::GpuExecutor *parent) +ROCMBlas::ROCMBlas(gpu::GpuExecutor* parent) : parent_(CHECK_NOTNULL(parent)), blas_(nullptr) {} ROCMBlas::~ROCMBlas() { @@ -1476,11 +1476,11 @@ bool ROCMBlas::DoBlasTrsv(Stream *stream, blas::UpperLower uplo, return false; } -bool ROCMBlas::DoBlasGemm(Stream *stream, blas::Transpose transa, +bool ROCMBlas::DoBlasGemm(Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, uint64 n, uint64 k, - float alpha, const DeviceMemory &a, - int lda, const DeviceMemory &b, int ldb, - float beta, DeviceMemory *c, int ldc) { + float alpha, const DeviceMemory& a, + int lda, const DeviceMemory& b, int ldb, + float beta, DeviceMemory* c, int ldc) { VLOG(1) << absl::StreamFormat( "doing rocBLAS SGEMM: at=%d bt=%d m=%u n=%u " "k=%llu alpha=%f a=%p lda=%d b=%p ldb=%d beta=%f " @@ -1514,11 +1514,11 @@ bool ROCMBlas::DoBlasGemm(Stream *stream, blas::Transpose transa, return DoBlasInternal( wrap::rocblas_hgemm, stream, true /* = pointer_mode_host */, ROCMBlasTranspose(transa), ROCMBlasTranspose(transb), m, n, k, - reinterpret_cast(&alpha_half), - reinterpret_cast(GpuMemory(a)), lda, - reinterpret_cast(GpuMemory(b)), ldb, - reinterpret_cast(&beta_half), - reinterpret_cast(GpuMemoryMutable(c)), ldc); + reinterpret_cast(&alpha_half), + reinterpret_cast(GpuMemory(a)), lda, + reinterpret_cast(GpuMemory(b)), ldb, + reinterpret_cast(&beta_half), + reinterpret_cast(GpuMemoryMutable(c)), ldc); } bool ROCMBlas::DoBlasGemm(Stream *stream, blas::Transpose transa, @@ -1731,12 +1731,12 @@ bool ROCMBlas::GetBlasGemmAlgorithms( } bool ROCMBlas::DoBlasGemmWithAlgorithm( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, - uint64 n, uint64 k, const HostOrDeviceScalar &alpha, - const DeviceMemory &a, int lda, const DeviceMemory &b, int ldb, - const HostOrDeviceScalar &beta, DeviceMemory *c, int ldc, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + uint64 n, uint64 k, const HostOrDeviceScalar& alpha, + const DeviceMemory& a, int lda, const DeviceMemory& b, int ldb, + const HostOrDeviceScalar& beta, DeviceMemory* c, int ldc, blas::ComputationType computation_type, blas::AlgorithmType algorithm, - blas::ProfileResult *output_profile_result) { + blas::ProfileResult* output_profile_result) { LOG(ERROR) << "rocBLAS does not currently support the GEMMwithAlgorithm operation " << "for the \"int8\" dataype"; @@ -1744,13 +1744,13 @@ bool ROCMBlas::DoBlasGemmWithAlgorithm( } bool ROCMBlas::DoBlasGemmWithAlgorithm( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, - uint64 n, uint64 k, const HostOrDeviceScalar &alpha, - const DeviceMemory &a, int lda, - const DeviceMemory &b, int ldb, - const HostOrDeviceScalar &beta, DeviceMemory *c, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + uint64 n, uint64 k, const HostOrDeviceScalar& alpha, + const DeviceMemory& a, int lda, + const DeviceMemory& b, int ldb, + const HostOrDeviceScalar& beta, DeviceMemory* c, int ldc, blas::ComputationType computation_type, - blas::AlgorithmType algorithm, blas::ProfileResult *output_profile_result) { + blas::AlgorithmType algorithm, blas::ProfileResult* output_profile_result) { LOG(ERROR) << "rocBLAS does not currently support the GEMMwithAlgorithm operation " << "for the \"half\" dataype"; @@ -1758,12 +1758,12 @@ bool ROCMBlas::DoBlasGemmWithAlgorithm( } bool ROCMBlas::DoBlasGemmWithAlgorithm( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, - uint64 n, uint64 k, const HostOrDeviceScalar &alpha, - const DeviceMemory &a, int lda, const DeviceMemory &b, - int ldb, const HostOrDeviceScalar &beta, DeviceMemory *c, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + uint64 n, uint64 k, const HostOrDeviceScalar& alpha, + const DeviceMemory& a, int lda, const DeviceMemory& b, + int ldb, const HostOrDeviceScalar& beta, DeviceMemory* c, int ldc, blas::ComputationType computation_type, - blas::AlgorithmType algorithm, blas::ProfileResult *output_profile_result) { + blas::AlgorithmType algorithm, blas::ProfileResult* output_profile_result) { LOG(ERROR) << "rocBLAS does not currently support the GEMMwithAlgorithm operation " << "for the \"float\" dataype"; @@ -1771,12 +1771,12 @@ bool ROCMBlas::DoBlasGemmWithAlgorithm( } bool ROCMBlas::DoBlasGemmWithAlgorithm( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, - uint64 n, uint64 k, const HostOrDeviceScalar &alpha, - const DeviceMemory &a, int lda, const DeviceMemory &b, - int ldb, const HostOrDeviceScalar &beta, DeviceMemory *c, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + uint64 n, uint64 k, const HostOrDeviceScalar& alpha, + const DeviceMemory& a, int lda, const DeviceMemory& b, + int ldb, const HostOrDeviceScalar& beta, DeviceMemory* c, int ldc, blas::ComputationType computation_type, - blas::AlgorithmType algorithm, blas::ProfileResult *output_profile_result) { + blas::AlgorithmType algorithm, blas::ProfileResult* output_profile_result) { LOG(ERROR) << "rocBLAS does not currently support the GEMMwithAlgorithm operation " << "for the \"double\" dataype"; @@ -1815,14 +1815,14 @@ bool ROCMBlas::DoBlasGemmWithAlgorithm( template port::Status ROCMBlas::AllocateStridedBuffer( - const std::vector::mapped_type *> - &raw_ptrs, - int batch_count, uint64_t batch_stride, ScratchAllocator *scratch_allocator, - Stream *stream, + const std::vector::mapped_type*>& + raw_ptrs, + int batch_count, uint64_t batch_stride, ScratchAllocator* scratch_allocator, + Stream* stream, std::unique_ptr::mapped_type>> *temp_memory, - DeviceMemory::mapped_type> - *device_memory) { + typename RocBlasTypeConversionHelper::mapped_type>>* temp_memory, + DeviceMemory::mapped_type>* + device_memory) { assert(device_memory != nullptr); using MAPPED_T = typename RocBlasTypeConversionHelper::mapped_type; @@ -1969,12 +1969,12 @@ port::Status ROCMBlas::DoBlasGemmBatchedInternal( } bool ROCMBlas::DoBlasGemmBatched( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, uint64 n, uint64 k, float alpha, - const port::ArraySlice *> &a, int lda, - const port::ArraySlice *> &b, int ldb, float beta, - const port::ArraySlice *> &c, int ldc, - int batch_count, ScratchAllocator *scratch_allocator) { + const port::ArraySlice*>& a, int lda, + const port::ArraySlice*>& b, int ldb, float beta, + const port::ArraySlice*>& c, int ldc, + int batch_count, ScratchAllocator* scratch_allocator) { const Eigen::half alpha_half(alpha); const Eigen::half beta_half(beta); @@ -2299,7 +2299,7 @@ bool ROCMBlas::DoBlasTrsm(Stream *stream, blas::Side side, return DoBlasInternal( wrap::rocblas_strsm, stream, true /* = pointer_mode_host */, ROCMBlasSide(side), ROCMBlasUpperLower(uplo), ROCMBlasTranspose(transa), - ROCMBlasDiagonal(diag), m, n, &alpha, const_cast(GpuMemory(a)), + ROCMBlasDiagonal(diag), m, n, &alpha, const_cast(GpuMemory(a)), lda, GpuMemoryMutable(b), ldb); } @@ -2311,7 +2311,7 @@ bool ROCMBlas::DoBlasTrsm(Stream *stream, blas::Side side, return DoBlasInternal( wrap::rocblas_dtrsm, stream, true /* = pointer_mode_host */, ROCMBlasSide(side), ROCMBlasUpperLower(uplo), ROCMBlasTranspose(transa), - ROCMBlasDiagonal(diag), m, n, &alpha, const_cast(GpuMemory(a)), + ROCMBlasDiagonal(diag), m, n, &alpha, const_cast(GpuMemory(a)), lda, GpuMemoryMutable(b), ldb); } @@ -2349,19 +2349,19 @@ bool ROCMBlas::DoBlasGemmStridedBatched( wrap::rocblas_hgemm_strided_batched, stream, false, /* pointer_mode_host */ ROCMBlasTranspose(transa), ROCMBlasTranspose(transb), m, n, k, - reinterpret_cast(&alpha_half), - reinterpret_cast(GpuMemory(a)), lda, stride_a, - reinterpret_cast(GpuMemory(b)), ldb, stride_b, - reinterpret_cast(&beta_half), - reinterpret_cast(GpuMemoryMutable(c)), ldc, stride_c, + reinterpret_cast(&alpha_half), + reinterpret_cast(GpuMemory(a)), lda, stride_a, + reinterpret_cast(GpuMemory(b)), ldb, stride_b, + reinterpret_cast(&beta_half), + reinterpret_cast(GpuMemoryMutable(c)), ldc, stride_c, batch_count); } bool ROCMBlas::DoBlasGemmStridedBatched( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, - uint64 n, uint64 k, float alpha, const DeviceMemory &a, int lda, - int64 stride_a, const DeviceMemory &b, int ldb, int64 stride_b, - float beta, DeviceMemory *c, int ldc, int64 stride_c, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + uint64 n, uint64 k, float alpha, const DeviceMemory& a, int lda, + int64 stride_a, const DeviceMemory& b, int ldb, int64 stride_b, + float beta, DeviceMemory* c, int ldc, int64 stride_c, int batch_count) { return DoBlasInternal(wrap::rocblas_sgemm_strided_batched, stream, false, /* pointer_mode_host */ @@ -2371,10 +2371,10 @@ bool ROCMBlas::DoBlasGemmStridedBatched( stride_c, batch_count); } bool ROCMBlas::DoBlasGemmStridedBatched( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, - uint64 n, uint64 k, double alpha, const DeviceMemory &a, int lda, - int64 stride_a, const DeviceMemory &b, int ldb, int64 stride_b, - double beta, DeviceMemory *c, int ldc, int64 stride_c, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + uint64 n, uint64 k, double alpha, const DeviceMemory& a, int lda, + int64 stride_a, const DeviceMemory& b, int ldb, int64 stride_b, + double beta, DeviceMemory* c, int ldc, int64 stride_c, int batch_count) { return DoBlasInternal(wrap::rocblas_dgemm_strided_batched, stream, false, /* pointer_mode_host */ @@ -2384,11 +2384,11 @@ bool ROCMBlas::DoBlasGemmStridedBatched( stride_c, batch_count); } bool ROCMBlas::DoBlasGemmStridedBatched( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, uint64 n, uint64 k, std::complex alpha, - const DeviceMemory> &a, int lda, int64 stride_a, - const DeviceMemory> &b, int ldb, int64 stride_b, - std::complex beta, DeviceMemory> *c, int ldc, + const DeviceMemory>& a, int lda, int64 stride_a, + const DeviceMemory>& b, int ldb, int64 stride_b, + std::complex beta, DeviceMemory>* c, int ldc, int64 stride_c, int batch_count) { LOG(ERROR) << "rocBLAS does not currently support the " "DoBlasGemmStridedBatched operation " @@ -2396,11 +2396,11 @@ bool ROCMBlas::DoBlasGemmStridedBatched( return false; } bool ROCMBlas::DoBlasGemmStridedBatched( - Stream *stream, blas::Transpose transa, blas::Transpose transb, uint64 m, + Stream* stream, blas::Transpose transa, blas::Transpose transb, uint64 m, uint64 n, uint64 k, std::complex alpha, - const DeviceMemory> &a, int lda, int64 stride_a, - const DeviceMemory> &b, int ldb, int64 stride_b, - std::complex beta, DeviceMemory> *c, int ldc, + const DeviceMemory>& a, int lda, int64 stride_a, + const DeviceMemory>& b, int ldb, int64 stride_b, + std::complex beta, DeviceMemory>* c, int ldc, int64 stride_c, int batch_count) { LOG(ERROR) << "rocBLAS does not currently support the " "DoBlasGemmStridedBatched operation " @@ -2418,10 +2418,10 @@ void initialize_rocblas() { PluginRegistry::Instance() ->RegisterFactory( rocm::kROCmPlatformId, gpu::kRocBlasPlugin, "rocBLAS", - [](internal::StreamExecutorInterface *parent) - -> blas::BlasSupport * { - gpu::GpuExecutor *rocm_executor = - dynamic_cast(parent); + [](internal::StreamExecutorInterface* parent) + -> blas::BlasSupport* { + gpu::GpuExecutor* rocm_executor = + dynamic_cast(parent); if (rocm_executor == nullptr) { LOG(ERROR) << "Attempting to initialize an instance of the " @@ -2430,7 +2430,7 @@ void initialize_rocblas() { return nullptr; } - gpu::ROCMBlas *blas = new gpu::ROCMBlas(rocm_executor); + gpu::ROCMBlas* blas = new gpu::ROCMBlas(rocm_executor); if (!blas->Init()) { // Note: Init() will log a more specific error. delete blas; diff --git a/tensorflow/stream_executor/rocm/rocm_blas.h b/tensorflow/stream_executor/rocm/rocm_blas.h index 1b73a356b88..87e7d6717f3 100644 --- a/tensorflow/stream_executor/rocm/rocm_blas.h +++ b/tensorflow/stream_executor/rocm/rocm_blas.h @@ -62,7 +62,7 @@ class GpuExecutor; // Thread-safe post-initialization. class ROCMBlas : public blas::BlasSupport { public: - explicit ROCMBlas(GpuExecutor *parent); + explicit ROCMBlas(GpuExecutor* parent); // Allocates a rocBLAS handle. bool Init(); @@ -98,7 +98,7 @@ class ROCMBlas : public blas::BlasSupport { // Convenience functions that call DoBlasInternalImpl with different values // for err_on_failure. template - bool DoBlasInternal(FuncT rocblas_func, Stream *stream, + bool DoBlasInternal(FuncT rocblas_func, Stream* stream, bool pointer_mode_host, Args... args) { return DoBlasInternalImpl(rocblas_func, stream, pointer_mode_host, /*err_on_failure=*/true, args...); @@ -114,14 +114,14 @@ class ROCMBlas : public blas::BlasSupport { // strided flavor template port::Status AllocateStridedBuffer( - const std::vector::mapped_type *> - &raw_ptrs, + const std::vector::mapped_type*>& + raw_ptrs, int batch_count, uint64_t batch_stride, - ScratchAllocator *scratch_allocator, Stream *stream, + ScratchAllocator* scratch_allocator, Stream* stream, std::unique_ptr::mapped_type>> *temp_memory, - DeviceMemory::mapped_type> - *device_memory); + typename RocBlasTypeConversionHelper::mapped_type>>* temp_memory, + DeviceMemory::mapped_type>* + device_memory); // A helper function to implement DoBlasGemmBatched interfaces for generic // types. @@ -185,7 +185,7 @@ class ROCMBlas : public blas::BlasSupport { // GpuExecutor which instantiated this ROCMBlas. // Immutable post-initialization. - GpuExecutor *parent_; + GpuExecutor* parent_; // rocBLAS library handle on the device. rocblas_handle blas_ GUARDED_BY(mu_); diff --git a/tensorflow/stream_executor/rocm/rocm_driver_wrapper.h b/tensorflow/stream_executor/rocm/rocm_driver_wrapper.h index ba803edaafb..bc5b6a87888 100644 --- a/tensorflow/stream_executor/rocm/rocm_driver_wrapper.h +++ b/tensorflow/stream_executor/rocm/rocm_driver_wrapper.h @@ -27,10 +27,6 @@ limitations under the License. #include "tensorflow/stream_executor/platform/dso_loader.h" #include "tensorflow/stream_executor/platform/port.h" -#if defined(TENSORFLOW_USE_ROCM) - -#endif - namespace tensorflow { namespace wrap { #ifdef PLATFORM_GOOGLE @@ -83,8 +79,8 @@ namespace wrap { __macro(hipDeviceTotalMem) \ __macro(hipDriverGetVersion) \ __macro(hipEventCreateWithFlags) \ - __macro(hipEventElapsedTime) \ __macro(hipEventDestroy) \ + __macro(hipEventElapsedTime) \ __macro(hipEventQuery) \ __macro(hipEventRecord) \ __macro(hipEventSynchronize) \ diff --git a/tensorflow/stream_executor/rocm/rocm_fft.cc b/tensorflow/stream_executor/rocm/rocm_fft.cc index 2af973309c0..8ad5cc19ded 100644 --- a/tensorflow/stream_executor/rocm/rocm_fft.cc +++ b/tensorflow/stream_executor/rocm/rocm_fft.cc @@ -48,7 +48,7 @@ namespace wrap { #define STREAM_EXECUTOR_ROCFFT_WRAP(__name) \ struct WrapperShim__##__name { \ template \ - hipfftResult operator()(GpuExecutor *parent, Args... args) { \ + hipfftResult operator()(GpuExecutor* parent, Args... args) { \ gpu::ScopedActivateExecutorContext sac{parent}; \ return ::__name(args...); \ } \ @@ -86,21 +86,33 @@ namespace wrap { #endif -#define ROCFFT_ROUTINE_EACH(__macro) \ - __macro(hipfftDestroy) __macro(hipfftSetStream) __macro(hipfftPlan1d) \ - __macro(hipfftPlan2d) __macro(hipfftPlan3d) __macro(hipfftPlanMany) \ - __macro(hipfftCreate) __macro(hipfftSetAutoAllocation) \ - __macro(hipfftSetWorkArea) __macro(hipfftGetSize1d) \ - __macro(hipfftMakePlan1d) __macro(hipfftGetSize2d) \ - __macro(hipfftMakePlan2d) __macro(hipfftGetSize3d) \ - __macro(hipfftMakePlan3d) __macro(hipfftGetSizeMany) \ - __macro(hipfftMakePlanMany) \ - __macro(hipfftExecD2Z) \ - __macro(hipfftExecZ2D) \ - __macro(hipfftExecC2C) \ - __macro(hipfftExecC2R) \ - __macro(hipfftExecZ2Z) \ - __macro(hipfftExecR2C) +// clang-format off +#define ROCFFT_ROUTINE_EACH(__macro) \ + __macro(hipfftDestroy) \ + __macro(hipfftSetStream) \ + __macro(hipfftPlan1d) \ + __macro(hipfftPlan2d) \ + __macro(hipfftPlan3d) \ + __macro(hipfftPlanMany) \ + __macro(hipfftCreate) \ + __macro(hipfftSetAutoAllocation) \ + __macro(hipfftSetWorkArea) \ + __macro(hipfftGetSize1d) \ + __macro(hipfftMakePlan1d) \ + __macro(hipfftGetSize2d) \ + __macro(hipfftMakePlan2d) \ + __macro(hipfftGetSize3d) \ + __macro(hipfftMakePlan3d) \ + __macro(hipfftGetSizeMany) \ + __macro(hipfftMakePlanMany) \ + __macro(hipfftExecD2Z) \ + __macro(hipfftExecZ2D) \ + __macro(hipfftExecC2C) \ + __macro(hipfftExecC2R) \ + __macro(hipfftExecZ2Z) \ + __macro(hipfftExecR2C) \ + +// clang-format on ROCFFT_ROUTINE_EACH(STREAM_EXECUTOR_ROCFFT_WRAP) @@ -515,7 +527,7 @@ bool ROCMFft::DoFftInternal(Stream *stream, fft::Plan *plan, FuncT hipfftExec, } auto ret = hipfftExec(parent_, rocm_fft_plan->GetPlan(), - GpuComplex(const_cast(GpuMemory(input))), + GpuComplex(const_cast(GpuMemory(input))), GpuComplex(GpuMemoryMutable(output))); if (ret != HIPFFT_SUCCESS) { @@ -542,7 +554,7 @@ bool ROCMFft::DoFftWithDirectionInternal(Stream *stream, fft::Plan *plan, } auto ret = hipfftExec(parent_, rocm_fft_plan->GetPlan(), - GpuComplex(const_cast(GpuMemory(input))), + GpuComplex(const_cast(GpuMemory(input))), GpuComplex(GpuMemoryMutable(output)), rocm_fft_plan->GetFftDirection()); @@ -556,21 +568,21 @@ bool ROCMFft::DoFftWithDirectionInternal(Stream *stream, fft::Plan *plan, #define STREAM_EXECUTOR_ROCM_DEFINE_FFT(__type, __fft_type1, __fft_type2, \ __fft_type3) \ - bool ROCMFft::DoFft(Stream *stream, fft::Plan *plan, \ - const DeviceMemory> &input, \ - DeviceMemory> *output) { \ + bool ROCMFft::DoFft(Stream* stream, fft::Plan* plan, \ + const DeviceMemory>& input, \ + DeviceMemory>* output) { \ return DoFftWithDirectionInternal( \ stream, plan, wrap::hipfftExec##__fft_type1, input, output); \ } \ - bool ROCMFft::DoFft(Stream *stream, fft::Plan *plan, \ - const DeviceMemory<__type> &input, \ - DeviceMemory> *output) { \ + bool ROCMFft::DoFft(Stream* stream, fft::Plan* plan, \ + const DeviceMemory<__type>& input, \ + DeviceMemory>* output) { \ return DoFftInternal(stream, plan, wrap::hipfftExec##__fft_type2, input, \ output); \ } \ - bool ROCMFft::DoFft(Stream *stream, fft::Plan *plan, \ - const DeviceMemory> &input, \ - DeviceMemory<__type> *output) { \ + bool ROCMFft::DoFft(Stream* stream, fft::Plan* plan, \ + const DeviceMemory>& input, \ + DeviceMemory<__type>* output) { \ return DoFftInternal(stream, plan, wrap::hipfftExec##__fft_type3, input, \ output); \ } @@ -590,9 +602,9 @@ void initialize_rocfft() { port::Status status = PluginRegistry::Instance()->RegisterFactory( rocm::kROCmPlatformId, gpu::kRocFftPlugin, "rocFFT", - [](internal::StreamExecutorInterface *parent) -> fft::FftSupport * { - gpu::GpuExecutor *rocm_executor = - dynamic_cast(parent); + [](internal::StreamExecutorInterface* parent) -> fft::FftSupport* { + gpu::GpuExecutor* rocm_executor = + dynamic_cast(parent); if (rocm_executor == nullptr) { LOG(ERROR) << "Attempting to initialize an instance of the rocFFT " diff --git a/tensorflow/stream_executor/rocm/rocm_rng.cc b/tensorflow/stream_executor/rocm/rocm_rng.cc index 38f4f8bb0c6..2492cc0e5d9 100644 --- a/tensorflow/stream_executor/rocm/rocm_rng.cc +++ b/tensorflow/stream_executor/rocm/rocm_rng.cc @@ -14,12 +14,11 @@ limitations under the License. ==============================================================================*/ #include "rocm/include/hiprand/hiprand.h" -#include "tensorflow/stream_executor/gpu/gpu_rng.h" - #include "tensorflow/stream_executor/device_memory.h" #include "tensorflow/stream_executor/gpu/gpu_activation.h" #include "tensorflow/stream_executor/gpu/gpu_executor.h" #include "tensorflow/stream_executor/gpu/gpu_helpers.h" +#include "tensorflow/stream_executor/gpu/gpu_rng.h" #include "tensorflow/stream_executor/gpu/gpu_stream.h" #include "tensorflow/stream_executor/lib/env.h" #include "tensorflow/stream_executor/lib/initialize.h" From 619f8bcc35c0efb0f3af9d092cbb08bb1dbb0267 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Tue, 2 Jul 2019 03:51:45 +0000 Subject: [PATCH 041/332] sync changes from the rocm repo --- .../stream_executor/rocm/rocm_gpu_executor.cc | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc b/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc index 98bc68929b9..e37d6d24232 100644 --- a/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc +++ b/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc @@ -110,6 +110,7 @@ GpuExecutor::~GpuExecutor() { if (context_ != nullptr) { GpuDriver::DestroyContext(context_); } + CHECK(kernel_to_gpu_binary_.empty()) << "GpuExecutor has live kernels."; CHECK(gpu_binary_to_module_.empty()) << "GpuExecutor has loaded modules."; } bool GpuExecutor::UnloadModule(ModuleHandle module_handle) { @@ -136,7 +137,19 @@ bool GpuExecutor::UnloadGpuBinary(const void* gpu_binary) { } void GpuExecutor::UnloadKernel(const KernelBase* kernel) { - LOG(FATAL) << "Feature not supported on ROCM platform (UnloadKernel)"; + VLOG(3) << "Unloading kernel " << kernel << " : " << kernel->name(); + + absl::MutexLock lock{&in_memory_modules_mu_}; + auto gpu_binary_it = kernel_to_gpu_binary_.find(kernel); + if (kernel_to_gpu_binary_.end() == gpu_binary_it) { + VLOG(3) << "Kernel " << kernel << " : " << kernel->name() + << " has never been loaded."; + return; // We've never seen this kernel. + } + VLOG(3) << "Kernel " << kernel << " : " << kernel->name() + << " has loaded GPU code " << gpu_binary_it->second; + UnloadGpuBinary(gpu_binary_it->second); + kernel_to_gpu_binary_.erase(gpu_binary_it); } port::Status GpuExecutor::Init(int device_ordinal, @@ -244,8 +257,8 @@ bool GpuExecutor::GetKernel(const MultiKernelLoaderSpec& spec, LOG(ERROR) << "failed to load HSACO\n"; return false; } - in_memory_modules_[hsaco] = module; } + kernel_to_gpu_binary_[kernel] = hsaco; } else { LOG(WARNING) << "no method of loading ROCM kernel provided"; return false; @@ -401,6 +414,7 @@ bool GpuExecutor::LoadModuleFromHsaco(const char* hsaco, hipModule_t* module) { return false; } module_refcount = 1; + in_memory_modules_[hsaco] = *module; VLOG(3) << "Loaded HSACO " << static_cast(hsaco) << " as module " << *module; } else { @@ -765,29 +779,6 @@ bool GpuExecutor::DeviceMemoryUsage(int64* free, int64* total) const { bool GpuExecutor::GetSymbol(const string& symbol_name, ModuleHandle module_handle, void** mem, size_t* bytes) { - { // give limited scope to lock - absl::MutexLock lock{&disk_modules_mu_}; - for (auto& it : disk_modules_) { - if (GpuDriver::GetModuleSymbol(context_, it.second, symbol_name.c_str(), - reinterpret_cast(mem), - bytes)) { - return true; - } - } - } - - { // give limited scope to lock - absl::MutexLock lock{&in_memory_modules_mu_}; - for (auto& it : in_memory_modules_) { - if (GpuDriver::GetModuleSymbol(context_, it.second, symbol_name.c_str(), - reinterpret_cast(mem), - bytes)) { - return true; - } - } - } - - { // give limited scope to lock absl::MutexLock lock{&in_memory_modules_mu_}; if (static_cast(module_handle)) { auto it = gpu_binary_to_module_.find(module_handle.id()); @@ -806,7 +797,6 @@ bool GpuExecutor::GetSymbol(const string& symbol_name, return true; } } - } LOG(INFO) << "Falied to find symbol in any modules: " << symbol_name; return false; From 1e24623dcabbd76336d5bca0d72e8fefad80ca32 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Tue, 2 Jul 2019 03:52:52 +0000 Subject: [PATCH 042/332] sync changes related to adding support for 3D convolutions --- tensorflow/stream_executor/rocm/rocm_dnn.cc | 87 +++++++++++---------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/tensorflow/stream_executor/rocm/rocm_dnn.cc b/tensorflow/stream_executor/rocm/rocm_dnn.cc index 4a0df0af171..efe49ddcf3f 100644 --- a/tensorflow/stream_executor/rocm/rocm_dnn.cc +++ b/tensorflow/stream_executor/rocm/rocm_dnn.cc @@ -115,7 +115,6 @@ class MIOpenHandle { namespace wrap { #ifdef PLATFORM_GOOGLE - #define STREAM_EXECUTOR_MIOPEN_WRAP(__name) \ struct WrapperShim__##__name { \ template \ @@ -162,6 +161,7 @@ namespace wrap { __macro(miopenBatchNormalizationForwardInference) \ __macro(miopenBatchNormalizationForwardTraining) \ __macro(miopenGetConvolutionForwardOutputDim) \ + __macro(miopenGetConvolutionNdForwardOutputDim) \ __macro(miopenFindConvolutionForwardAlgorithm) \ __macro(miopenCreateTensorDescriptor) \ __macro(miopenDestroyTensorDescriptor) \ @@ -183,7 +183,9 @@ namespace wrap { __macro(miopenConvolutionBackwardBias) \ __macro(miopenConvolutionForwardGetWorkSpaceSize) \ __macro(miopenInitConvolutionDescriptor) \ + __macro(miopenInitConvolutionNdDescriptor) \ __macro(miopenGetConvolutionDescriptor) \ + __macro(miopenGetConvolutionNdDescriptor) \ __macro(miopenSetConvolutionGroupCount) \ __macro(miopenSet4dTensorDescriptor) \ __macro(miopenGetTensorDescriptor) \ @@ -282,28 +284,29 @@ uint64 GetHashValue(miopenTensorDescriptor_t tensor_desc) { uint64 GetHashValue(miopenConvolutionDescriptor_t conv_desc) { miopenConvolutionMode_t c_mode = miopenConvolution; - int pad_h = 0, pad_w = 0, u = 0, v = 0, dilation_h = 0, dilation_w = 0; - wrap::miopenGetConvolutionDescriptor(conv_desc, &c_mode, &pad_h, &pad_w, &u, - &v, &dilation_h, &dilation_w); + int nd = 0; + wrap::miopenGetConvolutionNdDescriptor(conv_desc, 0, &nd, nullptr, nullptr, + nullptr, &c_mode); + + std::vector stride(nd); + std::vector pad(nd); + std::vector dilation(nd); + + wrap::miopenGetConvolutionNdDescriptor( + conv_desc, nd, &nd, pad.data(), stride.data(), dilation.data(), &c_mode); uint64 hash_value = tensorflow::hash()(c_mode); - hash_value = - tensorflow::Hash64Combine(hash_value, tensorflow::hash()(pad_h)); - hash_value = - tensorflow::Hash64Combine(hash_value, tensorflow::hash()(pad_w)); - hash_value = - tensorflow::Hash64Combine(hash_value, tensorflow::hash()(u)); - hash_value = - tensorflow::Hash64Combine(hash_value, tensorflow::hash()(v)); - hash_value = tensorflow::Hash64Combine(hash_value, - tensorflow::hash()(dilation_h)); - hash_value = tensorflow::Hash64Combine(hash_value, - tensorflow::hash()(dilation_w)); + auto hash64Combine = [&hash_value](int element) { + tensorflow::Hash64Combine(hash_value, tensorflow::hash()(element)); + }; + std::for_each(pad.begin(), pad.end(), hash64Combine); + std::for_each(stride.begin(), stride.end(), hash64Combine); + std::for_each(dilation.begin(), dilation.end(), hash64Combine); return hash_value; } -// Class to implement a cache of compiled fusion plans. +// Class to implement a cache of compiled fusion plans class CachedFusionPlans { public: // Check if we already have a fusion_plan corresponding to the given hash @@ -340,7 +343,7 @@ class CachedFusionPlans { return found_cached_plan; } - // Need to figure out the right place to call this routine. + // Need to figure out the right place to call this routine static void Clear() { absl::MutexLock lock{&cached_plans_mutex}; @@ -357,24 +360,24 @@ class CachedFusionPlans { unsupported_plans.clear(); } - // Is the Fusion plan corresponding to this hash unsupported. + // Is the Fusion plan corresponding to this hash unsupported static bool IsUnsupportedFusionPlan(uint64 hash) { absl::MutexLock lock{&cached_plans_mutex}; return unsupported_plans.count(hash) > 0; } - // Mark the given hash value as corresponding to an unsupported fusion plan. + // Mark the given hash value as corresponding to an unsupported fusion plan static void MarkFusionPlanUnsupported(uint64 hash) { absl::MutexLock lock{&cached_plans_mutex}; unsupported_plans.insert(hash); } private: - // Mutex to guard access to all data within this class. + // Mutex to guard access to all data within this class static absl::Mutex cached_plans_mutex; - // Map of hash-value to MIOpen Fusion plan descriptors. - // Need to be able share this across more than one stream and hence static. + // Map of hash-value to MIOpen Fusion plan descriptors + // Need to be able share this across more than one stream and hence static static std::map cached_plans; // Set of hash-values that correspond to MIOpen Fusion plans that will fail @@ -386,6 +389,10 @@ absl::Mutex CachedFusionPlans::cached_plans_mutex; std::map CachedFusionPlans::cached_plans; std::set CachedFusionPlans::unsupported_plans; +} // namespace + +namespace { + miopenHandle_t ToHandle(void* opaque_handle) { return static_cast(opaque_handle); } @@ -538,10 +545,6 @@ class ScopedTensorDescriptor { case dnn::DataLayout::kBatchYXDepth: case dnn::DataLayout::kBatchDepthYX: { const int nd = batch_descriptor.ndims() + 2; - if (nd != 4) { - LOG(FATAL) << "miopen only supports 4D tensors, dim=" << nd - << " not allowed"; - } // MIOpen requires the strides and dims to be ordered as BDYX. std::vector strides64 = @@ -556,8 +559,8 @@ class ScopedTensorDescriptor { &CheckedNarrowing); std::transform(dims64.cbegin(), dims64.cend(), dims.begin(), &CheckedNarrowing); - status = wrap::miopenSet4dTensorDescriptor(handle_, elem_type, dims[0], - dims[1], dims[2], dims[3]); + status = wrap::miopenSetTensorDescriptor(handle_, elem_type, nd, + dims.data(), strides.data()); if (status != miopenStatusSuccess) { LOG(FATAL) << "could not convert BatchDescriptor " @@ -604,19 +607,14 @@ class ScopedFilterDescriptor { const int nd = batch_descriptor.ndims() + 2; - if (nd != 4) { - LOG(FATAL) << "miopen only supports 4D filters, dim=" << nd - << "not allowed" << ToString(status); - } - std::vector dims(2 + filter_descriptor.ndims()); dims[0] = filter_descriptor.output_feature_map_count(); dims[1] = filter_descriptor.input_feature_map_count(); const auto& spatial_dims = filter_descriptor.input_filter_dims(); std::copy(spatial_dims.begin(), spatial_dims.end(), dims.begin() + 2); - status = wrap::miopenSet4dTensorDescriptor(handle_, elem_type, dims[0], - dims[1], dims[2], dims[3]); + status = wrap::miopenSetTensorDescriptor(handle_, elem_type, nd, + dims.data(), nullptr); if (status != miopenStatusSuccess) { LOG(FATAL) << "could not set miopen filter descriptor: " << ToString(status); @@ -667,11 +665,15 @@ class ScopedConvolutionDescriptor { &CheckedNarrowing); std::transform(padding64.cbegin(), padding64.cend(), padding.begin(), &CheckedNarrowing); - std::vector upscale(convolution_descriptor.ndims(), 1); - status = wrap::miopenInitConvolutionDescriptor( - handle_, miopenConvolution, padding[0], padding[1], strides[0], - strides[1], upscale[0], upscale[1]); + std::vector upscale(convolution_descriptor.ndims()); + const auto& dilations64 = convolution_descriptor.dilations(); + std::transform(dilations64.cbegin(), dilations64.cend(), upscale.begin(), + &CheckedNarrowing); + + status = wrap::miopenInitConvolutionNdDescriptor( + handle_, convolution_descriptor.ndims(), padding.data(), strides.data(), + upscale.data(), miopenConvolution); if (status != miopenStatusSuccess) { LOG(FATAL) << "could not set miopen convolution descriptor: " << ToString(status); @@ -4003,9 +4005,8 @@ bool MIOpenSupport::DeriveOutputBatchDescriptor( int dn = batch_descriptor.ndims() + 2; std::vector dims(dn); // in BDYX - auto status = wrap::miopenGetConvolutionForwardOutputDim( - conv.handle(), input_nd.handle(), filter.handle(), &dims[0], &dims[1], - &dims[2], &dims[3]); + auto status = wrap::miopenGetConvolutionNdForwardOutputDim( + conv.handle(), input_nd.handle(), filter.handle(), &dn, dims.data()); if (status != miopenStatusSuccess) { LOG(ERROR) << "could not get output tensor for convolution: " << ToString(status); From 3b8c945be172c164a1f3c5046126918c60bf34bc Mon Sep 17 00:00:00 2001 From: Rasmus Diederichsen Date: Tue, 2 Jul 2019 09:10:00 +0200 Subject: [PATCH 043/332] Remove empty lines --- tensorflow/contrib/makefile/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index e2e665e0479..090df268915 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -920,8 +920,6 @@ ifeq ($(TARGET),IOS) endif endif - - # Runs proto_text to generate C++ source files from protos. $(PROTO_TEXT): $(PROTO_TEXT_OBJS) $(PROTO_TEXT_PB_H_FILES) @mkdir -p $(dir $@) From 4eabf5215b66c584a659a1533247a7022a51d416 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Tue, 2 Jul 2019 10:45:23 -0700 Subject: [PATCH 044/332] Refactor CacheDatasetOps --- .../core/kernels/data/cache_dataset_ops.cc | 1707 +++++++++-------- .../core/kernels/data/cache_dataset_ops.h | 45 + tensorflow/core/kernels/data/name_utils.cc | 7 +- tensorflow/core/kernels/data/name_utils.h | 1 + 4 files changed, 916 insertions(+), 844 deletions(-) create mode 100644 tensorflow/core/kernels/data/cache_dataset_ops.h diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc index 341a02cc259..43a7bfcb2f5 100644 --- a/tensorflow/core/kernels/data/cache_dataset_ops.cc +++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc @@ -12,10 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/framework/dataset.h" +#include "tensorflow/core/kernels/data/cache_dataset_ops.h" + #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/resource_mgr.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/kernels/data/name_utils.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/env.h" @@ -23,117 +25,245 @@ limitations under the License. namespace tensorflow { namespace data { -namespace { // See documentation in ../../ops/dataset_ops.cc for a high-level description of // the following op. -class CacheDatasetOp : public UnaryDatasetOpKernel { +/* static */ constexpr const char* const CacheDatasetOp::kDatasetType; +/* static */ constexpr const char* const CacheDatasetOp::kInputDataset; +/* static */ constexpr const char* const CacheDatasetOp::kFileName; +/* static */ constexpr const char* const CacheDatasetOp::kOutputTypes; +/* static */ constexpr const char* const CacheDatasetOp::kOutputShapes; + +constexpr char kTensorStrFormat[] = "%%%zuzu_%%%zuzu"; +constexpr char kPaddingSizeStrFormat[] = "%zu"; +constexpr char kFileDatasetPrefix[] = "File"; +constexpr char kMode[] = "Mode"; +constexpr char kLockFileSuffix[] = ".lockfile"; +constexpr char kIterationCompleted[] = "iteration_completed"; +constexpr char kCurIndex[] = "cur_index"; +constexpr char kShardId[] = "shard_id"; +constexpr char kCreatedAt[] = "Created at"; +constexpr char kMemoryDatasetPrefix[] = "Memory"; +constexpr char kMemoryCache[] = "MemoryCache"; +constexpr char kTFData[] = "tf_data"; +constexpr char kCacheClaimed[] = "cache_claimed"; +constexpr char kCacheSize[] = "cache_size"; +constexpr char kCache[] = "cache"; +constexpr char kSizeSuffix[] = ".size"; +constexpr char kCacheCompleted[] = "cache_completed"; +constexpr char kIndex[] = "index"; +constexpr char kImpl[] = "Impl"; + +class CacheDatasetOp::FileDataset : public DatasetBase { public: - explicit CacheDatasetOp(OpKernelConstruction* ctx) - : UnaryDatasetOpKernel(ctx) {} + explicit FileDataset(OpKernelContext* ctx, const DatasetBase* input, + string filename, Env* env) + : DatasetBase(DatasetContext(ctx)), + input_(input), + filename_(std::move(filename)), + env_(env), + num_tensors_(input->output_dtypes().size()), + tensor_index_padding_size_(StringPaddingSize(num_tensors_)), + item_index_padding_size_(StringPaddingSize(kMaxItems)), + tensor_format_string_(strings::Printf(kTensorStrFormat, + item_index_padding_size_, + tensor_index_padding_size_)) { + input_->Ref(); + DCHECK_EQ(item_index_padding_size_, 7); + } - void MakeDataset(OpKernelContext* ctx, DatasetBase* input, - DatasetBase** output) override { - // Parse out the filenames tensor. - string filename; - OP_REQUIRES_OK(ctx, - ParseScalarArgument(ctx, "filename", &filename)); + ~FileDataset() override { input_->Unref(); } - if (filename.empty()) { - *output = new MemoryDataset(ctx, input); - } else { - *output = new FileDataset(ctx, input, filename, ctx->env()); - } + std::unique_ptr MakeIteratorInternal( + const string& prefix) const override { + name_utils::IteratorPrefixParams params; + params.dataset_prefix = kFileDatasetPrefix; + return absl::make_unique(FileIterator::Params{ + this, name_utils::IteratorPrefix(kDatasetType, prefix, params)}); + } + + const DataTypeVector& output_dtypes() const override { + return input_->output_dtypes(); + } + + const std::vector& output_shapes() const override { + return input_->output_shapes(); + } + + string DebugString() const override { + name_utils::DatasetDebugStringParams params; + params.dataset_prefix = kFileDatasetPrefix; + return name_utils::DatasetDebugString(kDatasetType, params); + } + + int64 Cardinality() const override { return input_->Cardinality(); } + + protected: + Status AsGraphDefInternal(SerializationContext* ctx, + DatasetGraphDefBuilder* b, + Node** output) const override { + Node* input_graph = nullptr; + TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph)); + Node* filename = nullptr; + TF_RETURN_IF_ERROR(b->AddScalar(filename_, &filename)); + TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph, filename}, output)); + return Status::OK(); } private: - class FileDataset : public DatasetBase { + static size_t StringPaddingSize(size_t num_tensors) { + return strings::Printf(kPaddingSizeStrFormat, num_tensors - 1).size(); + } + + string FormatName(size_t item_index, size_t tensor_index) const { + return strings::Printf(tensor_format_string_.c_str(), item_index, + tensor_index); + } + + class FileIterator : public DatasetIterator { public: - explicit FileDataset(OpKernelContext* ctx, const DatasetBase* input, - string filename, Env* env) - : DatasetBase(DatasetContext(ctx)), - input_(input), - filename_(std::move(filename)), - env_(env), - num_tensors_(input->output_dtypes().size()), - tensor_index_padding_size_(StringPaddingSize(num_tensors_)), - item_index_padding_size_(StringPaddingSize(kMaxItems)), - tensor_format_string_(strings::Printf("%%%zuzu_%%%zuzu", - item_index_padding_size_, - tensor_index_padding_size_)) { - input_->Ref(); - DCHECK_EQ(item_index_padding_size_, 7); + explicit FileIterator(const Params& params) + : DatasetIterator(params) { + if (params.dataset->env_ + ->FileExists(MetaFilename(params.dataset->filename_)) + .ok()) { + mode_ = Mode::read; + } else { + mode_ = Mode::write; + } + InitializeIterator(); } - ~FileDataset() override { input_->Unref(); } - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return absl::make_unique( - FileIterator::Params{this, strings::StrCat(prefix, "::FileCache")}); + Status Initialize(IteratorContext* ctx) override { + mutex_lock l(mu_); + return iterator_->Initialize(ctx); } - const DataTypeVector& output_dtypes() const override { - return input_->output_dtypes(); + Status GetNextInternal(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) override { + mutex_lock l(mu_); + return iterator_->GetNext(ctx, out_tensors, end_of_sequence); } - const std::vector& output_shapes() const override { - return input_->output_shapes(); - } - - string DebugString() const override { - return "CacheDatasetOp::FileDataset"; - } - - int64 Cardinality() const override { return input_->Cardinality(); } - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - Node* input_graph = nullptr; - TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph)); - Node* filename = nullptr; - TF_RETURN_IF_ERROR(b->AddScalar(filename_, &filename)); - TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph, filename}, output)); - return Status::OK(); + std::shared_ptr CreateNode( + IteratorContext* ctx, model::Node::Args args) const override { + return model::MakeKnownRatioNode(std::move(args), + /*ratio=*/1); + } + + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name(kMode), mode_)); + return SaveInput(writer, iterator_); + } + Status RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + { + int64 temp; + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name(kMode), &temp)); + mode_ = static_cast(temp); + } + if (mode_ == Mode::write && + dataset() + ->env_->FileExists(MetaFilename(dataset()->filename_)) + .ok()) { + // This could happen if the cache was completely written after the + // checkpoint was saved. + LOG(WARNING) + << "It looks like the cache was already completely written(" + << MetaFilename(dataset()->filename_) + << ") after the last checkpoint was saved. Attempting to read " + << "the cache instead of continuing to write. If this is a " + << "mistake, please remove the above file and try running again."; + mode_ = Mode::read; + } + InitializeIterator(); + TF_RETURN_IF_ERROR(iterator_->Initialize(ctx)); + return RestoreInput(ctx, reader, iterator_); } private: - static size_t StringPaddingSize(size_t num_tensors) { - return strings::Printf("%zu", num_tensors - 1).size(); - } - - string FormatName(size_t item_index, size_t tensor_index) const { - return strings::Printf(tensor_format_string_.c_str(), item_index, - tensor_index); - } - - class FileIterator : public DatasetIterator { + // FileWriterIterator passes through and caches items from the input + // FileDataset. + // + // This iterator is used when the cache directory is not found on disk. It + // creates the cache directory, and passes on the underlying iterator's + // elements. + // + // Caching is performed by writing the input tensors to disk using the + // `BundleWriter`. Note that the cache gets fully flushed to disk only + // after the input iterator has been fully exhausted. If the program + // exits, before completion of an epoch, the cached state would be lost. + // To ensure that the partial cache persists across sessions, one should + // checkpoint the input pipeline. On each call to `SaveInternal` the + // partial cache gets flushed to disk in files with prefix + // _ where shard_id is unique for each checkpoint. + // When all elements have been produced, these shards get coalesced. + class FileWriterIterator : public DatasetIterator { public: - explicit FileIterator(const Params& params) - : DatasetIterator(params) { - if (params.dataset->env_ - ->FileExists(MetaFilename(params.dataset->filename_)) - .ok()) { - mode_ = Mode::read; - } else { - mode_ = Mode::write; - } - InitializeIterator(); - } + explicit FileWriterIterator(const Params& params) + : DatasetIterator(params), + cur_index_(0), + shard_id_(0), + filename_( + strings::StrCat(params.dataset->filename_, "_", shard_id_)), + lockfile_(strings::StrCat(filename_, kLockFileSuffix)), + lockfile_created_(false), + iteration_completed_(false) {} Status Initialize(IteratorContext* ctx) override { - mutex_lock l(mu_); - return iterator_->Initialize(ctx); + return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_); } Status GetNextInternal(IteratorContext* ctx, std::vector* out_tensors, bool* end_of_sequence) override { mutex_lock l(mu_); - return iterator_->GetNext(ctx, out_tensors, end_of_sequence); + *end_of_sequence = false; + TF_RETURN_IF_ERROR(EnsureLockFileExists(end_of_sequence)); + if (*end_of_sequence) { + return Status::OK(); + } + TF_RETURN_IF_ERROR(writer_->status()); + if (cur_index_ >= kMaxItems) { + // As a courtesy, close the [truncated] cache file. + Status s = Finish(); + if (!s.ok()) { + LOG(ERROR) << s; + } + return errors::InvalidArgument( + "Upstream iterator is producing more than ", kMaxItems, + " items, which is more than the cache limit."); + } + + TF_RETURN_IF_ERROR( + input_impl_->GetNext(ctx, out_tensors, end_of_sequence)); + if (*end_of_sequence && out_tensors->empty()) { + TF_RETURN_IF_ERROR(Finish()); + cur_index_++; + return Status::OK(); + } + if (out_tensors->size() != dataset()->num_tensors_) { + return errors::Internal( + "Upstream iterator returned invalid number of tensors. " + "Expected ", + dataset()->num_tensors_, " got: ", out_tensors->size()); + } + size_t tensor_index = 0; + for (const Tensor& t : *out_tensors) { + DCHECK_LT(tensor_index, dataset()->num_tensors_); + string key = dataset()->FormatName(cur_index_, tensor_index++); + TF_RETURN_IF_ERROR(writer_->Add(key, t)); + } + if (*end_of_sequence) { + TF_RETURN_IF_ERROR(Finish()); + } + cur_index_++; + return Status::OK(); } protected: @@ -145,578 +275,219 @@ class CacheDatasetOp : public UnaryDatasetOpKernel { Status SaveInternal(IteratorStateWriter* writer) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("mode"), mode_)); - return SaveInput(writer, iterator_); - } - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - mutex_lock l(mu_); - { - int64 temp; - TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("mode"), &temp)); - mode_ = static_cast(temp); - } - if (mode_ == Mode::write && - dataset() - ->env_->FileExists(MetaFilename(dataset()->filename_)) - .ok()) { - // This could happen if the cache was completely written after the - // checkpoint was saved. - LOG(WARNING) - << "It looks like the cache was already completely written(" - << MetaFilename(dataset()->filename_) - << ") after the last checkpoint was saved. Attempting to read " - << "the cache instead of continuing to write. If this is a " - << "mistake, please remove the above file and try running again."; - mode_ = Mode::read; - } - InitializeIterator(); - TF_RETURN_IF_ERROR(iterator_->Initialize(ctx)); - return RestoreInput(ctx, reader, iterator_); - } - - private: - // FileWriterIterator passes through and caches items from the input - // FileDataset. - // - // This iterator is used when the cache directory is not found on disk. It - // creates the cache directory, and passes on the underlying iterator's - // elements. - // - // Caching is performed by writing the input tensors to disk using the - // `BundleWriter`. Note that the cache gets fully flushed to disk only - // after the input iterator has been fully exhausted. If the program - // exits, before completion of an epoch, the cached state would be lost. - // To ensure that the partial cache persists across sessions, one should - // checkpoint the input pipeline. On each call to `SaveInternal` the - // partial cache gets flushed to disk in files with prefix - // _ where shard_id is unique for each checkpoint. - // When all elements have been produced, these shards get coalesced. - class FileWriterIterator : public DatasetIterator { - public: - explicit FileWriterIterator(const Params& params) - : DatasetIterator(params), - cur_index_(0), - shard_id_(0), - filename_( - strings::StrCat(params.dataset->filename_, "_", shard_id_)), - lockfile_(strings::StrCat(filename_, ".lockfile")), - lockfile_created_(false), - iteration_completed_(false) {} - - Status Initialize(IteratorContext* ctx) override { - return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_); - } - - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - *end_of_sequence = false; - TF_RETURN_IF_ERROR(EnsureLockFileExists(end_of_sequence)); - if (*end_of_sequence) { - return Status::OK(); - } - TF_RETURN_IF_ERROR(writer_->status()); - if (cur_index_ >= kMaxItems) { - // As a courtesy, close the [truncated] cache file. - Status s = Finish(); - if (!s.ok()) { - LOG(ERROR) << s; - } - return errors::InvalidArgument( - "Upstream iterator is producing more than ", kMaxItems, - " items, which is more than the cache limit."); - } - + if (iteration_completed_) { TF_RETURN_IF_ERROR( - input_impl_->GetNext(ctx, out_tensors, end_of_sequence)); - if (*end_of_sequence && out_tensors->empty()) { - TF_RETURN_IF_ERROR(Finish()); - cur_index_++; - return Status::OK(); - } - if (out_tensors->size() != dataset()->num_tensors_) { - return errors::Internal( - "Upstream iterator returned invalid number of tensors. " - "Expected ", - dataset()->num_tensors_, " got: ", out_tensors->size()); - } - size_t tensor_index = 0; - for (const Tensor& t : *out_tensors) { - DCHECK_LT(tensor_index, dataset()->num_tensors_); - string key = dataset()->FormatName(cur_index_, tensor_index++); - TF_RETURN_IF_ERROR(writer_->Add(key, t)); - } - if (*end_of_sequence) { - TF_RETURN_IF_ERROR(Finish()); - } - cur_index_++; + writer->WriteScalar(full_name(kIterationCompleted), "")); return Status::OK(); } - protected: - std::shared_ptr CreateNode( - IteratorContext* ctx, model::Node::Args args) const override { - return model::MakeKnownRatioNode(std::move(args), - /*ratio=*/1); - } - - Status SaveInternal(IteratorStateWriter* writer) override { - mutex_lock l(mu_); - if (iteration_completed_) { - TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("iteration_completed"), "")); - return Status::OK(); - } - - // lockfile is created on the first call to GetNextInternal. The - // absence of a lockfile means that GetNextInternal was not called - // and hence nothing was written to cache. So we don't need to worry - // about flushing the current shard. This ensures that we never write - // empty shards. - if (lockfile_created_) { - // Flush the current bundle. - TF_RETURN_IF_ERROR(writer_->Finish()); - - // Note: We do not delete the lockfile here. We keep lockfiles of - // all shards around until the entire cache has been written to - // prevent concurrent iterators from corrupting any of the shards. - - // Start caching to a new shard. - shard_id_++; - filename_ = strings::StrCat(dataset()->filename_, "_", shard_id_); - lockfile_ = strings::StrCat(filename_, ".lockfile"); - lockfile_created_ = false; - } - TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_)); - TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("cur_index"), cur_index_)); - TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("shard_id"), shard_id_)); - return Status::OK(); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - mutex_lock l(mu_); - if (reader->Contains(full_name("iteration_completed"))) { - iteration_completed_ = true; - return Status::OK(); - } - - TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_)); - int64 temp; - // TODO(b/78048575): Update this when saving size_t tensors directly - // is supported. - { - TF_RETURN_IF_ERROR( - reader->ReadScalar(full_name("cur_index"), &temp)); - cur_index_ = static_cast(temp); - if (cur_index_ != temp) { - return errors::Internal("Invalid value for cur_index ", temp); - } - } - // TODO(b/78048575): Update this when saving size_t tensors directly - // is supported. - { - TF_RETURN_IF_ERROR( - reader->ReadScalar(full_name("shard_id"), &temp)); - shard_id_ = static_cast(temp); - if (shard_id_ != temp) { - return errors::Internal("Invalid value for shard_id ", temp); - } - } - filename_ = strings::StrCat(dataset()->filename_, "_", shard_id_); - lockfile_ = strings::StrCat(filename_, ".lockfile"); - writer_ = absl::make_unique(dataset()->env_, filename_); - return Status::OK(); - } - - private: - Status EnsureLockFileExists(bool* end_of_sequence) - EXCLUSIVE_LOCKS_REQUIRED(mu_) { - if (iteration_completed_) { - *end_of_sequence = true; - return Status::OK(); - } - if (lockfile_created_ && !iteration_completed_) return Status::OK(); - - // Perform rudimentary locking to help catch concurrent writes to the - // same cache files. - - // 1. Check that a checkpoint for the shard has not already been - // written. - if (dataset()->env_->FileExists(MetaFilename(filename_)).ok()) { - return errors::AlreadyExists("Existing cache files found: \n", - MetaFilename(filename_), "\n", - DataFilename(filename_, 0, 1), "\n", - "To continue delete the above files."); - } - - // 2. Check that there isn't a concurrent iterator that is writing - // to cache. - if (dataset()->env_->FileExists(lockfile_).ok()) { - // Attempt to read the contents of the lockfile. - char contents_scratch[151] = {0}; // Initialize all to 0. - StringPiece contents; - std::unique_ptr file; - if (dataset()->env_->NewRandomAccessFile(lockfile_, &file).ok()) { - file->Read(0, 150, &contents, contents_scratch).IgnoreError(); - } - return errors::AlreadyExists( - "There appears to be a concurrent caching iterator running - " - "cache lockfile already exists ('", - lockfile_, - "'). If you are sure no other running TF computations are " - "using this cache prefix, delete the lockfile and " - "re-initialize the iterator. Lockfile contents: ", - contents); - } - // Create the file, and write some basic contents. - std::unique_ptr lockfile; - TF_RETURN_IF_ERROR( - dataset()->env_->NewWritableFile(lockfile_, &lockfile)); - TF_RETURN_IF_ERROR(lockfile->Append( - strings::StrCat("Created at: ", dataset()->env_->NowSeconds()))); - - // At this point we know that - // 1. There is no conflicting checkpoint with prefix `filename_`. - // 2. There is no concurrent session that is trying to write a ckpt - // to filename. - // So it is safe to create a BundleWriter here. Note that it is - // unsafe to initialize the BundleWriter anywhere the above - // conditions are not met since BundleWriter's constructor creates - // new temp files which can delete the temp files created by a - // BundleWriter in another Session. - writer_ = absl::make_unique(dataset()->env_, filename_); - lockfile_created_ = true; - return Status::OK(); - } - - Status Finish() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - iteration_completed_ = true; + // lockfile is created on the first call to GetNextInternal. The + // absence of a lockfile means that GetNextInternal was not called + // and hence nothing was written to cache. So we don't need to worry + // about flushing the current shard. This ensures that we never write + // empty shards. + if (lockfile_created_) { // Flush the current bundle. TF_RETURN_IF_ERROR(writer_->Finish()); - // Merge all the bundles. - // Currently there are `shard_id_ + 1` bundles, one for each - // checkpoint. Each bundle has prefix _ where `id` is an - // integer starting at 0 an incremented by 1 for each new checkpoint. - // We merge all these bundles into a bundle with prefix so - // that the next call to `MakeIterator` can build a - // `FileReaderIterator`. - { - std::vector prefixes; - prefixes.reserve(shard_id_ + 1); - for (size_t i = 0; i <= shard_id_; ++i) { - prefixes.emplace_back( - strings::StrCat(dataset()->filename_, "_", i)); - } - TF_RETURN_IF_ERROR( - MergeBundles(dataset()->env_, prefixes, dataset()->filename_)); - } - // Delete all lockfiles. - for (size_t i = 0; i <= shard_id_; ++i) { - TF_RETURN_IF_ERROR(dataset()->env_->DeleteFile( - strings::StrCat(dataset()->filename_, "_", i, ".lockfile"))); - } + + // Note: We do not delete the lockfile here. We keep lockfiles of + // all shards around until the entire cache has been written to + // prevent concurrent iterators from corrupting any of the shards. + + // Start caching to a new shard. + shard_id_++; + filename_ = strings::StrCat(dataset()->filename_, "_", shard_id_); + lockfile_ = strings::StrCat(filename_, kLockFileSuffix); + lockfile_created_ = false; + } + TF_RETURN_IF_ERROR(SaveInput(writer, input_impl_)); + TF_RETURN_IF_ERROR( + writer->WriteScalar(full_name(kCurIndex), cur_index_)); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name(kShardId), shard_id_)); + return Status::OK(); + } + + Status RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + if (reader->Contains(full_name(kIterationCompleted))) { + iteration_completed_ = true; return Status::OK(); } - mutex mu_; - size_t cur_index_ GUARDED_BY(mu_); - // Index of the current shard. This gets incremented whenever a new - // cache shard is saved. - size_t shard_id_ GUARDED_BY(mu_); - std::unique_ptr input_impl_ GUARDED_BY(mu_); - // The current prefix for the cache file. This is equal to - // `StrCat(dataset()->filename_, "_", shard_id_)`. - string filename_; - std::unique_ptr writer_ GUARDED_BY(mu_); - string lockfile_ GUARDED_BY(mu_); - bool lockfile_created_ GUARDED_BY(mu_); - bool iteration_completed_ GUARDED_BY(mu_); - }; // FileWriterIterator - - class FileReaderIterator : public DatasetIterator { - public: - explicit FileReaderIterator(const Params& params) - : DatasetIterator(params), - cur_index_(0), - reader_(dataset()->env_, dataset()->filename_), - iterator_restored_(false) {} - - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - *end_of_sequence = false; - TF_RETURN_IF_ERROR(reader_.status()); - if (!reader_.Valid()) { - *end_of_sequence = true; - return Status::OK(); + TF_RETURN_IF_ERROR(RestoreInput(ctx, reader, input_impl_)); + int64 temp; + // TODO(b/78048575): Update this when saving size_t tensors directly + // is supported. + { + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name(kCurIndex), &temp)); + cur_index_ = static_cast(temp); + if (cur_index_ != temp) { + return errors::Internal("Invalid value for cur_index ", temp); } - out_tensors->clear(); - out_tensors->resize(dataset()->num_tensors_); - - for (size_t i = 0; i < dataset()->num_tensors_; ++i) { - // When the iterator is restored from the checkpoint, `reader_` is - // already pointing at `key` so we do not need to skip the header - // entry. - if (!iterator_restored_) { - reader_.Next(); // The first entry in the table is a header. - } else { - iterator_restored_ = false; - } - if (!reader_.Valid()) { - out_tensors->clear(); - *end_of_sequence = true; - return Status::OK(); - } - StringPiece key = reader_.key(); - DCHECK_EQ(key, dataset()->FormatName(cur_index_, i)); - TF_RETURN_IF_ERROR(reader_.ReadCurrent(&(*out_tensors)[i])); - TF_RETURN_IF_ERROR(reader_.status()); + } + // TODO(b/78048575): Update this when saving size_t tensors directly + // is supported. + { + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name(kShardId), &temp)); + shard_id_ = static_cast(temp); + if (shard_id_ != temp) { + return errors::Internal("Invalid value for shard_id ", temp); } - cur_index_++; - return Status::OK(); } - - protected: - std::shared_ptr CreateNode( - IteratorContext* ctx, model::Node::Args args) const override { - return model::MakeKnownRatioNode(std::move(args), - /*ratio=*/1); - } - - Status SaveInternal(IteratorStateWriter* writer) override { - mutex_lock l(mu_); - TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("cur_index"), cur_index_)); - return Status::OK(); - } - - Status RestoreInternal( - IteratorContext* ctx, - IteratorStateReader* iterator_state_reader) override { - mutex_lock l(mu_); - { - // TODO(b/78048575): Update this when saving size_t tensors directly - // is supported. - int64 temp; - TF_RETURN_IF_ERROR(iterator_state_reader->ReadScalar( - full_name("cur_index"), &temp)); - cur_index_ = static_cast(temp); - if (cur_index_ != temp) { - return errors::Internal("Invalid value for cur_index ", temp); - } - } - if (!reader_.Valid()) { - return errors::Internal("Error initializing BundleReader."); - } - reader_.Seek(dataset()->FormatName(cur_index_, 0)); - iterator_restored_ = true; - return Status::OK(); - } - - private: - mutex mu_; - size_t cur_index_ GUARDED_BY(mu_); - BundleReader reader_ GUARDED_BY(mu_); - bool iterator_restored_ GUARDED_BY(mu_); - }; // FileReaderIterator - - void InitializeIterator() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - // We intentionally use the same prefix for both `FileReaderIterator` - // and `FileWriterIterator`. Since at any time there will be at most - // one of them alive, there should be no conflicts. This allows both - // iterators to use a common key for `cur_index`. We leverage this - // in the corner case when this iterator is restored from an old - // checkpoint in `write` mode and the cache has been completely - // flushed to disk since then. In that case we simply build a - // `FileReaderIterator` and seek to the `cur_index`. - switch (mode_) { - case Mode::read: - iterator_ = absl::make_unique( - FileReaderIterator::Params{dataset(), - strings::StrCat(prefix(), "Impl")}); - break; - case Mode::write: - iterator_ = absl::make_unique( - FileWriterIterator::Params{dataset(), - strings::StrCat(prefix(), "Impl")}); - } - } - - mutex mu_; - enum Mode { read, write }; - Mode mode_ GUARDED_BY(mu_); - std::unique_ptr iterator_ GUARDED_BY(mu_); - }; // FileIterator - - const DatasetBase* const input_; - const string filename_; - Env* const env_; - const size_t num_tensors_; - const size_t tensor_index_padding_size_; - static const size_t kMaxItems = 10000000; // 10 million - const size_t item_index_padding_size_; - const string tensor_format_string_; - }; // FileDataset - - class MemoryDataset : public DatasetBase { - public: - explicit MemoryDataset(OpKernelContext* ctx, const DatasetBase* input) - : DatasetBase(DatasetContext(ctx)), input_(input) { - input->Ref(); - } - - ~MemoryDataset() override { input_->Unref(); } - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return absl::make_unique(MemoryIterator::Params{ - this, strings::StrCat(prefix, "::MemoryCache")}); - } - - const DataTypeVector& output_dtypes() const override { - return input_->output_dtypes(); - } - - const std::vector& output_shapes() const override { - return input_->output_shapes(); - } - - string DebugString() const override { - return "CacheDatasetOp::MemoryDataset"; - } - - int64 Cardinality() const override { return input_->Cardinality(); } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - Node* input_node = nullptr; - TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_node)); - Node* filename_node = nullptr; - TF_RETURN_IF_ERROR(b->AddScalar(string(""), &filename_node)); - TF_RETURN_IF_ERROR( - b->AddDataset(this, {input_node, filename_node}, output)); - return Status::OK(); - } - - private: - // A thread-safe data structure for caching dataset elements. - // - // The expected use is that a single `MemoryWriterIterator` populates the - // cache with dataset elements. Once all elements are cached, the cache can - // be used by one or more `MemoryReaderIterator`s. - class MemoryCache : public ResourceBase { - public: - MemoryCache() = default; - - string DebugString() const override { - return "CacheDataset::MemoryCache"; - } - - // Marks the cache as completed. - void Complete() { - mutex_lock l(mu_); - completed_ = true; - } - - // Returns whether the cache is claimed. - bool IsClaimed() { - tf_shared_lock l(mu_); - return claimed_; - } - - // Returns whether the cache is completed. - bool IsCompleted() { - tf_shared_lock l(mu_); - return completed_; - } - - // Attempts to claim the cache, returning whether the cache was claimed. - bool MaybeClaim() { - mutex_lock l(mu_); - if (!claimed_) { - claimed_ = true; - return true; - } - return false; - } - - // Resets the cache. - void Reset() { - mutex_lock l(mu_); - claimed_ = false; - completed_ = false; - cache_.clear(); - } - - // Returns the element at the given index. - const std::vector& at(int64 index) { - tf_shared_lock l(mu_); - DCHECK(index < cache_.size()); - return cache_[index]; - } - - // Adds the element to the cache. - void emplace_back(std::vector element) { - mutex_lock l(mu_); - cache_.emplace_back(std::move(element)); - } - - // Returns the size of the cache. - size_t size() { - tf_shared_lock l(mu_); - return cache_.size(); + filename_ = strings::StrCat(dataset()->filename_, "_", shard_id_); + lockfile_ = strings::StrCat(filename_, kLockFileSuffix); + writer_ = absl::make_unique(dataset()->env_, filename_); + return Status::OK(); } private: - mutex mu_; - // Determines whether a writer has claimed the cache. - bool claimed_ GUARDED_BY(mu_) = false; - // Determines whether all elements of the dataset have been cached. - bool completed_ GUARDED_BY(mu_) = false; - std::vector> cache_ GUARDED_BY(mu_); - }; - - class MemoryIterator : public DatasetIterator { - public: - explicit MemoryIterator(const Params& params) - : DatasetIterator(params) {} - - ~MemoryIterator() override { cache_->Unref(); } - - Status Initialize(IteratorContext* ctx) override { - mutex_lock l(mu_); - // Use the resource manager in the iterator context to get / create - // a cache. - ResourceMgr* mgr = ctx->resource_mgr(); - const string name = strings::StrCat( - prefix(), "::", dataset()->node_name(), "::MemoryCache"); - TF_RETURN_IF_ERROR(mgr->LookupOrCreate( - "tf_data", name, &cache_, [](MemoryCache** cache) { - *cache = new MemoryCache(); - return Status::OK(); - })); - mode_ = cache_->MaybeClaim() ? Mode::write : Mode::read; - InitializeIterator(); - if (mode_ == Mode::read && !cache_->IsCompleted()) { - return errors::Internal( - "Cache should only be read after it has been completed."); + Status EnsureLockFileExists(bool* end_of_sequence) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (iteration_completed_) { + *end_of_sequence = true; + return Status::OK(); } - return iterator_->Initialize(ctx); + if (lockfile_created_ && !iteration_completed_) return Status::OK(); + + // Perform rudimentary locking to help catch concurrent writes to the + // same cache files. + + // 1. Check that a checkpoint for the shard has not already been + // written. + if (dataset()->env_->FileExists(MetaFilename(filename_)).ok()) { + return errors::AlreadyExists("Existing cache files found: \n", + MetaFilename(filename_), "\n", + DataFilename(filename_, 0, 1), "\n", + "To continue delete the above files."); + } + + // 2. Check that there isn't a concurrent iterator that is writing + // to cache. + if (dataset()->env_->FileExists(lockfile_).ok()) { + // Attempt to read the contents of the lockfile. + char contents_scratch[151] = {0}; // Initialize all to 0. + StringPiece contents; + std::unique_ptr file; + if (dataset()->env_->NewRandomAccessFile(lockfile_, &file).ok()) { + file->Read(0, 150, &contents, contents_scratch).IgnoreError(); + } + return errors::AlreadyExists( + "There appears to be a concurrent caching iterator running - " + "cache lockfile already exists ('", + lockfile_, + "'). If you are sure no other running TF computations are " + "using this cache prefix, delete the lockfile and " + "re-initialize the iterator. Lockfile contents: ", + contents); + } + // Create the file, and write some basic contents. + std::unique_ptr lockfile; + TF_RETURN_IF_ERROR( + dataset()->env_->NewWritableFile(lockfile_, &lockfile)); + TF_RETURN_IF_ERROR(lockfile->Append( + strings::StrCat(kCreatedAt, ": ", dataset()->env_->NowSeconds()))); + + // At this point we know that + // 1. There is no conflicting checkpoint with prefix `filename_`. + // 2. There is no concurrent session that is trying to write a ckpt + // to filename. + // So it is safe to create a BundleWriter here. Note that it is + // unsafe to initialize the BundleWriter anywhere the above + // conditions are not met since BundleWriter's constructor creates + // new temp files which can delete the temp files created by a + // BundleWriter in another Session. + writer_ = absl::make_unique(dataset()->env_, filename_); + lockfile_created_ = true; + return Status::OK(); } + Status Finish() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + iteration_completed_ = true; + // Flush the current bundle. + TF_RETURN_IF_ERROR(writer_->Finish()); + // Merge all the bundles. + // Currently there are `shard_id_ + 1` bundles, one for each + // checkpoint. Each bundle has prefix _ where `id` is an + // integer starting at 0 an incremented by 1 for each new checkpoint. + // We merge all these bundles into a bundle with prefix so + // that the next call to `MakeIterator` can build a + // `FileReaderIterator`. + { + std::vector prefixes; + prefixes.reserve(shard_id_ + 1); + for (size_t i = 0; i <= shard_id_; ++i) { + prefixes.emplace_back( + strings::StrCat(dataset()->filename_, "_", i)); + } + TF_RETURN_IF_ERROR( + MergeBundles(dataset()->env_, prefixes, dataset()->filename_)); + } + // Delete all lockfiles. + for (size_t i = 0; i <= shard_id_; ++i) { + TF_RETURN_IF_ERROR(dataset()->env_->DeleteFile( + strings::StrCat(dataset()->filename_, "_", i, kLockFileSuffix))); + } + return Status::OK(); + } + + mutex mu_; + size_t cur_index_ GUARDED_BY(mu_); + // Index of the current shard. This gets incremented whenever a new + // cache shard is saved. + size_t shard_id_ GUARDED_BY(mu_); + std::unique_ptr input_impl_ GUARDED_BY(mu_); + // The current prefix for the cache file. This is equal to + // `StrCat(dataset()->filename_, "_", shard_id_)`. + string filename_; + std::unique_ptr writer_ GUARDED_BY(mu_); + string lockfile_ GUARDED_BY(mu_); + bool lockfile_created_ GUARDED_BY(mu_); + bool iteration_completed_ GUARDED_BY(mu_); + }; // FileWriterIterator + + class FileReaderIterator : public DatasetIterator { + public: + explicit FileReaderIterator(const Params& params) + : DatasetIterator(params), + cur_index_(0), + reader_(dataset()->env_, dataset()->filename_), + iterator_restored_(false) {} + Status GetNextInternal(IteratorContext* ctx, std::vector* out_tensors, bool* end_of_sequence) override { mutex_lock l(mu_); - return iterator_->GetNext(ctx, out_tensors, end_of_sequence); + *end_of_sequence = false; + TF_RETURN_IF_ERROR(reader_.status()); + if (!reader_.Valid()) { + *end_of_sequence = true; + return Status::OK(); + } + out_tensors->clear(); + out_tensors->resize(dataset()->num_tensors_); + + for (size_t i = 0; i < dataset()->num_tensors_; ++i) { + // When the iterator is restored from the checkpoint, `reader_` is + // already pointing at `key` so we do not need to skip the header + // entry. + if (!iterator_restored_) { + reader_.Next(); // The first entry in the table is a header. + } else { + iterator_restored_ = false; + } + if (!reader_.Valid()) { + out_tensors->clear(); + *end_of_sequence = true; + return Status::OK(); + } + StringPiece key = reader_.key(); + DCHECK_EQ(key, dataset()->FormatName(cur_index_, i)); + TF_RETURN_IF_ERROR(reader_.ReadCurrent(&(*out_tensors)[i])); + TF_RETURN_IF_ERROR(reader_.status()); + } + cur_index_++; + return Status::OK(); } protected: @@ -728,240 +499,494 @@ class CacheDatasetOp : public UnaryDatasetOpKernel { Status SaveInternal(IteratorStateWriter* writer) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("mode"), mode_)); - if (cache_->IsClaimed()) { + TF_RETURN_IF_ERROR( + writer->WriteScalar(full_name(kCurIndex), cur_index_)); + return Status::OK(); + } + + Status RestoreInternal( + IteratorContext* ctx, + IteratorStateReader* iterator_state_reader) override { + mutex_lock l(mu_); + { + // TODO(b/78048575): Update this when saving size_t tensors directly + // is supported. + int64 temp; TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("cache_claimed"), "")); - size_t cache_size = cache_->size(); - TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("cache_size"), cache_size)); - for (size_t i = 0; i < cache_size; i++) { - auto& element = cache_->at(i); - TF_RETURN_IF_ERROR(writer->WriteScalar( - full_name(strings::StrCat("cache[", i, "].size")), - element.size())); - for (size_t j = 0; j < element.size(); ++j) { - TF_RETURN_IF_ERROR(writer->WriteTensor( - full_name(strings::StrCat("cache[", i, "][", j, "]")), - element[j])); - } - } - if (cache_->IsCompleted()) { - TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("cache_completed"), "")); + iterator_state_reader->ReadScalar(full_name(kCurIndex), &temp)); + cur_index_ = static_cast(temp); + if (cur_index_ != temp) { + return errors::Internal("Invalid value for cur_index ", temp); } } - return SaveInput(writer, iterator_); + if (!reader_.Valid()) { + return errors::Internal("Error initializing BundleReader."); + } + reader_.Seek(dataset()->FormatName(cur_index_, 0)); + iterator_restored_ = true; + return Status::OK(); + } + + private: + mutex mu_; + size_t cur_index_ GUARDED_BY(mu_); + BundleReader reader_ GUARDED_BY(mu_); + bool iterator_restored_ GUARDED_BY(mu_); + }; // FileReaderIterator + + void InitializeIterator() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + // We intentionally use the same prefix for both `FileReaderIterator` + // and `FileWriterIterator`. Since at any time there will be at most + // one of them alive, there should be no conflicts. This allows both + // iterators to use a common key for `cur_index`. We leverage this + // in the corner case when this iterator is restored from an old + // checkpoint in `write` mode and the cache has been completely + // flushed to disk since then. In that case we simply build a + // `FileReaderIterator` and seek to the `cur_index`. + switch (mode_) { + case Mode::read: + iterator_ = + absl::make_unique(FileReaderIterator::Params{ + dataset(), strings::StrCat(prefix(), kImpl)}); + break; + case Mode::write: + iterator_ = + absl::make_unique(FileWriterIterator::Params{ + dataset(), strings::StrCat(prefix(), kImpl)}); + } + } + + mutex mu_; + enum Mode { read, write }; + Mode mode_ GUARDED_BY(mu_); + std::unique_ptr iterator_ GUARDED_BY(mu_); + }; // FileIterator + + const DatasetBase* const input_; + const string filename_; + Env* const env_; + const size_t num_tensors_; + const size_t tensor_index_padding_size_; + static const size_t kMaxItems = 10000000; // 10 million + const size_t item_index_padding_size_; + const string tensor_format_string_; +}; // FileDataset + +class CacheDatasetOp::MemoryDataset : public DatasetBase { + public: + explicit MemoryDataset(OpKernelContext* ctx, const DatasetBase* input) + : DatasetBase(DatasetContext(ctx)), input_(input) { + input->Ref(); + } + + ~MemoryDataset() override { input_->Unref(); } + + std::unique_ptr MakeIteratorInternal( + const string& prefix) const override { + name_utils::IteratorPrefixParams params; + params.dataset_prefix = kMemoryDatasetPrefix; + return absl::make_unique(MemoryIterator::Params{ + this, name_utils::IteratorPrefix(kDatasetType, prefix, params)}); + } + + const DataTypeVector& output_dtypes() const override { + return input_->output_dtypes(); + } + + const std::vector& output_shapes() const override { + return input_->output_shapes(); + } + + string DebugString() const override { + name_utils::DatasetDebugStringParams params; + params.dataset_prefix = kMemoryDatasetPrefix; + return name_utils::DatasetDebugString(kDatasetType, params); + } + + int64 Cardinality() const override { return input_->Cardinality(); } + + protected: + Status AsGraphDefInternal(SerializationContext* ctx, + DatasetGraphDefBuilder* b, + Node** output) const override { + Node* input_node = nullptr; + TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_node)); + Node* filename_node = nullptr; + TF_RETURN_IF_ERROR(b->AddScalar(string(""), &filename_node)); + TF_RETURN_IF_ERROR( + b->AddDataset(this, {input_node, filename_node}, output)); + return Status::OK(); + } + + private: + // A thread-safe data structure for caching dataset elements. + // + // The expected use is that a single `MemoryWriterIterator` populates the + // cache with dataset elements. Once all elements are cached, the cache can + // be used by one or more `MemoryReaderIterator`s. + class MemoryCache : public ResourceBase { + public: + MemoryCache() = default; + + string DebugString() const override { return "CacheDataset::MemoryCache"; } + + // Marks the cache as completed. + void Complete() { + mutex_lock l(mu_); + completed_ = true; + } + + // Returns whether the cache is claimed. + bool IsClaimed() { + tf_shared_lock l(mu_); + return claimed_; + } + + // Returns whether the cache is completed. + bool IsCompleted() { + tf_shared_lock l(mu_); + return completed_; + } + + // Attempts to claim the cache, returning whether the cache was claimed. + bool MaybeClaim() { + mutex_lock l(mu_); + if (!claimed_) { + claimed_ = true; + return true; + } + return false; + } + + // Resets the cache. + void Reset() { + mutex_lock l(mu_); + claimed_ = false; + completed_ = false; + cache_.clear(); + } + + // Returns the element at the given index. + const std::vector& at(int64 index) { + tf_shared_lock l(mu_); + DCHECK(index < cache_.size()); + return cache_[index]; + } + + // Adds the element to the cache. + void emplace_back(std::vector element) { + mutex_lock l(mu_); + cache_.emplace_back(std::move(element)); + } + + // Returns the size of the cache. + size_t size() { + tf_shared_lock l(mu_); + return cache_.size(); + } + + private: + mutex mu_; + // Determines whether a writer has claimed the cache. + bool claimed_ GUARDED_BY(mu_) = false; + // Determines whether all elements of the dataset have been cached. + bool completed_ GUARDED_BY(mu_) = false; + std::vector> cache_ GUARDED_BY(mu_); + }; + + class MemoryIterator : public DatasetIterator { + public: + explicit MemoryIterator(const Params& params) + : DatasetIterator(params) {} + + ~MemoryIterator() override { cache_->Unref(); } + + Status Initialize(IteratorContext* ctx) override { + mutex_lock l(mu_); + // Use the resource manager in the iterator context to get / create + // a cache. + ResourceMgr* mgr = ctx->resource_mgr(); + const string name = strings::StrCat(prefix(), name_utils::kDelimiter, + dataset()->node_name(), + name_utils::kDelimiter, kMemoryCache); + TF_RETURN_IF_ERROR(mgr->LookupOrCreate( + kTFData, name, &cache_, [](MemoryCache** cache) { + *cache = new MemoryCache(); + return Status::OK(); + })); + mode_ = cache_->MaybeClaim() ? Mode::write : Mode::read; + InitializeIterator(); + if (mode_ == Mode::read && !cache_->IsCompleted()) { + return errors::Internal( + "Cache should only be read after it has been completed."); + } + return iterator_->Initialize(ctx); + } + + Status GetNextInternal(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) override { + mutex_lock l(mu_); + return iterator_->GetNext(ctx, out_tensors, end_of_sequence); + } + + protected: + std::shared_ptr CreateNode( + IteratorContext* ctx, model::Node::Args args) const override { + return model::MakeKnownRatioNode(std::move(args), + /*ratio=*/1); + } + + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name(kMode), mode_)); + if (cache_->IsClaimed()) { + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name(kCacheClaimed), "")); + size_t cache_size = cache_->size(); + TF_RETURN_IF_ERROR( + writer->WriteScalar(full_name(kCacheSize), cache_size)); + for (size_t i = 0; i < cache_size; i++) { + auto& element = cache_->at(i); + TF_RETURN_IF_ERROR(writer->WriteScalar( + full_name(strings::StrCat(kCache, "[", i, "]", kSizeSuffix)), + element.size())); + for (size_t j = 0; j < element.size(); ++j) { + TF_RETURN_IF_ERROR(writer->WriteTensor( + full_name(strings::StrCat(kCache, "[", i, "][", j, "]")), + element[j])); + } + } + if (cache_->IsCompleted()) { + TF_RETURN_IF_ERROR( + writer->WriteScalar(full_name(kCacheCompleted), "")); + } + } + return SaveInput(writer, iterator_); + } + + Status RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + iterator_.reset(); + cache_->Reset(); + { + int64 temp; + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name(kMode), &temp)); + mode_ = static_cast(temp); + } + if (reader->Contains(full_name(kCacheClaimed))) { + CHECK(cache_->MaybeClaim()); + size_t cache_size; + { + int64 temp; + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name(kCacheSize), &temp)); + cache_size = static_cast(temp); + } + for (size_t i = 0; i < cache_size; ++i) { + std::vector element; + size_t element_size; + { + int64 temp; + TF_RETURN_IF_ERROR(reader->ReadScalar( + full_name(strings::StrCat(kCache, "[", i, "]", kSizeSuffix)), + &temp)); + element_size = static_cast(temp); + } + element.reserve(element_size); + for (size_t j = 0; j < element_size; ++j) { + element.emplace_back(); + TF_RETURN_IF_ERROR(reader->ReadTensor( + full_name(strings::StrCat(kCache, "[", i, "][", j, "]")), + &element.back())); + } + cache_->emplace_back(std::move(element)); + } + if (reader->Contains(full_name(kCacheCompleted))) { + cache_->Complete(); + } + } + InitializeIterator(); + TF_RETURN_IF_ERROR(iterator_->Initialize(ctx)); + return RestoreInput(ctx, reader, iterator_); + } + + private: + class MemoryWriterIterator : public DatasetIterator { + public: + explicit MemoryWriterIterator(const Params& params, MemoryCache* cache) + : DatasetIterator(params), cache_(cache) { + CHECK(cache_); + } + + ~MemoryWriterIterator() override { + mutex_lock l(mu_); + if (cache_->size() > 0 && !cache_->IsCompleted()) { + LOG(WARNING) + << "The calling iterator did not fully read the dataset being " + "cached. In order to avoid unexpected truncation of the " + "dataset, the partially cached contents of the dataset " + "will be discarded. This can happen if you have an input " + "pipeline similar to `dataset.cache().take(k).repeat()`. " + "You should use `dataset.take(k).cache().repeat()` instead."; + cache_->Reset(); + } + } + + Status Initialize(IteratorContext* ctx) override { + return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_); + } + + Status GetNextInternal(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR( + input_impl_->GetNext(ctx, out_tensors, end_of_sequence)); + if (*end_of_sequence) { + cache_->Complete(); + return Status::OK(); + } + RecordBufferEnqueue(ctx, *out_tensors); + cache_->emplace_back(*out_tensors); + return Status::OK(); + } + + protected: + std::shared_ptr CreateNode( + IteratorContext* ctx, model::Node::Args args) const override { + return model::MakeKnownRatioNode(std::move(args), + /*ratio=*/1); + } + + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + return SaveInput(writer, input_impl_); } Status RestoreInternal(IteratorContext* ctx, IteratorStateReader* reader) override { mutex_lock l(mu_); - iterator_.reset(); - cache_->Reset(); - { - int64 temp; - TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("mode"), &temp)); - mode_ = static_cast(temp); - } - if (reader->Contains(full_name("cache_claimed"))) { - CHECK(cache_->MaybeClaim()); - size_t cache_size; - { - int64 temp; - TF_RETURN_IF_ERROR( - reader->ReadScalar(full_name("cache_size"), &temp)); - cache_size = static_cast(temp); - } - for (size_t i = 0; i < cache_size; ++i) { - std::vector element; - size_t element_size; - { - int64 temp; - TF_RETURN_IF_ERROR(reader->ReadScalar( - full_name(strings::StrCat("cache[", i, "].size")), &temp)); - element_size = static_cast(temp); - } - element.reserve(element_size); - for (size_t j = 0; j < element_size; ++j) { - element.emplace_back(); - TF_RETURN_IF_ERROR(reader->ReadTensor( - full_name(strings::StrCat("cache[", i, "][", j, "]")), - &element.back())); - } - cache_->emplace_back(std::move(element)); - } - if (reader->Contains(full_name("cache_completed"))) { - cache_->Complete(); - } - } - InitializeIterator(); - TF_RETURN_IF_ERROR(iterator_->Initialize(ctx)); - return RestoreInput(ctx, reader, iterator_); + return RestoreInput(ctx, reader, input_impl_); } private: - class MemoryWriterIterator : public DatasetIterator { - public: - explicit MemoryWriterIterator(const Params& params, MemoryCache* cache) - : DatasetIterator(params), cache_(cache) { - CHECK(cache_); - } + mutex mu_; + std::unique_ptr input_impl_ GUARDED_BY(mu_); + MemoryCache* const cache_ GUARDED_BY(mu_); // not owned. + }; // MemoryWriterIterator - ~MemoryWriterIterator() override { - mutex_lock l(mu_); - if (cache_->size() > 0 && !cache_->IsCompleted()) { - LOG(WARNING) - << "The calling iterator did not fully read the dataset being " - "cached. In order to avoid unexpected truncation of the " - "dataset, the partially cached contents of the dataset " - "will be discarded. This can happen if you have an input " - "pipeline similar to `dataset.cache().take(k).repeat()`. " - "You should use `dataset.take(k).cache().repeat()` instead."; - cache_->Reset(); - } - } + class MemoryReaderIterator : public DatasetIterator { + public: + explicit MemoryReaderIterator(const Params& params, MemoryCache* cache) + : DatasetIterator(params), cache_(cache), index_(0) { + CHECK(cache); + } - Status Initialize(IteratorContext* ctx) override { - return dataset()->input_->MakeIterator(ctx, prefix(), &input_impl_); + Status Initialize(IteratorContext* ctx) override { + // The memory allocated for the cache is owned by the parent + // dataset but performance modeling uses the iterator abstraction and + // thus we record the memory allocated for the cache here. The caveat + // is that this is incorrect if there are concurrent instances of this + // iterator. + tf_shared_lock l(mu_); + for (size_t i = 0; i < cache_->size(); ++i) { + RecordBufferEnqueue(ctx, cache_->at(i)); } + return Status::OK(); + } - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - TF_RETURN_IF_ERROR( - input_impl_->GetNext(ctx, out_tensors, end_of_sequence)); - if (*end_of_sequence) { - cache_->Complete(); - return Status::OK(); - } - RecordBufferEnqueue(ctx, *out_tensors); - cache_->emplace_back(*out_tensors); + Status GetNextInternal(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) override { + mutex_lock l(mu_); + if (index_ < cache_->size()) { + const std::vector& cache_tensors = cache_->at(index_); + out_tensors->insert(out_tensors->begin(), cache_tensors.begin(), + cache_tensors.end()); + index_++; + *end_of_sequence = false; return Status::OK(); - } - - protected: - std::shared_ptr CreateNode( - IteratorContext* ctx, model::Node::Args args) const override { - return model::MakeKnownRatioNode(std::move(args), - /*ratio=*/1); - } - - Status SaveInternal(IteratorStateWriter* writer) override { - mutex_lock l(mu_); - return SaveInput(writer, input_impl_); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - mutex_lock l(mu_); - return RestoreInput(ctx, reader, input_impl_); - } - - private: - mutex mu_; - std::unique_ptr input_impl_ GUARDED_BY(mu_); - MemoryCache* const cache_ GUARDED_BY(mu_); // not owned. - }; // MemoryWriterIterator - - class MemoryReaderIterator : public DatasetIterator { - public: - explicit MemoryReaderIterator(const Params& params, MemoryCache* cache) - : DatasetIterator(params), cache_(cache), index_(0) { - CHECK(cache); - } - - Status Initialize(IteratorContext* ctx) override { - // The memory allocated for the cache is owned by the parent - // dataset but performance modeling uses the iterator abstraction and - // thus we record the memory allocated for the cache here. The caveat - // is that this is incorrect if there are concurrent instances of this - // iterator. - tf_shared_lock l(mu_); - for (size_t i = 0; i < cache_->size(); ++i) { - RecordBufferEnqueue(ctx, cache_->at(i)); - } + } else { + *end_of_sequence = true; return Status::OK(); } - - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - if (index_ < cache_->size()) { - const std::vector& cache_tensors = cache_->at(index_); - out_tensors->insert(out_tensors->begin(), cache_tensors.begin(), - cache_tensors.end()); - index_++; - *end_of_sequence = false; - return Status::OK(); - } else { - *end_of_sequence = true; - return Status::OK(); - } - } - - protected: - std::shared_ptr CreateNode( - IteratorContext* ctx, model::Node::Args args) const override { - return model::MakeKnownRatioNode(std::move(args), - /*ratio=*/1); - } - - Status SaveInternal(IteratorStateWriter* writer) override { - mutex_lock l(mu_); - TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("index"), index_)); - return Status::OK(); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - mutex_lock l(mu_); - { - int64 temp; - TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("index"), &temp)); - index_ = static_cast(temp); - } - return Status::OK(); - } - - private: - mutex mu_; - MemoryCache* const cache_ GUARDED_BY(mu_); // not owned. - size_t index_ GUARDED_BY(mu_); - }; // MemoryReaderIterator - - void InitializeIterator() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - switch (mode_) { - case Mode::read: - iterator_ = absl::make_unique( - MemoryReaderIterator::Params{dataset(), - strings::StrCat(prefix(), "Impl")}, - cache_); - break; - case Mode::write: - iterator_ = absl::make_unique( - MemoryWriterIterator::Params{dataset(), - strings::StrCat(prefix(), "Impl")}, - cache_); - } } + protected: + std::shared_ptr CreateNode( + IteratorContext* ctx, model::Node::Args args) const override { + return model::MakeKnownRatioNode(std::move(args), + /*ratio=*/1); + } + + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name(kIndex), index_)); + return Status::OK(); + } + + Status RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + { + int64 temp; + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name(kIndex), &temp)); + index_ = static_cast(temp); + } + return Status::OK(); + } + + private: mutex mu_; - MemoryCache* cache_ GUARDED_BY(mu_); // not owned. - enum Mode { read, write }; - Mode mode_ GUARDED_BY(mu_); - std::unique_ptr iterator_ GUARDED_BY(mu_); - }; // MemoryIterator + MemoryCache* const cache_ GUARDED_BY(mu_); // not owned. + size_t index_ GUARDED_BY(mu_); + }; // MemoryReaderIterator - const DatasetBase* const input_; - }; // MemoryDataset -}; // CacheDatasetOp + void InitializeIterator() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + switch (mode_) { + case Mode::read: + iterator_ = absl::make_unique( + MemoryReaderIterator::Params{dataset(), + strings::StrCat(prefix(), kImpl)}, + cache_); + break; + case Mode::write: + iterator_ = absl::make_unique( + MemoryWriterIterator::Params{dataset(), + strings::StrCat(prefix(), kImpl)}, + cache_); + } + } + mutex mu_; + MemoryCache* cache_ GUARDED_BY(mu_); // not owned. + enum Mode { read, write }; + Mode mode_ GUARDED_BY(mu_); + std::unique_ptr iterator_ GUARDED_BY(mu_); + }; // MemoryIterator + + const DatasetBase* const input_; +}; // MemoryDataset + +CacheDatasetOp::CacheDatasetOp(OpKernelConstruction* ctx) + : UnaryDatasetOpKernel(ctx) {} + +void CacheDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input, + DatasetBase** output) { + // Parse out the filenames tensor. + string filename; + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, kFileName, &filename)); + + if (filename.empty()) { + *output = new MemoryDataset(ctx, input); + } else { + *output = new FileDataset(ctx, input, filename, ctx->env()); + } +} + +namespace { REGISTER_KERNEL_BUILDER(Name("CacheDataset").Device(DEVICE_CPU), CacheDatasetOp); - } // namespace } // namespace data } // namespace tensorflow diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.h b/tensorflow/core/kernels/data/cache_dataset_ops.h new file mode 100644 index 00000000000..af023a60075 --- /dev/null +++ b/tensorflow/core/kernels/data/cache_dataset_ops.h @@ -0,0 +1,45 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_KERNELS_DATA_CACHE_DATASET_OP_H_ +#define TENSORFLOW_CORE_KERNELS_DATA_CACHE_DATASET_OP_H_ + +#include "tensorflow/core/framework/dataset.h" + +namespace tensorflow { +namespace data { + +class CacheDatasetOp : public UnaryDatasetOpKernel { + public: + static constexpr const char* const kDatasetType = "Cache"; + static constexpr const char* const kInputDataset = "input_dataset"; + static constexpr const char* const kFileName = "filename"; + static constexpr const char* const kOutputTypes = "output_types"; + static constexpr const char* const kOutputShapes = "output_shapes"; + + explicit CacheDatasetOp(OpKernelConstruction* ctx); + + protected: + void MakeDataset(OpKernelContext* ctx, DatasetBase* input, + DatasetBase** output) override; + + private: + class FileDataset; + class MemoryDataset; +}; + +} // namespace data +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_DATA_CACHE_DATASET_OP_H_ diff --git a/tensorflow/core/kernels/data/name_utils.cc b/tensorflow/core/kernels/data/name_utils.cc index 391f45014c8..b6404892fdb 100644 --- a/tensorflow/core/kernels/data/name_utils.cc +++ b/tensorflow/core/kernels/data/name_utils.cc @@ -65,10 +65,11 @@ string IteratorPrefix(const string& dataset_type, const string& prefix) { string IteratorPrefix(const string& dataset_type, const string& prefix, const IteratorPrefixParams& params) { if (params.op_version == 1) { - return strings::StrCat(prefix, kDelimiter, dataset_type); + return strings::StrCat(prefix, kDelimiter, params.dataset_prefix, + dataset_type); } - return strings::StrCat(prefix, kDelimiter, dataset_type, kVersion, - params.op_version); + return strings::StrCat(prefix, kDelimiter, params.dataset_prefix, + dataset_type, kVersion, params.op_version); } } // namespace name_utils diff --git a/tensorflow/core/kernels/data/name_utils.h b/tensorflow/core/kernels/data/name_utils.h index 0efa825ec5e..5171b8e05e3 100644 --- a/tensorflow/core/kernels/data/name_utils.h +++ b/tensorflow/core/kernels/data/name_utils.h @@ -44,6 +44,7 @@ struct DatasetDebugStringParams { struct IteratorPrefixParams { int op_version = 1; + string dataset_prefix = ""; }; // Merge the given args in the format of "(arg1, arg2, ..., argn)". From e00c5f66d9a7f697c9c400549171b0eb4efc9d4f Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Tue, 2 Jul 2019 10:50:16 -0700 Subject: [PATCH 045/332] Add tests for CacheDatasetOps --- tensorflow/core/kernels/data/BUILD | 19 + .../kernels/data/cache_dataset_ops_test.cc | 533 ++++++++++++++++++ 2 files changed, 552 insertions(+) create mode 100644 tensorflow/core/kernels/data/cache_dataset_ops_test.cc diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD index 5c8fbd7333d..4b332ed15e5 100644 --- a/tensorflow/core/kernels/data/BUILD +++ b/tensorflow/core/kernels/data/BUILD @@ -1063,7 +1063,9 @@ tf_kernel_library( tf_kernel_library( name = "cache_dataset_ops", srcs = ["cache_dataset_ops.cc"], + hdrs = ["cache_dataset_ops.h"], deps = [ + ":name_utils", "//tensorflow/core:dataset_ops_op_lib", "//tensorflow/core:framework", "//tensorflow/core:lib", @@ -1072,6 +1074,23 @@ tf_kernel_library( ], ) +tf_cc_test( + name = "cache_dataset_ops_test", + srcs = ["cache_dataset_ops_test.cc"], + deps = [ + ":cache_dataset_ops", + ":dataset_test_base", + ":dataset_utils", + ":iterator_ops", + ":tensor_slice_dataset_op", + "//tensorflow/core:framework", + "//tensorflow/core:ptr_util", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + tf_kernel_library( name = "optimize_dataset_op", srcs = ["optimize_dataset_op.cc"], diff --git a/tensorflow/core/kernels/data/cache_dataset_ops_test.cc b/tensorflow/core/kernels/data/cache_dataset_ops_test.cc new file mode 100644 index 00000000000..7e697d3f917 --- /dev/null +++ b/tensorflow/core/kernels/data/cache_dataset_ops_test.cc @@ -0,0 +1,533 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/kernels/data/cache_dataset_ops.h" + +#include "tensorflow/core/kernels/data/dataset_test_base.h" + +namespace tensorflow { +namespace data { +namespace { + +constexpr char kNodeName[] = "cache_dataset"; +constexpr char kIteratorPrefix[] = "Iterator"; +constexpr char kFileDatasetPrefix[] = "File"; +constexpr char kMemoryDatasetPrefix[] = "Memory"; + +class CacheDatasetOpTest : public DatasetOpsTestBase { + protected: + // Creates `TensorSliceDataset` variant tensor from the input vector of + // tensors. + Status CreateTensorSliceDatasetTensor( + std::vector* const tensor_vector, Tensor* dataset_tensor) { + DatasetBase* tensor_slice_dataset; + TF_RETURN_IF_ERROR(CreateTensorSliceDataset( + "tensor_slice_node", tensor_vector, &tensor_slice_dataset)); + TF_RETURN_IF_ERROR( + StoreDatasetInVariantTensor(tensor_slice_dataset, dataset_tensor)); + return Status::OK(); + } + + // Create a new `CacheDataset` op kernel. + Status CreateCacheDatasetOpKernel( + const DataTypeVector& output_types, + const std::vector& output_shapes, + std::unique_ptr* cache_dataset_op_kernel) { + NodeDef node_def = test::function::NDef( + kNodeName, name_utils::OpName(CacheDatasetOp::kDatasetType), + {CacheDatasetOp::kInputDataset, CacheDatasetOp::kFileName}, + {{CacheDatasetOp::kOutputTypes, output_types}, + {CacheDatasetOp::kOutputShapes, output_shapes}}); + TF_RETURN_IF_ERROR(CreateOpKernel(node_def, cache_dataset_op_kernel)); + return Status::OK(); + } + + // Create a new `CacheDataset` op kernel context. + Status CreateCacheDatasetContext( + OpKernel* const op_kernel, + gtl::InlinedVector* const inputs, + std::unique_ptr* context) { + TF_RETURN_IF_ERROR(CheckOpKernelInput(*op_kernel, *inputs)); + TF_RETURN_IF_ERROR(CreateOpKernelContext(op_kernel, inputs, context)); + return Status::OK(); + } +}; + +struct TestCase { + std::vector input_tensors; + string file_name; + std::vector expected_outputs; + DataTypeVector expected_output_dtypes; + std::vector expected_output_shapes; + int64 expected_cardinality; + std::vector breakpoints; +}; + +// Test case 1: cache data in file. +TestCase TestCase1() { + return { + /*input_tensors*/ {DatasetOpsTestBase::CreateTensor( + TensorShape{3, 3, 1}, {0, 1, 2, 3, 4, 5, 6, 7, 8})}, + /*file_name*/ "cache_data", + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape{3, 1}, {0, 1, 2}), + DatasetOpsTestBase::CreateTensor(TensorShape{3, 1}, {3, 4, 5}), + DatasetOpsTestBase::CreateTensor(TensorShape{3, 1}, {6, 7, 8})}, + /*expected_output_dtypes*/ {DT_INT64}, + /*expected_output_shapes*/ {PartialTensorShape({3, 1})}, + /*expected_cardinality*/ 3, + /*breakpoints*/ {0, 4, 11}}; +} + +// Test case 2: cache empty data in file. +TestCase TestCase2() { + return {/*input_tensors*/ { + DatasetOpsTestBase::CreateTensor(TensorShape{0}, {})}, + /*file_name*/ "empty_cache_data", + /*expected_outputs*/ {}, + /*expected_output_dtypes*/ {DT_INT64}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ 0, + /*breakpoints*/ {0, 4, 11}}; +} + +// Test case 3: cache data in memory. +TestCase TestCase3() { + return { + /*input_tensors*/ {DatasetOpsTestBase::CreateTensor( + TensorShape{3, 3, 1}, {0, 1, 2, 3, 4, 5, 6, 7, 8})}, + /*file_name*/ "", + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape{3, 1}, {0, 1, 2}), + DatasetOpsTestBase::CreateTensor(TensorShape{3, 1}, {3, 4, 5}), + DatasetOpsTestBase::CreateTensor(TensorShape{3, 1}, {6, 7, 8})}, + /*expected_output_dtypes*/ {DT_INT64}, + /*expected_output_shapes*/ {PartialTensorShape({3, 1})}, + /*expected_cardinality*/ 3, + /*breakpoints*/ {0, 4, 11}}; +} + +// Test case 4: cache empty data in memory. +TestCase TestCase4() { + return {/*input_tensors*/ { + DatasetOpsTestBase::CreateTensor(TensorShape{0}, {})}, + /*file_name*/ "", + /*expected_outputs*/ {}, + /*expected_output_dtypes*/ {DT_INT64}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ 0, + /*breakpoints*/ {0, 4, 11}}; +} + +class ParameterizedCacheDatasetOpTest + : public CacheDatasetOpTest, + public ::testing::WithParamInterface {}; + +TEST_P(ParameterizedCacheDatasetOpTest, GetNext) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(cache_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(cache_dataset->MakeIterator(iterator_ctx.get(), kIteratorPrefix, + &iterator)); + + // Test the write mode. + bool end_of_sequence = false; + std::vector out_tensors; + while (!end_of_sequence) { + std::vector next; + TF_EXPECT_OK( + iterator->GetNext(iterator_ctx.get(), &next, &end_of_sequence)); + out_tensors.insert(out_tensors.end(), next.begin(), next.end()); + } + TF_EXPECT_OK(ExpectEqual(out_tensors, test_case.expected_outputs, + /*compare_order*/ true)); + + // Test the read mode. + TF_ASSERT_OK(cache_dataset->MakeIterator(iterator_ctx.get(), kIteratorPrefix, + &iterator)); + end_of_sequence = false; + out_tensors.clear(); + while (!end_of_sequence) { + std::vector next; + TF_EXPECT_OK( + iterator->GetNext(iterator_ctx.get(), &next, &end_of_sequence)); + out_tensors.insert(out_tensors.end(), next.begin(), next.end()); + } + TF_EXPECT_OK(ExpectEqual(out_tensors, test_case.expected_outputs, + /*compare_order*/ true)); +} + +TEST_F(CacheDatasetOpTest, DatasetNodeName) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = TestCase1(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + EXPECT_EQ(cache_dataset->node_name(), kNodeName); +} + +TEST_P(ParameterizedCacheDatasetOpTest, DatasetTypeString) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + EXPECT_EQ(cache_dataset->type_string(), + name_utils::OpName(CacheDatasetOp::kDatasetType)); +} + +TEST_P(ParameterizedCacheDatasetOpTest, DatasetOutputDtypes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + TF_EXPECT_OK(VerifyTypesMatch(cache_dataset->output_dtypes(), + test_case.expected_output_dtypes)); +} + +TEST_P(ParameterizedCacheDatasetOpTest, DatasetOutputShapes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + TF_EXPECT_OK(VerifyShapesCompatible(cache_dataset->output_shapes(), + test_case.expected_output_shapes)); +} + +TEST_P(ParameterizedCacheDatasetOpTest, Cardinality) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + EXPECT_EQ(cache_dataset->Cardinality(), test_case.expected_cardinality); +} + +TEST_P(ParameterizedCacheDatasetOpTest, DatasetSave) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + std::unique_ptr serialization_context; + TF_ASSERT_OK(CreateSerializationContext(&serialization_context)); + VariantTensorData data; + VariantTensorDataWriter writer(&data); + TF_ASSERT_OK(cache_dataset->Save(serialization_context.get(), &writer)); + TF_ASSERT_OK(writer.Flush()); +} + +TEST_P(ParameterizedCacheDatasetOpTest, IteratorOutputShapes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(cache_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(cache_dataset->MakeIterator(iterator_ctx.get(), kIteratorPrefix, + &iterator)); + + TF_EXPECT_OK(VerifyTypesMatch(iterator->output_dtypes(), + test_case.expected_output_dtypes)); +} + +TEST_P(ParameterizedCacheDatasetOpTest, IteratorOutputPrefix) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(cache_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(cache_dataset->MakeIterator(iterator_ctx.get(), kIteratorPrefix, + &iterator)); + + name_utils::IteratorPrefixParams params; + params.dataset_prefix = + test_case.file_name.empty() ? kMemoryDatasetPrefix : kFileDatasetPrefix; + EXPECT_EQ(iterator->prefix(), + name_utils::IteratorPrefix(CacheDatasetOp::kDatasetType, + kIteratorPrefix, params)); +} + +TEST_P(ParameterizedCacheDatasetOpTest, Roundtrip) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::unique_ptr cache_dataset_kernel; + TF_ASSERT_OK(CreateCacheDatasetOpKernel(test_case.expected_output_dtypes, + test_case.expected_output_shapes, + &cache_dataset_kernel)); + Tensor tensor_slice_dataset_tensor(DT_VARIANT, TensorShape({})); + std::vector inputs_for_tensor_slice_dataset = test_case.input_tensors; + TF_ASSERT_OK(CreateTensorSliceDatasetTensor(&inputs_for_tensor_slice_dataset, + &tensor_slice_dataset_tensor)); + Tensor file_name = CreateTensor(TensorShape{}, {test_case.file_name}); + gtl::InlinedVector inputs( + {TensorValue(&tensor_slice_dataset_tensor), TensorValue(&file_name)}); + std::unique_ptr cache_dataset_context; + TF_ASSERT_OK(CreateCacheDatasetContext(cache_dataset_kernel.get(), &inputs, + &cache_dataset_context)); + DatasetBase* cache_dataset; + TF_ASSERT_OK(CreateDataset(cache_dataset_kernel.get(), + cache_dataset_context.get(), &cache_dataset)); + core::ScopedUnref scoped_unref(cache_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(cache_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(cache_dataset->MakeIterator(iterator_ctx.get(), kIteratorPrefix, + &iterator)); + + bool end_of_sequence = false; + std::vector out_tensors; + // For MemoryIterator in the read mode, the cache needs to be completed before + // it has been read. + if (test_case.file_name.empty()) { + while (!end_of_sequence) { + TF_EXPECT_OK(iterator->GetNext(iterator_ctx.get(), &out_tensors, + &end_of_sequence)); + } + end_of_sequence = false; + out_tensors.clear(); + TF_ASSERT_OK(cache_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + } + + std::unique_ptr serialization_ctx; + TF_ASSERT_OK(CreateSerializationContext(&serialization_ctx)); + int cur_iteration = 0; + auto expected_outputs_it = test_case.expected_outputs.begin(); + for (int breakpoint : test_case.breakpoints) { + VariantTensorData data; + VariantTensorDataWriter writer(&data); + TF_EXPECT_OK(iterator->Save(serialization_ctx.get(), &writer)); + TF_EXPECT_OK(writer.Flush()); + VariantTensorDataReader reader(&data); + TF_EXPECT_OK(RestoreIterator(iterator_ctx.get(), &reader, kIteratorPrefix, + *cache_dataset, &iterator)); + + while (cur_iteration <= breakpoint) { + out_tensors.clear(); + TF_EXPECT_OK(iterator->GetNext(iterator_ctx.get(), &out_tensors, + &end_of_sequence)); + if (!end_of_sequence) { + EXPECT_LT(expected_outputs_it, test_case.expected_outputs.end()); + TF_EXPECT_OK(ExpectEqual(out_tensors.back(), *expected_outputs_it)); + expected_outputs_it++; + } + cur_iteration++; + } + + if (breakpoint >= test_case.expected_cardinality) { + EXPECT_TRUE(end_of_sequence); + EXPECT_EQ(expected_outputs_it, test_case.expected_outputs.end()); + } else { + EXPECT_FALSE(end_of_sequence); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + CacheDatasetOpTest, ParameterizedCacheDatasetOpTest, + ::testing::ValuesIn(std::vector({TestCase1(), TestCase2(), + TestCase3(), TestCase4()}))); + +} // namespace +} // namespace data +} // namespace tensorflow From 63730603361314b7260778f2690f9a08451e4e49 Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Tue, 2 Jul 2019 19:48:16 +0000 Subject: [PATCH 046/332] Rename nvptx_constants to target_constants and add AMDGPU triple and datalayout --- tensorflow/compiler/xla/service/gpu/BUILD | 8 +++---- .../xla/service/gpu/gpu_transfer_manager.cc | 4 ++-- .../xla/service/gpu/nvptx_compiler.cc | 8 +++---- .../{nvptx_constants.h => target_constants.h} | 21 ++++++++++++++++--- 4 files changed, 28 insertions(+), 13 deletions(-) rename tensorflow/compiler/xla/service/gpu/{nvptx_constants.h => target_constants.h} (60%) diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 9858fe6cfc6..384e0befa0f 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -927,8 +927,8 @@ tf_cc_test( ) cc_library( - name = "nvptx_constants", - hdrs = ["nvptx_constants.h"], + name = "target_constants", + hdrs = ["target_constants.h"], ) cc_library( @@ -937,7 +937,7 @@ cc_library( hdrs = ["gpu_transfer_manager.h"], deps = [ ":infeed_manager", - ":nvptx_constants", + ":target_constants", ":outfeed_manager", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:literal_util", @@ -985,10 +985,10 @@ cc_library( ":ir_emission_utils", ":ir_emitter", ":multi_output_fusion", - ":nvptx_constants", ":partition_assignment", ":stream_assignment", ":stream_executor_util", + ":target_constants", ":variadic_op_splitter", "//tensorflow/compiler/xla:protobuf_util", "//tensorflow/compiler/xla:status_macros", diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc index ec1a093a31c..ba25f7db0f8 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc @@ -23,7 +23,7 @@ limitations under the License. #include "llvm/IR/DataLayout.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/literal_util.h" -#include "tensorflow/compiler/xla/service/gpu/nvptx_constants.h" +#include "tensorflow/compiler/xla/service/gpu/target_constants.h" #include "tensorflow/compiler/xla/service/gpu/outfeed_manager.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -181,7 +181,7 @@ Status GpuTransferManager::TransferLiteralFromOutfeed( static std::unique_ptr CreateNVPTXTransferManager() { return absl::make_unique( /*id=*/stream_executor::cuda::kCudaPlatformId, - /*pointer_size=*/llvm::DataLayout(xla::gpu::kDataLayout) + /*pointer_size=*/llvm::DataLayout(xla::gpu::nvptx::kDataLayout) .getPointerSize(0 /* default address space */)); } diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc index 48df4e4f3b8..bc25527238a 100644 --- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc @@ -66,10 +66,10 @@ limitations under the License. #include "tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h" #include "tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/nvptx_backend_lib.h" #include "tensorflow/compiler/xla/service/gpu/multi_output_fusion.h" -#include "tensorflow/compiler/xla/service/gpu/nvptx_constants.h" #include "tensorflow/compiler/xla/service/gpu/partition_assignment.h" #include "tensorflow/compiler/xla/service/gpu/stream_assignment.h" #include "tensorflow/compiler/xla/service/gpu/stream_executor_util.h" +#include "tensorflow/compiler/xla/service/gpu/target_constants.h" #include "tensorflow/compiler/xla/service/gpu/thunk_schedule.h" #include "tensorflow/compiler/xla/service/gpu/variadic_op_splitter.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" @@ -497,7 +497,7 @@ void WarnIfBadDriverJITVersion() { } // namespace NVPTXCompiler::NVPTXCompiler() - : pointer_size_(llvm::DataLayout(kDataLayout) + : pointer_size_(llvm::DataLayout(nvptx::kDataLayout) .getPointerSize(0 /* default address space */)) {} StatusOr> NVPTXCompiler::RunHloPasses( @@ -536,8 +536,8 @@ StatusOr> NVPTXCompiler::RunBackend( llvm::Module llvm_module(module->name().c_str(), llvm_context); // Set the target triple and the data layout. - llvm_module.setTargetTriple(kTargetTriple); - llvm_module.setDataLayout(kDataLayout); + llvm_module.setTargetTriple(nvptx::kTargetTriple); + llvm_module.setDataLayout(nvptx::kDataLayout); // Determine the HLO schedule, which is an ordering of HLO instructions. This // is used by buffer assignment to enable buffer reuse, and the same ordering diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_constants.h b/tensorflow/compiler/xla/service/gpu/target_constants.h similarity index 60% rename from tensorflow/compiler/xla/service/gpu/nvptx_constants.h rename to tensorflow/compiler/xla/service/gpu/target_constants.h index 67fa0020aa0..cb2d682d693 100644 --- a/tensorflow/compiler/xla/service/gpu/nvptx_constants.h +++ b/tensorflow/compiler/xla/service/gpu/target_constants.h @@ -13,20 +13,35 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_NVPTX_CONSTANTS_H_ -#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_NVPTX_CONSTANTS_H_ +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TARGET_CONSTANTS_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TARGET_CONSTANTS_H_ namespace xla { namespace gpu { +namespace nvptx { // The triple that represents our target. constexpr char kTargetTriple[] = "nvptx64-nvidia-cuda"; // The data layout of the emitted module. Copied from computeDataLayout in // NVPTXTargetMachine.cpp. constexpr char kDataLayout[] = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"; +} // namespace nvptx + +namespace amdgpu { + + // The triple that represents our target on LLVM AMDGPU backend. +constexpr char kTargetTriple[] = "amdgcn-amd-amdhsa"; + + // The data layout of the emitted module. +constexpr char kDataLayout[] = + "e-p:64:64-p1:64:64-p2:64:64-p3:32:32-p4:32:32-p5:32:32" + "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128" + "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-A5"; + + } // namespace amdgpu } // namespace gpu } // namespace xla -#endif // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_NVPTX_CONSTANTS_H_ +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_TARGET_CONSTANTS_H_ From 47324c000e52d8f91206099e0d1526d3488b13e9 Mon Sep 17 00:00:00 2001 From: Frederic Bastien Date: Wed, 26 Jun 2019 11:36:13 -0700 Subject: [PATCH 047/332] Consider integer division/remainder by constant scalar as cheap. LLVM optimize that to faster instruction. --- tensorflow/compiler/xla/service/gpu/BUILD | 1 + .../xla/service/gpu/instruction_fusion.cc | 29 +++++++--- .../service/gpu/instruction_fusion_test.cc | 57 +++++++++++++++++++ 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 9858fe6cfc6..99017ce1a25 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -730,6 +730,7 @@ cc_library( "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/compiler/xla/service:hlo_query", "//tensorflow/compiler/xla/service:instruction_fusion", "//tensorflow/compiler/xla/service:pattern_matcher", "//tensorflow/compiler/xla/service/llvm_ir:fused_ir_emitter", diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc index a696c220147..46739880de6 100644 --- a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/gpu/gpu_fusible.h" #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/service/hlo_query.h" #include "tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h" #include "tensorflow/compiler/xla/service/pattern_matcher.h" #include "tensorflow/compiler/xla/shape_util.h" @@ -42,15 +43,27 @@ bool IsIEEEFloatingPointScalarConstant(const HloInstruction* constant) { /*static*/ bool GpuInstructionFusion::IsExpensive( const HloInstruction& instruction) { - switch (instruction.opcode()) { - // We say that floating-point division is cheap on the GPU. - case HloOpcode::kDivide: - return !ShapeUtil::ElementIsFloating(instruction.shape()) && - InstructionFusion::IsExpensive(instruction); - - default: - return InstructionFusion::IsExpensive(instruction); + // We say that floating-point division is cheap on the GPU. + if (instruction.opcode() == HloOpcode::kDivide && + ShapeUtil::ElementIsFloating(instruction.shape())) { + return false; } + // LLVM optimizes the integer division/remainder by a + // constant scalar to a few fast operations. + if ((instruction.opcode() == HloOpcode::kDivide || + instruction.opcode() == HloOpcode::kRemainder) && + ShapeUtil::ElementIsIntegral(instruction.shape())) { + auto* operand1 = instruction.operands()[1]; + if (hlo_query::IsScalarConstant(operand1)) { + return false; + } + // Broadcasted scalar is also being optimized. + if (operand1->opcode() == HloOpcode::kBroadcast && + hlo_query::IsScalarConstant(operand1->operands()[0])) { + return false; + } + } + return InstructionFusion::IsExpensive(instruction); } bool GpuInstructionFusion::ShouldFuseInexpensiveChecks(HloInstruction* consumer, diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc index bf65ec520cc..48c6170d1e4 100644 --- a/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc +++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion_test.cc @@ -581,5 +581,62 @@ TEST_F(InstructionFusionTest, FuseReverse) { op::Reverse(op::Add(op::Parameter(), op::Parameter()))); } +TEST_F(InstructionFusionTest, GpuIsExpensiveF32) { + auto m = CreateNewVerifiedModule(); + Shape r0f32 = ShapeUtil::MakeShape(F32, {}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r0f32, "param0")); + + HloInstruction* one = builder.AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(1.0f))); + HloInstruction* div = builder.AddInstruction( + HloInstruction::CreateBinary(r0f32, HloOpcode::kDivide, param0, one)); + HloInstruction* rem = builder.AddInstruction( + HloInstruction::CreateBinary(r0f32, HloOpcode::kRemainder, param0, one)); + + EXPECT_FALSE(GpuInstructionFusion::IsExpensive(*div)); + EXPECT_TRUE(GpuInstructionFusion::IsExpensive(*rem)); +} + +TEST_F(InstructionFusionTest, GpuIsExpensiveS32) { + auto m = CreateNewVerifiedModule(); + Shape r0s32 = ShapeUtil::MakeShape(S32, {}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r0s32, "param0")); + + HloInstruction* one = builder.AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(1.0f))); + HloInstruction* div = builder.AddInstruction( + HloInstruction::CreateBinary(r0s32, HloOpcode::kDivide, param0, one)); + HloInstruction* rem = builder.AddInstruction( + HloInstruction::CreateBinary(r0s32, HloOpcode::kRemainder, param0, one)); + + EXPECT_FALSE(GpuInstructionFusion::IsExpensive(*div)); + EXPECT_FALSE(GpuInstructionFusion::IsExpensive(*rem)); +} + +TEST_F(InstructionFusionTest, GpuIsExpensiveBroadcastS32) { + auto m = CreateNewVerifiedModule(); + Shape r1s32 = ShapeUtil::MakeShape(S32, {10}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r1s32, "param0")); + + HloInstruction* one = builder.AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::CreateR0(1.0f))); + HloInstruction* one_broad = builder.AddInstruction( + HloInstruction::CreateBroadcast(r1s32, one, {})); + + HloInstruction* div = builder.AddInstruction( + HloInstruction::CreateBinary(r1s32, HloOpcode::kDivide, param0, one_broad)); + HloInstruction* rem = builder.AddInstruction( + HloInstruction::CreateBinary(r1s32, HloOpcode::kRemainder, param0, one_broad)); + + EXPECT_FALSE(GpuInstructionFusion::IsExpensive(*div)); + EXPECT_FALSE(GpuInstructionFusion::IsExpensive(*rem)); +} + } // namespace gpu } // namespace xla From e6ed5c576f0ca02a7a38ece5ae6b0e35dd67622f Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Tue, 2 Jul 2019 22:21:03 +0000 Subject: [PATCH 048/332] Address do_buildifier comments --- tensorflow/compiler/xla/service/gpu/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 384e0befa0f..90fe70d3d35 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -937,8 +937,8 @@ cc_library( hdrs = ["gpu_transfer_manager.h"], deps = [ ":infeed_manager", - ":target_constants", ":outfeed_manager", + ":target_constants", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:shape_tree", From e6593c186bed1e740efd4e65463542a1ca3a63d2 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Tue, 2 Jul 2019 18:25:44 -0700 Subject: [PATCH 049/332] Revise the variable name --- tensorflow/core/kernels/data/cache_dataset_ops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc index 43a7bfcb2f5..9b1fed90463 100644 --- a/tensorflow/core/kernels/data/cache_dataset_ops.cc +++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc @@ -35,7 +35,7 @@ namespace data { /* static */ constexpr const char* const CacheDatasetOp::kOutputTypes; /* static */ constexpr const char* const CacheDatasetOp::kOutputShapes; -constexpr char kTensorStrFormat[] = "%%%zuzu_%%%zuzu"; +constexpr char kKeyStrFormat[] = "%%%zuzu_%%%zuzu"; constexpr char kPaddingSizeStrFormat[] = "%zu"; constexpr char kFileDatasetPrefix[] = "File"; constexpr char kMode[] = "Mode"; @@ -66,7 +66,7 @@ class CacheDatasetOp::FileDataset : public DatasetBase { num_tensors_(input->output_dtypes().size()), tensor_index_padding_size_(StringPaddingSize(num_tensors_)), item_index_padding_size_(StringPaddingSize(kMaxItems)), - tensor_format_string_(strings::Printf(kTensorStrFormat, + tensor_format_string_(strings::Printf(kKeyStrFormat, item_index_padding_size_, tensor_index_padding_size_)) { input_->Ref(); From 9109af0b86c1cf503fe11dd6fc578ce4f23cdd8a Mon Sep 17 00:00:00 2001 From: Siju Date: Wed, 3 Jul 2019 11:30:12 +0530 Subject: [PATCH 050/332] [LITE]BugFix non-string type causes memleak for two outputs --- tensorflow/lite/models/smartreply/ops/extract_feature.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/lite/models/smartreply/ops/extract_feature.cc b/tensorflow/lite/models/smartreply/ops/extract_feature.cc index f9d29229457..8ec05885976 100644 --- a/tensorflow/lite/models/smartreply/ops/extract_feature.cc +++ b/tensorflow/lite/models/smartreply/ops/extract_feature.cc @@ -59,8 +59,6 @@ bool IsValidNgram(const tflite::StringRef& strref) { } TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { - TfLiteIntArray* outputSize1 = TfLiteIntArrayCreate(1); - TfLiteIntArray* outputSize2 = TfLiteIntArrayCreate(1); const TfLiteTensor* input = GetInput(context, node, 0); int dim = input->dims->data[0]; if (dim == 0) { @@ -68,6 +66,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { dim = 1; } TF_LITE_ENSURE_EQ(context, input->type, kTfLiteString); + TfLiteIntArray* outputSize1 = TfLiteIntArrayCreate(1); + TfLiteIntArray* outputSize2 = TfLiteIntArrayCreate(1); outputSize1->data[0] = dim; outputSize2->data[0] = dim; context->ResizeTensor(context, GetOutput(context, node, 0), outputSize1); From b47c1aafac45396ad1b3c620ac87029d45801477 Mon Sep 17 00:00:00 2001 From: wenxizhu Date: Wed, 3 Jul 2019 14:46:34 +0800 Subject: [PATCH 051/332] Fix clang-format issues. --- tensorflow/core/kernels/mkl_conv_ops.h | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h index 8222619deca..a314f76b83d 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.h +++ b/tensorflow/core/kernels/mkl_conv_ops.h @@ -196,9 +196,8 @@ class MklDnnConvUtil { filter_shape.DebugString())); for (int i = 0; i < ((strides_.size() == 4) ? 3 : 5); i++) { - OP_REQUIRES(context_, - FastBoundsCheck(filter_shape.dim_size(i), - std::numeric_limits::max()), + OP_REQUIRES(context_, FastBoundsCheck(filter_shape.dim_size(i), + std::numeric_limits::max()), errors::InvalidArgument("filter too large")); } @@ -413,14 +412,16 @@ class MklDnnConvUtil { } else { OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerboseV2( input_planes, filter_planes, dilation_planes, - stride_planes, padding_, &out_planes, &pad_D1, - &pad_D2)); - OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerboseV2( - input_rows, filter_rows, dilation_rows, stride_rows, - padding_, &out_rows, &pad_top, &pad_bottom)); - OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerboseV2( - input_cols, filter_cols, dilation_cols, stride_cols, - padding_, &out_cols, &pad_left, &pad_right)); + stride_planes, padding_, &out_planes, + &pad_D1, &pad_D2)); + OP_REQUIRES_OK(context_, + GetWindowedOutputSizeVerboseV2( + input_rows, filter_rows, dilation_rows, stride_rows, + padding_, &out_rows, &pad_top, &pad_bottom)); + OP_REQUIRES_OK(context_, + GetWindowedOutputSizeVerboseV2( + input_cols, filter_cols, dilation_cols, stride_cols, + padding_, &out_cols, &pad_left, &pad_right)); } if (is_conv2d) { From ea353f8fc9f20efdf792a99dd1df5919883e403d Mon Sep 17 00:00:00 2001 From: Rasmus Diederichsen Date: Wed, 3 Jul 2019 11:04:43 +0200 Subject: [PATCH 052/332] Check for host OS arch for linking CoreFoundation framework --- tensorflow/contrib/makefile/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 090df268915..dd484a90e50 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -911,8 +911,10 @@ $(PROTO_TEXT_OBJS) : $(PROTO_TEXT_PB_H_FILES) # Ensures we link CoreFoundation as it is used for time library when building # for Mac. ifeq ($(TARGET),OSX) - HOST_LDOPTS += -framework CoreFoundation - LIBS += -framework CoreFoundation + ifeq($(HOST_ARCH), x86_64) + HOST_LDOPTS += -framework CoreFoundation + LIBS += -framework CoreFoundation + endif() endif ifeq ($(TARGET),IOS) ifeq ($(IOS_ARCH),X86_64) From cf75208dfe89bd5af51c944f423ba417a041ead7 Mon Sep 17 00:00:00 2001 From: Rasmus Diederichsen Date: Wed, 3 Jul 2019 16:39:44 +0200 Subject: [PATCH 053/332] Fix endif-error --- tensorflow/contrib/makefile/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index dd484a90e50..922d837541f 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -914,7 +914,7 @@ ifeq ($(TARGET),OSX) ifeq($(HOST_ARCH), x86_64) HOST_LDOPTS += -framework CoreFoundation LIBS += -framework CoreFoundation - endif() + endif endif ifeq ($(TARGET),IOS) ifeq ($(IOS_ARCH),X86_64) From 2a40f531ffadacb602071c77938fed0eef81f6d4 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 3 Jul 2019 18:45:00 +0100 Subject: [PATCH 054/332] Update gpu.Dockerfile added missing /usr/local/cuda/lib64 in LD_LIBRARY_PATH --- tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile index 270524ae298..a6ff1a5ccea 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu.Dockerfile @@ -58,7 +58,7 @@ RUN [ ${ARCH} = ppc64le ] || (apt-get update && \ && rm -rf /var/lib/apt/lists/*) # For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} From a9bbd58220c6bf027a5c18d1bbf15a282a3b2e1e Mon Sep 17 00:00:00 2001 From: "srinivasan.narayanamoorthy" Date: Wed, 3 Jul 2019 13:47:29 -0700 Subject: [PATCH 055/332] Parallelize scatter update operation --- tensorflow/core/kernels/scatter_functor.h | 41 +++++++++++++++------- tensorflow/core/kernels/scatter_op_test.cc | 38 +++++++++++++++++--- 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/tensorflow/core/kernels/scatter_functor.h b/tensorflow/core/kernels/scatter_functor.h index 755f8f8dc55..f3ed40df75e 100644 --- a/tensorflow/core/kernels/scatter_functor.h +++ b/tensorflow/core/kernels/scatter_functor.h @@ -18,14 +18,15 @@ limitations under the License. #include -#include "third_party/eigen3/Eigen/Core" -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/kernels/dense_update_functor.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/work_sharder.h" +#include "third_party/eigen3/Eigen/Core" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" namespace tensorflow { @@ -205,16 +206,32 @@ struct ScatterFunctorBase { // indices and params sizes were validated in DoCompute(). const Index N = static_cast(indices.size()); const Index limit = static_cast(params.dimension(0)); - for (Index i = 0; i < N; i++) { - // Grab the index and check its validity. Do this carefully, - // to avoid checking the value and grabbing it again from - // memory a second time (a security risk since it may change in between). - const Index index = ::tensorflow::internal::SubtleMustCopy(indices(i)); - if (!FastBoundsCheck(index, limit)) return i; - // Copy last Ndim-1 dimensions of updates[i] to params[index] - scatter_op::internal::Assign::Run(params.template chip<0>(index), - updates.template chip<0>(i)); - } + mutex mu_; + Index bad_index GUARDED_BY(mu_); + bool found_bad_index = false GUARDED_BY(mu_); + auto ParallelScatter = [&](Index start, Index end) LOCKS_EXCLUDED(mu_){ + for (Index i = start; i < end; i++) { + // Grab the index and check its validity. Do this carefully, + // to avoid checking the value and grabbing it again from + // memory a second time (a security risk since it may change in + // between). + const Index index = ::tensorflow::internal::SubtleMustCopy(indices(i)); + if (!FastBoundsCheck(index, limit)) { + mutex_lock lock(mu_); + bad_index = i; + found_bad_index = true; + return; + } + // Copy last Ndim-1 dimensions of updates[i] to params[index] + scatter_op::internal::Assign::Run(params.template chip<0>(index), + updates.template chip<0>(i)); + } + }; + const DeviceBase::CpuWorkerThreads& worker_threads = + *(c->device()->tensorflow_cpu_worker_threads()); + Shard(worker_threads.num_threads, worker_threads.workers, N, 35.0, + ParallelScatter); // Cost is arbitrary for now. + if (found_bad_index) return bad_index; return -1; } }; diff --git a/tensorflow/core/kernels/scatter_op_test.cc b/tensorflow/core/kernels/scatter_op_test.cc index ae6548e9ef2..d1924a00f58 100644 --- a/tensorflow/core/kernels/scatter_op_test.cc +++ b/tensorflow/core/kernels/scatter_op_test.cc @@ -47,6 +47,17 @@ class ScatterUpdateOpTest : public OpsTestBase { TF_ASSERT_OK(InitOp()); } }; +class ScatterSubOpTest : public OpsTestBase { + protected: + void MakeOp(DataType variable_ref_type, DataType index_type) { + TF_ASSERT_OK(NodeDefBuilder("myop", "ScatterSub") + .Input(FakeInput(variable_ref_type)) + .Input(FakeInput(index_type)) + .Input(FakeInput(RemoveRefType(variable_ref_type))) + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + } +}; TEST_F(ScatterUpdateOpTest, Simple_StringType) { MakeOp(DT_STRING_REF, DT_INT32); @@ -175,6 +186,19 @@ TEST_F(ScatterUpdateOpTest, Error_IndexOutOfRange) { << s; } +TEST_F(ScatterSubOpTest, Error_IndexOutOfRange) { + MakeOp(DT_FLOAT_REF, DT_INT32); + // Feed and run + AddInputFromArray(TensorShape({14}), + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + AddInputFromArray(TensorShape({3}), {0, 1, 99}); + AddInputFromArray(TensorShape({3}), {100, 101, 102}); + Status s = RunOpKernel(); + EXPECT_TRUE( + absl::StrContains(s.ToString(), "indices[2] = 99 is not in [0, 14)")) + << s; +} + TEST_F(ScatterUpdateOpTest, Error_WrongDimsIndices) { MakeOp(DT_FLOAT_REF, DT_INT32); @@ -238,7 +262,8 @@ class ScatterUpdateBM : public ScatterUpdateOpTest { }; template -static void BM_ScatterHelper(int iters, int embedding_size, const char* op) { +static void BM_ScatterHelper(int iters, int embedding_size, const char* op, + bool big_num_updates = false) { testing::StopTiming(); const int kRows = 10000000 / embedding_size; std::vector values; @@ -246,7 +271,7 @@ static void BM_ScatterHelper(int iters, int embedding_size, const char* op) { for (int i = 0; i < kRows * embedding_size; i++) { values.push_back(i); } - const int kNumUpdates = 1000; + int kNumUpdates = big_num_updates ? 1000000 : 1000; random::PhiloxRandom philox(301, 17); random::SimplePhilox rnd(&philox); std::vector indices; @@ -282,7 +307,9 @@ static void BM_ScatterUpdateInt64(int iters, int embedding_size) { static void BM_ScatterAddInt32(int iters, int embedding_size) { BM_ScatterHelper(iters, embedding_size, "ScatterAdd"); + BM_ScatterHelper(iters, embedding_size, "ScatterAdd", true); } + static void BM_ScatterAddInt64(int iters, int embedding_size) { BM_ScatterHelper(iters, embedding_size, "ScatterAdd"); } @@ -314,7 +341,7 @@ static void BM_ScatterMaxInt32(int iters, int embedding_size) { static void BM_ScatterMaxInt64(int iters, int embedding_size) { BM_ScatterHelper(iters, embedding_size, "ScatterMax"); } - +/* BENCHMARK(BM_ScatterUpdateInt32) ->Arg(1) ->Arg(10) @@ -337,8 +364,9 @@ BENCHMARK(BM_ScatterUpdateInt64) ->Arg(256) ->Arg(1024) ->Arg(100000); - +*/ BENCHMARK(BM_ScatterAddInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); +/* BENCHMARK(BM_ScatterAddInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); BENCHMARK(BM_ScatterMulInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); @@ -352,6 +380,6 @@ BENCHMARK(BM_ScatterMinInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); BENCHMARK(BM_ScatterMaxInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); BENCHMARK(BM_ScatterMaxInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); - +*/ } // namespace } // namespace tensorflow From aa62e702455d5c3979574658ab29d90de98daa5a Mon Sep 17 00:00:00 2001 From: "srinivasan.narayanamoorthy" Date: Wed, 3 Jul 2019 14:26:20 -0700 Subject: [PATCH 056/332] minor fix --- tensorflow/core/kernels/scatter_op_test.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/kernels/scatter_op_test.cc b/tensorflow/core/kernels/scatter_op_test.cc index d1924a00f58..1ea777b8a84 100644 --- a/tensorflow/core/kernels/scatter_op_test.cc +++ b/tensorflow/core/kernels/scatter_op_test.cc @@ -341,7 +341,7 @@ static void BM_ScatterMaxInt32(int iters, int embedding_size) { static void BM_ScatterMaxInt64(int iters, int embedding_size) { BM_ScatterHelper(iters, embedding_size, "ScatterMax"); } -/* + BENCHMARK(BM_ScatterUpdateInt32) ->Arg(1) ->Arg(10) @@ -364,9 +364,9 @@ BENCHMARK(BM_ScatterUpdateInt64) ->Arg(256) ->Arg(1024) ->Arg(100000); -*/ + BENCHMARK(BM_ScatterAddInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); -/* + BENCHMARK(BM_ScatterAddInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); BENCHMARK(BM_ScatterMulInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); @@ -380,6 +380,6 @@ BENCHMARK(BM_ScatterMinInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); BENCHMARK(BM_ScatterMaxInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); BENCHMARK(BM_ScatterMaxInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); -*/ + } // namespace } // namespace tensorflow From a582fb3e406a15053ca5cb07da16201151b6c02c Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Wed, 3 Jul 2019 19:57:58 +0000 Subject: [PATCH 057/332] Construct GPU transfer manager for both NVPTX and AMDGPU --- .../compiler/xla/service/gpu/gpu_transfer_manager.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc index ba25f7db0f8..e1f82cf5bd8 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_transfer_manager.cc @@ -185,9 +185,18 @@ static std::unique_ptr CreateNVPTXTransferManager() { .getPointerSize(0 /* default address space */)); } +static std::unique_ptr CreateAMDGPUTransferManager() { + return absl::make_unique( + /*id=*/stream_executor::rocm::kROCmPlatformId, + /*pointer_size=*/llvm::DataLayout(xla::gpu::amdgpu::kDataLayout) + .getPointerSize(0 /* default address space */)); +} + static bool InitModule() { xla::TransferManager::RegisterTransferManager( stream_executor::cuda::kCudaPlatformId, &CreateNVPTXTransferManager); + xla::TransferManager::RegisterTransferManager( + stream_executor::rocm::kROCmPlatformId, &CreateAMDGPUTransferManager); return true; } static bool module_initialized = InitModule(); From 78f8e24c255fe37152722a80e3b14ad5ada644bd Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Wed, 3 Jul 2019 22:17:29 +0000 Subject: [PATCH 058/332] Register ROCm platform on XLA computation placer --- tensorflow/compiler/xla/service/computation_placer.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/compiler/xla/service/computation_placer.cc b/tensorflow/compiler/xla/service/computation_placer.cc index dd3a6ad4067..f4dfb48168c 100644 --- a/tensorflow/compiler/xla/service/computation_placer.cc +++ b/tensorflow/compiler/xla/service/computation_placer.cc @@ -188,6 +188,8 @@ static bool InitModule() { stream_executor::host::kHostPlatformId, &CreateComputationPlacer); xla::ComputationPlacer::RegisterComputationPlacer( stream_executor::cuda::kCudaPlatformId, &CreateComputationPlacer); + xla::ComputationPlacer::RegisterComputationPlacer( + stream_executor::rocm::kROCmPlatformId, &CreateComputationPlacer); return true; } static bool module_initialized = InitModule(); From 9d2e777314012100b8119951c54875c91a519466 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Sat, 29 Jun 2019 11:20:17 +0800 Subject: [PATCH 059/332] make mlir related stuff build on macos 1. exp10() is not a standard C function, macos has __exp10() 2. include "mlir/StandardOps/Ops.h" earlier to avoid TRUE and FALSE being defined in --- tensorflow/compiler/mlir/lite/ir/tfl_traits.h | 5 +++++ .../compiler/mlir/tensorflow/translate/export_graphdef.h | 1 + .../compiler/mlir/tensorflow/translate/import_graphdef.h | 1 + .../compiler/mlir/tensorflow/translate/mlir_roundtrip_pass.h | 1 + tensorflow/compiler/mlir/tensorflow/utils/export_utils.h | 1 + 5 files changed, 9 insertions(+) diff --git a/tensorflow/compiler/mlir/lite/ir/tfl_traits.h b/tensorflow/compiler/mlir/lite/ir/tfl_traits.h index c9174dfdd07..8a61a72d169 100644 --- a/tensorflow/compiler/mlir/lite/ir/tfl_traits.h +++ b/tensorflow/compiler/mlir/lite/ir/tfl_traits.h @@ -22,6 +22,11 @@ limitations under the License. #include "mlir/Support/LLVM.h" // TF:local_config_mlir #include "tensorflow/compiler/mlir/lite/utils/quantization_utils.h" +// exp10() is not a standard function +#if defined(__APPLE__) +#define exp10(x) __exp10(x) +#endif + namespace mlir { namespace OpTrait { namespace TFL { diff --git a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h index bfdceac9de0..ada401f9224 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h +++ b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h @@ -20,6 +20,7 @@ limitations under the License. #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Module.h" // TF:local_config_mlir #include "mlir/IR/Operation.h" // TF:local_config_mlir +#include "mlir/StandardOps/Ops.h" // TF:local_config_mlir #include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" #include "tensorflow/core/framework/function.h" #include "tensorflow/core/framework/graph.pb.h" diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h index c494526bb4d..97b89914cd1 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h @@ -18,6 +18,7 @@ limitations under the License. #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Module.h" // TF:local_config_mlir +#include "mlir/StandardOps/Ops.h" // TF:local_config_mlir #include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" #include "tensorflow/core/framework/function.h" #include "tensorflow/core/framework/graph.pb.h" diff --git a/tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_pass.h b/tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_pass.h index 6e8547f3dcd..96a66d4eab3 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_pass.h +++ b/tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_pass.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_MLIR_ROUNDTRIP_PASS_H_ #define TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSLATE_MLIR_ROUNDTRIP_PASS_H_ +#include "mlir/StandardOps/Ops.h" // TF:local_config_mlir #include "tensorflow/core/common_runtime/optimization_registry.h" #include "tensorflow/core/lib/core/status.h" diff --git a/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h b/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h index 4c6d8ade04a..ab2db3d57e6 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h +++ b/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h @@ -27,6 +27,7 @@ limitations under the License. #include "mlir/IR/Location.h" // TF:local_config_mlir #include "mlir/IR/Operation.h" // TF:local_config_mlir #include "mlir/IR/Types.h" // TF:local_config_mlir +#include "mlir/StandardOps/Ops.h" // TF:local_config_mlir #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/function.h" #include "tensorflow/core/framework/node_def.pb.h" From 8440545a81631acd4a58528ba22215bb0e2385a0 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 2 Jul 2019 08:42:11 +0800 Subject: [PATCH 060/332] replace exp10(X) with portable standard pow(10.0, X) --- tensorflow/compiler/mlir/lite/ir/tfl_traits.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/ir/tfl_traits.h b/tensorflow/compiler/mlir/lite/ir/tfl_traits.h index 8a61a72d169..807c1100b71 100644 --- a/tensorflow/compiler/mlir/lite/ir/tfl_traits.h +++ b/tensorflow/compiler/mlir/lite/ir/tfl_traits.h @@ -22,11 +22,6 @@ limitations under the License. #include "mlir/Support/LLVM.h" // TF:local_config_mlir #include "tensorflow/compiler/mlir/lite/utils/quantization_utils.h" -// exp10() is not a standard function -#if defined(__APPLE__) -#define exp10(x) __exp10(x) -#endif - namespace mlir { namespace OpTrait { namespace TFL { @@ -80,7 +75,7 @@ class FixedResultUniformScale { Builder builder(op->getContext()); IntegerType storage_type = builder.getIntegerType(BitWidth); const double scale = static_cast(ScaleMantissa) * - ::exp10(static_cast(ScaleExp)); + ::pow(10.0, static_cast(ScaleExp)); return UniformQuantizedType::getChecked( Sign, storage_type, result_type.getElementType(), scale, ZeroPoint, StorageTypeMin, StorageTypeMax, builder.getUnknownLoc()); From 028628376ac6175213db2c8dca350b44137ac976 Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Thu, 4 Jul 2019 10:35:32 +0800 Subject: [PATCH 061/332] revert "mlir/StandardOps/Ops.h" chagnes remove additional "mlir/StandardOps/Ops.h" chagnes since there is no boolean problem anymore --- tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h | 1 - tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h | 1 - tensorflow/compiler/mlir/tensorflow/utils/export_utils.h | 1 - 3 files changed, 3 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h index ada401f9224..bfdceac9de0 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h +++ b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h @@ -20,7 +20,6 @@ limitations under the License. #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Module.h" // TF:local_config_mlir #include "mlir/IR/Operation.h" // TF:local_config_mlir -#include "mlir/StandardOps/Ops.h" // TF:local_config_mlir #include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" #include "tensorflow/core/framework/function.h" #include "tensorflow/core/framework/graph.pb.h" diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h index 97b89914cd1..c494526bb4d 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.h @@ -18,7 +18,6 @@ limitations under the License. #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Module.h" // TF:local_config_mlir -#include "mlir/StandardOps/Ops.h" // TF:local_config_mlir #include "tensorflow/compiler/mlir/tensorflow/translate/mlir_roundtrip_flags.h" #include "tensorflow/core/framework/function.h" #include "tensorflow/core/framework/graph.pb.h" diff --git a/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h b/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h index ab2db3d57e6..4c6d8ade04a 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h +++ b/tensorflow/compiler/mlir/tensorflow/utils/export_utils.h @@ -27,7 +27,6 @@ limitations under the License. #include "mlir/IR/Location.h" // TF:local_config_mlir #include "mlir/IR/Operation.h" // TF:local_config_mlir #include "mlir/IR/Types.h" // TF:local_config_mlir -#include "mlir/StandardOps/Ops.h" // TF:local_config_mlir #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/function.h" #include "tensorflow/core/framework/node_def.pb.h" From 00d36b9ba7547146cb2bfc69332e921da1c260c1 Mon Sep 17 00:00:00 2001 From: Rasmus Diederichsen Date: Thu, 4 Jul 2019 09:05:17 +0200 Subject: [PATCH 062/332] spaces->tabs --- tensorflow/contrib/makefile/Makefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 922d837541f..6339a9f0ea4 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -909,17 +909,17 @@ $(HOST_OBJDIR)%.pb.o: $(HOST_GENDIR)%.pb.cc $(PROTO_TEXT_OBJS) : $(PROTO_TEXT_PB_H_FILES) # Ensures we link CoreFoundation as it is used for time library when building -# for Mac. +# for Mac and iOS ifeq ($(TARGET),OSX) - ifeq($(HOST_ARCH), x86_64) - HOST_LDOPTS += -framework CoreFoundation - LIBS += -framework CoreFoundation - endif + ifeq ($(HOST_ARCH),x86_64) + HOST_LDOPTS += -framework CoreFoundation + LIBS += -framework CoreFoundation + endif endif ifeq ($(TARGET),IOS) - ifeq ($(IOS_ARCH),X86_64) - HOST_LDOPTS += -framework CoreFoundation - endif + ifeq ($(IOS_ARCH),X86_64) + HOST_LDOPTS += -framework CoreFoundation + endif endif # Runs proto_text to generate C++ source files from protos. From 8d3ffddb308405c2f959adaf88df2901cd6725aa Mon Sep 17 00:00:00 2001 From: Rasmus Diederichsen Date: Thu, 4 Jul 2019 10:41:07 +0200 Subject: [PATCH 063/332] Indent nested ifeqs w/ spaces to avoid make passing them to the shell --- tensorflow/contrib/makefile/Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 6339a9f0ea4..43d0ef4c80d 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -911,15 +911,15 @@ $(PROTO_TEXT_OBJS) : $(PROTO_TEXT_PB_H_FILES) # Ensures we link CoreFoundation as it is used for time library when building # for Mac and iOS ifeq ($(TARGET),OSX) - ifeq ($(HOST_ARCH),x86_64) - HOST_LDOPTS += -framework CoreFoundation - LIBS += -framework CoreFoundation - endif + ifeq ($(HOST_ARCH),x86_64) + HOST_LDOPTS += -framework CoreFoundation + LIBS += -framework CoreFoundation + endif endif ifeq ($(TARGET),IOS) - ifeq ($(IOS_ARCH),X86_64) - HOST_LDOPTS += -framework CoreFoundation - endif + ifeq ($(IOS_ARCH),X86_64) + HOST_LDOPTS += -framework CoreFoundation + endif endif # Runs proto_text to generate C++ source files from protos. From 26ef37bef3b04d18698b29f361f10f25f14539cc Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Fri, 5 Jul 2019 09:53:27 +0530 Subject: [PATCH 064/332] deprecated keep_prob removed from dropout in nn_test.py --- tensorflow/python/ops/nn_test.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py index df07721e5d3..18d0de0b9b6 100644 --- a/tensorflow/python/ops/nn_test.py +++ b/tensorflow/python/ops/nn_test.py @@ -313,7 +313,7 @@ class DropoutTest(test_lib.TestCase): num_iter = 10 for keep_prob in [0.1, 0.5, 0.8]: t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - dropout = nn_ops.dropout(t, keep_prob) + dropout = nn_ops.dropout(t, rate=(1 - keep_prob)) final_count = 0 self.assertEqual([x_dim, y_dim], dropout.get_shape()) for _ in xrange(0, num_iter): @@ -340,7 +340,7 @@ class DropoutTest(test_lib.TestCase): num_iter = 10 for keep_prob in [0.1, 0.5, 0.8]: t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1]) + dropout = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) self.assertEqual([x_dim, y_dim], dropout.get_shape()) final_count = 0 for _ in xrange(0, num_iter): @@ -364,7 +364,7 @@ class DropoutTest(test_lib.TestCase): num_iter = 10 for keep_prob in [0.1, 0.5, 0.8]: t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) - dropout = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1]) + dropout = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) self.assertEqual([x_dim, y_dim], dropout.get_shape()) for _ in xrange(0, num_iter): value = self.evaluate(dropout) @@ -409,7 +409,7 @@ class DropoutTest(test_lib.TestCase): keep_prob = 0.5 x = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) dropout_x = nn_ops.dropout( - x, keep_prob, noise_shape=array_ops.placeholder(dtypes.int32)) + x, rate=(1- keep_prob), noise_shape=array_ops.placeholder(dtypes.int32)) self.assertEqual(x.get_shape(), dropout_x.get_shape()) def testPartialShapedDropout(self): @@ -419,7 +419,7 @@ class DropoutTest(test_lib.TestCase): for keep_prob in [0.1, 0.5, 0.8]: t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) # Set noise_shape=[None, 1] which means [x_dim, 1]. - dropout = nn_ops.dropout(t, keep_prob, noise_shape=[None, 1]) + dropout = nn_ops.dropout(t, rate=(1- keep_prob), noise_shape=[None, 1]) self.assertEqual([x_dim, y_dim], dropout.get_shape()) final_count = 0 for _ in xrange(0, num_iter): @@ -478,22 +478,22 @@ class DropoutTest(test_lib.TestCase): keep_prob = 0.5 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) with self.assertRaises(ValueError): - _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, y_dim + 10]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim + 10]) with self.assertRaises(ValueError): - _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, y_dim, 5]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim, 5]) with self.assertRaises(ValueError): - _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim + 3]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim + 3]) with self.assertRaises(ValueError): - _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim]) # test that broadcasting proceeds - _ = nn_ops.dropout(t, keep_prob, noise_shape=[y_dim]) - _ = nn_ops.dropout(t, keep_prob, noise_shape=[1, y_dim]) - _ = nn_ops.dropout(t, keep_prob, noise_shape=[x_dim, 1]) - _ = nn_ops.dropout(t, keep_prob, noise_shape=[1, 1]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[y_dim]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[1, y_dim]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, 1]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[1, 1]) def testNoDropoutFast(self): x = array_ops.zeros((5,)) - y = nn_ops.dropout(x, keep_prob=1) + y = nn_ops.dropout(x, rate=0) self.assertTrue(x is y) y = nn_ops.dropout_v2(x, rate=0) From 0fe684b5ebfc59d1cf5a519e3f128ad6eea07da8 Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Fri, 5 Jul 2019 09:57:23 +0530 Subject: [PATCH 065/332] deprecated keep_prob removed from dropout in tutorials/mnist/mnist_with_summaries.py --- tensorflow/examples/tutorials/mnist/mnist_with_summaries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py b/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py index 3485e7afbf1..efe35ca096f 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py +++ b/tensorflow/examples/tutorials/mnist/mnist_with_summaries.py @@ -103,7 +103,7 @@ def train(): with tf.name_scope('dropout'): keep_prob = tf.placeholder(tf.float32) tf.summary.scalar('dropout_keep_probability', keep_prob) - dropped = tf.nn.dropout(hidden1, keep_prob) + dropped = tf.nn.dropout(hidden1, rate=(1 - keep_prob)) # Do not apply softmax activation yet, see below. y = nn_layer(dropped, 500, 10, 'layer2', act=tf.identity) From e6a655127a54f0f4361c7b44fd2ac4f125e85f29 Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Fri, 5 Jul 2019 09:57:37 +0530 Subject: [PATCH 066/332] deprecated keep_prob removed from dropout in grappler/hierarchical_controller.py --- tensorflow/python/grappler/hierarchical_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/grappler/hierarchical_controller.py b/tensorflow/python/grappler/hierarchical_controller.py index c8f2f4245bb..e39988d96b5 100644 --- a/tensorflow/python/grappler/hierarchical_controller.py +++ b/tensorflow/python/grappler/hierarchical_controller.py @@ -883,7 +883,7 @@ class HierarchicalController(Controller): actions.read(i - 1)) ) if self.hparams.keep_prob is not None: - signal = nn_ops.dropout(signal, self.hparams.keep_prob) + signal = nn_ops.dropout(signal, rate=(1 - self.hparams.keep_prob)) next_c, next_h = lstm(signal, prev_c, prev_h, w_lstm, forget_bias) query = math_ops.matmul(next_h, attn_w_2) query = array_ops.reshape( From 27865f5d0a537b7e187d02923d886a0a2fb070a3 Mon Sep 17 00:00:00 2001 From: Taehoon Lee Date: Fri, 5 Jul 2019 13:56:35 +0900 Subject: [PATCH 067/332] Fix typos --- .../mlir/tensorflow/translate/tf_mlir_translate.cc | 2 +- .../xla/service/dynamic_dimension_inference.cc | 6 +++--- .../core/api_def/base_api/api_def_LeakyReluGrad.pbtxt | 2 +- tensorflow/go/op/wrappers.go | 2 +- .../keras/distribute/distribute_strategy_test.py | 2 +- tensorflow/python/keras/engine/training_distributed.py | 10 +++++----- tensorflow/python/tpu/tpu_feed.py | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc b/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc index 14141ed84a5..6f42dbc1f7f 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc @@ -119,7 +119,7 @@ mlir::OwningModuleRef GraphdefToSplattedMlirTranslateFunction( break; default: inst.emitWarning() - << "Skipping splat converstion for " + << "Skipping splat conversation for " << "an unsupported attribute type " << element_type; continue; } diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc index cd48f9e336d..076046afba4 100644 --- a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc +++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc @@ -742,7 +742,7 @@ Status DynamicDimensionInferenceVisitor::HandleGather(HloInstruction* hlo) { if (operand_index != 1) { return Unimplemented( "Detects a dynamic dimension on the data input of gather, which " - "is not suported: %s", + "is not supported: %s", hlo->ToString()); } // A mapping from output to input batch dim number. -1 means not a batch @@ -785,7 +785,7 @@ Status DynamicDimensionInferenceVisitor::HandleScatter(HloInstruction* hlo) { if (operand_index == 0) { return Unimplemented( "Detects a dynamic dimension on the data input of scatter, which " - "is not suported: %s", + "is not supported: %s", hlo->ToString()); } @@ -802,7 +802,7 @@ Status DynamicDimensionInferenceVisitor::HandleScatter(HloInstruction* hlo) { dimension)) { return Unimplemented( "Dynamic dimension of update window dims is not supported " - "is not suported: %s", + "is not supported: %s", hlo->ToString()); } // The dynamic dimension is collapsed and won't show up in the output. diff --git a/tensorflow/core/api_def/base_api/api_def_LeakyReluGrad.pbtxt b/tensorflow/core/api_def/base_api/api_def_LeakyReluGrad.pbtxt index e4275266020..022f950f630 100644 --- a/tensorflow/core/api_def/base_api/api_def_LeakyReluGrad.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_LeakyReluGrad.pbtxt @@ -17,7 +17,7 @@ END out_arg { name: "backprops" description: < 0) + alpha * gradients * (featurs <= 0)`. +`gradients * (features > 0) + alpha * gradients * (features <= 0)`. END } summary: "Computes rectified linear gradients for a LeakyRelu operation." diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 800d03fc70e..67b1fea194f 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -31867,7 +31867,7 @@ func LeakyReluGradAlpha(value float32) LeakyReluGradAttr { // features: The features passed as input to the corresponding LeakyRelu operation, // OR the outputs of that operation (both work equivalently). // -// Returns `gradients * (features > 0) + alpha * gradients * (featurs <= 0)`. +// Returns `gradients * (features > 0) + alpha * gradients * (features <= 0)`. func LeakyReluGrad(scope *Scope, gradients tf.Output, features tf.Output, optional ...LeakyReluGradAttr) (backprops tf.Output) { if scope.Err() != nil { return diff --git a/tensorflow/python/keras/distribute/distribute_strategy_test.py b/tensorflow/python/keras/distribute/distribute_strategy_test.py index f3d8be62a1a..85d13afd0e7 100644 --- a/tensorflow/python/keras/distribute/distribute_strategy_test.py +++ b/tensorflow/python/keras/distribute/distribute_strategy_test.py @@ -1008,7 +1008,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase, predict_with_numpy, predict_with_ds, atol=1e-4, rtol=1e-4) with self.assertRaisesRegexp(ValueError, - 'Number of steps could not be infered'): + 'Number of steps could not be inferred'): model.fit(dataset, epochs=1) @combinations.generate(all_strategy_combinations_plus_cloning()) diff --git a/tensorflow/python/keras/engine/training_distributed.py b/tensorflow/python/keras/engine/training_distributed.py index 5d01a4ab5e7..18851d5ee0c 100644 --- a/tensorflow/python/keras/engine/training_distributed.py +++ b/tensorflow/python/keras/engine/training_distributed.py @@ -374,7 +374,7 @@ def experimental_tpu_test_loop(model, if steps is not None: target_steps = steps else: - raise ValueError('Number of steps could not be infered from the data, ' + raise ValueError('Number of steps could not be inferred from the data, ' 'please pass the steps argument.') current_step = 0 @@ -519,7 +519,7 @@ def experimental_tpu_predict_loop(model, if steps is not None: target_steps = steps else: - raise ValueError('Number of steps could not be infered from the data, ' + raise ValueError('Number of steps could not be inferred from the data, ' 'please pass the steps argument.') current_step = 0 @@ -647,7 +647,7 @@ class DistributionSingleWorkerTrainingLoop(training_utils.TrainingLoop): steps_per_epoch = training_utils.infer_steps_for_dataset( dataset, steps_per_epoch, epochs, steps_name='steps_per_epoch') if steps_per_epoch is None: - raise ValueError('Number of steps could not be infered from the data, ' + raise ValueError('Number of steps could not be inferred from the data, ' 'please pass the steps_per_epoch argument.') if not context.executing_eagerly(): @@ -704,7 +704,7 @@ class DistributionSingleWorkerTrainingLoop(training_utils.TrainingLoop): steps = training_utils.infer_steps_for_dataset( dataset, steps, steps_name='steps') if steps is None: - raise ValueError('Number of steps could not be infered from the data, ' + raise ValueError('Number of steps could not be inferred from the data, ' 'please pass the steps argument.') if not context.executing_eagerly(): @@ -741,7 +741,7 @@ class DistributionSingleWorkerTrainingLoop(training_utils.TrainingLoop): steps = training_utils.infer_steps_for_dataset( dataset, steps, steps_name='steps') if steps is None: - raise ValueError('Number of steps could not be infered from the data, ' + raise ValueError('Number of steps could not be inferred from the data, ' 'please pass the steps argument.') if not context.executing_eagerly(): return experimental_tpu_predict_loop( diff --git a/tensorflow/python/tpu/tpu_feed.py b/tensorflow/python/tpu/tpu_feed.py index 54a77a14655..2b01eeb3934 100644 --- a/tensorflow/python/tpu/tpu_feed.py +++ b/tensorflow/python/tpu/tpu_feed.py @@ -908,7 +908,7 @@ class _PartitionedInfeedQueue(InfeedQueue): if dims.prod() != self._device_assignment.num_cores_per_replica: raise ValueError( - "The product of each input parition dim should equal to " + "The product of each input partition dim should equal to " "num_cores_per_replica. (dim = {}, num_cores_per_replica " "= {})".format(dims, self._device_assignment.num_cores_per_replica)) if dims.shape[0] != tensor.shape.ndims: From 048209aab9f61556f84491b020eff134e7544969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Sandsmark?= Date: Fri, 5 Jul 2019 10:16:12 +0200 Subject: [PATCH 068/332] Add missing closing paren in optimizer_v2 docs --- tensorflow/python/keras/optimizer_v2/optimizer_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py index 9cc358bb0cb..46ebdb6bf70 100644 --- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py +++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py @@ -108,7 +108,7 @@ class OptimizerV2(trackable.Trackable): opt = tf.keras.optimizers.SGD(learning_rate=0.1) model = tf.keras.Sequential() model.add(tf.keras.layers.Dense(num_hidden, activation='relu')) - model.add(tf.keras.layers.Dense(num_classes, activation='sigmoid') + model.add(tf.keras.layers.Dense(num_classes, activation='sigmoid')) loss_fn = lambda: tf.keras.losses.mse(model(input), output) var_list_fn = lambda: model.trainable_weights for input, output in data: From 22e53cc8c4c24616419eaf7c8018f624e67f79ef Mon Sep 17 00:00:00 2001 From: Karl Lessard Date: Sun, 16 Jun 2019 22:11:57 -0400 Subject: [PATCH 069/332] Fix eager session testing with GC --- .../java/org/tensorflow/EagerSession.java | 27 ++-- .../java/org/tensorflow/EagerSessionTest.java | 117 +++++++++++------- 2 files changed, 87 insertions(+), 57 deletions(-) diff --git a/tensorflow/java/src/main/java/org/tensorflow/EagerSession.java b/tensorflow/java/src/main/java/org/tensorflow/EagerSession.java index d3bb43a8958..cda6156be33 100644 --- a/tensorflow/java/src/main/java/org/tensorflow/EagerSession.java +++ b/tensorflow/java/src/main/java/org/tensorflow/EagerSession.java @@ -179,7 +179,12 @@ public final class EagerSession implements ExecutionEnvironment, AutoCloseable { /** Builds an eager session with the selected options. */ public EagerSession build() { - return new EagerSession(this); + return new EagerSession(this, new ReferenceQueue()); + } + + // For garbage-collection tests only + EagerSession buildForGcTest(ReferenceQueue gcQueue) { + return new EagerSession(this, gcQueue); } private boolean async; @@ -344,6 +349,10 @@ public final class EagerSession implements ExecutionEnvironment, AutoCloseable { return nativeHandle; } + ResourceCleanupStrategy resourceCleanupStrategy() { + return resourceCleanupStrategy; + } + /** * A reference to one or more allocated native resources. * @@ -411,6 +420,10 @@ public final class EagerSession implements ExecutionEnvironment, AutoCloseable { * longer needed. */ private static class NativeResourceCollector { + + NativeResourceCollector(ReferenceQueue garbageQueue) { + this.garbageQueue = garbageQueue; + } void attach(NativeReference nativeRef) { synchronized (nativeRefs) { @@ -484,17 +497,18 @@ public final class EagerSession implements ExecutionEnvironment, AutoCloseable { private final ExecutorService cleanupService = Executors.newSingleThreadExecutor(); private final Map nativeRefs = new IdentityHashMap<>(); - private final ReferenceQueue garbageQueue = new ReferenceQueue<>(); + private final ReferenceQueue garbageQueue; private volatile boolean cleanupInBackground = false; } private static volatile EagerSession defaultSession = null; - private final NativeResourceCollector nativeResources = new NativeResourceCollector(); + private final NativeResourceCollector nativeResources; private final ResourceCleanupStrategy resourceCleanupStrategy; private long nativeHandle; - private EagerSession(Options options) { + private EagerSession(Options options, ReferenceQueue garbageQueue) { + this.nativeResources = new NativeResourceCollector(garbageQueue); this.nativeHandle = allocate(options.async, options.devicePlacementPolicy.code, options.config); this.resourceCleanupStrategy = options.resourceCleanupStrategy; @@ -509,11 +523,6 @@ public final class EagerSession implements ExecutionEnvironment, AutoCloseable { } } - // For tests - ResourceCleanupStrategy resourceCleanupStrategy() { - return resourceCleanupStrategy; - } - private static native long allocate(boolean async, int devicePlacementPolicy, byte[] config); private static native void delete(long handle); diff --git a/tensorflow/java/src/test/java/org/tensorflow/EagerSessionTest.java b/tensorflow/java/src/test/java/org/tensorflow/EagerSessionTest.java index 7db1cecb943..b4f50c6e7c6 100644 --- a/tensorflow/java/src/test/java/org/tensorflow/EagerSessionTest.java +++ b/tensorflow/java/src/test/java/org/tensorflow/EagerSessionTest.java @@ -21,8 +21,13 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.lang.ref.Reference; +import java.lang.ref.ReferenceQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import org.junit.Ignore; + import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -40,74 +45,67 @@ public class EagerSessionTest { @Test public void cleanupResourceOnSessionClose() { - AtomicBoolean deleted = new AtomicBoolean(); - + TestReference ref; try (EagerSession s = EagerSession.options() .resourceCleanupStrategy(ResourceCleanupStrategy.ON_SESSION_CLOSE) .build()) { + ref = new TestReference(s, new Object()); + assertFalse(ref.isDeleted()); - new TestReference(s, new Object(), deleted); - - assertFalse(deleted.get()); - runGC(); - assertFalse(deleted.get()); - + // check that reaching safe point did not release resources buildOp(s); - assertFalse(deleted.get()); // reaching safe point did not release resources + assertFalse(ref.isDeleted()); } - assertTrue(deleted.get()); + assertTrue(ref.isDeleted()); } - // TODO(b/135541743): Re-enable once fixed. - // Disabled due to flakiness with -c opt --config=cuda - @Ignore + @Test public void cleanupResourceOnSafePoints() { - AtomicBoolean deleted = new AtomicBoolean(); - + TestGarbageCollectorQueue gcQueue = new TestGarbageCollectorQueue(); try (EagerSession s = EagerSession.options() .resourceCleanupStrategy(ResourceCleanupStrategy.ON_SAFE_POINTS) - .build()) { + .buildForGcTest(gcQueue)) { - new TestReference(s, new Object(), deleted); - - assertFalse(deleted.get()); - runGC(); - assertFalse(deleted.get()); - - buildOp(s); - assertTrue(deleted.get()); // reaching safe point released resources + TestReference ref = new TestReference(s, new Object()); + assertFalse(ref.isDeleted()); + + // garbage collecting the reference won't release until we reached safe point + gcQueue.collect(ref); + assertFalse(ref.isDeleted()); + buildOp(s); // safe point + assertTrue(ref.isDeleted()); + assertTrue(gcQueue.isEmpty()); } } @Test public void cleanupResourceInBackground() { - AtomicBoolean deleted = new AtomicBoolean(); - + TestGarbageCollectorQueue gcQueue = new TestGarbageCollectorQueue(); try (EagerSession s = EagerSession.options() .resourceCleanupStrategy(ResourceCleanupStrategy.IN_BACKGROUND) - .build()) { + .buildForGcTest(gcQueue)) { - new TestReference(s, new Object(), deleted); + TestReference ref = new TestReference(s, new Object()); + assertFalse(ref.isDeleted()); - assertFalse(deleted.get()); - runGC(); + gcQueue.collect(ref); sleep(50); // allow some time to the background thread for cleaning up resources - assertTrue(deleted.get()); + assertTrue(ref.isDeleted()); + assertTrue(gcQueue.isEmpty()); } } @Test public void clearedResourcesAreNotCleanedUp() { - AtomicBoolean deleted = new AtomicBoolean(); - + TestReference ref; try (EagerSession s = EagerSession.create()) { - TestReference ref = new TestReference(s, new Object(), deleted); + ref = new TestReference(s, new Object()); ref.clear(); } - assertFalse(deleted.get()); + assertFalse(ref.isDeleted()); } @Test @@ -127,7 +125,7 @@ public class EagerSessionTest { EagerSession s = EagerSession.create(); s.close(); try { - new TestReference(s, new Object(), new AtomicBoolean()); + new TestReference(s, new Object()); fail(); } catch (IllegalStateException e) { // ok @@ -158,9 +156,8 @@ public class EagerSessionTest { private static class TestReference extends EagerSession.NativeReference { - TestReference(EagerSession session, Object referent, AtomicBoolean deleted) { + TestReference(EagerSession session, Object referent) { super(session, referent); - this.deleted = deleted; } @Override @@ -169,8 +166,40 @@ public class EagerSessionTest { fail("Reference was deleted more than once"); } } + + boolean isDeleted() { + return deleted.get(); + } + + private final AtomicBoolean deleted = new AtomicBoolean(); + } + + private static class TestGarbageCollectorQueue extends ReferenceQueue { - private final AtomicBoolean deleted; + @Override + public Reference poll() { + return garbage.poll(); + } + + @Override + public Reference remove() throws InterruptedException { + return garbage.take(); + } + + @Override + public Reference remove(long timeout) throws IllegalArgumentException, InterruptedException { + return garbage.poll(timeout, TimeUnit.MILLISECONDS); + } + + void collect(TestReference ref) { + garbage.add(ref); + } + + boolean isEmpty() { + return garbage.isEmpty(); + } + + private final BlockingQueue garbage = new LinkedBlockingQueue<>(); } private static void buildOp(EagerSession s) { @@ -182,14 +211,6 @@ public class EagerSessionTest { } } - private static void runGC() { - // Warning: There is no way to force the garbage collector to run, so here we simply to our best - // to get it triggered but it might be sufficient on some platforms. Adjust accordingly if some - // cleanup tests start to fail. - System.gc(); - System.runFinalization(); - } - private static void sleep(int millis) { try { Thread.sleep(millis); From 430acadce34e73ed4130c068a1a45e5f4411be3e Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Mon, 8 Jul 2019 08:39:42 +0530 Subject: [PATCH 070/332] PyLint issue fixed --- tensorflow/python/ops/nn_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py index 18d0de0b9b6..ca91b603049 100644 --- a/tensorflow/python/ops/nn_test.py +++ b/tensorflow/python/ops/nn_test.py @@ -478,7 +478,7 @@ class DropoutTest(test_lib.TestCase): keep_prob = 0.5 t = constant_op.constant(1.0, shape=[x_dim, y_dim], dtype=dtypes.float32) with self.assertRaises(ValueError): - _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim + 10]) + _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim+10]) with self.assertRaises(ValueError): _ = nn_ops.dropout(t, rate=(1 - keep_prob), noise_shape=[x_dim, y_dim, 5]) with self.assertRaises(ValueError): From 668bd2a76fb45172b32a956c93682aaefd9323dc Mon Sep 17 00:00:00 2001 From: Dayananda-V Date: Thu, 4 Jul 2019 19:32:00 +0530 Subject: [PATCH 071/332] [Lite] num_ranks option brings to user through cmd line parameter num_ranks optional parameters provides to user to calculate accuracy ranks on out file. --- tensorflow/lite/tools/accuracy/ilsvrc/README.md | 4 ++++ .../lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/README.md b/tensorflow/lite/tools/accuracy/ilsvrc/README.md index 6e27c8570f3..4d61aa1d854 100644 --- a/tensorflow/lite/tools/accuracy/ilsvrc/README.md +++ b/tensorflow/lite/tools/accuracy/ilsvrc/README.md @@ -56,6 +56,10 @@ and the following optional parameters: Optionally, the computed accuracies can be output to a file as a string-serialized instance of tflite::evaluation::TopkAccuracyEvalMetrics. +* `num_ranks`: `int` (default=10) \ + The number of top-K accuracies to return. For example, if num_ranks=5, + top-1 to top-5 accuracy fractions are returned. + The following optional parameters can be used to modify the inference runtime: * `num_interpreter_threads`: `int` (default=1) \ diff --git a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc index d7230976961..f296b89b583 100644 --- a/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc +++ b/tensorflow/lite/tools/accuracy/ilsvrc/imagenet_model_evaluator.cc @@ -49,6 +49,7 @@ constexpr char kInterpreterThreadsFlag[] = "num_interpreter_threads"; constexpr char kDelegateFlag[] = "delegate"; constexpr char kNnapiDelegate[] = "nnapi"; constexpr char kGpuDelegate[] = "gpu"; +constexpr char kNumRanksFlag[] = "num_ranks"; template std::vector GetFirstN(const std::vector& v, int n) { @@ -144,6 +145,9 @@ class CompositeObserver : public ImagenetModelEvaluator::Observer { tflite::Flag::CreateFlag(kDelegateFlag, ¶ms.delegate, "Delegate to use for inference, if available. " "Must be one of {'nnapi', 'gpu'}"), + tflite::Flag::CreateFlag(kNumRanksFlag, ¶ms.num_ranks, + "Generates the top-1 to top-k accuracy values" + "where k = num_ranks. Default: 10"), }; tflite::Flags::Parse(&argc, const_cast(argv), flag_list); From b1fca4d98c2fe2bb5a697100707fe6f82386fac7 Mon Sep 17 00:00:00 2001 From: wenxizhu Date: Mon, 8 Jul 2019 15:47:58 +0800 Subject: [PATCH 072/332] Add 'test_util.IsMklEnabled()' to guard tests to only mkl. --- .../python/kernel_tests/conv_ops_3d_test.py | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py index bdb8bc514c9..deb465a0474 100644 --- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py @@ -32,6 +32,7 @@ from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import nn_ops import tensorflow.python.ops.nn_grad # pylint: disable=unused-import from tensorflow.python.platform import test +from tensorflow.python.framework import test_util def GetTestConfigs(): @@ -220,12 +221,13 @@ class Conv3DTest(test.TestCase): expected=expected_output) def testConv3D1x1x1Filter2x1x1Dilation(self): - self._VerifyDilatedConvValues( - tensor_in_sizes=[1, 3, 6, 1, 1], - filter_in_sizes=[1, 1, 1, 1, 1], - stride=1, - padding="VALID", - dilations=[2, 1, 1]) + if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled(): + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 3, 6, 1, 1], + filter_in_sizes=[1, 1, 1, 1, 1], + stride=1, + padding="VALID", + dilations=[2, 1, 1]) # Expected values computed using scipy's correlate function. def testConv3D2x2x2Filter(self): @@ -244,12 +246,13 @@ class Conv3DTest(test.TestCase): expected=expected_output) def testConv3D2x2x2Filter1x2x1Dilation(self): - self._VerifyDilatedConvValues( - tensor_in_sizes=[1, 4, 6, 3, 1], - filter_in_sizes=[2, 2, 2, 1, 1], - stride=1, - padding="VALID", - dilations=[1, 2, 1]) + if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled(): + self._VerifyDilatedConvValues( + tensor_in_sizes=[1, 4, 6, 3, 1], + filter_in_sizes=[2, 2, 2, 1, 1], + stride=1, + padding="VALID", + dilations=[1, 2, 1]) def testConv3DStrides(self): expected_output = [ From 2d9851f9b0ea64e92cfef332d810e865f143beec Mon Sep 17 00:00:00 2001 From: Abdullah Selek Date: Fri, 28 Jun 2019 13:13:52 +0100 Subject: [PATCH 073/332] Create an internal undeprecated function to check checkpoint exists. --- .../python/training/checkpoint_management.py | 33 ++++++++++++++----- tensorflow/python/training/saver.py | 2 +- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/tensorflow/python/training/checkpoint_management.py b/tensorflow/python/training/checkpoint_management.py index 32b9c023aae..c18e1395b45 100644 --- a/tensorflow/python/training/checkpoint_management.py +++ b/tensorflow/python/training/checkpoint_management.py @@ -347,6 +347,30 @@ def latest_checkpoint(checkpoint_dir, latest_filename=None): return None +def checkpoint_exists_internal(checkpoint_prefix): + """Checks whether a V1 or V2 checkpoint exists with the specified prefix. + + This is an internal function to check if a checkpoint exists, + since it takes into account the naming difference between V1 and V2 formats. + + Args: + checkpoint_prefix: the prefix of a V1 or V2 checkpoint, with V2 taking + priority. Typically the result of `Saver.save()` or that of + `tf.train.latest_checkpoint()`, regardless of sharded/non-sharded or + V1/V2. + Returns: + A bool, true iff a checkpoint referred to by `checkpoint_prefix` exists. + """ + pathname = _prefix_to_checkpoint_path(checkpoint_prefix, + saver_pb2.SaverDef.V2) + if file_io.get_matching_files(pathname): + return True + elif file_io.get_matching_files(checkpoint_prefix): + return True + else: + return False + + @deprecation.deprecated( date=None, instructions="Use standard file APIs to check for files with this prefix.") @@ -365,14 +389,7 @@ def checkpoint_exists(checkpoint_prefix): Returns: A bool, true iff a checkpoint referred to by `checkpoint_prefix` exists. """ - pathname = _prefix_to_checkpoint_path(checkpoint_prefix, - saver_pb2.SaverDef.V2) - if file_io.get_matching_files(pathname): - return True - elif file_io.get_matching_files(checkpoint_prefix): - return True - else: - return False + return checkpoint_exists_internal(checkpoint_exists) @deprecation.deprecated( diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index 7b502bffa38..4a30e667be9 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -1276,7 +1276,7 @@ class Saver(object): if save_path is None: raise ValueError("Can't load save_path when it is None.") - if not checkpoint_management.checkpoint_exists(compat.as_text(save_path)): + if not checkpoint_management.checkpoint_exists_internal(compat.as_text(save_path)): raise ValueError("The passed save_path is not a valid checkpoint: " + compat.as_text(save_path)) From fe5d8a5cadcf1a5d55d34ab0b37afdbf4ea36b5f Mon Sep 17 00:00:00 2001 From: Abdullah Selek Date: Wed, 3 Jul 2019 19:05:28 +0100 Subject: [PATCH 074/332] Fix parameter and typos. --- tensorflow/python/training/checkpoint_management.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/training/checkpoint_management.py b/tensorflow/python/training/checkpoint_management.py index c18e1395b45..cb84185db02 100644 --- a/tensorflow/python/training/checkpoint_management.py +++ b/tensorflow/python/training/checkpoint_management.py @@ -359,7 +359,7 @@ def checkpoint_exists_internal(checkpoint_prefix): `tf.train.latest_checkpoint()`, regardless of sharded/non-sharded or V1/V2. Returns: - A bool, true iff a checkpoint referred to by `checkpoint_prefix` exists. + A bool, true if a checkpoint referred to by `checkpoint_prefix` exists. """ pathname = _prefix_to_checkpoint_path(checkpoint_prefix, saver_pb2.SaverDef.V2) @@ -387,9 +387,9 @@ def checkpoint_exists(checkpoint_prefix): `tf.train.latest_checkpoint()`, regardless of sharded/non-sharded or V1/V2. Returns: - A bool, true iff a checkpoint referred to by `checkpoint_prefix` exists. + A bool, true if a checkpoint referred to by `checkpoint_prefix` exists. """ - return checkpoint_exists_internal(checkpoint_exists) + return checkpoint_exists_internal(checkpoint_prefix) @deprecation.deprecated( From 8cc20365f748e9cada5d4a8ef25732eb15281b28 Mon Sep 17 00:00:00 2001 From: Abdullah Selek Date: Thu, 4 Jul 2019 09:09:33 +0100 Subject: [PATCH 075/332] Fix sanity errors. --- tensorflow/python/training/saver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index 4a30e667be9..d65297fb30d 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -1276,11 +1276,12 @@ class Saver(object): if save_path is None: raise ValueError("Can't load save_path when it is None.") - if not checkpoint_management.checkpoint_exists_internal(compat.as_text(save_path)): + checkpoint_prefix = compat.as_text(save_path) + if not checkpoint_management.checkpoint_exists_internal(checkpoint_prefix): raise ValueError("The passed save_path is not a valid checkpoint: " + - compat.as_text(save_path)) + checkpoint_prefix) - logging.info("Restoring parameters from %s", compat.as_text(save_path)) + logging.info("Restoring parameters from %s", checkpoint_prefix) try: if context.executing_eagerly(): self._build_eager(save_path, build_save=False, build_restore=True) From 360f99cbc3cf10e4b14f0ad934b53b2d6b68be9b Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 14:18:54 +0000 Subject: [PATCH 076/332] [ROCm] Skipping subtests that check GPU stream tracing/profiling. ROCm platform currently does not support the ability to do GPU stream level tracing / profiling. This commit skip subtests (within python unit-tests) that this functionality. The "skip" is guarded by the call to "is_buil_with_rocm()", and hence these unit-tests will not be affected in any way when running with TF which was not built with ROCm support (i.e. `--config=rocm`) --- tensorflow/python/client/timeline_test.py | 5 ++++- .../python/profiler/internal/run_metadata_test.py | 11 +++++++++-- tensorflow/python/profiler/profile_context_test.py | 7 +++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py index e7d60de6905..90ed4d18771 100644 --- a/tensorflow/python/client/timeline_test.py +++ b/tensorflow/python/client/timeline_test.py @@ -104,7 +104,10 @@ class TimelineTest(test.TestCase): step_stats = run_metadata.step_stats devices = [d.device for d in step_stats.dev_stats] self.assertTrue('/job:localhost/replica:0/task:0/device:GPU:0' in devices) - self.assertTrue('/device:GPU:0/stream:all' in devices) + if not test.is_built_with_rocm(): + # skip this check for the ROCm platform + # stream level tracing is not yet supported on the ROCm platform + self.assertTrue('/device:GPU:0/stream:all' in devices) tl = timeline.Timeline(step_stats) ctf = tl.generate_chrome_trace_format() self._validateTrace(ctf) diff --git a/tensorflow/python/profiler/internal/run_metadata_test.py b/tensorflow/python/profiler/internal/run_metadata_test.py index daaff0ab601..ee0db8d3db3 100644 --- a/tensorflow/python/profiler/internal/run_metadata_test.py +++ b/tensorflow/python/profiler/internal/run_metadata_test.py @@ -129,7 +129,10 @@ class RunMetadataTest(test.TestCase): ret = _extract_node(run_meta, 'MatMul') self.assertEqual(len(ret['gpu:0']), 1) - self.assertEqual(len(ret['gpu:0/stream:all']), 1, '%s' % run_meta) + if not test.is_built_with_rocm(): + # skip this check for the ROCm platform + # stream level tracing is not yet supported on the ROCm platform + self.assertEqual(len(ret['gpu:0/stream:all']), 1, '%s' % run_meta) @test_util.run_deprecated_v1 def testAllocationHistory(self): @@ -234,7 +237,11 @@ class RunMetadataTest(test.TestCase): for node in ret['gpu:0']: total_cpu_execs += node.op_end_rel_micros - self.assertGreaterEqual(len(ret['gpu:0/stream:all']), 4, '%s' % run_meta) + if not test.is_built_with_rocm(): + # skip this check for the ROCm platform + # stream level tracing is not yet supported on the ROCm platform + self.assertGreaterEqual(len(ret['gpu:0/stream:all']), + 4, '%s' % run_meta) if __name__ == '__main__': diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py index 885f08ca4b9..f4051ed7b7d 100644 --- a/tensorflow/python/profiler/profile_context_test.py +++ b/tensorflow/python/profiler/profile_context_test.py @@ -69,6 +69,13 @@ class ProfilerContextTest(test.TestCase): os.path.join(test.get_temp_dir(), "profile_100")) as profiler: profiler.profile_operations(options=opts) with gfile.Open(outfile, "r") as f: + + if test.is_built_with_rocm(): + # The profiler output for ROCm mode, includes an extra warning + # related to the lack of stream tracing in ROCm mode. + # Need to skip this warning when doing the diff + profile_str = "\n".join(profile_str.split("\n")[7:]) + self.assertEqual(profile_str, f.read()) @test_util.run_deprecated_v1 From cd75ae6e676420f18fda5d6fb91a46abd762a149 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 14:45:06 +0000 Subject: [PATCH 077/332] [ROCm] Skipping subtests that check support for the Cholesky and QR ops ROCm platform currently does not support the Cholesky and QR ops (on the GPU) This commit skip subtests (within python unit-tests) that test this functionality. The "skip" is guarded by the call to "is_built_with_rocm()", and hence these unit-tests will not be affected in any way when running with TF which was not built with ROCm support (i.e. `--config=rocm`) --- tensorflow/python/kernel_tests/BUILD | 4 ++++ tensorflow/python/ops/control_flow_ops_test.py | 5 +++++ tensorflow/python/ops/init_ops_test.py | 4 ++++ tensorflow/python/ops/init_ops_v2_test.py | 4 ++++ 4 files changed, 17 insertions(+) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 6491a540801..dff0ccdb424 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -235,6 +235,7 @@ cuda_py_test( shard_count = 5, tags = [ "no_gpu", # TODO(b/131773093): Re-enable. + "no_rocm", # TODO(rocm): feature not supported on ROCm platform "nomsan", # TODO(b/131773093): Re-enable. ], xla_enable_strict_auto_jit = True, @@ -3503,6 +3504,9 @@ cuda_py_test( "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", ], + tags = [ + "no_rocm", # TODO(rocm): feature not supported on ROCm platform + ], shard_count = 20, xla_enable_strict_auto_jit = True, ) diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py index c08548712fc..9d0f991d893 100644 --- a/tensorflow/python/ops/control_flow_ops_test.py +++ b/tensorflow/python/ops/control_flow_ops_test.py @@ -51,6 +51,7 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables import tensorflow.python.ops.tensor_array_grad # pylint: disable=unused-import from tensorflow.python.platform import googletest +from tensorflow.python.platform import test from tensorflow.python.training import momentum from tensorflow.python.util import nest @@ -1081,6 +1082,10 @@ class IndexedCaseTest(test_util.TensorFlowTestCase, parameterized.TestCase): @test_util.disable_xla("Wants RunMetadata") def testParallelExecution(self): """Verify disjoint branches across while iterations are run in parallel.""" + if test.is_built_with_rocm(): + self.skipTest( + 'Disable subtest on ROCm due to missing Cholesky op support') + with ops.Graph().as_default() as g: nbranches = 7 matrices = array_ops.unstack( # Ensure all are ready before while. diff --git a/tensorflow/python/ops/init_ops_test.py b/tensorflow/python/ops/init_ops_test.py index 2955a0bf69f..ae8bfbdbdd0 100644 --- a/tensorflow/python/ops/init_ops_test.py +++ b/tensorflow/python/ops/init_ops_test.py @@ -178,6 +178,10 @@ class InitializersTest(test.TestCase): @test_util.run_gpu_only def testVariablePlacementWithOrthogonalInitializer(self): + + if test.is_built_with_rocm(): + self.skipTest('Disable subtest on ROCm due to missing QR op support') + with ops.Graph().as_default() as g: with ops.device('gpu:0'): variable_scope.get_variable( diff --git a/tensorflow/python/ops/init_ops_v2_test.py b/tensorflow/python/ops/init_ops_v2_test.py index fceba1d04a2..2bd13835638 100644 --- a/tensorflow/python/ops/init_ops_v2_test.py +++ b/tensorflow/python/ops/init_ops_v2_test.py @@ -366,6 +366,10 @@ class OrthogonalInitializerTest(InitializersTest): @test_util.run_in_graph_and_eager_modes def testShapesValues(self): + + if test.is_built_with_rocm(): + self.skipTest('Disable subtest on ROCm due to missing QR op support') + for shape in [(10, 10), (10, 9, 8), (100, 5, 5), (50, 40), (40, 50)]: init = init_ops_v2.Orthogonal() tol = 1e-5 From f4094a3abffdca011e9e9087a6b15e01e36febdd Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 18:39:38 +0000 Subject: [PATCH 078/332] [ROCm] Skipping subtests that check support for Pooling Ops with 3D tensors ROCm platform currently does not support Pooling Ops with 3D Tensors This commit skips subtests (within python unit-tests) that test this functionality. The "skip" is guarded by the call to "is_built_with_rocm()", and hence these unit-tests will not be affected in any way when running with TF which was not built with ROCm support (i.e. `--config=rocm`) --- tensorflow/python/eager/backprop_test.py | 2 ++ tensorflow/python/keras/backend_test.py | 2 ++ tensorflow/python/keras/layers/pooling_test.py | 4 ++++ tensorflow/python/kernel_tests/pool_test.py | 7 +++++++ tensorflow/python/ops/nn_test.py | 8 ++++++++ .../python/ops/parallel_for/control_flow_ops_test.py | 2 ++ 6 files changed, 25 insertions(+) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 3bebc8341ca..479ae52a7d8 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -1334,6 +1334,8 @@ class BackpropTest(test.TestCase): @test_util.run_in_graph_and_eager_modes def testMaxPooling3DGradient(self): + if test.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") def forward(a): r = max_pooling3d(a, pool_size=pool_size, strides=strides, padding='SAME') return r diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py index e3bc5467261..82965b4b0de 100644 --- a/tensorflow/python/keras/backend_test.py +++ b/tensorflow/python/keras/backend_test.py @@ -789,6 +789,8 @@ class BackendNNOpsTest(test.TestCase, parameterized.TestCase): y = keras.backend.pool2d(x, (2, 2), strides=(2, 2), pool_mode='other') def test_pool3d(self): + if test.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") val = np.random.random((10, 3, 10, 10, 10)) x = keras.backend.variable(val) y = keras.backend.pool3d(x, (2, 2, 2), strides=(1, 1, 1), diff --git a/tensorflow/python/keras/layers/pooling_test.py b/tensorflow/python/keras/layers/pooling_test.py index 67df4d7a256..13d08d953fd 100644 --- a/tensorflow/python/keras/layers/pooling_test.py +++ b/tensorflow/python/keras/layers/pooling_test.py @@ -144,6 +144,8 @@ class Pooling3DTest(test.TestCase): @tf_test_util.run_in_graph_and_eager_modes def test_maxpooling_3d(self): + if test.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") pool_size = (3, 3, 3) testing_utils.layer_test( keras.layers.MaxPooling3D, @@ -163,6 +165,8 @@ class Pooling3DTest(test.TestCase): @tf_test_util.run_in_graph_and_eager_modes def test_averagepooling_3d(self): + if test.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") pool_size = (3, 3, 3) testing_utils.layer_test( keras.layers.AveragePooling3D, diff --git a/tensorflow/python/kernel_tests/pool_test.py b/tensorflow/python/kernel_tests/pool_test.py index 78e786f01ca..0f0eaa25402 100644 --- a/tensorflow/python/kernel_tests/pool_test.py +++ b/tensorflow/python/kernel_tests/pool_test.py @@ -219,6 +219,8 @@ class PoolingTest(test.TestCase): strides=strides) def testPool3D(self): + if test.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") with self.session(use_gpu=test.is_gpu_available()): for padding in ["SAME", "VALID"]: for pooling_type in ["MAX", "AVG"]: @@ -274,6 +276,9 @@ class PoolingTest(test.TestCase): strides=[1, 2], dilation_rate=[1, 1], data_format="NCHW") + if test.is_built_with_rocm(): + # Pooling with 3D tensors is not supported in ROCm + continue self._test( input_shape=[2, 2, 7, 5, 3], window_shape=[2, 2, 2], @@ -358,6 +363,8 @@ class PoolingTest(test.TestCase): @test_util.run_deprecated_v1 def testGradient3D(self): + if test.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") with self.session(use_gpu=test.is_gpu_available()): for padding in ["SAME", "VALID"]: for pooling_type in ["AVG", "MAX"]: diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py index df07721e5d3..e8236547c97 100644 --- a/tensorflow/python/ops/nn_test.py +++ b/tensorflow/python/ops/nn_test.py @@ -1291,6 +1291,8 @@ class AvgPoolTest(test_lib.TestCase): self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) def test3DTensor(self): + if test_lib.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") x = array_ops.ones([3, 7, 6, 6, 5]) ksize = 2 strides = 2 @@ -1301,6 +1303,8 @@ class AvgPoolTest(test_lib.TestCase): self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) def test3DNumpy(self): + if test_lib.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") x = np.ones([3, 7, 6, 6, 5], dtype=np.float32) ksize = 2 strides = 2 @@ -1355,6 +1359,8 @@ class MaxPoolTest(test_lib.TestCase): self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) def test3DTensor(self): + if test_lib.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") x = array_ops.ones([3, 7, 6, 6, 5]) ksize = 2 strides = 2 @@ -1365,6 +1371,8 @@ class MaxPoolTest(test_lib.TestCase): self.assertAllEqual(self.evaluate(y1), self.evaluate(y2)) def test3DNumpy(self): + if test_lib.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") x = np.ones([3, 7, 6, 6, 5], dtype=np.float32) ksize = 2 strides = 2 diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py index f795b7c7a2f..ac45a89473f 100644 --- a/tensorflow/python/ops/parallel_for/control_flow_ops_test.py +++ b/tensorflow/python/ops/parallel_for/control_flow_ops_test.py @@ -354,6 +354,8 @@ class NNTest(PForTestCase): self._test_loop_fn(loop_fn, 3, loop_fn_dtypes=[dtypes.float32] * 3) def test_max_pool3d(self): + if test.is_built_with_rocm(): + self.skipTest("Pooling with 3D tensors is not supported in ROCm") with backprop.GradientTape(persistent=True) as g: x = random_ops.random_uniform([3, 3, 2, 12, 12, 3]) g.watch(x) From 78d1654db1260c0b98d3e51338185bd051e98325 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 18:59:07 +0000 Subject: [PATCH 079/332] [ROCm] Skipping subtests that check support for complex datatype in BLAS calls ROCm platform currently does not support complex datatype in BLAS calls This commit skips subtests (within python unit-tests) that test this functionality. The "skip" is guarded by the call to "is_built_with_rocm()", and hence these unit-tests will not be affected in any way when running with TF which was not built with ROCm support (i.e. `--config=rocm`) --- .../kernel_tests/batch_matmul_op_test.py | 8 ++- .../linalg/linear_operator_adjoint_test.py | 4 ++ tensorflow/python/kernel_tests/lu_op_test.py | 56 +++++++++++-------- .../python/kernel_tests/matmul_op_test.py | 7 ++- .../matrix_exponential_op_test.py | 4 ++ .../kernel_tests/matrix_inverse_op_test.py | 40 +++++++------ .../kernel_tests/matrix_logarithm_op_test.py | 8 +++ .../kernel_tests/matrix_solve_ls_op_test.py | 8 ++- .../matrix_square_root_op_test.py | 18 +++--- .../matrix_triangular_solve_op_test.py | 4 ++ .../kernel_tests/self_adjoint_eig_op_test.py | 7 ++- tensorflow/python/kernel_tests/svd_op_test.py | 9 ++- .../python/kernel_tests/tensordot_op_test.py | 6 +- 13 files changed, 117 insertions(+), 62 deletions(-) diff --git a/tensorflow/python/kernel_tests/batch_matmul_op_test.py b/tensorflow/python/kernel_tests/batch_matmul_op_test.py index 4cc427970b0..b4db7003bd0 100644 --- a/tensorflow/python/kernel_tests/batch_matmul_op_test.py +++ b/tensorflow/python/kernel_tests/batch_matmul_op_test.py @@ -265,9 +265,11 @@ class BatchMatMulBenchmark(test.Benchmark): if __name__ == "__main__": - for dtype_ in [ - np.float16, np.float32, np.float64, np.complex64, np.complex128, np.int32 - ]: + dtypes_to_test = [np.float16, np.float32, np.float64, np.int32] + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + dtypes_to_test += [np.complex64, np.complex128] + for dtype_ in dtypes_to_test: for adjoint_a_ in False, True: for adjoint_b_ in False, True: name = "%s_%s_%s" % (dtype_.__name__, adjoint_a_, adjoint_b_) diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py index d305277b5f4..1687054398b 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_adjoint_test.py @@ -138,6 +138,8 @@ class LinearOperatorAdjointTest( full_matrix2, adjoint=True, adjoint_arg=True).to_dense())) def test_matmul_adjoint_complex_operator(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") matrix1 = np.random.randn(4, 4) + 1j * np.random.randn(4, 4) matrix2 = np.random.randn(4, 4) + 1j * np.random.randn(4, 4) full_matrix1 = linalg.LinearOperatorFullMatrix(matrix1) @@ -188,6 +190,8 @@ class LinearOperatorAdjointTest( full_matrix2, adjoint=True, adjoint_arg=True).to_dense())) def test_solve_adjoint_complex_operator(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") matrix1 = self.evaluate(linear_operator_test_util.random_tril_matrix( [4, 4], dtype=dtypes.complex128, force_well_conditioned=True) + 1j * linear_operator_test_util.random_tril_matrix( diff --git a/tensorflow/python/kernel_tests/lu_op_test.py b/tensorflow/python/kernel_tests/lu_op_test.py index 1c0280c3ce6..875a3768602 100644 --- a/tensorflow/python/kernel_tests/lu_op_test.py +++ b/tensorflow/python/kernel_tests/lu_op_test.py @@ -130,12 +130,14 @@ class LuOpTest(test.TestCase): for output_idx_type in (dtypes.int32, dtypes.int64): self._verifyLu(data.astype(dtype), output_idx_type=output_idx_type) - for dtype in (np.complex64, np.complex128): - for output_idx_type in (dtypes.int32, dtypes.int64): - complex_data = np.tril(1j * data, -1).astype(dtype) - complex_data += np.triu(-1j * data, 1).astype(dtype) - complex_data += data - self._verifyLu(complex_data, output_idx_type=output_idx_type) + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + for dtype in (np.complex64, np.complex128): + for output_idx_type in (dtypes.int32, dtypes.int64): + complex_data = np.tril(1j * data, -1).astype(dtype) + complex_data += np.triu(-1j * data, 1).astype(dtype) + complex_data += data + self._verifyLu(complex_data, output_idx_type=output_idx_type) def testPivoting(self): # This matrix triggers partial pivoting because the first diagonal entry @@ -150,15 +152,17 @@ class LuOpTest(test.TestCase): # Make sure p_val is not the identity permutation. self.assertNotAllClose(np.arange(3), p_val) - for dtype in (np.complex64, np.complex128): - complex_data = np.tril(1j * data, -1).astype(dtype) - complex_data += np.triu(-1j * data, 1).astype(dtype) - complex_data += data - self._verifyLu(complex_data) - _, p = linalg_ops.lu(data) - p_val = self.evaluate([p]) - # Make sure p_val is not the identity permutation. - self.assertNotAllClose(np.arange(3), p_val) + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + for dtype in (np.complex64, np.complex128): + complex_data = np.tril(1j * data, -1).astype(dtype) + complex_data += np.triu(-1j * data, 1).astype(dtype) + complex_data += data + self._verifyLu(complex_data) + _, p = linalg_ops.lu(data) + p_val = self.evaluate([p]) + # Make sure p_val is not the identity permutation. + self.assertNotAllClose(np.arange(3), p_val) def testInvalidMatrix(self): # LU factorization gives an error when the input is singular. @@ -191,11 +195,13 @@ class LuOpTest(test.TestCase): matrices = np.random.rand(batch_size, 5, 5) self._verifyLu(matrices) - # Generate random complex valued matrices. - np.random.seed(52) - matrices = np.random.rand(batch_size, 5, - 5) + 1j * np.random.rand(batch_size, 5, 5) - self._verifyLu(matrices) + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + # Generate random complex valued matrices. + np.random.seed(52) + matrices = np.random.rand(batch_size, 5, + 5) + 1j * np.random.rand(batch_size, 5, 5) + self._verifyLu(matrices) def testLargeMatrix(self): # Generate random matrices. @@ -204,10 +210,12 @@ class LuOpTest(test.TestCase): data = np.random.rand(n, n) self._verifyLu(data) - # Generate random complex valued matrices. - np.random.seed(129) - data = np.random.rand(n, n) + 1j * np.random.rand(n, n) - self._verifyLu(data) + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + # Generate random complex valued matrices. + np.random.seed(129) + data = np.random.rand(n, n) + 1j * np.random.rand(n, n) + self._verifyLu(data) @test_util.run_v1_only("b/120545219") def testEmpty(self): diff --git a/tensorflow/python/kernel_tests/matmul_op_test.py b/tensorflow/python/kernel_tests/matmul_op_test.py index a3dd7dbf2af..f123492ff15 100644 --- a/tensorflow/python/kernel_tests/matmul_op_test.py +++ b/tensorflow/python/kernel_tests/matmul_op_test.py @@ -225,10 +225,13 @@ class MatMulInfixOperatorTest(test_lib.TestCase): if __name__ == "__main__": sizes = [1, 3, 5] trans_options = [[False, False], [True, False], [False, True]] + dtypes_to_test = [np.int32, np.int64, np.float16, np.float32, np.float64] + if not test_lib.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + dtypes_to_test += [np.complex64, np.complex128] # TF2 does not support placeholders under eager so we skip it for use_static_shape in set([True, tf2.enabled()]): - for dtype in (np.int32, np.int64, np.float16, np.float32, np.float64, - np.complex64, np.complex128): + for dtype in dtypes_to_test: if not use_static_shape and (dtype == np.int32 or dtype == np.int64): # TODO(rmlarsen): Re-enable this test when we have fixed the underlying # bug in Windows (b/35935459). diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py index 705f25b4fcd..520e4d3eb8d 100644 --- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py @@ -91,6 +91,8 @@ class ExponentialOpTest(test.TestCase): @test_util.run_deprecated_v1 def testNonsymmetricComplex(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") matrix1 = np.array([[1., 2.], [3., 4.]]) matrix2 = np.array([[1., 3.], [3., 5.]]) matrix1 = matrix1.astype(np.complex64) @@ -112,6 +114,8 @@ class ExponentialOpTest(test.TestCase): self._verifyExponentialReal(self._makeBatch(matrix1, matrix2)) def testSymmetricPositiveDefiniteComplex(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") matrix1 = np.array([[2., 1.], [1., 2.]]) matrix2 = np.array([[3., -1.], [-1., 3.]]) matrix1 = matrix1.astype(np.complex64) diff --git a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py index 60603f62112..56a242c0234 100644 --- a/tensorflow/python/kernel_tests/matrix_inverse_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_inverse_op_test.py @@ -74,15 +74,17 @@ class InverseOpTest(test.TestCase): self._verifyInverseReal(matrix2) # A multidimensional batch of 2x2 matrices self._verifyInverseReal(self._makeBatch(matrix1, matrix2)) - # Complex - matrix1 = matrix1.astype(np.complex64) - matrix1 += 1j * matrix1 - matrix2 = matrix2.astype(np.complex64) - matrix2 += 1j * matrix2 - self._verifyInverseComplex(matrix1) - self._verifyInverseComplex(matrix2) - # Complex batch - self._verifyInverseComplex(self._makeBatch(matrix1, matrix2)) + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + # Complex + matrix1 = matrix1.astype(np.complex64) + matrix1 += 1j * matrix1 + matrix2 = matrix2.astype(np.complex64) + matrix2 += 1j * matrix2 + self._verifyInverseComplex(matrix1) + self._verifyInverseComplex(matrix2) + # Complex batch + self._verifyInverseComplex(self._makeBatch(matrix1, matrix2)) def testSymmetricPositiveDefinite(self): # 2x2 matrices @@ -92,15 +94,17 @@ class InverseOpTest(test.TestCase): self._verifyInverseReal(matrix2) # A multidimensional batch of 2x2 matrices self._verifyInverseReal(self._makeBatch(matrix1, matrix2)) - # Complex - matrix1 = matrix1.astype(np.complex64) - matrix1 += 1j * matrix1 - matrix2 = matrix2.astype(np.complex64) - matrix2 += 1j * matrix2 - self._verifyInverseComplex(matrix1) - self._verifyInverseComplex(matrix2) - # Complex batch - self._verifyInverseComplex(self._makeBatch(matrix1, matrix2)) + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + # Complex + matrix1 = matrix1.astype(np.complex64) + matrix1 += 1j * matrix1 + matrix2 = matrix2.astype(np.complex64) + matrix2 += 1j * matrix2 + self._verifyInverseComplex(matrix1) + self._verifyInverseComplex(matrix2) + # Complex batch + self._verifyInverseComplex(self._makeBatch(matrix1, matrix2)) @test_util.deprecated_graph_mode_only def testNonSquareMatrix(self): diff --git a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py index 82f249a6444..ee6e3bb464f 100644 --- a/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_logarithm_op_test.py @@ -60,6 +60,8 @@ class LogarithmOpTest(test.TestCase): @test_util.run_v1_only("b/120545219") def testNonsymmetric(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") # 2x2 matrices matrix1 = np.array([[1., 2.], [3., 4.]]) matrix2 = np.array([[1., 3.], [3., 5.]]) @@ -74,6 +76,8 @@ class LogarithmOpTest(test.TestCase): @test_util.run_v1_only("b/120545219") def testSymmetricPositiveDefinite(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") # 2x2 matrices matrix1 = np.array([[2., 1.], [1., 2.]]) matrix2 = np.array([[3., -1.], [-1., 3.]]) @@ -108,6 +112,8 @@ class LogarithmOpTest(test.TestCase): @test_util.run_v1_only("b/120545219") def testRandomSmallAndLargeComplex64(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") np.random.seed(42) for batch_dims in [(), (1,), (3,), (2, 2)]: for size in 8, 31, 32: @@ -119,6 +125,8 @@ class LogarithmOpTest(test.TestCase): @test_util.run_v1_only("b/120545219") def testRandomSmallAndLargeComplex128(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") np.random.seed(42) for batch_dims in [(), (1,), (3,), (2, 2)]: for size in 8, 31, 32: diff --git a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py index 463477a6a2c..b99c8f6d256 100644 --- a/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py @@ -353,7 +353,11 @@ class MatrixSolveLsBenchmark(test_lib.Benchmark): if __name__ == "__main__": - for dtype_ in [np.float32, np.float64, np.complex64, np.complex128]: + dtypes_to_test = [np.float32, np.float64] + if not test_lib.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + dtypes_to_test += [np.complex64, np.complex128] + for dtype_ in dtypes_to_test: # TF2 does not support placeholders under eager so we skip it for use_placeholder_ in set([False, not tf2.enabled()]): for fast_ in [True, False]: @@ -368,7 +372,7 @@ if __name__ == "__main__": l2_regularizer_) _AddTest(MatrixSolveLsOpTest, "MatrixSolveLsOpTest", name, test_case) - for dtype_ in [np.float32, np.float64, np.complex64, np.complex128]: + for dtype_ in dtypes_to_test: for test_case in _GetLargeMatrixSolveLsOpTests(dtype_, False, True, 0.0): name = "%s_%s" % (test_case.__name__, dtype_.__name__) _AddTest(MatrixSolveLsOpTest, "MatrixSolveLsOpTest", name, test_case) diff --git a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py index 51a90e8f337..2a761140b0a 100644 --- a/tensorflow/python/kernel_tests/matrix_square_root_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_square_root_op_test.py @@ -59,14 +59,16 @@ class SquareRootOpTest(test.TestCase): self._verifySquareRootReal(matrix1) self._verifySquareRootReal(matrix2) self._verifySquareRootReal(self._makeBatch(matrix1, matrix2)) - # Complex - matrix1 = matrix1.astype(np.complex64) - matrix2 = matrix2.astype(np.complex64) - matrix1 += 1j * matrix1 - matrix2 += 1j * matrix2 - self._verifySquareRootComplex(matrix1) - self._verifySquareRootComplex(matrix2) - self._verifySquareRootComplex(self._makeBatch(matrix1, matrix2)) + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + # Complex + matrix1 = matrix1.astype(np.complex64) + matrix2 = matrix2.astype(np.complex64) + matrix1 += 1j * matrix1 + matrix2 += 1j * matrix2 + self._verifySquareRootComplex(matrix1) + self._verifySquareRootComplex(matrix2) + self._verifySquareRootComplex(self._makeBatch(matrix1, matrix2)) def testSymmetricPositiveDefinite(self): matrix1 = np.array([[2., 1.], [1., 2.]]) diff --git a/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py b/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py index 2d0427cad94..a8eda0f4fe8 100644 --- a/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_triangular_solve_op_test.py @@ -110,6 +110,8 @@ class MatrixTriangularSolveOpTest(test.TestCase): @test_util.run_deprecated_v1 def testSolveComplex(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") # 1x1 matrix, single rhs. matrix = np.array([[0.1 + 1j * 0.1]]) rhs0 = np.array([[1. + 1j]]) @@ -136,6 +138,8 @@ class MatrixTriangularSolveOpTest(test.TestCase): @test_util.run_deprecated_v1 def testSolveBatchComplex(self): + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support BLAS operations for complex types") matrix = np.array([[1., 2.], [3., 4.]]).astype(np.complex64) matrix += 1j * matrix rhs = np.array([[1., 0., 1.], [0., 1., 1.]]).astype(np.complex64) diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py index 47b22ec2967..a42d7922bfb 100644 --- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py +++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py @@ -240,9 +240,12 @@ def _GetSelfAdjointEigGradTest(dtype_, shape_, compute_v_): if __name__ == "__main__": + dtypes_to_test = [dtypes_lib.float32, dtypes_lib.float64] + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + dtypes_to_test += [dtypes_lib.complex64, dtypes_lib.complex128] for compute_v in True, False: - for dtype in (dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.complex64, - dtypes_lib.complex128): + for dtype in dtypes_to_test: for size in 1, 2, 5, 10: for batch_dims in [(), (3,)] + [(3, 2)] * (max(size, size) < 10): shape = batch_dims + (size, size) diff --git a/tensorflow/python/kernel_tests/svd_op_test.py b/tensorflow/python/kernel_tests/svd_op_test.py index 36f3e95dd7e..0ec4c620e80 100644 --- a/tensorflow/python/kernel_tests/svd_op_test.py +++ b/tensorflow/python/kernel_tests/svd_op_test.py @@ -365,9 +365,13 @@ class SVDBenchmark(test.Benchmark): if __name__ == "__main__": + dtypes_to_test = [np.float32, np.float64] + if not test.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + dtypes_to_test += [np.complex64, np.complex128] for compute_uv in False, True: for full_matrices in False, True: - for dtype in np.float32, np.float64, np.complex64, np.complex128: + for dtype in dtypes_to_test: for rows in 1, 2, 5, 10, 32, 100: for cols in 1, 2, 5, 10, 32, 100: for batch_dims in [(), (3,)] + [(3, 2)] * (max(rows, cols) < 10): @@ -383,7 +387,8 @@ if __name__ == "__main__": for compute_uv in False, True: for full_matrices in False, True: dtypes = ([np.float32, np.float64] - + [np.complex64, np.complex128] * (not compute_uv)) + + [np.complex64, np.complex128] + * (not compute_uv) * (not test.is_built_with_rocm())) for dtype in dtypes: mat_shapes = [(10, 11), (11, 10), (11, 11), (2, 2, 2, 3)] if not full_matrices or not compute_uv: diff --git a/tensorflow/python/kernel_tests/tensordot_op_test.py b/tensorflow/python/kernel_tests/tensordot_op_test.py index febfe23b16d..635a76323f6 100644 --- a/tensorflow/python/kernel_tests/tensordot_op_test.py +++ b/tensorflow/python/kernel_tests/tensordot_op_test.py @@ -221,7 +221,11 @@ def _get_tensordot_tests(dtype_, rank_a_, rank_b_, num_dims_, dynamic_shape_): if __name__ == "__main__": - for dtype in np.float16, np.float32, np.float64, np.complex64, np.complex128: + dtypes_to_test = [np.float16, np.float32, np.float64] + if not test_lib.is_built_with_rocm(): + # ROCm does not support BLAS operations for complex types + dtypes_to_test += [np.complex64, np.complex128] + for dtype in dtypes_to_test: for rank_a in 1, 2, 4, 5: for rank_b in 1, 2, 4, 5: for num_dims in range(0, min(rank_a, rank_b) + 1): From e74eb611d28da3117911fafbc30241b9be64c979 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 19:14:55 +0000 Subject: [PATCH 080/332] (temporarily) skipping a minor subtest on the ROCm platform because it is the sole failure in the entire unit-test. --- tensorflow/python/kernel_tests/linalg_grad_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/python/kernel_tests/linalg_grad_test.py b/tensorflow/python/kernel_tests/linalg_grad_test.py index 1494329f806..64ecd491b3c 100644 --- a/tensorflow/python/kernel_tests/linalg_grad_test.py +++ b/tensorflow/python/kernel_tests/linalg_grad_test.py @@ -161,6 +161,13 @@ if __name__ == '__main__': for lower in True, False: name = '%s_low_%s' % (name, lower) + if (name == 'float32_10_10_adj_False_low_True') and \ + test_lib.is_built_with_rocm(): + # Skip this one particular subtest on the ROCm platform + # It will fail because of 1 element in 10,000 mismatch, + # and the mismatch is minor (tolerance is 0.20, mismtach is 0,22) + # TODO(rocm) : investigate cause of mistmach and fix + continue _AddTest(MatrixBinaryFunctorGradientTest, 'MatrixTriangularSolveGradient', name, _GetMatrixBinaryFunctorGradientTest( From 7c5e10524cb2979d9f5e1964c69680c9a062bab6 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 19:31:36 +0000 Subject: [PATCH 081/332] [ROCm] Skipping subtests that check support for NAN propagation in the NN ops ROCm platform currently does not support the NAN propagation in the NN ops This commit skips subtests (within python unit-tests) that test this functionality. The "skip" is guarded by the call to "is_built_with_rocm()", and hence these unit-tests will not be affected in any way when running with TF which was not built with ROCm support (i.e. `--config=rocm`) --- tensorflow/python/kernel_tests/pooling_ops_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py index 68b23a20bcf..cd698b725ac 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_test.py @@ -1439,6 +1439,13 @@ class PoolingTest(test.TestCase): if not test.is_gpu_available(): return + # The functionality associated with TF_ENABLE_NANPROP is currently + # not supported on the ROCm platform, so skip this part of the test + # NANs in input lead to non-deterministic results, and hence skipping + # the remaining tests altogeher on the ROCm platform + if test.is_built_with_rocm(): + return + # Test the GPU implementation that uses cudnn for now. saved_nanprop = os.environ.get("TF_ENABLE_MAXPOOL_NANPROP") # Do not propagate the diff in cases of NaNs @@ -1519,6 +1526,13 @@ class PoolingTest(test.TestCase): if not test.is_gpu_available(): return + # The functionality associated with TF_ENABLE_NANPROP is currently + # not supported on the ROCm platform, so skip this part of the test + # NANs in input lead to non-deterministic results, and hence skipping + # the remaining tests altogeher on the ROCm platform + if test.is_built_with_rocm(): + return + # Test the GPU implementation that uses cudnn for now. saved_nanprop = os.environ.get("TF_ENABLE_MAXPOOL_NANPROP") # Do not propagate the diff in cases of NaNs From 3451c7de5244d1e6783d3e37ba84e9c45284b0c1 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 19:34:32 +0000 Subject: [PATCH 082/332] fixing what seems to be a formatting bug in the call to tf_logging.info --- tensorflow/python/kernel_tests/pooling_ops_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py index cd698b725ac..b51e6571042 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_test.py @@ -1014,7 +1014,7 @@ class PoolingTest(test.TestCase): output_sizes, x_init_value=x_init_value, delta=1e-2) - tf_logging.info("%s gradient error = " % func_name, err) + tf_logging.info("%s gradient error = %.4f" % (func_name, err)) self.assertLess(err, err_tolerance) def _ConstructAndTestSecondGradient(self, @@ -1091,7 +1091,7 @@ class PoolingTest(test.TestCase): input_sizes, x_init_value=x_init_value, delta=1e-2) - tf_logging.info("%s second-order gradient error = " % func_name, err) + tf_logging.info("%s second-order gradient error = %.4f" % (func_name, err)) self.assertLess(err, err_tolerance) def _testMaxPoolGradValidPadding1_1(self, data_format, use_gpu): From 965dbc92642f481f3ae26fbf127b40d5852348b7 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 20:34:42 +0000 Subject: [PATCH 083/332] [ROCm] Skipping subtests that check support for int8x4 type (on the GPU) ROCm platform currently does not support int8x4 type (on the GPU) This commit skips subtests (within python unit-tests) that test this functionality. The "skip" is guarded by the call to "is_built_with_rocm()", and hence these unit-tests will not be affected in any way when running with TF which was not built with ROCm support (i.e. `--config=rocm`) --- tensorflow/python/kernel_tests/relu_op_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/python/kernel_tests/relu_op_test.py b/tensorflow/python/kernel_tests/relu_op_test.py index 737024d871f..eed8bd7d258 100644 --- a/tensorflow/python/kernel_tests/relu_op_test.py +++ b/tensorflow/python/kernel_tests/relu_op_test.py @@ -80,6 +80,8 @@ class ReluTest(test.TestCase): def testReluInt8x4GoodShape(self): if not test.is_gpu_available(cuda_only=True): self.skipTest("No GPU available") + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support int8x4 type") inputs = np.array([[-50, 7, 23, 0], [-1, -5, 6, 11]]) np_relu = self._npRelu(inputs) tf_relu = nn_ops.relu(constant_op.constant(inputs, dtypes.qint8)) @@ -90,6 +92,8 @@ class ReluTest(test.TestCase): def testReluInt8x4BadShape(self): if not test.is_gpu_available(cuda_only=True): self.skipTest("No GPU available") + if test.is_built_with_rocm(): + self.skipTest("ROCm does not support int8x4 type") inputs = constant_op.constant( np.array([[-50, 7, 23], [0, 1, -5], [6, -2, 11]]), dtypes.qint8) with self.assertRaisesRegexp( From e48f20bd9dcb9ec242a6b3e43daff236373e53c7 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Mon, 8 Jul 2019 20:53:48 +0000 Subject: [PATCH 084/332] [ROCm] Skipping subtests that check bit pattern result from zero division. ROCm platform currently does not produce the same (as CUDA platform) bit pattern result from zero division. This commit skips subtests (within python unit-tests) that test this functionality. The "skip" is guarded by the call to "is_built_with_rocm()", and hence these unit-tests will not be affected in any way when running with TF which was not built with ROCm support (i.e. `--config=rocm`) --- tensorflow/python/kernel_tests/zero_division_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/kernel_tests/zero_division_test.py b/tensorflow/python/kernel_tests/zero_division_test.py index 1220be45733..d2424986982 100644 --- a/tensorflow/python/kernel_tests/zero_division_test.py +++ b/tensorflow/python/kernel_tests/zero_division_test.py @@ -54,7 +54,12 @@ class ZeroDivisionTest(test.TestCase): # # XLA constant folds integer division by zero to 1. self.assertTrue(test.is_gpu_available()) - self.assertIn(result, (-1, 1, 0xff, 0xffffffff)) + if not test.is_built_with_rocm(): + # division by zero yields a different pattern on AMD GPUs + # TODO(rocm) : investigate whether the resulting bit pattern on + # AMD GPUs is deterministic + self.assertIn(result, (-1, 1, 0xff, 0xffffffff)) + if __name__ == '__main__': From 7a1c5db5942ced68a73bb956db2db0e18c6686e0 Mon Sep 17 00:00:00 2001 From: Bhavani Subramanian Date: Mon, 8 Jul 2019 13:49:59 -0700 Subject: [PATCH 085/332] Addressed review comments. --- tensorflow/tensorflow.bzl | 2 +- third_party/mkl/build_defs.bzl | 2 +- third_party/mkl_dnn/build_defs.bzl | 8 +++- third_party/mkl_dnn/mkldnn.BUILD | 73 ++++-------------------------- 4 files changed, 16 insertions(+), 69 deletions(-) diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 0adb6383d25..86761618589 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -293,7 +293,7 @@ def tf_copts(android_optimization_level_override = "-O2", is_external = False): if_tensorrt(["-DGOOGLE_TENSORRT=1"]) + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"]) + if_mkl_open_source_only(["-DINTEL_MKL_DNN_ONLY"]) + - if_mkl_v1_open_source_only(["-DENABLE_MKLDNN_v1"]) + + if_mkl_v1_open_source_only(["-DENABLE_MKLDNN_V1"]) + if_enable_mkl(["-DENABLE_MKL"]) + if_ngraph(["-DINTEL_NGRAPH=1"]) + if_mkl_lnx_x64(["-fopenmp"]) + diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl index 00789019a8b..9f16fdd124e 100644 --- a/third_party/mkl/build_defs.bzl +++ b/third_party/mkl/build_defs.bzl @@ -107,7 +107,7 @@ def mkl_deps(): """ return select({ str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_only")): ["@mkl_dnn"], - str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_v1_only")): ["@mkl_dnn_v1"], + str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_v1_only")): ["@mkl_dnn_v1//:mkl_dnn"], str(Label("//third_party/mkl:build_with_mkl_ml_only")): ["//third_party/mkl:intel_binary_blob"], str(Label("//third_party/mkl:build_with_mkl")): [ "//third_party/mkl:intel_binary_blob", diff --git a/third_party/mkl_dnn/build_defs.bzl b/third_party/mkl_dnn/build_defs.bzl index ed3f0619623..384b528c273 100644 --- a/third_party/mkl_dnn/build_defs.bzl +++ b/third_party/mkl_dnn/build_defs.bzl @@ -1,5 +1,7 @@ def if_mkl_open_source_only(if_true, if_false = []): - """Shorthand for select()'ing on whether we're building with + """Returns `if_true` if MKL-DNN v0.x is used. + + Shorthand for select()'ing on whether we're building with MKL-DNN v0.x open source library only, without depending on MKL binary form. Returns a select statement which evaluates to if_true if we're building @@ -13,7 +15,9 @@ def if_mkl_open_source_only(if_true, if_false = []): }) def if_mkl_v1_open_source_only(if_true, if_false = []): - """Shorthand for select()'ing on whether we're building with + """Returns `if_true` if MKL-DNN v1.x is used. + + Shorthand for select()'ing on whether we're building with MKL-DNN v1.x open source library only, without depending on MKL binary form. Returns a select statement which evaluates to if_true if we're building diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index cc17dd2b4c0..cbf1874448d 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -3,6 +3,7 @@ exports_files(["LICENSE"]) load( "@org_tensorflow//third_party/mkl_dnn:build_defs.bzl", "if_mkl_open_source_only", + "if_mkl_v1_open_source_only", ) load( "@org_tensorflow//third_party:common.bzl", @@ -64,7 +65,12 @@ cc_library( "src/cpu/rnn/*.cpp", "src/cpu/rnn/*.hpp", "src/cpu/xbyak/*.h", - ]) + [":mkldnn_version_h"], + ]) + if_mkl_v1_open_source_only([ + "src/cpu/jit_utils/jit_utils.cpp", + "src/cpu/jit_utils/jit_utils.hpp", + ]) + [":mkldnn_version_h"] + if_mkl_v1_open_source_only([ + ":mkldnn_config_h", + ]), hdrs = glob(["include/*"]), copts = [ "-fexceptions", @@ -73,70 +79,7 @@ cc_library( ] + if_mkl_open_source_only([ "-UUSE_MKL", "-UUSE_CBLAS", - ]) + select({ - "@org_tensorflow//tensorflow:linux_x86_64": [ - "-fopenmp", # only works with gcc - ], - # TODO(ibiryukov): enable openmp with clang by including libomp as a - # dependency. - ":clang_linux_x86_64": [], - "//conditions:default": [], - }), - includes = [ - "include", - "src", - "src/common", - "src/cpu", - "src/cpu/gemm", - "src/cpu/xbyak", - ], - nocopts = "-fno-exceptions", - visibility = ["//visibility:public"], - deps = select({ - "@org_tensorflow//tensorflow:linux_x86_64": [ - "@mkl_linux//:mkl_headers", - "@mkl_linux//:mkl_libs_linux", - ], - "@org_tensorflow//tensorflow:macos": [ - "@mkl_darwin//:mkl_headers", - "@mkl_darwin//:mkl_libs_darwin", - ], - "@org_tensorflow//tensorflow:windows": [ - "@mkl_windows//:mkl_headers", - "@mkl_windows//:mkl_libs_windows", - ], - "//conditions:default": [], - }), -) - -cc_library( - name = "mkl_dnn_v1", - srcs = glob([ - "src/common/*.cpp", - "src/common/*.hpp", - "src/cpu/*.cpp", - "src/cpu/*.hpp", - "src/cpu/gemm/*.cpp", - "src/cpu/gemm/*.hpp", - "src/cpu/gemm/f32/*.cpp", - "src/cpu/gemm/f32/*.hpp", - "src/cpu/gemm/s8x8s32/*.cpp", - "src/cpu/gemm/s8x8s32/*.hpp", - "src/cpu/jit_utils/*.cpp", # newly added - "src/cpu/jit_utils/*.hpp", # newly added - "src/cpu/rnn/*.cpp", - "src/cpu/rnn/*.hpp", - "src/cpu/xbyak/*.h", - ]) + [ - ":mkldnn_version_h", - ":mkldnn_config_h", # newly added - ], - hdrs = glob(["include/*"]), - copts = [ - "-fexceptions", - "-DUSE_MKL", - "-DUSE_CBLAS", - ] + if_mkl_open_source_only([ + ]) + if_mkl_v1_open_source_only([ "-UUSE_MKL", "-UUSE_CBLAS", ]) + select({ From 847845c8359193f0dd25435368fc7480245c418e Mon Sep 17 00:00:00 2001 From: I-Hong Date: Mon, 8 Jul 2019 14:41:48 -0700 Subject: [PATCH 086/332] add test value of test_default_value_saved_as_tuple and test_dtype_should_be_string_or_integer --- .../python/feature_column/feature_column_v2_test.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/feature_column/feature_column_v2_test.py b/tensorflow/python/feature_column/feature_column_v2_test.py index 4031589cf2b..7752164b0e6 100644 --- a/tensorflow/python/feature_column/feature_column_v2_test.py +++ b/tensorflow/python/feature_column/feature_column_v2_test.py @@ -265,11 +265,13 @@ class NumericColumnTest(test.TestCase): self.assertEqual(((3., 2.),), a.default_value) def test_shape_and_default_value_compatibility(self): - fc.numeric_column('aaa', shape=[2], default_value=[1, 2.]) + a = fc.numeric_column('aaa', shape=[2], default_value=[1, 2.]) + self.assertEqual((1, 2.), a.default_value) with self.assertRaisesRegexp(ValueError, 'The shape of default_value'): fc.numeric_column('aaa', shape=[2], default_value=[1, 2, 3.]) - fc.numeric_column( + a = fc.numeric_column( 'aaa', shape=[3, 2], default_value=[[2, 3], [1, 2], [2, 3.]]) + self.assertEqual(((2, 3), (1, 2), (2, 3.)), a.default_value) with self.assertRaisesRegexp(ValueError, 'The shape of default_value'): fc.numeric_column( 'aaa', shape=[3, 1], default_value=[[2, 3], [1, 2], [2, 3.]]) @@ -858,8 +860,11 @@ class HashedCategoricalColumnTest(test.TestCase): fc.categorical_column_with_hash_bucket('aaa', 0) def test_dtype_should_be_string_or_integer(self): - fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.string) - fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32) + a = fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.string) + b = fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.int32) + self.assertEqual(dtypes.string, a.dtype) + self.assertEqual(dtypes.int32, b.dtype) + with self.assertRaisesRegexp(ValueError, 'dtype must be string or integer'): fc.categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32) From 76448f175f2a7b7a3eea03d07d335af48aee2e6a Mon Sep 17 00:00:00 2001 From: Bhavani Subramanian Date: Mon, 8 Jul 2019 15:06:46 -0700 Subject: [PATCH 087/332] Minor changes. --- third_party/mkl_dnn/mkldnn.BUILD | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index cbf1874448d..042c6fe4e0e 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -35,7 +35,8 @@ template_rule( # set to "version_major.version_minor.version_patch". The git hash version can # be set to NA. # TODO(agramesh1) Automatically get the version numbers from CMakeLists.txt. -# TODO(bhavanis): MKL-DNN version needs to be updated for MKL-DNN v1.x +# TODO(bhavanis): MKL-DNN minor version needs to be updated for MKL-DNN v1.x. +# The current version numbers will work only if MKL-DNN v0.18 is used. template_rule( name = "mkldnn_version_h", @@ -68,9 +69,8 @@ cc_library( ]) + if_mkl_v1_open_source_only([ "src/cpu/jit_utils/jit_utils.cpp", "src/cpu/jit_utils/jit_utils.hpp", - ]) + [":mkldnn_version_h"] + if_mkl_v1_open_source_only([ ":mkldnn_config_h", - ]), + ]) + [":mkldnn_version_h"], hdrs = glob(["include/*"]), copts = [ "-fexceptions", From 645b7c373129883160979f51a39dbc00169466e3 Mon Sep 17 00:00:00 2001 From: Bhavani Subramanian Date: Mon, 8 Jul 2019 16:04:12 -0700 Subject: [PATCH 088/332] Minor change. --- third_party/mkl_dnn/mkldnn.BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index 042c6fe4e0e..120da20f560 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -67,9 +67,9 @@ cc_library( "src/cpu/rnn/*.hpp", "src/cpu/xbyak/*.h", ]) + if_mkl_v1_open_source_only([ + ":mkldnn_config_h", "src/cpu/jit_utils/jit_utils.cpp", "src/cpu/jit_utils/jit_utils.hpp", - ":mkldnn_config_h", ]) + [":mkldnn_version_h"], hdrs = glob(["include/*"]), copts = [ From 57a924910f89ad7c3896db30944f353950591e83 Mon Sep 17 00:00:00 2001 From: "srinivasan.narayanamoorthy" Date: Mon, 8 Jul 2019 21:36:22 -0700 Subject: [PATCH 089/332] Review comments. --- tensorflow/core/kernels/scatter_functor.h | 7 ++----- tensorflow/core/kernels/scatter_op_test.cc | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/kernels/scatter_functor.h b/tensorflow/core/kernels/scatter_functor.h index f3ed40df75e..5051b508f06 100644 --- a/tensorflow/core/kernels/scatter_functor.h +++ b/tensorflow/core/kernels/scatter_functor.h @@ -207,8 +207,7 @@ struct ScatterFunctorBase { const Index N = static_cast(indices.size()); const Index limit = static_cast(params.dimension(0)); mutex mu_; - Index bad_index GUARDED_BY(mu_); - bool found_bad_index = false GUARDED_BY(mu_); + Index bad_index = -1 GUARDED_BY(mu_); auto ParallelScatter = [&](Index start, Index end) LOCKS_EXCLUDED(mu_){ for (Index i = start; i < end; i++) { // Grab the index and check its validity. Do this carefully, @@ -219,7 +218,6 @@ struct ScatterFunctorBase { if (!FastBoundsCheck(index, limit)) { mutex_lock lock(mu_); bad_index = i; - found_bad_index = true; return; } // Copy last Ndim-1 dimensions of updates[i] to params[index] @@ -231,8 +229,7 @@ struct ScatterFunctorBase { *(c->device()->tensorflow_cpu_worker_threads()); Shard(worker_threads.num_threads, worker_threads.workers, N, 35.0, ParallelScatter); // Cost is arbitrary for now. - if (found_bad_index) return bad_index; - return -1; + return bad_index; } }; diff --git a/tensorflow/core/kernels/scatter_op_test.cc b/tensorflow/core/kernels/scatter_op_test.cc index 1ea777b8a84..27286521091 100644 --- a/tensorflow/core/kernels/scatter_op_test.cc +++ b/tensorflow/core/kernels/scatter_op_test.cc @@ -47,6 +47,7 @@ class ScatterUpdateOpTest : public OpsTestBase { TF_ASSERT_OK(InitOp()); } }; + class ScatterSubOpTest : public OpsTestBase { protected: void MakeOp(DataType variable_ref_type, DataType index_type) { @@ -271,7 +272,7 @@ static void BM_ScatterHelper(int iters, int embedding_size, const char* op, for (int i = 0; i < kRows * embedding_size; i++) { values.push_back(i); } - int kNumUpdates = big_num_updates ? 1000000 : 1000; + const int kNumUpdates = big_num_updates ? 1000000 : 1000; random::PhiloxRandom philox(301, 17); random::SimplePhilox rnd(&philox); std::vector indices; From 0c9d4b5ea1ad7d948f006fcbd1398420a48904b9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 8 Jul 2019 23:57:01 -0700 Subject: [PATCH 090/332] [Grappler]: Canonicalize expression trees containing addition and subtraction, or multiplication and division, for improved constant folding. PiperOrigin-RevId: 257132026 --- .../grappler/optimizers/constant_folding.cc | 233 ++++++++++++------ .../optimizers/constant_folding_test.cc | 105 ++++++-- .../core/grappler/optimizers/remapper.cc | 1 - 3 files changed, 246 insertions(+), 93 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index 2a593be9635..8711d57dd24 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -989,11 +989,10 @@ bool ConstantFolding::IsFoldable(const NodeDef& node, } } - // No need to (and don't) fold nodes that have no outgoing edges except - // whitelisted nodes. Such nodes could be introduced by an earlier constant - // folding pass and are preserved in case users want to fetch their values; - // re-processing them would lead to an error of adding a duplicated node - // to graph. + // Don't fold nodes that have no outgoing edges except whitelisted nodes. + // Such nodes could be introduced by an earlier constant folding pass and are + // preserved in case users want to fetch their values; re-processing them + // would lead to an error of adding a duplicated node to graph. const auto& outputs = node_map_->GetOutputs(node.name()); if (outputs.empty() && nodes_whitelist_.find(node.name()) == nodes_whitelist_.end()) { @@ -1029,6 +1028,7 @@ bool ConstantFolding::IsFoldable(const NodeDef& node, return false; } } + if (is_merge && !merge_has_constant_input) return false; // If we know the output shapes, make sure that the outputs are small enough // to materialize. @@ -1050,7 +1050,7 @@ bool ConstantFolding::IsFoldable(const NodeDef& node, } } - return !is_merge || merge_has_constant_input; + return true; } namespace { @@ -2827,83 +2827,162 @@ bool ConstantFolding::ConstantPushDown(GraphDef* optimized_graph, // / \ / \ // X Y C Y = leaves // - // where C is constant and X is non-constant, and '+' denotes an - // associative and commutative operator like addition or multiplication. - // This optimization pushes constants down in the tree to canonicalize it. - // Moreoever, in cases where the child node has a second constant input Y - // we will create a leaf node that can be folded, e.g. + // where C is constant, X is non-constant, Y may be constant or non-constant, + // and '+' denotes an associative and commutative operator like addition or + // multiplication. This optimization pushes constants down in the tree to + // canonicalize it. Moreoever, in cases where the child node has a second + // constant input Y we will create a leaf node that can be folded, e.g. // // Add(C1, Add(C2, X)) -> Add(X, Add(C1, C2)) -> Add(X, C1 + C2) // - // TODO(rmlarsen): Handle non-associative/non-commutative operators like - // subtraction and division, as well as mixed subtraction/addition, - // division/multiplication. - // Don't touch BiasAdd since they can't handle vectors as their first + // We also handle the non-commutative cases of subtraction and division + // by rotating the tree locally, e.g. + // Sub(C, Add(X, Y)) -> Sub(Sub(C, Y), X) + // Mul(C, Div(X, Y)) -> Mul(X, Div(C, Y)). + // + // Note: Don't touch BiasAdd since they can't handle vectors as their first // inputs. - if (has_fetch_ && (IsAdd(*node) || IsMul(*node)) && - NumNonControlInputs(*node) == 2) { - NodeDef* left_child = node_map_->GetNode(node->input(0)); - NodeDef* right_child = node_map_->GetNode(node->input(1)); - // One child must be constant, and the other the same op as the parent. - if (node->op() != left_child->op() && node->op() != right_child->op()) { - return false; - } - const bool left_child_is_constant = IsReallyConstant(*left_child); - const bool right_child_is_constant = IsReallyConstant(*right_child); - if (!left_child_is_constant && !right_child_is_constant) { - return false; - } - if (node->device() != left_child->device() || - node->device() != right_child->device()) { - return false; - } - NodeDef* op_child_node = left_child_is_constant ? right_child : left_child; - NodeDef* const_child_node = - left_child_is_constant ? left_child : right_child; - // Make sure that it is safe to change the value of the child node-> - if (op_child_node->input_size() < 2 || - nodes_to_preserve_.find(op_child_node->name()) != - nodes_to_preserve_.end() || - NumNonControlOutputs(*op_child_node, *node_map_) > 1) { - return false; - } - // Identify the nodes to swap. - NodeDef* left_leaf = node_map_->GetNode(op_child_node->input(0)); - NodeDef* right_leaf = node_map_->GetNode(op_child_node->input(1)); - const bool left_leaf_is_constant = IsReallyConstant(*left_leaf); - const bool right_leaf_is_constant = IsReallyConstant(*right_leaf); - if (left_leaf_is_constant && right_leaf_is_constant) { - // Child is already foldable, leave it alone. - return false; - } - const int non_const_leaf_input = left_leaf_is_constant ? 1 : 0; - const int parent_const_input = left_child_is_constant ? 0 : 1; - const auto& child_output = node_map_->GetOutputs(op_child_node->name()); - if (child_output.find(const_child_node) != child_output.end()) { - // If there is a control edge from the child op to C, the transformation - // would create a cycle in the graph. We know that it must be a control - // edge. We can replace such a control edge with a control edge from A - // to C. - CHECK(MaybeRemoveControlInput(op_child_node->name(), const_child_node, - optimized_graph, node_map_.get())); - string other_leaf_input = left_leaf_is_constant ? op_child_node->input(0) - : op_child_node->input(1); - MaybeAddControlInput(other_leaf_input, const_child_node, optimized_graph, - node_map_.get()); - } - - // Swap the constant child with a non-constant leaf node. - node_map_->UpdateInput(node->name(), node->input(parent_const_input), - op_child_node->input(non_const_leaf_input)); - node_map_->UpdateInput(op_child_node->name(), - op_child_node->input(non_const_leaf_input), - node->input(parent_const_input)); - std::swap(*node->mutable_input(parent_const_input), - *op_child_node->mutable_input(non_const_leaf_input)); - return true; + // Get parent op type. + const bool is_add = IsAdd(*node); + const bool is_mul = IsMul(*node); + const bool is_sub = IsSub(*node); + const bool is_div = IsDiv(*node); + const bool is_symmetric = is_add || is_mul; + if (!has_fetch_ || !(is_add || is_sub || is_mul || is_div) || + NumNonControlInputs(*node) != 2) { + return false; } - return false; + + NodeDef* left_child = node_map_->GetNode(node->input(0)); + NodeDef* right_child = node_map_->GetNode(node->input(1)); + + const bool left_child_is_constant = IsReallyConstant(*left_child); + const bool right_child_is_constant = IsReallyConstant(*right_child); + if (!left_child_is_constant && !right_child_is_constant) { + return false; + } + // Don't move nodes across devices. + if (node->device() != left_child->device() || + node->device() != right_child->device()) { + return false; + } + NodeDef* op_child = left_child_is_constant ? right_child : left_child; + NodeDef* const_child = left_child_is_constant ? left_child : right_child; + // Don't rewrite the tree if it might create cycles. + // TODO(rmlarsen): Add back handling of control dependency from op to C. + const auto& child_output = node_map_->GetOutputs(op_child->name()); + if (child_output.find(const_child) != child_output.end()) { + return false; + } + // Get child op type. + const bool is_child_add = IsAdd(*op_child); + const bool is_child_mul = IsMul(*op_child); + const bool is_child_sub = IsSub(*op_child); + const bool is_child_div = IsDiv(*op_child); + const bool is_add_sub = (is_add || is_sub) && (is_child_add || is_child_sub); + const bool is_mul_div = (is_mul || is_div) && (is_child_mul || is_child_div); + if (!is_add_sub && !is_mul_div) { + return false; + } + const bool is_child_symmetric = is_child_add || is_child_mul; + // Make sure that it is safe to change the value of the child node result. + if (op_child->input_size() < 2 || + nodes_to_preserve_.find(op_child->name()) != nodes_to_preserve_.end() || + NumNonControlOutputs(*op_child, *node_map_) > 1) { + return false; + } + // Do not rewrite integer expressions with subtraction or division. + if (!CheckAttrExists(*node, "T").ok()) return false; + DataType dtype = node->attr().at("T").type(); + if (!(is_symmetric && is_child_symmetric) && + !(DataTypeIsFloating(dtype) || DataTypeIsComplex(dtype))) { + return false; + } + + // Identify the nodes to swap. + NodeDef* left_leaf = node_map_->GetNode(op_child->input(0)); + NodeDef* right_leaf = node_map_->GetNode(op_child->input(1)); + const bool left_leaf_is_constant = IsReallyConstant(*left_leaf); + const bool right_leaf_is_constant = IsReallyConstant(*right_leaf); + if (left_leaf_is_constant && right_leaf_is_constant) { + // Child is already foldable, leave it alone. + return false; + } + // Don't move nodes across devices. + if (node->device() != left_leaf->device() || + node->device() != right_leaf->device()) { + return false; + } + // Get the node names corresponding to X, Y, and C. + const string input_x = + left_leaf_is_constant ? op_child->input(1) : op_child->input(0); + const string input_y = + input_x == op_child->input(0) ? op_child->input(1) : op_child->input(0); + const string input_c = + left_child_is_constant ? node->input(0) : node->input(1); + const string input_op = + left_child_is_constant ? node->input(1) : node->input(0); + + // Now we have identified the nodes to swap (non_const_leaf_input and + // const_child). + node_map_->UpdateInput(node->name(), input_c, input_x); + node_map_->AddOutput(input_c, op_child->name()); + if (input_x != input_y) { + node_map_->RemoveOutput(input_x, op_child->name()); + } + + if (is_symmetric && is_child_symmetric) { + // Easy case (only commutative ops). We always write this as one of + // + + // / \ + // X + + // / \ + // C Y + node->set_input(0, input_x); + node->set_input(1, input_op); + op_child->set_input(0, input_c); + op_child->set_input(1, input_y); + } else { + // More complicated case: When there are non-commutative operations like + // subtractions or divisions involved, we may have to rotate the tree + // and/or change op types. There are 6 non-trivial cases depending on + // the effective generalized "sign" of each of the three terms C, Y, and X. + // Here are the final trees we want to generate for those 6 cases: + // + // (CYX signs): ++- +-- -+- --+ +-+ -++ + // - - - - + + + // / \ / \ / \ / \ / \ / \ + // + X - X - X X + X - X - + // / \ / \ / \ / \ / \ / \ + // C Y C Y Y C Y C C Y Y C + // + NodeDef* non_const_leaf = left_leaf_is_constant ? right_leaf : left_leaf; + NodeDef* maybe_const_leaf = + non_const_leaf == right_leaf ? left_leaf : right_leaf; + + // First, let's determine the effective sign of each term in the original + // expression + auto is_leaf_negated = [&](const NodeDef* node) -> bool { + bool leaf_negated = !is_child_symmetric && (node == right_leaf); + bool child_negated = !is_symmetric && (op_child == right_child); + return leaf_negated != child_negated; + }; + const string symmetric_op = (is_add || is_sub) ? "Add" : "Mul"; + const string nonsymmetric_op = (is_add || is_sub) ? "Sub" : "Div"; + bool neg_c = !is_symmetric && (const_child == right_child); + bool neg_x = is_leaf_negated(non_const_leaf); + bool neg_y = is_leaf_negated(maybe_const_leaf); + // Rewrite the parent node. + node->set_op((neg_x || (neg_c && neg_y)) ? nonsymmetric_op : symmetric_op); + node->set_input(0, neg_x ? input_op : input_x); + node->set_input(1, neg_x ? input_x : input_op); + // Rewrite the child node. + op_child->set_op(neg_c != neg_y ? nonsymmetric_op : symmetric_op); + op_child->set_input(0, neg_c ? input_y : input_c); + op_child->set_input(1, neg_c ? input_c : input_y); + } + return true; } bool ConstantFolding::MulConvPushDown(GraphDef* optimized_graph, NodeDef* node, diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index 3928fdff9ff..3eba75c4214 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -255,20 +255,19 @@ TEST_F(ConstantFoldingTest, SimpleFolding) { TEST_F(ConstantFoldingTest, AddTree) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output c1 = ops::Const(s.WithOpName("c1"), 1.0f, {1}); Output c2 = ops::Const(s.WithOpName("c2"), 2.0f, {2}); Output c3 = ops::Const(s.WithOpName("c3"), 3.0f, {2}); Output x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT, ops::Placeholder::Shape(TensorShape({2, 2}))); Output add_child = ops::Add(s.WithOpName("add_child"), c2, x); - Output c1 = ops::Const(s.WithOpName("c1").WithControlDependencies(add_child), - 1.0f, {1}); Output add_parent = ops::Add(s.WithOpName("add_parent"), c1, add_child); - Output y = ops::Placeholder(s.WithOpName("y"), DT_FLOAT, - ops::Placeholder::Shape(TensorShape({2, 2}))); Output c4 = ops::Const(s.WithOpName("c4"), 4.0f, {2}); Output c5 = ops::Const(s.WithOpName("c5"), 5.0f, {2}); Output c20 = ops::Const(s.WithOpName("c20"), 20.0f, {2}); + Output y = ops::Placeholder(s.WithOpName("y"), DT_FLOAT, + ops::Placeholder::Shape(TensorShape({2, 2}))); Output mul_child = ops::Mul(s.WithOpName("mul_child"), c4, y); Output mul_parent = ops::Mul(s.WithOpName("mul_parent"), c5, mul_child); Output addmul_child = ops::Add(s.WithOpName("addmul_child"), c4, x); @@ -298,16 +297,16 @@ TEST_F(ConstantFoldingTest, AddTree) { // / \ / \ // 5.0 y 4.0 5.0 - EXPECT_EQ(11, output.node_size()); + EXPECT_EQ(10, output.node_size()); for (const auto& node : output.node()) { if (node.name() == "add_child") { EXPECT_EQ("Const", node.op()); TensorProto t = node.attr().at("value").tensor(); - EXPECT_EQ(1, t.tensor_shape().dim_size()); + ASSERT_EQ(1, t.tensor_shape().dim_size()); EXPECT_EQ(2, t.tensor_shape().dim(0).size()); } else if (node.name() == "add_parent") { EXPECT_EQ("Add", node.op()); - EXPECT_EQ(2, node.input_size()); + ASSERT_EQ(2, node.input_size()); EXPECT_EQ("x", node.input(0)); EXPECT_EQ("add_child", node.input(1)); } else if (node.name() == "mul_child") { @@ -317,30 +316,106 @@ TEST_F(ConstantFoldingTest, AddTree) { EXPECT_EQ(2, t.tensor_shape().dim(0).size()); } else if (node.name() == "mul_parent") { EXPECT_EQ("Mul", node.op()); - EXPECT_EQ(2, node.input_size()); + ASSERT_EQ(2, node.input_size()); EXPECT_EQ("y", node.input(0)); EXPECT_EQ("mul_child", node.input(1)); } else if (node.name() == "addmul_child") { // Unchanged. EXPECT_EQ("Add", node.op()); - EXPECT_EQ(2, node.input_size()); + ASSERT_EQ(2, node.input_size()); EXPECT_EQ("c4", node.input(0)); EXPECT_EQ("x", node.input(1)); } } // Check that the result nodes have the expected value. - std::vector fetch = {"c3", "c20"}; - auto tensor_expected = EvaluateNodes(item.graph, fetch); - EXPECT_EQ(fetch.size(), tensor_expected.size()); - fetch = {"add_child", "mul_child"}; - auto tensors = EvaluateNodes(output, fetch); - EXPECT_EQ(fetch.size(), tensors.size()); + auto x_t = GenerateRandomTensor(TensorShape({2, 2})); + auto y_t = GenerateRandomTensor(TensorShape({2, 2})); + + std::vector fetch = {"add_parent", "mul_parent"}; + auto tensor_expected = + EvaluateNodes(item.graph, fetch, {{"x", x_t}, {"y", y_t}}); + ASSERT_EQ(fetch.size(), tensor_expected.size()); + fetch = {"add_parent", "mul_parent"}; + auto tensors = EvaluateNodes(output, fetch, {{"x", x_t}, {"y", y_t}}); + ASSERT_EQ(fetch.size(), tensors.size()); for (int i = 0; i < fetch.size(); i++) { test::ExpectTensorEqual(tensor_expected[i], tensors[i]); } } +TEST_F(ConstantFoldingTest, TreeCanonicalization) { + for (int is_add : {true, false}) { + for (int is_parent_commutative : {true, false}) { + for (int is_child_commutative : {true, false}) { + for (int is_left_child_const : {true, false}) { + for (int is_left_leaf_const : {true, false}) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output c2 = ops::Const(s.WithOpName("c2"), 2.0f, {2}); + Output c3 = ops::Const(s.WithOpName("c3"), 3.0f, {2}); + Output x = + ops::Placeholder(s.WithOpName("x"), DT_FLOAT, + ops::Placeholder::Shape(TensorShape({2, 2}))); + + auto get_op = [&](bool is_commutative, bool is_left_arg_cont, + const string& name, const Output& const_arg, + const Output non_const_arg) -> Output { + if (is_add) { + if (is_commutative) { + return ops::Add(s.WithOpName(name), + is_left_arg_cont ? const_arg : non_const_arg, + is_left_arg_cont ? non_const_arg : const_arg); + } else { + return ops::Sub(s.WithOpName(name), + is_left_arg_cont ? const_arg : non_const_arg, + is_left_arg_cont ? non_const_arg : const_arg); + } + } else { + if (is_commutative) { + return ops::Mul(s.WithOpName(name), + is_left_arg_cont ? const_arg : non_const_arg, + is_left_arg_cont ? non_const_arg : const_arg); + } else { + return ops::Div(s.WithOpName(name), + is_left_arg_cont ? const_arg : non_const_arg, + is_left_arg_cont ? non_const_arg : const_arg); + } + } + }; + + Output child = get_op(is_child_commutative, is_left_leaf_const, + "child", c2, x); + Output parent = get_op(is_parent_commutative, is_left_child_const, + "parent", c3, child); + GrapplerItem item; + item.fetch = {"parent"}; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + + ConstantFolding optimizer(/*cpu_device=*/nullptr); + GraphDef output; + Status status = + optimizer.Optimize(/*cluster=*/nullptr, item, &output); + TF_EXPECT_OK(status); + + // Check that the result nodes have the expected value. + auto x_t = GenerateRandomTensor(TensorShape({2, 2})); + std::vector fetch = {"parent"}; + auto tensor_expected = + EvaluateNodes(item.graph, fetch, {{"x", x_t}}); + ASSERT_EQ(fetch.size(), tensor_expected.size()); + fetch = {"parent"}; + auto tensors = EvaluateNodes(output, fetch, {{"x", x_t}}); + ASSERT_EQ(fetch.size(), tensors.size()); + for (int i = 0; i < fetch.size(); i++) { + test::ExpectTensorEqual(tensor_expected[i], tensors[i]); + } + } + } + } + } + } +} + TEST_F(ConstantFoldingTest, MulConvPushDownTest_Conv2D_ScalarConst) { for (string data_format : { "NHWC", diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc index 3cfebdadcda..662dd08c523 100644 --- a/tensorflow/core/grappler/optimizers/remapper.cc +++ b/tensorflow/core/grappler/optimizers/remapper.cc @@ -1634,7 +1634,6 @@ Status Remapper::Optimize(Cluster* cluster, const GrapplerItem& item, // Infer properties lazily in case they are not needed. if (!ctx.inferred_graph_properties && RequiresInferredShapes(ctx, i)) { const bool assume_valid_feeds = opt_level_ == RewriterConfig::AGGRESSIVE; - // TODO(rmlarsen): Get rid of tensor value copies. TF_RETURN_IF_ERROR(ctx.graph_properties.InferStatically( assume_valid_feeds, /*aggressive_shape_inference=*/false, From 557a724035bdf10c9a868e931aaead39d4d696b8 Mon Sep 17 00:00:00 2001 From: Thomas Joerg Date: Tue, 9 Jul 2019 00:45:20 -0700 Subject: [PATCH 091/332] [XLA] Clean up dangling HLO ops after multi-output fusion. PiperOrigin-RevId: 257138119 --- tensorflow/compiler/xla/service/multi_output_fusion.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/compiler/xla/service/multi_output_fusion.cc b/tensorflow/compiler/xla/service/multi_output_fusion.cc index c0db149e340..12a4a5ad5e9 100644 --- a/tensorflow/compiler/xla/service/multi_output_fusion.cc +++ b/tensorflow/compiler/xla/service/multi_output_fusion.cc @@ -128,6 +128,8 @@ HloInstruction* MultiOutputFusion::Fuse(HloInstruction* instr1, remaining->MergeFusionInstructionIntoMultiOutput(fused); } else { remaining->FuseInstructionIntoMultiOutput(fused); + CHECK_EQ(0, fused->user_count()); + TF_CHECK_OK(computation()->RemoveInstruction(fused)); } return remaining; } From 523ec85946e4f7362e2e74c59f55d668668cf3ca Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Tue, 9 Jul 2019 01:57:23 -0700 Subject: [PATCH 092/332] Delete the deprecated ParseHloString functions. Migrate the last remaining users to ParseAndReturnUnverifiedModule. PiperOrigin-RevId: 257147482 --- .../service/gpu/tests/gpu_convolution_regression_test.cc | 2 +- tensorflow/compiler/xla/service/hlo_parser.cc | 9 --------- tensorflow/compiler/xla/service/hlo_parser.h | 7 ------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_convolution_regression_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_convolution_regression_test.cc index 4451ab8ccb9..9e5485e3c95 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/gpu_convolution_regression_test.cc +++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_convolution_regression_test.cc @@ -28,7 +28,7 @@ class GpuConvolutionRegressionTest : public HloTestBase { HloModuleConfig config; config.set_debug_options(GetDebugOptionsFromFlags()); (void)backend().compiler()->RunHloPasses( - ParseHloString(hlo_string, config).ConsumeValueOrDie(), + ParseAndReturnUnverifiedModule(hlo_string, config).ConsumeValueOrDie(), backend().default_stream_executor(), backend().memory_allocator()); } }; diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index 57cd453faf7..6a9eeb94d48 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -4235,15 +4235,6 @@ StatusOr> ParseAndReturnUnverifiedModule( return ParseAndReturnUnverifiedModule(str, HloModuleConfig()); } -StatusOr> ParseHloString( - absl::string_view str, const HloModuleConfig& config) { - return ParseAndReturnUnverifiedModule(str, config); -} - -StatusOr> ParseHloString(absl::string_view str) { - return ParseAndReturnUnverifiedModule(str); -} - Status ParseHloString(absl::string_view str, HloModule* module) { TF_RET_CHECK(module->computation_count() == 0); HloParser parser(str); diff --git a/tensorflow/compiler/xla/service/hlo_parser.h b/tensorflow/compiler/xla/service/hlo_parser.h index 2bfabd6ab20..d9343e71189 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.h +++ b/tensorflow/compiler/xla/service/hlo_parser.h @@ -45,13 +45,6 @@ StatusOr> ParseAndReturnUnverifiedModule( StatusOr> ParseAndReturnUnverifiedModule( absl::string_view str); -ABSL_DEPRECATED("Use ParseAndReturnUnverifiedModule instead") -StatusOr> ParseHloString( - absl::string_view str, const HloModuleConfig& config); - -ABSL_DEPRECATED("Use ParseAndReturnUnverifiedModule instead") -StatusOr> ParseHloString(absl::string_view str); - // Given a string in the HloModule::ToString() format, parses the string and // builds the HloModule in place at the given module pointer. 'module' must // point to an empty module (no computations). From 7f7ea5aaa564135df016ffa7e10dd24ebf865c0d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 02:01:49 -0700 Subject: [PATCH 093/332] Change DynamicLossScale to return a tensor. This is instead of a resource variable so it's consistent with the FixedLossScale. PiperOrigin-RevId: 257147955 --- tensorflow/python/training/experimental/loss_scale.py | 2 +- .../python/training/experimental/loss_scale_test.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/training/experimental/loss_scale.py b/tensorflow/python/training/experimental/loss_scale.py index 711ec91369e..ebe80050b73 100644 --- a/tensorflow/python/training/experimental/loss_scale.py +++ b/tensorflow/python/training/experimental/loss_scale.py @@ -324,7 +324,7 @@ class DynamicLossScale(LossScale): return self._multiplier def __call__(self): - return self._current_loss_scale + return ops.convert_to_tensor(self._current_loss_scale) def update(self, grads): """Updates loss scale based on if gradients are finite in current step.""" diff --git a/tensorflow/python/training/experimental/loss_scale_test.py b/tensorflow/python/training/experimental/loss_scale_test.py index 3d2d5ba8aa3..7891156539e 100644 --- a/tensorflow/python/training/experimental/loss_scale_test.py +++ b/tensorflow/python/training/experimental/loss_scale_test.py @@ -87,6 +87,11 @@ class FixedLossScaleTest(test.TestCase): loss_scale = loss_scale_module.FixedLossScale.from_config(config) self.assertEqual(self.evaluate(loss_scale()), 123.) + @test_util.run_in_graph_and_eager_modes + def test_call_type(self): + scalar = loss_scale_module.FixedLossScale(123) + self.assertIsInstance(scalar(), ops.Tensor) + def _get_example_iter(inputs): dataset = dataset_ops.Dataset.from_tensor_slices(inputs) @@ -283,6 +288,10 @@ class DynamicLossScaleTest(test.TestCase, parameterized.TestCase): self.assertEqual(scalar.increment_period, scalar2.increment_period) self.assertEqual(scalar.multiplier, scalar2.multiplier) + @test_util.run_in_graph_and_eager_modes + def test_call_type(self): + scalar = loss_scale_module.DynamicLossScale() + self.assertIsInstance(scalar(), ops.Tensor) if __name__ == '__main__': test.main() From 3da9b87c5f1c66370e4583ec596b25444298c746 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 02:02:16 -0700 Subject: [PATCH 094/332] Update GraphDef version to 91. PiperOrigin-RevId: 257148220 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index c6f515fee13..f6bb1cb66fd 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 90 // Updated: 2019/7/8 +#define TF_GRAPH_DEF_VERSION 91 // Updated: 2019/7/9 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From 73bdfb3d42a201e709f3544a143bade418ad193d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 02:02:19 -0700 Subject: [PATCH 095/332] compat: Update forward compatibility horizon to 2019-07-09 PiperOrigin-RevId: 257148233 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 6a161c4bee9..db35b0790ae 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -27,7 +27,7 @@ import datetime from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 7, 8) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 7, 9) @tf_export("compat.forward_compatible") From 7ad65ad9024f5a6a46a108383080235963928520 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 02:07:26 -0700 Subject: [PATCH 096/332] Add a grappler optimization (CPU) for `pow(x, 3)` -> `mul(x, square(x))`. Benchmarks show mul+square to be about 13x to 50x faster than pow on CPUs: name time/op BM_cpu_Cube_CubeWithPow3/4k 362?s ? 0% BM_cpu_Cube_CubeWithPow3/64k 782?s ? 0% BM_cpu_Cube_CubeWithPow3/1M 8.59ms ? 0% BM_cpu_Cube_CubeWithTwoMuls/4k 11.4?s ? 3% BM_cpu_Cube_CubeWithTwoMuls/64k 61.5?s ?12% BM_cpu_Cube_CubeWithTwoMuls/1M 172?s ? 1% BM_cpu_Cube_CubeWithMulSquare/4k 13.7?s ? 3% BM_cpu_Cube_CubeWithMulSquare/64k 57.5?s ? 2% BM_cpu_Cube_CubeWithMulSquare/1M 173?s ? 1% PiperOrigin-RevId: 257149180 --- .../optimizers/arithmetic_optimizer.cc | 50 +++++++++------ .../optimizers/arithmetic_optimizer_test.cc | 17 ++++-- tensorflow/core/kernels/cwise_ops_test.cc | 61 +++++++++++++++++++ 3 files changed, 106 insertions(+), 22 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index cb6f77efd1a..273460050fc 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -190,6 +190,13 @@ bool GetElementUnexhaustive(const Tensor& t, int i, const std::set& dtypes, } } +bool NodeIsOnCpu(const NodeDef& node) { + string task; + string device; + return DeviceNameUtils::SplitDeviceName(node.device(), &task, &device) && + absl::StrContains(device, DEVICE_CPU); +} + // Graph optimizer context extension specific to ArithmeticOptimizer. struct ArithmeticOptimizerContext { explicit ArithmeticOptimizerContext(SetVector* nodes_to_simplify) @@ -2361,13 +2368,7 @@ class ReplaceMulWithSquare : public ArithmeticOptimizerStage { const DataType type = GetDataTypeFromAttr(*node, "T"); bool is_complex = (type == DT_COMPLEX64) || (type == DT_COMPLEX128); - string task; - string device; - bool is_on_cpu = - DeviceNameUtils::SplitDeviceName(node->device(), &task, &device) && - absl::StrContains(device, DEVICE_CPU); - - if (!is_complex || is_on_cpu) { + if (!is_complex || NodeIsOnCpu(*node)) { NodeDef* new_square_node = AddCopyNode(optimized_node_name, node); new_square_node->set_op("Square"); for (int i = 1; i < new_square_node->input_size(); ++i) { @@ -2528,6 +2529,30 @@ class ConvertPowStage : public ArithmeticOptimizerStage { node->set_input(1, AsControlDependency(y->name())); AddToOptimizationQueue(node); AddToOptimizationQueue(y); + } else if (curr == complex128(3, 0)) { + // TODO(courbet): Use 'Cube' when it's added to TF ops. + if (NodeIsOnCpu(*node)) { + // We create an inner square node: inner_square = square(x) + const NodeScopeAndName scope_and_name = + ParseNodeScopeAndName(node->name()); + const string inner_square_name = + OptimizedNodeName(scope_and_name, "_inner"); + NodeDef* inner_square_node = ctx().node_map->GetNode(inner_square_name); + if (inner_square_node == nullptr) { + inner_square_node = AddCopyNode(inner_square_name, node); + inner_square_node->set_op("Square"); + inner_square_node->mutable_input()->RemoveLast(); + } + ctx().node_map->AddOutput(x->name(), inner_square_node->name()); + // We modify `node`: node = mul(x, inner_square); + node->set_op("Mul"); + node->set_input(1, inner_square_node->name()); + node->add_input(AsControlDependency(y->name())); + + AddToOptimizationQueue(node); + AddToOptimizationQueue(inner_square_node); + AddToOptimizationQueue(y); + } } else if (curr == complex128(1, 0) && ShapesSymbolicallyEqual(value_props.shape(), output_shape)) { // Pow could be used to broadcast, so make sure the shapes of the two @@ -2985,17 +3010,6 @@ class UnaryOpsComposition : public ArithmeticOptimizerStage { DrivesControlDependency(node)); } - // UnaryOpsComposition is defined only for CPU. - bool NodeIsOnCpu(const NodeDef& node) const { - using absl::StartsWith; - - string task; - string device; - - return DeviceNameUtils::SplitDeviceName(node.device(), &task, &device) && - StartsWith(device, DEVICE_CPU); - } - bool NodeIsAlreadyFused(const NodeDef& node) const { return fused_nodes_.count(node.name()) > 0; } diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc index d9ce9f66b7a..ae3da034212 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc @@ -2728,6 +2728,7 @@ TEST_F(ArithmeticOptimizerTest, ConvertPow) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); auto x = ops::Const(s.WithOpName("x"), {1.0f, 2.0f}, {1, 2}); auto y2 = ops::Const(s.WithOpName("y2"), {2.0f, 2.0f}, {1, 2}); + auto y3 = ops::Const(s.WithOpName("y3"), {3.0f, 3.0f}, {1, 2}); auto y1 = ops::Const(s.WithOpName("y1"), {1.0f, 1.0f}, {1, 2}); auto yPoint5 = ops::Const(s.WithOpName("y.5"), {0.5f, 0.5f}, {1, 2}); auto y0 = ops::Const(s.WithOpName("y0"), {0.0f, 0.0f}, {1, 2}); @@ -2738,6 +2739,8 @@ TEST_F(ArithmeticOptimizerTest, ConvertPow) { auto ones = ops::Const(s.WithOpName("ones"), {1.0f, 1.0f, 1.0f}, {1, 3}); auto zeros = ops::Const(s.WithOpName("zeros"), {0.0f, 0.0f, 0.0f}, {1, 3}); Output out2 = ops::Pow(s.WithOpName("out2"), x, y2); + Output out3 = + ops::Pow(s.WithOpName("out3").WithDevice("/device:CPU:0"), x, y3); Output out1 = ops::Pow(s.WithOpName("out1"), x, y1); Output outPoint5 = ops::Pow(s.WithOpName("out.5"), x, yPoint5); Output out0 = ops::Pow(s.WithOpName("out0"), x, y0); @@ -2748,18 +2751,18 @@ TEST_F(ArithmeticOptimizerTest, ConvertPow) { Output out_bcast2 = ops::Pow(s.WithOpName("out_bcast2"), z, zeros); GrapplerItem item; - item.fetch = {"out2", "out1", "out.5", "out0", "out_.5", - "out_1", "out", "out_bcast1", "out_bcast2"}; + item.fetch = {"out2", "out3", "out1", "out.5", "out0", + "out_.5", "out_1", "out", "out_bcast1", "out_bcast2"}; TF_CHECK_OK(s.ToGraphDef(&item.graph)); auto tensors_expected = EvaluateNodes(item.graph, item.fetch); - ASSERT_EQ(tensors_expected.size(), 9); + ASSERT_EQ(tensors_expected.size(), 10); GraphDef got; ArithmeticOptimizer optimizer; EnableOnlyConvertPow(&optimizer); OptimizeAndPrune(&optimizer, &item, &got); auto tensors = EvaluateNodes(got, item.fetch); - ASSERT_EQ(tensors.size(), 9); + ASSERT_EQ(tensors.size(), 10); for (int i = 0; i < tensors.size(); ++i) { EXPECT_EQ(tensors[i].NumElements(), tensors_expected[i].NumElements()); @@ -2773,6 +2776,12 @@ TEST_F(ArithmeticOptimizerTest, ConvertPow) { AddNode("ones", "Const", {}, {}, &want); AddNode("zeros", "Const", {}, {}, &want); AddNode("out2", "Square", {"x"}, {}, &want); + AddNode("ArithmeticOptimizer/ConvertPow__inner_out3", "Square", {"x"}, {}, + &want) + ->set_device("/device:CPU:0"); + AddNode("out3", "Mul", {"x", "ArithmeticOptimizer/ConvertPow__inner_out3"}, + {}, &want) + ->set_device("/device:CPU:0"); AddNode("out1", "Identity", {"x"}, {}, &want); AddNode("out.5", "Sqrt", {"x"}, {}, &want); AddNode("out0", "Const", {AsControlDependency("x")}, {}, &want); diff --git a/tensorflow/core/kernels/cwise_ops_test.cc b/tensorflow/core/kernels/cwise_ops_test.cc index d6ce0f1cfa5..739ccf7730a 100644 --- a/tensorflow/core/kernels/cwise_ops_test.cc +++ b/tensorflow/core/kernels/cwise_ops_test.cc @@ -147,6 +147,67 @@ BM_BINARY_SCALAR(sycl, DivNoNan); #undef BM_BINARY_SCALAR +// Three implementations of x^3. +Graph* CubeWithPow3(int num) { + Graph* g = new Graph(OpRegistry::Global()); + Tensor lhs(DT_FLOAT, TensorShape({64, 64, num / (64 * 64)})); + lhs.flat().setRandom(); + Tensor rhs(DT_FLOAT, TensorShape({})); + rhs.flat().setConstant(3); + test::graph::Binary(g, "Pow", test::graph::Constant(g, lhs), + test::graph::Constant(g, rhs)); + return g; +} + +Graph* CubeWithTwoMuls(int num) { + Graph* g = new Graph(OpRegistry::Global()); + Tensor lhs(DT_FLOAT, TensorShape({64, 64, num / (64 * 64)})); + lhs.flat().setRandom(); + auto* x = test::graph::Constant(g, lhs); + auto* inner = test::graph::Binary(g, "Mul", x, x); + test::graph::Binary(g, "Mul", x, inner); + return g; +} + +Graph* CubeWithMulSquare(int num) { + Graph* g = new Graph(OpRegistry::Global()); + Tensor lhs(DT_FLOAT, TensorShape({64, 64, num / (64 * 64)})); + lhs.flat().setRandom(); + auto* x = test::graph::Constant(g, lhs); + auto* inner = test::graph::Unary(g, "Square", x); + test::graph::Binary(g, "Mul", test::graph::Constant(g, lhs), inner); + return g; +} + +#define BM_CUBE(DEVICE, Impl) \ + void BM_##DEVICE##_Cube_##Impl(int iters, int num) { \ + const int64 tot = static_cast(iters) * num; \ + testing::UseRealTime(); \ + testing::ItemsProcessed(tot); \ + testing::BytesProcessed(tot * sizeof(float)); \ + test::Benchmark(#DEVICE, Impl(num)).Run(iters); \ + } \ + BENCHMARK(BM_##DEVICE##_Cube_##Impl) \ + ->Arg(1 << 12) /* must >= 4096 */ \ + ->Arg(1 << 16) \ + ->Arg(1 << 20); + +BM_CUBE(cpu, CubeWithPow3); +BM_CUBE(cpu, CubeWithTwoMuls); +BM_CUBE(cpu, CubeWithMulSquare); +#if GOOGLE_CUDA +BM_CUBE(gpu, CubeWithPow3); +BM_CUBE(gpu, CubeWithTwoMuls); +BM_CUBE(gpu, CubeWithMulSquare); +#endif // GOOGLE_CUDA +#ifdef TENSORFLOW_USE_SYCL +BM_CUBE(sycl, CubeWithPow3); +BM_CUBE(sycl, CubeWithTwoMuls); +BM_CUBE(sycl, CubeWithMulSquare); +#endif // TENSORFLOW_USE_SYCL + +#undef BM_CUBE + template Graph* BiasAdd(int rows, int cols, DataType type) { Graph* g = new Graph(OpRegistry::Global()); From 87a5371a12e4fa873f28a6ec5848185ffe30ca3e Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 10:37:22 +0530 Subject: [PATCH 097/332] Removed Depricated API from the file. --- .../python/ops/distributions/special_math.py | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tensorflow/python/ops/distributions/special_math.py b/tensorflow/python/ops/distributions/special_math.py index ccc667cae3e..836606a8398 100644 --- a/tensorflow/python/ops/distributions/special_math.py +++ b/tensorflow/python/ops/distributions/special_math.py @@ -149,11 +149,11 @@ def _ndtr(x): 0.5 * np.sqrt(2.), dtype=x.dtype, name="half_sqrt_2") w = x * half_sqrt_2 z = math_ops.abs(w) - y = array_ops.where(math_ops.less(z, half_sqrt_2), - 1. + math_ops.erf(w), - array_ops.where(math_ops.greater(w, 0.), - 2. - math_ops.erfc(z), - math_ops.erfc(z))) + y = array_ops.where_v2(math_ops.less(z, half_sqrt_2), + 1. + math_ops.erf(w), + array_ops.where_v2(math_ops.greater(w, 0.), + 2. - math_ops.erfc(z), + math_ops.erfc(z))) return 0.5 * y @@ -250,11 +250,11 @@ def _ndtri(p): return array_ops.zeros_like(var) return coeffs[0] + _create_polynomial(var, coeffs[1:]) * var - maybe_complement_p = array_ops.where(p > -np.expm1(-2.), 1. - p, p) + maybe_complement_p = array_ops.where_v2(p > -np.expm1(-2.), 1. - p, p) # Write in an arbitrary value in place of 0 for p since 0 will cause NaNs # later on. The result from the computation when p == 0 is not used so any # number that doesn't result in NaNs is fine. - sanitized_mcp = array_ops.where( + sanitized_mcp = array_ops.where_v2( maybe_complement_p <= 0., array_ops.fill(array_ops.shape(p), np.array(0.5, p.dtype.as_numpy_dtype)), maybe_complement_p) @@ -280,15 +280,16 @@ def _ndtri(p): x_for_small_p = first_term - second_term_small_p x_otherwise = first_term - second_term_otherwise - x = array_ops.where(sanitized_mcp > np.exp(-2.), - x_for_big_p, - array_ops.where(z >= 8.0, x_for_small_p, x_otherwise)) + x = array_ops.where_v2(sanitized_mcp > np.exp(-2.), + x_for_big_p, + array_ops.where_v2(z >= 8.0, x_for_small_p, + x_otherwise)) - x = array_ops.where(p > 1. - np.exp(-2.), x, -x) + x = array_ops.where_v2(p > 1. - np.exp(-2.), x, -x) infinity_scalar = constant_op.constant(np.inf, dtype=p.dtype) infinity = array_ops.fill(array_ops.shape(p), infinity_scalar) - x_nan_replaced = array_ops.where( - p <= 0.0, -infinity, array_ops.where(p >= 1.0, infinity, x)) + x_nan_replaced = array_ops.where_v2( + p <= 0.0, -infinity, array_ops.where_v2(p >= 1.0, infinity, x)) return x_nan_replaced @@ -375,13 +376,14 @@ def log_ndtr(x, series_order=3, name="log_ndtr"): # the gradient of a select involves the calculation 1*dy+0*(-inf)=nan # regardless of whether dy is finite. Note that the minimum is a NOP if # the branch is chosen. - return array_ops.where( + return array_ops.where_v2( math_ops.greater(x, upper_segment), -_ndtr(-x), # log(1-x) ~= -x, x << 1 - array_ops.where(math_ops.greater(x, lower_segment), - math_ops.log(_ndtr(math_ops.maximum(x, lower_segment))), - _log_ndtr_lower(math_ops.minimum(x, lower_segment), - series_order))) + array_ops.where_v2(math_ops.greater(x, lower_segment), + math_ops.log(_ndtr(math_ops.maximum(x, + lower_segment))), + _log_ndtr_lower(math_ops.minimum(x, lower_segment), + series_order))) def _log_ndtr_lower(x, series_order): @@ -484,4 +486,4 @@ def log_cdf_laplace(x, name="log_cdf_laplace"): # internally by log1p, rather than being done explicitly here. upper_solution = math_ops.log1p(-0.5 * safe_exp_neg_x) - return array_ops.where(x < 0., lower_solution, upper_solution) + return array_ops.where_v2(x < 0., lower_solution, upper_solution) From e88d95ffaca6c68b24f3ffc416a5c7618da33f49 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 10:43:05 +0530 Subject: [PATCH 098/332] Removed Depricated API from the file. --- tensorflow/python/ops/distributions/dirichlet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/distributions/dirichlet.py b/tensorflow/python/ops/distributions/dirichlet.py index 971ce46efbc..a9a01c69742 100644 --- a/tensorflow/python/ops/distributions/dirichlet.py +++ b/tensorflow/python/ops/distributions/dirichlet.py @@ -293,7 +293,7 @@ class Dirichlet(distribution.Distribution): array_ops.shape(mode), np.array(np.nan, dtype=self.dtype.as_numpy_dtype()), name="nan") - return array_ops.where( + return array_ops.where_v2( math_ops.reduce_all(self.concentration > 1., axis=-1), mode, nan) return control_flow_ops.with_dependencies([ From 014d4b5417b7a361c6b9102bf80455ea4b44e4b3 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 10:44:58 +0530 Subject: [PATCH 099/332] Removed Depricated API from the file. --- tensorflow/python/ops/distributions/gamma.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/distributions/gamma.py b/tensorflow/python/ops/distributions/gamma.py index 8b956993ed3..6fb105c2cbe 100644 --- a/tensorflow/python/ops/distributions/gamma.py +++ b/tensorflow/python/ops/distributions/gamma.py @@ -267,7 +267,7 @@ class Gamma(distribution.Distribution): self.batch_shape_tensor(), np.array(np.nan, dtype=self.dtype.as_numpy_dtype()), name="nan") - return array_ops.where(self.concentration > 1., mode, nan) + return array_ops.where_v2(self.concentration > 1., mode, nan) else: return control_flow_ops.with_dependencies([ check_ops.assert_less( From 76babf68f4799faaa3157494f968515398dd1645 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 10:47:45 +0530 Subject: [PATCH 100/332] Removed Depricated API from the file. --- tensorflow/python/ops/distributions/student_t.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/ops/distributions/student_t.py b/tensorflow/python/ops/distributions/student_t.py index 351f5605e24..ea8e83d4b83 100644 --- a/tensorflow/python/ops/distributions/student_t.py +++ b/tensorflow/python/ops/distributions/student_t.py @@ -281,7 +281,7 @@ class StudentT(distribution.Distribution): y = (x - self.loc) / math_ops.abs(self.scale) x_t = self.df / (y**2. + self.df) neg_cdf = 0.5 * math_ops.betainc(0.5 * self.df, 0.5, x_t) - return array_ops.where(math_ops.less(y, 0.), neg_cdf, 1. - neg_cdf) + return array_ops.where_v2(math_ops.less(y, 0.), neg_cdf, 1. - neg_cdf) def _entropy(self): v = array_ops.ones(self.batch_shape_tensor(), @@ -304,7 +304,7 @@ class StudentT(distribution.Distribution): dtype=self.dtype) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) - return array_ops.where( + return array_ops.where_v2( math_ops.greater( self.df, array_ops.ones(self.batch_shape_tensor(), dtype=self.dtype)), @@ -332,22 +332,22 @@ class StudentT(distribution.Distribution): def _variance(self): # We need to put the tf.where inside the outer tf.where to ensure we never # hit a NaN in the gradient. - denom = array_ops.where(math_ops.greater(self.df, 2.), - self.df - 2., - array_ops.ones_like(self.df)) + denom = array_ops.where_v2(math_ops.greater(self.df, 2.), + self.df - 2., + array_ops.ones_like(self.df)) # Abs(scale) superfluous. var = (array_ops.ones(self.batch_shape_tensor(), dtype=self.dtype) * math_ops.square(self.scale) * self.df / denom) # When 1 < df <= 2, variance is infinite. inf = np.array(np.inf, dtype=self.dtype.as_numpy_dtype()) - result_where_defined = array_ops.where( + result_where_defined = array_ops.where_v2( self.df > array_ops.fill(self.batch_shape_tensor(), 2.), var, array_ops.fill(self.batch_shape_tensor(), inf, name="inf")) if self.allow_nan_stats: nan = np.array(np.nan, dtype=self.dtype.as_numpy_dtype()) - return array_ops.where( + return array_ops.where_v2( math_ops.greater( self.df, array_ops.ones(self.batch_shape_tensor(), dtype=self.dtype)), From f967fa7835aea9f1d5af8a78c245c4902dd3c8f0 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 10:57:46 +0530 Subject: [PATCH 101/332] Removed Depricated API from the file. --- tensorflow/python/ops/distributions/util.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/ops/distributions/util.py b/tensorflow/python/ops/distributions/util.py index 71d84770ba7..97bf0599e5f 100644 --- a/tensorflow/python/ops/distributions/util.py +++ b/tensorflow/python/ops/distributions/util.py @@ -647,7 +647,7 @@ def rotate_transpose(x, shift, name="rotate_transpose"): # Finally, we transform shift by modulo length so it can be specified # independently from the array upon which it operates (like python). ndims = array_ops.rank(x) - shift = array_ops.where( + shift = array_ops.where_v2( math_ops.less(shift, 0), math_ops.mod(-shift, ndims), ndims - math_ops.mod(shift, ndims)) first = math_ops.range(0, shift) @@ -699,7 +699,7 @@ def pick_vector(cond, true_vector, false_vector, name="pick_vector"): n = array_ops.shape(true_vector)[0] return array_ops.slice( array_ops.concat([true_vector, false_vector], 0), - [array_ops.where(cond, 0, n)], [array_ops.where(cond, n, -1)]) + [array_ops.where_v2(cond, 0, n)], [array_ops.where(cond, n, -1)]) def prefer_static_broadcast_shape(shape1, @@ -1125,7 +1125,7 @@ def reduce_weighted_logsumexp(logx, # off the max. We do this because otherwise we'd get `inf - inf = NaN`. That # this is ok follows from the fact that we're actually free to subtract any # value we like, so long as we add it back after taking the `log(sum(...))`. - max_log_absw_x = array_ops.where( + max_log_absw_x = array_ops.where_v2( math_ops.is_inf(max_log_absw_x), array_ops.zeros_like(max_log_absw_x), max_log_absw_x) wx_over_max_absw_x = ( @@ -1191,12 +1191,13 @@ def softplus_inverse(x, name=None): too_large_value = x # This `where` will ultimately be a NOP because we won't select this # codepath whenever we used the surrogate `ones_like`. - x = array_ops.where( + x = array_ops.where_v2( math_ops.logical_or(is_too_small, is_too_large), array_ops.ones_like(x), x) y = x + math_ops.log(-math_ops.expm1(-x)) # == log(expm1(x)) - return array_ops.where(is_too_small, too_small_value, - array_ops.where(is_too_large, too_large_value, y)) + return array_ops.where_v2(is_too_small, too_small_value, + array_ops.where_v2(is_too_large, + too_large_value, y)) # TODO(b/35290280): Add unit-tests. @@ -1332,7 +1333,7 @@ def pad(x, axis, front=False, back=False, value=0, count=1, name=None): else: final_shape = None else: - axis = array_ops.where(axis < 0, ndims + axis, axis) + axis = array_ops.where_v2(axis < 0, ndims + axis, axis) final_shape = None x = array_ops.pad( x, From 8f9ac682a2319c86191d98e6841891b04f80ac14 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 11:09:51 +0530 Subject: [PATCH 102/332] Removed Depricated API from the file. --- tensorflow/python/ops/distributions/transformed_distribution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/distributions/transformed_distribution.py b/tensorflow/python/ops/distributions/transformed_distribution.py index eada3cc6b9a..61397f1e4f2 100644 --- a/tensorflow/python/ops/distributions/transformed_distribution.py +++ b/tensorflow/python/ops/distributions/transformed_distribution.py @@ -91,7 +91,7 @@ def _pick_scalar_condition(pred, cond_true, cond_false): # tf.select even though we use tf.select to implement it. pred_ = _static_value(pred) if pred_ is None: - return array_ops.where(pred, cond_true, cond_false) + return array_ops.where_v2(pred, cond_true, cond_false) return cond_true if pred_ else cond_false From c838a718f3f245750f8c66df236208880e943314 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 11:11:39 +0530 Subject: [PATCH 103/332] Removed Depricated API from the file. --- tensorflow/python/ops/distributions/beta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/distributions/beta.py b/tensorflow/python/ops/distributions/beta.py index 1d1a666317f..c1ec6ed6c69 100644 --- a/tensorflow/python/ops/distributions/beta.py +++ b/tensorflow/python/ops/distributions/beta.py @@ -312,7 +312,7 @@ class Beta(distribution.Distribution): name="nan") is_defined = math_ops.logical_and(self.concentration1 > 1., self.concentration0 > 1.) - return array_ops.where(is_defined, mode, nan) + return array_ops.where_v2(is_defined, mode, nan) return control_flow_ops.with_dependencies([ check_ops.assert_less( array_ops.ones([], dtype=self.dtype), From a433ee86f03afb113a81a52a94c6218f87ab5fca Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 11:13:21 +0530 Subject: [PATCH 104/332] Removed Depricated API from the file. --- tensorflow/python/ops/distributions/uniform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/ops/distributions/uniform.py b/tensorflow/python/ops/distributions/uniform.py index 8fac0167778..d72b65dfc86 100644 --- a/tensorflow/python/ops/distributions/uniform.py +++ b/tensorflow/python/ops/distributions/uniform.py @@ -177,10 +177,10 @@ class Uniform(distribution.Distribution): def _prob(self, x): broadcasted_x = x * array_ops.ones( self.batch_shape_tensor(), dtype=x.dtype) - return array_ops.where( + return array_ops.where_v2( math_ops.is_nan(broadcasted_x), broadcasted_x, - array_ops.where( + array_ops.where_v2( math_ops.logical_or(broadcasted_x < self.low, broadcasted_x >= self.high), array_ops.zeros_like(broadcasted_x), @@ -192,9 +192,9 @@ class Uniform(distribution.Distribution): zeros = array_ops.zeros(broadcast_shape, dtype=self.dtype) ones = array_ops.ones(broadcast_shape, dtype=self.dtype) broadcasted_x = x * ones - result_if_not_big = array_ops.where( + result_if_not_big = array_ops.where_v2( x < self.low, zeros, (broadcasted_x - self.low) / self.range()) - return array_ops.where(x >= self.high, ones, result_if_not_big) + return array_ops.where_v2(x >= self.high, ones, result_if_not_big) def _entropy(self): return math_ops.log(self.range()) From f4910d2721bcfa097ea3714b5134ce72524ce3c5 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 9 Jul 2019 15:53:27 +0530 Subject: [PATCH 105/332] Removed Depricated API from the file. --- tensorflow/python/ops/linalg_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index b822519ed57..914e5748534 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -605,7 +605,7 @@ def norm(tensor, perm_after = map_fn.map_fn( lambda i: math_ops.cast( array_ops.squeeze( - array_ops.where(math_ops.equal(perm_before, i))), + array_ops.where_v2(math_ops.equal(perm_before, i))), dtype=dtypes.int32), axes) permed = array_ops.transpose(tensor, perm=perm_before) matrix_2_norm = array_ops.expand_dims( From 0a9ea1fec2c3588464ea78cbadb884f280d902ef Mon Sep 17 00:00:00 2001 From: Taehoon Lee Date: Tue, 9 Jul 2019 20:48:13 +0900 Subject: [PATCH 106/332] Address the reviewer comments and resolve conflicts --- .../compiler/mlir/tensorflow/translate/tf_mlir_translate.cc | 2 +- tensorflow/go/op/wrappers.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc b/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc index 6f42dbc1f7f..3ca4aea9123 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc @@ -119,7 +119,7 @@ mlir::OwningModuleRef GraphdefToSplattedMlirTranslateFunction( break; default: inst.emitWarning() - << "Skipping splat conversation for " + << "Skipping splat conversion for " << "an unsupported attribute type " << element_type; continue; } diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 67b1fea194f..800d03fc70e 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -31867,7 +31867,7 @@ func LeakyReluGradAlpha(value float32) LeakyReluGradAttr { // features: The features passed as input to the corresponding LeakyRelu operation, // OR the outputs of that operation (both work equivalently). // -// Returns `gradients * (features > 0) + alpha * gradients * (features <= 0)`. +// Returns `gradients * (features > 0) + alpha * gradients * (featurs <= 0)`. func LeakyReluGrad(scope *Scope, gradients tf.Output, features tf.Output, optional ...LeakyReluGradAttr) (backprops tf.Output) { if scope.Err() != nil { return From 5bb817c1e4341a11d60be1888f0aadf7e0fc47ef Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Tue, 9 Jul 2019 04:48:52 -0700 Subject: [PATCH 107/332] Delete unused header include. PiperOrigin-RevId: 257167499 --- tensorflow/compiler/xla/service/BUILD | 1 - tensorflow/compiler/xla/service/hlo_parser.h | 1 - 2 files changed, 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 6457a49e0e8..2f74a1378cb 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -3946,7 +3946,6 @@ cc_library( "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "@com_google_absl//absl/algorithm:container", - "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", diff --git a/tensorflow/compiler/xla/service/hlo_parser.h b/tensorflow/compiler/xla/service/hlo_parser.h index d9343e71189..e4214c1e6b5 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.h +++ b/tensorflow/compiler/xla/service/hlo_parser.h @@ -16,7 +16,6 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_PARSER_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_PARSER_H_ -#include "absl/base/macros.h" #include "absl/memory/memory.h" #include "absl/strings/string_view.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" From dffe85688b3f2376c7c4d5a34760feec8e82449d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 05:13:56 -0700 Subject: [PATCH 108/332] Log output for incorrect set of flags updated. PiperOrigin-RevId: 257170572 --- .../xla/tools/interactive_graphviz.cc | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tensorflow/compiler/xla/tools/interactive_graphviz.cc b/tensorflow/compiler/xla/tools/interactive_graphviz.cc index 5652d303f02..01555fa3cff 100644 --- a/tensorflow/compiler/xla/tools/interactive_graphviz.cc +++ b/tensorflow/compiler/xla/tools/interactive_graphviz.cc @@ -52,7 +52,7 @@ namespace xla { namespace tools { namespace { -bool ReadLine(const char *prompt, string *line) { +bool ReadLine(const char* prompt, string* line) { #if defined(PLATFORM_GOOGLE) return util::ReadLine(prompt, line); #else @@ -628,28 +628,22 @@ void InteractiveDumpGraphs(const Options& opts, const HloModule& module) { } } -void CheckFlags(const Options &opts) { - std::vector nonempty_proto_flags; +void CheckFlags(const Options& opts) { + int nonempty_flags_amount = 0; if (!opts.hlo_proto.empty()) { - nonempty_proto_flags.push_back("--hlo_proto"); + ++nonempty_flags_amount; } if (!opts.hlo_snapshot.empty()) { - nonempty_proto_flags.push_back("--hlo_snapshot"); + ++nonempty_flags_amount; } if (!opts.hlo_text.empty()) { - nonempty_proto_flags.push_back("--hlo_text"); + ++nonempty_flags_amount; } - switch (nonempty_proto_flags.size()) { - case 1: - // We're good to go. - break; - case 0: - LOG(FATAL) << "Need one of the following options: " - << absl::StrJoin(nonempty_proto_flags, ", "); - default: - LOG(FATAL) << "Can only specify one of " - << absl::StrJoin(nonempty_proto_flags, ", "); + if (nonempty_flags_amount == 1) { + return; } + LOG(FATAL) << "Can only specify one and only one of '--hlo_proto', " + "'--hlo_snapshot', '--hlo_text' flags."; } void RealMain(const Options& opts) { @@ -726,8 +720,7 @@ int main(int argc, char** argv) { "Platform to compile for: CPU, CUDA, etc"), tensorflow::Flag("browser", &opts.browser, "Path to web browser used to display produced graphs."), - tensorflow::Flag("help", &need_help, - "Prints this help message"), + tensorflow::Flag("help", &need_help, "Prints this help message"), }; xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list); bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list); From ec91d46bc5d0cab5d906b3bd523761247de1a92e Mon Sep 17 00:00:00 2001 From: "Wen-Heng (Jack) Chung" Date: Wed, 3 Jul 2019 22:00:08 +0000 Subject: [PATCH 109/332] Emit llvm::Intrinsic::nvvm_atomic_load_add_f32 only on NVPTX --- tensorflow/compiler/xla/service/gpu/ir_emitter.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc index f90e2716869..a82b6119258 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc @@ -205,8 +205,9 @@ bool IrEmitter::MaybeEmitDirectAtomicOperation( } if (root_opcode == HloOpcode::kAdd) { + llvm::Triple target_triple = llvm::Triple(module_->getTargetTriple()); // NVPTX supports atomicAdd on F32 and integer types. - if (element_type == F32) { + if (target_triple.isNVPTX() && element_type == F32) { // F32 + F32 llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_atomic_load_add_f32, {output_address, source}, From 8305f9319f44b60cc52d378c4fccf253d8e1e428 Mon Sep 17 00:00:00 2001 From: Peter Buchlovsky Date: Tue, 9 Jul 2019 08:36:17 -0700 Subject: [PATCH 110/332] Boosted Trees: delay switching to version 2 cond (even when running in v1). This is breaking some tests. Delaying until we have a fix. PiperOrigin-RevId: 257199240 --- tensorflow/python/ops/data_flow_ops.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index bf7314d918d..9c49fc85270 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -1217,7 +1217,7 @@ class ConditionalAccumulatorBase(object): if name is None: name = "%s_NumAccumulated" % self._name - if compat.forward_compatible(2019, 7, 8): + if compat.forward_compatible(2019, 8, 8): return gen_data_flow_ops.resource_accumulator_num_accumulated( self._accumulator_ref, name=name) @@ -1237,7 +1237,7 @@ class ConditionalAccumulatorBase(object): Returns: Operation that sets the accumulator's time step. """ - if compat.forward_compatible(2019, 7, 8): + if compat.forward_compatible(2019, 8, 8): return gen_data_flow_ops.resource_accumulator_set_global_step( self._accumulator_ref, math_ops.cast(ops.convert_to_tensor(new_global_step), _dtypes.int64), @@ -1276,7 +1276,7 @@ class ConditionalAccumulator(ConditionalAccumulatorBase): name: Optional name for the accumulator. reduction_type: Reduction type to use when taking the gradient. """ - if compat.forward_compatible(2019, 7, 8): + if compat.forward_compatible(2019, 8, 8): accumulator_ref = gen_data_flow_ops.resource_conditional_accumulator( dtype=dtype, shape=shape, @@ -1316,7 +1316,7 @@ class ConditionalAccumulator(ConditionalAccumulatorBase): grad.get_shape().assert_is_compatible_with(self._shape) local_step = math_ops.cast(ops.convert_to_tensor(local_step), _dtypes.int64) - if compat.forward_compatible(2019, 7, 8): + if compat.forward_compatible(2019, 8, 8): return gen_data_flow_ops.resource_accumulator_apply_gradient( self._accumulator_ref, local_step=local_step, @@ -1347,7 +1347,7 @@ class ConditionalAccumulator(ConditionalAccumulatorBase): Raises: InvalidArgumentError: If num_required < 1 """ - if compat.forward_compatible(2019, 7, 8): + if compat.forward_compatible(2019, 8, 8): out = gen_data_flow_ops.resource_accumulator_take_gradient( self._accumulator_ref, num_required, dtype=self._dtype, name=name) else: From 8db90846fce0557545a896e0a7196fd56d186079 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Tue, 9 Jul 2019 08:49:06 -0700 Subject: [PATCH 111/332] Fix missing build dep. PiperOrigin-RevId: 257201388 --- tensorflow/compiler/mlir/tensorflow/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/mlir/tensorflow/BUILD b/tensorflow/compiler/mlir/tensorflow/BUILD index add4dbc9e67..f9457659887 100644 --- a/tensorflow/compiler/mlir/tensorflow/BUILD +++ b/tensorflow/compiler/mlir/tensorflow/BUILD @@ -280,6 +280,7 @@ cc_library( "//tensorflow/core:lib", "//tensorflow/core:protos_all_proto_cc", "@local_config_mlir//:IR", + "@local_config_mlir//:StandardOps", ], ) From 45925ad3d0c749db7a08abee38c0200a14728c6c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 09:29:29 -0700 Subject: [PATCH 112/332] Add single tensor gradient support for DynamicLossScale PiperOrigin-RevId: 257209230 --- tensorflow/python/training/experimental/loss_scale.py | 10 ++++++---- .../python/training/experimental/loss_scale_test.py | 9 +++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/training/experimental/loss_scale.py b/tensorflow/python/training/experimental/loss_scale.py index ebe80050b73..bbbd0cd7ec4 100644 --- a/tensorflow/python/training/experimental/loss_scale.py +++ b/tensorflow/python/training/experimental/loss_scale.py @@ -31,6 +31,7 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.training.tracking import base as trackable from tensorflow.python.ops import variable_scope +from tensorflow.python.util import nest from tensorflow.python.util.tf_export import tf_export @@ -93,10 +94,10 @@ class LossScale(trackable.Trackable): cross-replica context. Args: - grads: A list of unscaled gradients, each which is the gradient of the - loss with respect to a weight. The gradients should have already been - divided by the loss scale being before passed to this function. 'None' - gradients are accepted, and are ignored. + grads: A nested structure of unscaled gradients, each which is the + gradient of the loss with respect to a weight. The gradients should have + already been divided by the loss scale being before passed to this + function. 'None' gradients are accepted, and are ignored. Returns: update_op: In eager mode, None. In graph mode, an op to update the loss @@ -328,6 +329,7 @@ class DynamicLossScale(LossScale): def update(self, grads): """Updates loss scale based on if gradients are finite in current step.""" + grads = nest.flatten(grads) if distribution_strategy_context.has_strategy(): distribution = distribution_strategy_context.get_cross_replica_context() diff --git a/tensorflow/python/training/experimental/loss_scale_test.py b/tensorflow/python/training/experimental/loss_scale_test.py index 7891156539e..c3e18a18422 100644 --- a/tensorflow/python/training/experimental/loss_scale_test.py +++ b/tensorflow/python/training/experimental/loss_scale_test.py @@ -264,6 +264,15 @@ class DynamicLossScaleTest(test.TestCase, parameterized.TestCase): expected_outputs = [2, 2, 4, 4, 2, 2, 1, 1, 2, 2, 1] self._test_helper(inputs, expected_outputs, init_loss_scale) + @parameterized.named_parameters(*TESTCASES) + @test_util.run_in_graph_and_eager_modes + def test_single_tensor_gradient(self, strategy_fn): + with strategy_fn().scope(): + loss_scale = loss_scale_module.DynamicLossScale() + grad = constant_op.constant(4.0) + _, should_apply = loss_scale.update(grad) + self.assertTrue(self.evaluate(should_apply)) + @test_util.run_in_graph_and_eager_modes def test_serialization(self): loss_scale = loss_scale_module.DynamicLossScale( From 9c7ddffd970e8e72585c66c4a2c7cf8bb42ed608 Mon Sep 17 00:00:00 2001 From: Eugene Zhulenev Date: Tue, 9 Jul 2019 09:41:20 -0700 Subject: [PATCH 113/332] Optimize FusedBatchNormGrad on CPU device. 20% speedup in tensorflow_models/official/resnet/keras:keras_cifar_main: BEFORE: {'num_batches':200, 'time_taken': 19.408517,'examples_per_second': 329.752141} {'num_batches':300, 'time_taken': 19.280430,'examples_per_second': 331.942807} {'num_batches':400, 'time_taken': 19.173295,'examples_per_second': 333.797607} AFTER: {'num_batches':200, 'time_taken': 16.136061,'examples_per_second': 396.627158} {'num_batches':300, 'time_taken': 15.969341,'examples_per_second': 400.767946} {'num_batches':400, 'time_taken': 15.745600,'examples_per_second': 406.462758} PiperOrigin-RevId: 257211709 --- tensorflow/core/kernels/BUILD | 1 + tensorflow/core/kernels/bias_op.cc | 5 +- .../core/kernels/fused_batch_norm_op.cc | 192 +++++++++++++++--- .../core/kernels/fused_batch_norm_op.cu.cc | 77 +++++++ tensorflow/core/kernels/fused_batch_norm_op.h | 62 +----- .../core/kernels/fused_batch_norm_op_test.cc | 16 ++ tensorflow/core/kernels/redux_functor.h | 60 +++--- 7 files changed, 295 insertions(+), 118 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 364a05bd901..35d6a81751f 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -4348,6 +4348,7 @@ tf_kernel_library( prefix = "fused_batch_norm_op", deps = NN_DEPS + [ ":fill_functor", + ":redux_functor", ] + if_cuda([ "//tensorflow/core:stream_executor", ]), diff --git a/tensorflow/core/kernels/bias_op.cc b/tensorflow/core/kernels/bias_op.cc index 08979666d2f..6a407f5551a 100644 --- a/tensorflow/core/kernels/bias_op.cc +++ b/tensorflow/core/kernels/bias_op.cc @@ -18,6 +18,7 @@ limitations under the License. #define EIGEN_USE_THREADS #include "tensorflow/core/kernels/bias_op.h" + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/numeric_op.h" @@ -273,7 +274,7 @@ class BiasGradOp : public OpKernel { using AccumT = typename AccumulatorType::type; if (data_format_ == FORMAT_NCHW) { const functor::ReduceMiddleDimensions< - T, AccumT, Eigen::internal::scalar_sum_op, + T, AccumT, T, Eigen::internal::scalar_sum_op, Eigen::internal::SumReducer> redux; Eigen::DSizes three_dims(batch, channel, @@ -282,7 +283,7 @@ class BiasGradOp : public OpKernel { output, 1); } else { const functor::ReduceOuterDimensions< - T, AccumT, Eigen::internal::scalar_sum_op> + T, AccumT, T, Eigen::internal::scalar_sum_op> redux; Eigen::DSizes two_dims(batch * height * width * depth, diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cc b/tensorflow/core/kernels/fused_batch_norm_op.cc index 4179f17deee..70bd659be66 100644 --- a/tensorflow/core/kernels/fused_batch_norm_op.cc +++ b/tensorflow/core/kernels/fused_batch_norm_op.cc @@ -33,6 +33,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/kernels/fill_functor.h" #include "tensorflow/core/kernels/fused_batch_norm_op.h" +#include "tensorflow/core/kernels/redux_functor.h" #include "tensorflow/core/util/env_var.h" #include "tensorflow/core/util/tensor_format.h" @@ -358,7 +359,6 @@ struct FusedBatchNormGrad { typename TTypes::ConstVec mean(mean_input.vec()); typename TTypes::ConstVec variance(variance_input.vec()); typename TTypes::Tensor x_backprop(x_backprop_output->tensor()); - typename TTypes::Vec scale_backprop(scale_backprop_output->vec()); typename TTypes::Vec offset_backprop(offset_backprop_output->vec()); // Note: the following formulas are used to compute the gradients for @@ -378,12 +378,10 @@ struct FusedBatchNormGrad { #if !defined(EIGEN_HAS_INDEX_LIST) Eigen::DSizes one_by_depth(1, depth); - Eigen::array reduce_dims({0}); Eigen::array bcast_spec({rest_size, 1}); #else Eigen::IndexList, Eigen::Index> one_by_depth; one_by_depth.set(1, depth); - Eigen::IndexList> reduce_dims; Eigen::IndexList> bcast_spec; bcast_spec.set(0, rest_size); #endif @@ -391,41 +389,182 @@ struct FusedBatchNormGrad { auto x_rest_by_depth = x.reshape(rest_by_depth).template cast(); U rest_size_inv = static_cast(1.0f / static_cast(rest_size)); + // Eigen is notoriously bad at reducing outer dimension, so we materialize + // all temporary tensors that require reduction, and then use Eigen redux + // functor, that is optimized for this particular task. + // + // All reductions are of this type: [rest_size, depth] -> [depth]. + using ScalarSum = Eigen::internal::scalar_sum_op; + const functor::ReduceOuterDimensions redux_sum_t; + const functor::ReduceOuterDimensions redux_sum_u; + + auto scratch_dtype = DataTypeToEnum::value; + + // Allocate a temporary workspace of [depth] shape. + Tensor scratch_one_by_depth; + OP_REQUIRES_OK(context, context->allocate_temp(scratch_dtype, {depth}, + &scratch_one_by_depth)); + + // Maybe allocate a temporary workspace of [rest_size, depth] shape. + Tensor scratch_rest_by_depth; + if (std::is_same::value) { + OP_REQUIRES(context, + scratch_rest_by_depth.CopyFrom(*x_backprop_output, + {rest_size, depth}), + errors::Internal("Failed to copy a tensor")); + } else { + OP_REQUIRES_OK(context, + context->allocate_temp(scratch_dtype, {rest_size, depth}, + &scratch_rest_by_depth)); + } + + typename TTypes::Tensor scratch_tensor( + scratch_rest_by_depth.tensor()); + typename TTypes::Vec scratch_vector(scratch_one_by_depth.vec()); + auto x_mean_rest_by_depth = mean.reshape(one_by_depth).broadcast(bcast_spec); - auto x_centered = (x_rest_by_depth - x_mean_rest_by_depth).eval(); + auto x_centered = (x_rest_by_depth - x_mean_rest_by_depth); auto coef0 = (variance + epsilon).rsqrt(); auto coef0_rest_by_depth = - coef0.eval().reshape(one_by_depth).broadcast(bcast_spec); + coef0.reshape(one_by_depth).broadcast(bcast_spec); auto x_scaled = x_centered * coef0_rest_by_depth; auto y_backprop_rest_by_depth = - y_backprop.eval().reshape(rest_by_depth).template cast(); - scale_backprop.device(d) = - (y_backprop_rest_by_depth * x_scaled).sum(reduce_dims); - auto y_backprop_sum = y_backprop_rest_by_depth.sum(reduce_dims); - offset_backprop.device(d) = y_backprop_sum; + y_backprop.reshape(rest_by_depth).template cast(); - auto y_backprop_sum_one_by_depth = - y_backprop_sum.eval().reshape(one_by_depth); + // Compute `scale_backprop_output`: + // scale_backprop = + // (y_backprop_rest_by_depth * x_scaled).sum(reduce_dims) + scratch_tensor.device(d) = y_backprop_rest_by_depth * x_scaled; + redux_sum_u(d, rest_by_depth, scratch_rest_by_depth, scale_backprop_output); + + // Compute 'offset_backprop_output': + // offset_backprop = + // y_backprop_rest_by_depth.sum(reduce_dims) + redux_sum_t(d, rest_by_depth, y_backprop_input, offset_backprop_output); + auto y_backprop_sum = offset_backprop; + + auto y_backprop_sum_one_by_depth = y_backprop_sum.reshape(one_by_depth); auto y_backprop_mean_one_by_depth = y_backprop_sum_one_by_depth * rest_size_inv; auto y_backprop_mean_rest_by_depth = y_backprop_mean_one_by_depth.broadcast(bcast_spec); auto y_backprop_centered = y_backprop_rest_by_depth - y_backprop_mean_rest_by_depth; - auto coef1 = - (scale * coef0).eval().reshape(one_by_depth).broadcast(bcast_spec); - auto coef2 = (coef0.square() * - (y_backprop_rest_by_depth * x_centered).mean(reduce_dims)) - .eval() + + // Compute expression: + // y_backprop_centered_mean = + // (y_backprop_rest_by_depth * x_centered).mean(reduce_dims) + scratch_tensor.device(d) = y_backprop_rest_by_depth * x_centered; + redux_sum_u(d, rest_by_depth, scratch_rest_by_depth, &scratch_one_by_depth); + auto y_backprop_centered_mean = scratch_vector / static_cast(rest_size); + + auto coef1 = (scale * coef0).reshape(one_by_depth).broadcast(bcast_spec); + auto coef2 = (coef0.square() * y_backprop_centered_mean) .reshape(one_by_depth) + .eval() .broadcast(bcast_spec); + x_backprop.reshape(rest_by_depth).device(d) = (coef1 * (y_backprop_centered - x_centered * coef2)).template cast(); } }; +template +struct FusedBatchNormFreezeGrad { + void operator()(OpKernelContext* context, const Tensor& y_backprop_input, + const Tensor& x_input, const Tensor& scale_input, + const Tensor& pop_mean_input, + const Tensor& pop_variance_input, U epsilon, + Tensor* x_backprop_output, Tensor* scale_backprop_output, + Tensor* offset_backprop_output) { + typename TTypes::ConstTensor y_backprop( + y_backprop_input.tensor()); + typename TTypes::ConstTensor input(x_input.tensor()); + typename TTypes::ConstVec scale(scale_input.vec()); + typename TTypes::ConstVec pop_mean(pop_mean_input.vec()); + typename TTypes::ConstVec pop_var(pop_variance_input.vec()); + typename TTypes::Tensor x_backprop(x_backprop_output->tensor()); + typename TTypes::Vec scale_backprop(scale_backprop_output->vec()); + + const int depth = pop_mean.dimension(0); + const int rest_size = input.size() / depth; + + const CPUDevice& d = context->eigen_device(); + + // Allocate two temporary workspaces of [depth] shape. + Tensor scratch1_vec, scratch2_vec; + OP_REQUIRES_OK(context, context->allocate_temp(DataTypeToEnum::value, + {depth}, &scratch1_vec)); + OP_REQUIRES_OK(context, context->allocate_temp(DataTypeToEnum::value, + {depth}, &scratch2_vec)); + + // Maybe allocate a temporary workspace of [rest_size, depth] shape. + Tensor scratch3_tensor; + if (std::is_same::value) { + OP_REQUIRES( + context, + scratch3_tensor.CopyFrom(*x_backprop_output, {rest_size, depth}), + errors::Internal("Failed to copy a tensor")); + } else { + OP_REQUIRES_OK(context, context->allocate_temp(DataTypeToEnum::value, + {rest_size, depth}, + &scratch3_tensor)); + } + + typename TTypes::Vec scratch1(scratch1_vec.vec()); + typename TTypes::Vec scratch2(scratch2_vec.vec()); + typename TTypes::Tensor scratch3(scratch3_tensor.tensor()); + + Eigen::DSizes rest_by_depth(rest_size, depth); +#if !defined(EIGEN_HAS_INDEX_LIST) + Eigen::DSizes one_by_depth(1, depth); + Eigen::array rest_by_one({rest_size, 1}); +#else + Eigen::IndexList, Eigen::Index> one_by_depth; + one_by_depth.set(1, depth); + Eigen::IndexList> rest_by_one; + rest_by_one.set(0, rest_size); +#endif + + // Sum reduction along the 0th dimension using custom CPU functor. + using ScalarSum = Eigen::internal::scalar_sum_op; + const functor::ReduceOuterDimensions redux_sum_t; + const functor::ReduceOuterDimensions redux_sum_u; + + // offset_backprop = sum(y_backprop) + // scale_backprop = y_backprop * ((x - pop_mean) * rsqrt(pop_var + epsilon)) + // x_backprop = y_backprop * (scale * rsqrt(pop_var + epsilon)) + + // NOTE: DEFAULT DEVICE comment is added to expression assignments that + // we don't want to be executed in a thread pool. + + auto y_backprop_rest_by_depth = + y_backprop.reshape(rest_by_depth).template cast(); + auto input_rest_by_depth = input.reshape(rest_by_depth).template cast(); + + // offset_backprop = sum(y_backprop) + redux_sum_t(d, rest_by_depth, y_backprop_input, offset_backprop_output); + + // scratch1 = rsqrt(pop_var + epsilon) + scratch1 = (pop_var + pop_var.constant(epsilon)).rsqrt(); // DEFAULT DEVICE + + // scratch2 = sum(y_backprop * (x - mean)) + scratch3.device(d) = + y_backprop_rest_by_depth * + (input_rest_by_depth - + pop_mean.reshape(one_by_depth).broadcast(rest_by_one)); + redux_sum_u(d, rest_by_depth, scratch3_tensor, &scratch2_vec); + + x_backprop.reshape(rest_by_depth).device(d) = + (y_backprop_rest_by_depth * + ((scratch1 * scale).reshape(one_by_depth).broadcast(rest_by_one))) + .template cast(); + scale_backprop = scratch2 * scratch1; // DEFAULT DEVICE + } +}; + #if !GOOGLE_CUDA && !TENSORFLOW_USE_ROCM namespace { // See implementation under GOOGLE_CUDA #ifdef below. @@ -827,12 +966,11 @@ struct FusedBatchNormGrad { #define DECLARE_GPU_SPEC(T, U) \ template <> \ void FusedBatchNormFreezeGrad::operator()( \ - const GPUDevice& d, const Tensor& y_backprop_input, \ + OpKernelContext* context, const Tensor& y_backprop_input, \ const Tensor& x_input, const Tensor& scale_input, \ const Tensor& mean_input, const Tensor& variance_input, U epsilon, \ Tensor* x_backprop_output, Tensor* scale_backprop_output, \ - Tensor* offset_backprop_output, typename TTypes::Vec scratch1, \ - typename TTypes::Vec scratch2); \ + Tensor* offset_backprop_output); \ extern template struct FusedBatchNormFreezeGrad; \ template <> \ void FusedBatchNormInferenceFunctor::operator()( \ @@ -1152,18 +1290,10 @@ class FusedBatchNormGradOpBase : public OpKernel { << "The implementation of FusedBatchNormGrad with is_training=False " "only support " << "NHWC tensor format for now."; - Tensor scratch1, scratch2; - OP_REQUIRES_OK(context, - context->allocate_temp(DataTypeToEnum::value, - scale_offset_shape, &scratch1)); - OP_REQUIRES_OK(context, - context->allocate_temp(DataTypeToEnum::value, - scale_offset_shape, &scratch2)); functor::FusedBatchNormFreezeGrad()( - context->eigen_device(), y_backprop, x, scale, - saved_mean_or_pop_mean, saved_maybe_inv_var_or_pop_var, epsilon_, - x_backprop, scale_backprop, offset_backprop, scratch1.vec(), - scratch2.vec()); + context, y_backprop, x, scale, saved_mean_or_pop_mean, + saved_maybe_inv_var_or_pop_var, epsilon_, x_backprop, scale_backprop, + offset_backprop); } } diff --git a/tensorflow/core/kernels/fused_batch_norm_op.cu.cc b/tensorflow/core/kernels/fused_batch_norm_op.cu.cc index ff088bd6f88..0d2c1c4015d 100644 --- a/tensorflow/core/kernels/fused_batch_norm_op.cu.cc +++ b/tensorflow/core/kernels/fused_batch_norm_op.cu.cc @@ -26,6 +26,83 @@ typedef Eigen::GpuDevice GPUDevice; namespace functor { +// TODO(ezhulenev): Use CUB reductions on GPU. +template +struct FusedBatchNormFreezeGrad { + void operator()(OpKernelContext* context, const Tensor& y_backprop_input, + const Tensor& x_input, const Tensor& scale_input, + const Tensor& pop_mean_input, + const Tensor& pop_variance_input, U epsilon, + Tensor* x_backprop_output, Tensor* scale_backprop_output, + Tensor* offset_backprop_output) { + typename TTypes::ConstTensor y_backprop( + y_backprop_input.tensor()); + typename TTypes::ConstTensor input(x_input.tensor()); + typename TTypes::ConstVec scale(scale_input.vec()); + typename TTypes::ConstVec pop_mean(pop_mean_input.vec()); + typename TTypes::ConstVec pop_var(pop_variance_input.vec()); + typename TTypes::Tensor x_backprop(x_backprop_output->tensor()); + typename TTypes::Vec scale_backprop(scale_backprop_output->vec()); + typename TTypes::Vec offset_backprop(offset_backprop_output->vec()); + + const int depth = pop_mean.dimension(0); + const int rest_size = input.size() / depth; + + // Allocate two temporary workspaces of [depth] shape. + Tensor scratch1_vec, scratch2_vec; + OP_REQUIRES_OK(context, context->allocate_temp(DataTypeToEnum::value, + {depth}, &scratch1_vec)); + OP_REQUIRES_OK(context, context->allocate_temp(DataTypeToEnum::value, + {depth}, &scratch2_vec)); + + typename TTypes::Vec scratch1(scratch1_vec.vec()); + typename TTypes::Vec scratch2(scratch2_vec.vec()); + + const GPUDevice& d = context->eigen_device(); + + Eigen::DSizes rest_by_depth(rest_size, depth); +#if !defined(EIGEN_HAS_INDEX_LIST) + Eigen::DSizes one_by_depth(1, depth); + Eigen::array reduction_axis{0}; + Eigen::array rest_by_one({rest_size, 1}); +#else + Eigen::IndexList, Eigen::Index> one_by_depth; + one_by_depth.set(1, depth); + Eigen::IndexList > reduction_axis; + Eigen::IndexList > rest_by_one; + rest_by_one.set(0, rest_size); +#endif + + // offset_backprop = sum(y_backprop) + // scale_backprop = y_backprop * ((x - pop_mean) * rsqrt(pop_var + epsilon)) + // x_backprop = y_backprop * (scale * rsqrt(pop_var + epsilon)) + + auto y_backprop_rest_by_depth = + y_backprop.reshape(rest_by_depth).template cast(); + auto input_rest_by_depth = input.reshape(rest_by_depth).template cast(); + + offset_backprop.device(d) = y_backprop_rest_by_depth.sum(reduction_axis); + + // scratch1 = rsqrt(pop_var + epsilon) + scratch1.device(d) = (pop_var + pop_var.constant(epsilon)).rsqrt(); + + // scratch2 = sum(y_backprop * (x - mean)) + scratch2.device(d) = + (y_backprop_rest_by_depth * + (input_rest_by_depth - + pop_mean.reshape(one_by_depth).broadcast(rest_by_one))) + .sum(reduction_axis); + + x_backprop.reshape(rest_by_depth).device(d) = + (y_backprop_rest_by_depth * ((scratch1 * scale) + .eval() + .reshape(one_by_depth) + .broadcast(rest_by_one))) + .template cast(); + scale_backprop.device(d) = scratch2 * scratch1; + } +}; + template struct FusedBatchNormFreezeGrad; template struct FusedBatchNormFreezeGrad; diff --git a/tensorflow/core/kernels/fused_batch_norm_op.h b/tensorflow/core/kernels/fused_batch_norm_op.h index 2cb19e15ddb..4936192377c 100644 --- a/tensorflow/core/kernels/fused_batch_norm_op.h +++ b/tensorflow/core/kernels/fused_batch_norm_op.h @@ -85,71 +85,15 @@ struct FusedBatchNormInferenceFunctor { #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM // Functor used by FusedBatchNormGradOp to do the computations when -// is_training=False. Both CPU and GPU will use this functor. +// is_training=False. template struct FusedBatchNormFreezeGrad { - void operator()(const Device& d, const Tensor& y_backprop_input, + void operator()(OpKernelContext* context, const Tensor& y_backprop_input, const Tensor& x_input, const Tensor& scale_input, const Tensor& pop_mean_input, const Tensor& pop_variance_input, U epsilon, Tensor* x_backprop_output, Tensor* scale_backprop_output, - Tensor* offset_backprop_output, - typename TTypes::Vec scratch1, - typename TTypes::Vec scratch2) { - typename TTypes::ConstTensor y_backprop( - y_backprop_input.tensor()); - typename TTypes::ConstTensor input(x_input.tensor()); - typename TTypes::ConstVec scale(scale_input.vec()); - typename TTypes::ConstVec pop_mean(pop_mean_input.vec()); - typename TTypes::ConstVec pop_var(pop_variance_input.vec()); - typename TTypes::Tensor x_backprop(x_backprop_output->tensor()); - typename TTypes::Vec scale_backprop(scale_backprop_output->vec()); - typename TTypes::Vec offset_backprop(offset_backprop_output->vec()); - - const int depth = pop_mean.dimension(0); - const int rest_size = input.size() / depth; - - Eigen::DSizes rest_by_depth(rest_size, depth); -#if !defined(EIGEN_HAS_INDEX_LIST) - Eigen::DSizes one_by_depth(1, depth); - Eigen::array reduction_axis{0}; - Eigen::array rest_by_one({rest_size, 1}); -#else - Eigen::IndexList, Eigen::Index> one_by_depth; - one_by_depth.set(1, depth); - Eigen::IndexList > reduction_axis; - Eigen::IndexList > rest_by_one; - rest_by_one.set(0, rest_size); -#endif - - // offset_backprop = sum(y_backprop) - // scale_backprop = y_backprop * ((x - pop_mean) * rsqrt(pop_var + epsilon)) - // x_backprop = y_backprop * (scale * rsqrt(pop_var + epsilon)) - - auto y_backprop_rest_by_depth = - y_backprop.reshape(rest_by_depth).template cast(); - auto input_rest_by_depth = input.reshape(rest_by_depth).template cast(); - - offset_backprop.device(d) = y_backprop_rest_by_depth.sum(reduction_axis); - - // scratch1 = rsqrt(pop_var + epsilon) - scratch1.device(d) = (pop_var + pop_var.constant(epsilon)).rsqrt(); - - // scratch2 = sum(y_backprop * (x - mean)) - scratch2.device(d) = - (y_backprop_rest_by_depth * - (input_rest_by_depth - - pop_mean.reshape(one_by_depth).broadcast(rest_by_one))) - .sum(reduction_axis); - - x_backprop.reshape(rest_by_depth).device(d) = - (y_backprop_rest_by_depth * ((scratch1 * scale) - .eval() - .reshape(one_by_depth) - .broadcast(rest_by_one))) - .template cast(); - scale_backprop.device(d) = scratch2 * scratch1; - } + Tensor* offset_backprop_output) {} }; } // namespace functor diff --git a/tensorflow/core/kernels/fused_batch_norm_op_test.cc b/tensorflow/core/kernels/fused_batch_norm_op_test.cc index f765a3ee43d..5297d3ee138 100644 --- a/tensorflow/core/kernels/fused_batch_norm_op_test.cc +++ b/tensorflow/core/kernels/fused_batch_norm_op_test.cc @@ -269,6 +269,22 @@ BM_FusedBatchNorm(64, 14, 14, 256, fp16, true, NCHW, gpu); BENCHMARK(BM_NAME(FusedBatchNormGrad, N, H, W, C, T, IS_TRAINING, FORMAT, \ DEVICE)); +#define BM_FusedBatchNormGradResnetShapes(T, IS_TRAINING, FORMAT, DEVICE) \ + BM_FusedBatchNormGrad(64, 56, 56, 64, T, IS_TRAINING, FORMAT, DEVICE); \ + BM_FusedBatchNormGrad(64, 56, 56, 128, T, IS_TRAINING, FORMAT, DEVICE); \ + BM_FusedBatchNormGrad(64, 56, 56, 256, T, IS_TRAINING, FORMAT, DEVICE); \ + \ + BM_FusedBatchNormGrad(64, 28, 28, 128, T, IS_TRAINING, FORMAT, DEVICE); \ + BM_FusedBatchNormGrad(64, 28, 28, 256, T, IS_TRAINING, FORMAT, DEVICE); \ + BM_FusedBatchNormGrad(64, 28, 28, 512, T, IS_TRAINING, FORMAT, DEVICE); \ + \ + BM_FusedBatchNormGrad(64, 14, 14, 128, T, IS_TRAINING, FORMAT, DEVICE); \ + BM_FusedBatchNormGrad(64, 14, 14, 256, T, IS_TRAINING, FORMAT, DEVICE); \ + BM_FusedBatchNormGrad(64, 14, 14, 1024, T, IS_TRAINING, FORMAT, DEVICE) + +BM_FusedBatchNormGradResnetShapes(fp32, true, NHWC, cpu); +BM_FusedBatchNormGradResnetShapes(fp32, false, NHWC, cpu); + #ifdef GOOGLE_CUDA BM_FusedBatchNormGrad(64, 14, 14, 256, fp32, true, NHWC, gpu); BM_FusedBatchNormGrad(64, 14, 14, 256, fp16, true, NHWC, gpu); diff --git a/tensorflow/core/kernels/redux_functor.h b/tensorflow/core/kernels/redux_functor.h index 24dc876ef8e..30038c62dbd 100644 --- a/tensorflow/core/kernels/redux_functor.h +++ b/tensorflow/core/kernels/redux_functor.h @@ -35,16 +35,18 @@ namespace functor { // input: [D1, D2, ... , DN] // -> // output: [Di, ... , DN] where i belongs to set [1,N] -template +template struct ReduceOuterDimensions { - ReduceOuterDimensions(){}; + ReduceOuterDimensions() {} + template void operator()(const CPUDevice& device, const Eigen::DSizes& input_dims, const Tensor& input, Tensor* output) const { // Compute inner and outer dim after reshaping into 2d tensor. const int num_output_dims = output->dims(); - auto output_dims = output->template flat().dimensions(); + auto output_dims = output->template flat().dimensions(); Eigen::Index inner_dim = 1, outer_dim = 1; for (int i = 0; i < num_dims - num_output_dims; ++i) @@ -54,8 +56,8 @@ struct ReduceOuterDimensions { if (1 == outer_dim) { // Nothing to do but passing input to output. - output->template flat() = - input.template flat().reshape(output_dims); + output->template flat() = + input.template flat().reshape(output_dims); return; } @@ -63,13 +65,15 @@ struct ReduceOuterDimensions { const Eigen::Index num_threads = device.numThreads(); // If the inner dim parallelism is large enough - if (inner_dim > num_threads * 16) { + // TODO(ezhulenev): There seems to be no benefits in going this route. Check + // if this can be improved, or use better heuristic? + if (inner_dim > num_threads * 32) { // Do not create more blocks than there are threads in a pool. const Eigen::Index num_blocks = num_threads; // Block size along the outer dimension. const Eigen::Index inner_block_size = Eigen::divup(inner_dim, num_blocks); - const T* input_data = input.template flat().data(); + const InputT* input_data = input.template flat().data(); // Allocate temporary buffer for partial reductions. Eigen::Tensor buffer( @@ -82,7 +86,7 @@ struct ReduceOuterDimensions { Eigen::Unaligned>; using Input = Eigen::TensorMap< - Eigen::Tensor, + Eigen::Tensor, Eigen::Unaligned>; const auto compute = [inner_dim, outer_dim, num_blocks, inner_block_size, @@ -94,7 +98,7 @@ struct ReduceOuterDimensions { inner_dim_limit = std::min(inner_dim, inner_dim_limit); Eigen::Index my_job_len = inner_dim_limit - inner_dim_start; - const T* my_job_start = input_data + inner_dim_start; + const InputT* my_job_start = input_data + inner_dim_start; Buffer buf(buffer_data + inner_dim_start, my_job_len); for (Eigen::Index i = 0; i < outer_dim; ++i) { @@ -107,7 +111,7 @@ struct ReduceOuterDimensions { // Compute cost of reducing a single block. const Eigen::Index compute_size = outer_dim * inner_block_size; - const Eigen::Index compute_input_bytes = compute_size * sizeof(T); + const Eigen::Index compute_input_bytes = compute_size * sizeof(InputT); const Eigen::TensorOpCost cost( compute_input_bytes, 0, // We'll be mostly writing to L1, assume store cost is 0 @@ -116,8 +120,8 @@ struct ReduceOuterDimensions { device.parallelFor(num_blocks, cost, compute); // Write final result to the output. - output->template flat() = - buffer.template cast().reshape(output_dims); + output->template flat() = + buffer.template cast().reshape(output_dims); } else { // Compute block size along the outer dimension for efficiency. const Eigen::Index parallel_cell_size = inner_dim; @@ -136,7 +140,7 @@ struct ReduceOuterDimensions { // Block size along the outer dimension. const Eigen::Index outer_block_size = Eigen::divup(outer_dim, num_blocks); - const T* input_data = input.template flat().data(); + const InputT* input_data = input.template flat().data(); // Allocate temporary buffer for partial reductions. Tensor buffer(DataTypeToEnum::v(), {num_blocks, inner_dim}); @@ -148,7 +152,7 @@ struct ReduceOuterDimensions { Eigen::Unaligned>; using Input = Eigen::TensorMap< - Eigen::Tensor, + Eigen::Tensor, Eigen::Unaligned>; const auto compute = [inner_dim, num_blocks, outer_block_size, @@ -170,7 +174,7 @@ struct ReduceOuterDimensions { // Compute cost of reducing a single block. const Eigen::Index compute_size = outer_block_size * inner_dim; - const Eigen::Index compute_input_bytes = compute_size * sizeof(T); + const Eigen::Index compute_input_bytes = compute_size * sizeof(InputT); const Eigen::TensorOpCost cost( compute_input_bytes, 0, // We'll be mostly writing to L1, assume store cost is 0 @@ -187,7 +191,8 @@ struct ReduceOuterDimensions { const decltype(buf)>(buf0, buf); } // Write final result to the output. - output->template flat() = buf0.template cast().reshape(output_dims); + output->template flat() = + buf0.template cast().reshape(output_dims); } } }; @@ -197,9 +202,11 @@ struct ReduceOuterDimensions { // input: [D1, D2, ... , DN] // -> // output: [Di, ... , Dj] where i & j belongs to set [1,N]. -template +template struct ReduceMiddleDimensions { - ReduceMiddleDimensions(){}; + ReduceMiddleDimensions() {} + template void operator()(const CPUDevice& device, const Eigen::DSizes& input_dims, @@ -207,7 +214,7 @@ struct ReduceMiddleDimensions { const int axis_begin_dim) const { // Compute dims after reshaping into 3d tensor. const int num_output_dims = output->dims(); - auto output_dims = output->template flat().dimensions(); + auto output_dims = output->template flat().dimensions(); Eigen::Index inner_dim = 1, middle_dim = 1, outer_dim = 1; for (int i = 0; i < axis_begin_dim; ++i) outer_dim *= input_dims[i]; @@ -218,12 +225,12 @@ struct ReduceMiddleDimensions { if ((1 == inner_dim * outer_dim)) { // Nothing to do. - output->template flat() = - input.template flat().reshape(output_dims); + output->template flat() = + input.template flat().reshape(output_dims); return; } else if (1 == inner_dim) { // Equivalent to ReduceOuterDimensions. - const ReduceOuterDimensions redux; + const ReduceOuterDimensions redux; redux(device, input_dims, input, output); return; } @@ -247,7 +254,7 @@ struct ReduceMiddleDimensions { const Eigen::Index outer_block_size = Eigen::divup(total_workload, num_blocks); - const T* input_data = input.template flat().data(); + const InputT* input_data = input.template flat().data(); // Allocate temporary buffer for partial reductions. Eigen::Tensor buffer(num_blocks, middle_dim); @@ -255,7 +262,7 @@ struct ReduceMiddleDimensions { AccumT* buffer_data = buffer.data(); using Buffer = Eigen::TensorMap>; - using Input = Eigen::TensorMap>; + using Input = Eigen::TensorMap>; Eigen::array reduction_axis = {0}; Reducer reducer; @@ -301,7 +308,7 @@ struct ReduceMiddleDimensions { // Compute cost of reducing a single block. const Eigen::Index compute_size = outer_block_size * inner_dim; - const Eigen::Index compute_input_bytes = compute_size * sizeof(T); + const Eigen::Index compute_input_bytes = compute_size * sizeof(InputT); const Eigen::TensorOpCost cost( compute_input_bytes, 0, // We'll be mostly writing to L1, assume store cost is 0 @@ -322,7 +329,8 @@ struct ReduceMiddleDimensions { } // Write final result to the output. - output->template flat() = buf0.template cast().reshape(output_dims); + output->template flat() = + buf0.template cast().reshape(output_dims); } }; From 6df6fd298cecb7e55ae224c2ef17d4c4d2ccc0af Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Tue, 9 Jul 2019 09:44:18 -0700 Subject: [PATCH 114/332] [XLA] Simplify (A % B) % B => A % B. PiperOrigin-RevId: 257212321 --- .../compiler/xla/service/algebraic_simplifier.cc | 5 +++++ .../xla/service/algebraic_simplifier_test.cc | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 0290d0cf26b..10030ce6491 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -2660,6 +2660,11 @@ Status AlgebraicSimplifierVisitor::HandleRemainder(HloInstruction* remainder) { HloInstruction *a, *b; CHECK(Match(remainder, m::Remainder(m::Op(&a), m::Op(&b)))); + // (A % B) % B == A % B. + if (Match(a, m::Remainder(m::Op(), m::Op().Is(b)))) { + return ReplaceInstruction(remainder, a); + } + // A % B => A & (B - 1) if B is a power of 2. switch (remainder->shape().element_type()) { case S8: diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index 31b0e16c2a0..05d57cf3ba2 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -5503,5 +5503,20 @@ TEST_F(AlgebraicSimplifierTest, RemainderOfNPlusIotaOverflow) { ASSERT_FALSE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie()); } +TEST_F(AlgebraicSimplifierTest, RepeatedRemainder) { + const char* kModuleStr = R"( + HloModule m + test { + p = s32[1000] parameter(0) + q = s32[1000] parameter(1) + r = s32[1000] remainder(p, q) + ROOT rr = s32[1000] remainder(r, q) + })"; + TF_ASSERT_OK_AND_ASSIGN(auto m, ParseAndReturnVerifiedModule(kModuleStr)); + ASSERT_TRUE(AlgebraicSimplifier(default_options_).Run(m.get()).ValueOrDie()); + EXPECT_THAT(m->entry_computation()->root_instruction(), + GmockMatch(m::Remainder(m::Parameter(), m::Parameter()))); +} + } // namespace } // namespace xla From 67fc29f8fbfa433aa040f45b5b997adeda225ef3 Mon Sep 17 00:00:00 2001 From: Feng Liu Date: Tue, 9 Jul 2019 09:50:10 -0700 Subject: [PATCH 115/332] Create a generic quantization pattern for quantization graph rewrite This pattern assumes that the quantization parameter propagation pass has propagated the quantization parameters to all the quantizable ops. Instead of defining table gen pattern for each of the quantizable ops, we can use this single pattern to match all the ops. The tests are mainly the existing ones, plus some manual tests. PiperOrigin-RevId: 257213495 --- .../compiler/mlir/lite/transforms/quantize.cc | 39 +------- .../mlir/lite/transforms/quantize_patterns.td | 99 ------------------- .../mlir/lite/utils/quantization_utils.h | 54 ++++++++++ 3 files changed, 57 insertions(+), 135 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/transforms/quantize.cc b/tensorflow/compiler/mlir/lite/transforms/quantize.cc index 7d98487ded2..6e7d060cca5 100644 --- a/tensorflow/compiler/mlir/lite/transforms/quantize.cc +++ b/tensorflow/compiler/mlir/lite/transforms/quantize.cc @@ -50,52 +50,19 @@ struct QuantizePass : public FunctionPass { #include "tensorflow/compiler/mlir/lite/transforms/generated_quantize.inc" -struct QuantizeConcatOp : public RewritePattern { - explicit QuantizeConcatOp(MLIRContext* context) - : RewritePattern(QuantizeOp::getOperationName(), 1, context) {} - - PatternMatchResult matchAndRewrite(Operation* op, - PatternRewriter& rewriter) const override; -}; - -PatternMatchResult mlir::TFL::QuantizeConcatOp::matchAndRewrite( - Operation* op, PatternRewriter& rewriter) const { - auto quantize_op = cast(op); - auto concat_op = - dyn_cast_or_null(quantize_op.input()->getDefiningOp()); - if (!concat_op) { - return matchFailure(); - } - - SmallVector values; - values.reserve(concat_op.getNumOperands()); - for (auto operand : concat_op.values()) { - if (auto opInst = - dyn_cast_or_null(operand->getDefiningOp())) { - values.push_back(opInst.input()); - } else { - return matchFailure(); - } - } - rewriter.replaceOpWithNewOp( - op, quantize_op.output()->getType(), values, - rewriter.getI32IntegerAttr(concat_op.axis().getZExtValue()), - rewriter.getStringAttr(concat_op.fused_activation_function())); - return matchSuccess(); -} - void QuantizePass::runOnFunction() { OwningRewritePatternList patterns; auto func = getFunction(); auto* ctx = func.getContext(); TFL::populateWithGenerated(ctx, &patterns); - mlir::RewriteListBuilder::build(patterns, ctx); + mlir::RewriteListBuilder>::build(patterns, ctx); applyPatternsGreedily(func, std::move(patterns)); } } // namespace // Creates an instance of the TensorFlow Lite dialect QuantizeTFL pass. -FunctionPassBase *CreateQuantizePass() { return new QuantizePass(); } +FunctionPassBase* CreateQuantizePass() { return new QuantizePass(); } static PassRegistration pass( "tfl-quantize", "Apply quantization on models in TensorFlow Lite dialect"); diff --git a/tensorflow/compiler/mlir/lite/transforms/quantize_patterns.td b/tensorflow/compiler/mlir/lite/transforms/quantize_patterns.td index 756fae3a4cd..7fcf926d89f 100644 --- a/tensorflow/compiler/mlir/lite/transforms/quantize_patterns.td +++ b/tensorflow/compiler/mlir/lite/transforms/quantize_patterns.td @@ -22,10 +22,6 @@ include "tensorflow/compiler/mlir/lite/ir/tfl_ops.td" // Quantize attribute $0 by using quantization parameter from %1. def QuantizeByQuantizedType : NativeCodeCall<"Quantize($0, $1.getValue())">; -// Call the generic builder of `op`. Use the result type of $0 in the new op. -class ReplaceWith : NativeCodeCall<"$_builder.create<" # op # - ">($0->getLoc(), $0->getResult(0)->getType(), $1, $2, $3)">; - // Squash tfl.dequantize and tfl.quantize pairs. // TODO(fengliuai): Compare the scale of input and output. This can also be // squashed to a requantize op if the scales are different. @@ -39,98 +35,3 @@ def : Pat<(TFL_QuantizeOp (TFL_QConstOp $qtype, (QuantizeByQuantizedType $value, $qtype))>; - -// Quantize the AddOp if both inputs are dequantized and the output is -// quantized. -def : Pat<(TFL_QuantizeOp:$q - (TFL_AddOp (TFL_DequantizeOp $lhs), (TFL_DequantizeOp $rhs), - $fused_activation_function), - $output_type), - (ReplaceWith<"TFL::AddOp"> $q, $lhs, $rhs, - $fused_activation_function)>; - -// Quantize the Conv2DOp if the input and weight are dequantized. The scale of -// the bias input is determined by the scales of input and weight operands. -def : Pat<(TFL_QuantizeOp - (TFL_Conv2DOp - (TFL_DequantizeOp $in), - (TFL_DequantizeOp $weight), - (TFL_DequantizeOp $bias), - $dilation_h_factor, - $dilation_w_factor, - $fused_activation_function, - $padding, - $stride_h, - $stride_w), - $output_type), - (TFL_Conv2DOp - $in, - $weight, - $bias, - $dilation_h_factor, - $dilation_w_factor, - $fused_activation_function, - $padding, - $stride_h, - $stride_w)>; - -// Quantize the DepthwiseConv2DOp if the input and weight are dequantized. The -// scale of the bias input is determined by the scales of input and weight -// operands. -def : Pat<(TFL_QuantizeOp - (TFL_DepthwiseConv2DOp - (TFL_DequantizeOp $in), - (TFL_DequantizeOp $weight), - (TFL_DequantizeOp $bias), - $dilation_h_factor, - $dilation_w_factor, - $fused_activation_function, - $padding, - $stride_h, - $stride_w, - $multiplier), - $output_type), - (TFL_DepthwiseConv2DOp - $in, - $weight, - $bias, - $dilation_h_factor, - $dilation_w_factor, - $fused_activation_function, - $padding, - $stride_h, - $stride_w, - $multiplier)>; - -// Quantize the ReshapeOp if the input is dequantized and output is quantized. -// The pre-quantize pass can guarantee both quantization parameters are the -// same. -def : Pat<(TFL_QuantizeOp (TFL_ReshapeOp (TFL_DequantizeOp $in)), $output_type), - (TFL_ReshapeOp $in)>; - -// Quantize the ReshapeOp if the input is dequantized and output is quantized. -// The pre-quantize pass has set the output quantization parameters to a -// pre-defined value. -def : Pat<(TFL_QuantizeOp (TFL_SoftmaxOp (TFL_DequantizeOp $in), $beta), - $output_type), - (TFL_SoftmaxOp $in, $beta)>; - -// Quantize the AveragePool2DOp if the input is dequantized and output is -// quantized. The pre-quantize pass can guarantee both quantization parameters -// are the same. -def : Pat<(TFL_QuantizeOp (TFL_AveragePool2DOp (TFL_DequantizeOp $in), - $filter_height, $filter_width, $fused_activation_function, - $padding, $stride_h, $stride_w), $output_type), - (TFL_AveragePool2DOp $in, - $filter_height, $filter_width, $fused_activation_function, - $padding, $stride_h, $stride_w)>; - -// Quantize the MaxPool2DOp if the input is dequantized and output is -// quantized. The pre-quantize pass can guarantee both quantization parameters -// are the same. -def : Pat<(TFL_QuantizeOp (TFL_MaxPool2DOp (TFL_DequantizeOp $in), - $padding, $stride_w, $tride_h, $stride_width, $stride_height, - $fused_activation_function), $output_type), - (TFL_MaxPool2DOp $in, - $padding, $stride_w, $tride_h, $stride_width, $stride_height, - $fused_activation_function)>; diff --git a/tensorflow/compiler/mlir/lite/utils/quantization_utils.h b/tensorflow/compiler/mlir/lite/utils/quantization_utils.h index cee00e6be38..33fe39cb05c 100644 --- a/tensorflow/compiler/mlir/lite/utils/quantization_utils.h +++ b/tensorflow/compiler/mlir/lite/utils/quantization_utils.h @@ -20,12 +20,66 @@ limitations under the License. #define TENSORFLOW_COMPILER_MLIR_LITE_UTILS_QUANTIZATION_UTILS_H_ #include "mlir/Dialect/QuantOps/QuantTypes.h" // TF:local_config_mlir +#include "mlir/IR/BlockAndValueMapping.h" // TF:local_config_mlir +#include "mlir/IR/PatternMatch.h" // TF:local_config_mlir #include "mlir/IR/StandardTypes.h" // TF:local_config_mlir #include "mlir/StandardOps/Ops.h" // TF:local_config_mlir namespace mlir { namespace TFL { +// A generic rewrite pattern which matches any N-in-1-out operations with +// quantization parameters propagated to all the operands and results values. +// The quantization parameters are annotated by the Q/DQ op pairs. Each matched +// pattern are rewritten by its quantized alternatives. +// +// This pattern assumes all the matched ops are quantizable. This assumption is +// always right, except when a "Q" op is used as a requantize op. For non-"Q" +// ops, quantization parameters should be propagated to their result. +// +// This pattern only matches ops which only have one result. +template +struct GenericFullQuantizationPattern : public RewritePattern { + explicit GenericFullQuantizationPattern(MLIRContext* context) + : RewritePattern(Q::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite(Operation* op, + PatternRewriter& rewriter) const override { + if (op->getNumResults() != 1) { + return matchFailure(); + } + auto quantize_op = cast(op); + auto quantized_op = quantize_op.input()->getDefiningOp(); + // If it is a block argument, requantize op, or has more than one result, we + // shouldn't rewrite this op. + if (!quantized_op || llvm::isa(quantized_op) || + llvm::isa(quantized_op) || quantized_op->getNumResults() != 1) { + return matchFailure(); + } + + // Collect all the quantized inputs and "clone" the matched op by these + // inputs. + SmallVector inputs; + inputs.reserve(quantized_op->getNumOperands()); + for (int i = 0, e = quantized_op->getNumOperands(); i != e; ++i) { + auto* operand = quantized_op->getOperand(i); + if (auto op_inst = dyn_cast_or_null(operand->getDefiningOp())) { + inputs.push_back(op_inst.input()); + } else { + return matchFailure(); + } + } + // Use OpBuilder so we can use op name to create the new op. + OpBuilder builder(quantized_op); + OperationState new_state( + quantized_op->getLoc(), quantized_op->getName().getStringRef(), inputs, + op->getResult(0)->getType(), quantized_op->getAttrs()); + Operation* new_op = builder.createOperation(new_state); + rewriter.replaceOp(op, {new_op->getResult(0)}); + return matchSuccess(); + } +}; + // Converts the min/max/storage_type/narrow_range information to a // QuantizedType, and then returns the attribute containing the QuantizedType. TypeAttr GetQuantizedTypeAttr(Builder builder, Type input_type, FloatAttr min, From a13443841dabc86deaaf6050bfd04f42b68281f2 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Tue, 9 Jul 2019 09:51:48 -0700 Subject: [PATCH 116/332] [XLA] Don't print LLVM IR to stdout if the user specifies just --xla_dump_hlo_as_url. This flag alone means you probably just want the rendered HLO graphs printed to stdout and nothing more. If you do want the LLVM IR printed to stdout, just add --xla_dump_hlo_as_text. PiperOrigin-RevId: 257213859 --- tensorflow/compiler/xla/service/dump.cc | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/xla/service/dump.cc b/tensorflow/compiler/xla/service/dump.cc index d251c828bcd..6a4837211e8 100644 --- a/tensorflow/compiler/xla/service/dump.cc +++ b/tensorflow/compiler/xla/service/dump.cc @@ -48,22 +48,27 @@ struct CanonicalDebugOptions { // function we treat this struct's members as write-only, and read only from // `opts`. - // If dump_to is empty, default to dumping to stdout. - if (opts.xla_dump_to().empty()) { - dump_to = "-"; - } - // Did the user specifiy an explicit format for dumping? - bool output_format_specified = + bool output_format_other_than_url_specified = opts.xla_dump_hlo_as_text() || opts.xla_dump_hlo_as_proto() || opts.xla_dump_hlo_as_dot() || opts.xla_dump_hlo_as_html() || - opts.xla_dump_hlo_as_url() || opts.xla_dump_hlo_snapshots(); + opts.xla_dump_hlo_snapshots(); + bool output_format_specified = + output_format_other_than_url_specified || opts.xla_dump_hlo_as_url(); // If we haven't specified an output format, default to dumping as text. if (!output_format_specified) { dump_as_text = true; } + // If dump_to is empty, default to dumping to stdout, so long as some dump + // format other than dump-as-url was specified. If the user only specified + // --xla_dump_hlo_as_url, then don't dump to stdout, that is likely noise + // they don't want. + if (opts.xla_dump_to().empty() && output_format_other_than_url_specified) { + dump_to = "-"; + } + // If we specified a regular expression restricting which modules to dump, // respect that. // @@ -143,6 +148,10 @@ void DumpToFileInDirImpl(string_view filename, string_view contents, return; } + if (opts.dump_to.empty()) { + return; + } + const string& dir = opts.dump_to; VLOG(1) << "Dumping " << filename << " to " << dir; From 8c04b3e41d17107f8bd76b491cd58c0788f5c6a8 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Tue, 9 Jul 2019 10:17:01 -0700 Subject: [PATCH 117/332] Refactor TextLineDatasetOp --- tensorflow/core/kernels/data/BUILD | 34 +++ .../core/kernels/data/reader_dataset_ops.cc | 243 --------------- .../core/kernels/data/text_line_dataset_op.cc | 281 ++++++++++++++++++ .../core/kernels/data/text_line_dataset_op.h | 42 +++ 4 files changed, 357 insertions(+), 243 deletions(-) create mode 100644 tensorflow/core/kernels/data/text_line_dataset_op.cc create mode 100644 tensorflow/core/kernels/data/text_line_dataset_op.h diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD index 5c8fbd7333d..250d3221db3 100644 --- a/tensorflow/core/kernels/data/BUILD +++ b/tensorflow/core/kernels/data/BUILD @@ -984,6 +984,39 @@ tf_kernel_library( ], ) +tf_kernel_library( + name = "text_line_dataset_op", + srcs = ["text_line_dataset_op.cc"], + hdrs = ["text_line_dataset_op.h"], + deps = [ + ":name_utils", + "//tensorflow/core:core_cpu_internal", + "//tensorflow/core:dataset_ops_op_lib", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + ], +) + +tf_cc_test( + name = "text_line_dataset_op_test", + size = "small", + srcs = ["text_line_dataset_op_test.cc"], + deps = [ + ":dataset_test_base", + ":dataset_utils", + ":iterator_ops", + ":text_line_dataset_op", + "//tensorflow/core:core_cpu_internal", + "//tensorflow/core:dataset_ops_op_lib", + "//tensorflow/core:framework", + "//tensorflow/core:lib_internal", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + tf_kernel_library( name = "iterator_ops", srcs = ["iterator_ops.cc"], @@ -1164,6 +1197,7 @@ tf_kernel_library( ":take_dataset_op", ":tensor_dataset_op", ":tensor_slice_dataset_op", + ":text_line_dataset_op", ":window_dataset_op", ":zip_dataset_op", "//tensorflow/core:array_ops_op_lib", diff --git a/tensorflow/core/kernels/data/reader_dataset_ops.cc b/tensorflow/core/kernels/data/reader_dataset_ops.cc index 9ab687c0d7d..9b6e987ea6a 100644 --- a/tensorflow/core/kernels/data/reader_dataset_ops.cc +++ b/tensorflow/core/kernels/data/reader_dataset_ops.cc @@ -30,249 +30,6 @@ namespace { // See documentation in ../../ops/dataset_ops.cc for a high-level // description of the following ops. -constexpr char kTextLineDatasetName[] = "TextLine"; - -class TextLineDatasetOp : public DatasetOpKernel { - public: - using DatasetOpKernel::DatasetOpKernel; - - void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override { - const Tensor* filenames_tensor; - OP_REQUIRES_OK(ctx, ctx->input("filenames", &filenames_tensor)); - OP_REQUIRES( - ctx, filenames_tensor->dims() <= 1, - errors::InvalidArgument("`filenames` must be a scalar or a vector.")); - - string compression_type; - OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "compression_type", - &compression_type)); - - int64 buffer_size = -1; - OP_REQUIRES_OK( - ctx, ParseScalarArgument(ctx, "buffer_size", &buffer_size)); - OP_REQUIRES( - ctx, buffer_size >= 0, - errors::InvalidArgument("`buffer_size` must be >= 0 (0 == default)")); - - io::ZlibCompressionOptions zlib_compression_options = - io::ZlibCompressionOptions::DEFAULT(); - if (compression_type == "ZLIB") { - zlib_compression_options = io::ZlibCompressionOptions::DEFAULT(); - } else if (compression_type == "GZIP") { - zlib_compression_options = io::ZlibCompressionOptions::GZIP(); - } else { - OP_REQUIRES(ctx, compression_type.empty(), - errors::InvalidArgument("Unsupported compression_type.")); - } - - if (buffer_size != 0) { - // Set the override size. - zlib_compression_options.input_buffer_size = buffer_size; - } - - std::vector filenames; - filenames.reserve(filenames_tensor->NumElements()); - for (int i = 0; i < filenames_tensor->NumElements(); ++i) { - filenames.push_back(filenames_tensor->flat()(i)); - } - - *output = new Dataset(ctx, std::move(filenames), compression_type, - zlib_compression_options); - } - - private: - class Dataset : public DatasetBase { - public: - Dataset(OpKernelContext* ctx, std::vector filenames, - const string& compression_type, - const io::ZlibCompressionOptions& options) - : DatasetBase(DatasetContext(ctx)), - filenames_(std::move(filenames)), - compression_type_(compression_type), - use_compression_(!compression_type.empty()), - options_(options) {} - - std::unique_ptr MakeIteratorInternal( - const string& prefix) const override { - return absl::make_unique(Iterator::Params{ - this, strings::StrCat(prefix, "::", kTextLineDatasetName)}); - } - - const DataTypeVector& output_dtypes() const override { - static DataTypeVector* dtypes = new DataTypeVector({DT_STRING}); - return *dtypes; - } - - const std::vector& output_shapes() const override { - static std::vector* shapes = - new std::vector({{}}); - return *shapes; - } - - string DebugString() const override { return "TextLineDatasetOp::Dataset"; } - - protected: - Status AsGraphDefInternal(SerializationContext* ctx, - DatasetGraphDefBuilder* b, - Node** output) const override { - Node* filenames = nullptr; - Node* compression_type = nullptr; - Node* buffer_size = nullptr; - TF_RETURN_IF_ERROR(b->AddVector(filenames_, &filenames)); - TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); - TF_RETURN_IF_ERROR( - b->AddScalar(options_.input_buffer_size, &buffer_size)); - TF_RETURN_IF_ERROR(b->AddDataset( - this, {filenames, compression_type, buffer_size}, output)); - return Status::OK(); - } - - private: - class Iterator : public DatasetIterator { - public: - explicit Iterator(const Params& params) - : DatasetIterator(params) {} - - Status GetNextInternal(IteratorContext* ctx, - std::vector* out_tensors, - bool* end_of_sequence) override { - mutex_lock l(mu_); - do { - // We are currently processing a file, so try to read the next line. - if (buffered_input_stream_) { - string line_contents; - Status s = buffered_input_stream_->ReadLine(&line_contents); - - if (s.ok()) { - // Produce the line as output. - metrics::RecordTFDataBytesRead(kTextLineDatasetName, - line_contents.size()); - out_tensors->emplace_back(ctx->allocator({}), DT_STRING, - TensorShape({})); - out_tensors->back().scalar()() = std::move(line_contents); - *end_of_sequence = false; - return Status::OK(); - } else if (!errors::IsOutOfRange(s)) { - // Report non-EOF errors to the caller. - return s; - } - // We have reached the end of the current file, so maybe - // move on to next file. - ResetStreamsLocked(); - ++current_file_index_; - } - - // Iteration ends when there are no more files to process. - if (current_file_index_ == dataset()->filenames_.size()) { - *end_of_sequence = true; - return Status::OK(); - } - - TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); - } while (true); - } - - protected: - std::shared_ptr CreateNode( - IteratorContext* ctx, model::Node::Args args) const override { - return model::MakeSourceNode(std::move(args)); - } - - Status SaveInternal(IteratorStateWriter* writer) override { - mutex_lock l(mu_); - TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_file_index"), - current_file_index_)); - - // `buffered_input_stream_` is empty if - // 1. GetNext has not been called even once. - // 2. All files have been read and iterator has been exhausted. - if (buffered_input_stream_) { - TF_RETURN_IF_ERROR(writer->WriteScalar( - full_name("current_pos"), buffered_input_stream_->Tell())); - } - return Status::OK(); - } - - Status RestoreInternal(IteratorContext* ctx, - IteratorStateReader* reader) override { - mutex_lock l(mu_); - ResetStreamsLocked(); - int64 current_file_index; - TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("current_file_index"), - ¤t_file_index)); - current_file_index_ = size_t(current_file_index); - // The key "current_pos" is written only if the iterator was saved - // with an open file. - if (reader->Contains(full_name("current_pos"))) { - int64 current_pos; - TF_RETURN_IF_ERROR( - reader->ReadScalar(full_name("current_pos"), ¤t_pos)); - - TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); - TF_RETURN_IF_ERROR(buffered_input_stream_->Seek(current_pos)); - } - return Status::OK(); - } - - private: - // Sets up reader streams to read from the file at `current_file_index_`. - Status SetupStreamsLocked(Env* env) EXCLUSIVE_LOCKS_REQUIRED(mu_) { - if (current_file_index_ >= dataset()->filenames_.size()) { - return errors::InvalidArgument( - "current_file_index_:", current_file_index_, - " >= filenames_.size():", dataset()->filenames_.size()); - } - - // Actually move on to next file. - TF_RETURN_IF_ERROR(env->NewRandomAccessFile( - dataset()->filenames_[current_file_index_], &file_)); - input_stream_ = - absl::make_unique(file_.get(), false); - - if (dataset()->use_compression_) { - zlib_input_stream_ = absl::make_unique( - input_stream_.get(), dataset()->options_.input_buffer_size, - dataset()->options_.input_buffer_size, dataset()->options_); - buffered_input_stream_ = absl::make_unique( - zlib_input_stream_.get(), dataset()->options_.input_buffer_size, - false); - } else { - buffered_input_stream_ = absl::make_unique( - input_stream_.get(), dataset()->options_.input_buffer_size, - false); - } - return Status::OK(); - } - - // Resets all reader streams. - void ResetStreamsLocked() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - input_stream_.reset(); - zlib_input_stream_.reset(); - buffered_input_stream_.reset(); - file_.reset(); - } - - mutex mu_; - std::unique_ptr input_stream_ - GUARDED_BY(mu_); - std::unique_ptr zlib_input_stream_ GUARDED_BY(mu_); - std::unique_ptr buffered_input_stream_ - GUARDED_BY(mu_); - size_t current_file_index_ GUARDED_BY(mu_) = 0; - std::unique_ptr file_ - GUARDED_BY(mu_); // must outlive input_stream_ - }; - - const std::vector filenames_; - const string compression_type_; - const bool use_compression_; - const io::ZlibCompressionOptions options_; - }; -}; - -REGISTER_KERNEL_BUILDER(Name("TextLineDataset").Device(DEVICE_CPU), - TextLineDatasetOp); - constexpr char kFixedLengthRecordDatasetName[] = "FixedLengthRecord"; class FixedLengthRecordDatasetOp : public DatasetOpKernel { diff --git a/tensorflow/core/kernels/data/text_line_dataset_op.cc b/tensorflow/core/kernels/data/text_line_dataset_op.cc new file mode 100644 index 00000000000..b8302b890c8 --- /dev/null +++ b/tensorflow/core/kernels/data/text_line_dataset_op.cc @@ -0,0 +1,281 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/kernels/data/text_line_dataset_op.h" + +#include "tensorflow/core/common_runtime/metrics.h" +#include "tensorflow/core/framework/partial_tensor_shape.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/kernels/data/name_utils.h" +#include "tensorflow/core/lib/io/buffered_inputstream.h" +#include "tensorflow/core/lib/io/inputbuffer.h" +#include "tensorflow/core/lib/io/random_inputstream.h" +#include "tensorflow/core/lib/io/zlib_compression_options.h" +#include "tensorflow/core/lib/io/zlib_inputstream.h" + +namespace tensorflow { +namespace data { + +/* static */ constexpr const char* const TextLineDatasetOp::kDatasetType; +/* static */ constexpr const char* const TextLineDatasetOp::kFileNames; +/* static */ constexpr const char* const TextLineDatasetOp::kCompressionType; +/* static */ constexpr const char* const TextLineDatasetOp::kBufferSize; + +constexpr char kZLIB[] = "ZLIB"; +constexpr char kGZIP[] = "GZIP"; +constexpr char kCurrentFileIndex[] = "current_file_index"; +constexpr char kCurrentPos[] = "current_pos"; + +class TextLineDatasetOp::Dataset : public DatasetBase { + public: + Dataset(OpKernelContext* ctx, std::vector filenames, + const string& compression_type, + const io::ZlibCompressionOptions& options) + : DatasetBase(DatasetContext(ctx)), + filenames_(std::move(filenames)), + compression_type_(compression_type), + use_compression_(!compression_type.empty()), + options_(options) {} + + std::unique_ptr MakeIteratorInternal( + const string& prefix) const override { + return absl::make_unique(Iterator::Params{ + this, + name_utils::IteratorPrefix(TextLineDatasetOp::kDatasetType, prefix)}); + } + + const DataTypeVector& output_dtypes() const override { + static DataTypeVector* dtypes = new DataTypeVector({DT_STRING}); + return *dtypes; + } + + const std::vector& output_shapes() const override { + static std::vector* shapes = + new std::vector({{}}); + return *shapes; + } + + string DebugString() const override { + return name_utils::DatasetDebugString(kDatasetType); + } + + protected: + Status AsGraphDefInternal(SerializationContext* ctx, + DatasetGraphDefBuilder* b, + Node** output) const override { + Node* filenames = nullptr; + Node* compression_type = nullptr; + Node* buffer_size = nullptr; + TF_RETURN_IF_ERROR(b->AddVector(filenames_, &filenames)); + TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); + TF_RETURN_IF_ERROR(b->AddScalar(options_.input_buffer_size, &buffer_size)); + TF_RETURN_IF_ERROR(b->AddDataset( + this, {filenames, compression_type, buffer_size}, output)); + return Status::OK(); + } + + private: + class Iterator : public DatasetIterator { + public: + explicit Iterator(const Params& params) + : DatasetIterator(params) {} + + Status GetNextInternal(IteratorContext* ctx, + std::vector* out_tensors, + bool* end_of_sequence) override { + mutex_lock l(mu_); + do { + // We are currently processing a file, so try to read the next line. + if (buffered_input_stream_) { + string line_contents; + Status s = buffered_input_stream_->ReadLine(&line_contents); + + if (s.ok()) { + // Produce the line as output. + metrics::RecordTFDataBytesRead( + name_utils::OpName(TextLineDatasetOp::kDatasetType), + line_contents.size()); + out_tensors->emplace_back(ctx->allocator({}), DT_STRING, + TensorShape({})); + out_tensors->back().scalar()() = std::move(line_contents); + *end_of_sequence = false; + return Status::OK(); + } else if (!errors::IsOutOfRange(s)) { + // Report non-EOF errors to the caller. + return s; + } + // We have reached the end of the current file, so maybe + // move on to next file. + ResetStreamsLocked(); + ++current_file_index_; + } + + // Iteration ends when there are no more files to process. + if (current_file_index_ == dataset()->filenames_.size()) { + *end_of_sequence = true; + return Status::OK(); + } + + TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); + } while (true); + } + + protected: + std::shared_ptr CreateNode( + IteratorContext* ctx, model::Node::Args args) const override { + return model::MakeSourceNode(std::move(args)); + } + + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name(kCurrentFileIndex), + current_file_index_)); + // `buffered_input_stream_` is empty if + // 1. GetNext has not been called even once. + // 2. All files have been read and iterator has been exhausted. + if (buffered_input_stream_) { + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name(kCurrentPos), + buffered_input_stream_->Tell())); + } + return Status::OK(); + } + + Status RestoreInternal(IteratorContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + ResetStreamsLocked(); + int64 current_file_index; + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name(kCurrentFileIndex), + ¤t_file_index)); + current_file_index_ = size_t(current_file_index); + // The key "current_pos" is written only if the iterator was saved + // with an open file. + if (reader->Contains(full_name(kCurrentPos))) { + int64 current_pos; + TF_RETURN_IF_ERROR( + reader->ReadScalar(full_name(kCurrentPos), ¤t_pos)); + + TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); + TF_RETURN_IF_ERROR(buffered_input_stream_->Seek(current_pos)); + } + return Status::OK(); + } + + private: + // Sets up reader streams to read from the file at `current_file_index_`. + Status SetupStreamsLocked(Env* env) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (current_file_index_ >= dataset()->filenames_.size()) { + return errors::InvalidArgument( + "current_file_index_:", current_file_index_, + " >= filenames_.size():", dataset()->filenames_.size()); + } + + // Actually move on to next file. + TF_RETURN_IF_ERROR(env->NewRandomAccessFile( + dataset()->filenames_[current_file_index_], &file_)); + input_stream_ = + absl::make_unique(file_.get(), false); + + if (dataset()->use_compression_) { + zlib_input_stream_ = absl::make_unique( + input_stream_.get(), dataset()->options_.input_buffer_size, + dataset()->options_.input_buffer_size, dataset()->options_); + buffered_input_stream_ = absl::make_unique( + zlib_input_stream_.get(), dataset()->options_.input_buffer_size, + false); + } else { + buffered_input_stream_ = absl::make_unique( + input_stream_.get(), dataset()->options_.input_buffer_size, false); + } + return Status::OK(); + } + + // Resets all reader streams. + void ResetStreamsLocked() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + input_stream_.reset(); + zlib_input_stream_.reset(); + buffered_input_stream_.reset(); + file_.reset(); + } + + mutex mu_; + std::unique_ptr input_stream_ GUARDED_BY(mu_); + std::unique_ptr zlib_input_stream_ GUARDED_BY(mu_); + std::unique_ptr buffered_input_stream_ + GUARDED_BY(mu_); + size_t current_file_index_ GUARDED_BY(mu_) = 0; + std::unique_ptr file_ + GUARDED_BY(mu_); // must outlive input_stream_ + }; + + const std::vector filenames_; + const string compression_type_; + const bool use_compression_; + const io::ZlibCompressionOptions options_; +}; + +TextLineDatasetOp::TextLineDatasetOp(OpKernelConstruction* ctx) + : DatasetOpKernel(ctx) {} + +void TextLineDatasetOp::MakeDataset(OpKernelContext* ctx, + DatasetBase** output) { + const Tensor* filenames_tensor; + OP_REQUIRES_OK(ctx, ctx->input(kFileNames, &filenames_tensor)); + OP_REQUIRES( + ctx, filenames_tensor->dims() <= 1, + errors::InvalidArgument("`filenames` must be a scalar or a vector.")); + + string compression_type; + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, kCompressionType, + &compression_type)); + + int64 buffer_size = -1; + OP_REQUIRES_OK(ctx, + ParseScalarArgument(ctx, kBufferSize, &buffer_size)); + OP_REQUIRES( + ctx, buffer_size >= 0, + errors::InvalidArgument("`buffer_size` must be >= 0 (0 == default)")); + + io::ZlibCompressionOptions zlib_compression_options = + io::ZlibCompressionOptions::DEFAULT(); + if (compression_type == kZLIB) { + zlib_compression_options = io::ZlibCompressionOptions::DEFAULT(); + } else if (compression_type == kGZIP) { + zlib_compression_options = io::ZlibCompressionOptions::GZIP(); + } else { + OP_REQUIRES(ctx, compression_type.empty(), + errors::InvalidArgument("Unsupported compression_type.")); + } + + if (buffer_size != 0) { + // Set the override size. + zlib_compression_options.input_buffer_size = buffer_size; + } + + std::vector filenames; + filenames.reserve(filenames_tensor->NumElements()); + for (int i = 0; i < filenames_tensor->NumElements(); ++i) { + filenames.push_back(filenames_tensor->flat()(i)); + } + + *output = new Dataset(ctx, std::move(filenames), compression_type, + zlib_compression_options); +} + +namespace { +REGISTER_KERNEL_BUILDER(Name("TextLineDataset").Device(DEVICE_CPU), + TextLineDatasetOp); +} // namespace +} // namespace data +} // namespace tensorflow diff --git a/tensorflow/core/kernels/data/text_line_dataset_op.h b/tensorflow/core/kernels/data/text_line_dataset_op.h new file mode 100644 index 00000000000..3621b57ada2 --- /dev/null +++ b/tensorflow/core/kernels/data/text_line_dataset_op.h @@ -0,0 +1,42 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_KERNELS_DATA_TEXT_LINE_DATASET_OP_H_ +#define TENSORFLOW_CORE_KERNELS_DATA_TEXT_LINE_DATASET_OP_H_ + +#include "tensorflow/core/framework/dataset.h" + +namespace tensorflow { +namespace data { + +class TextLineDatasetOp : public DatasetOpKernel { + public: + static constexpr const char* const kDatasetType = "TextLine"; + static constexpr const char* const kFileNames = "filenames"; + static constexpr const char* const kCompressionType = "compression_type"; + static constexpr const char* const kBufferSize = "buffer_size"; + + explicit TextLineDatasetOp(OpKernelConstruction* ctx); + + protected: + void MakeDataset(OpKernelContext* ctx, DatasetBase** output) override; + + private: + class Dataset; +}; + +} // namespace data +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_DATA_TEXT_LINE_DATASET_OP_H_ From af8772fa69fb5593ec4d3e297602723705c922e2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 10:03:22 -0700 Subject: [PATCH 118/332] [XLA:Python] Fix potential issue in CopyToDevice where source device buffer might get deallocated before transfer completes. PiperOrigin-RevId: 257216762 --- tensorflow/compiler/xla/python/local_client.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index d7f27d00f76..2057788c12d 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -652,6 +652,10 @@ StatusOr> PyLocalBuffer::CopyToDevice( output_buffer)); } + // We hold on to the `src_device_buffer` until the transfer is finished. + src_device.ThenRelease(src_device_to_device_stream, + std::move(src_device_buffer)); + // Write new tuple buffers. The destination buffers have different addresses, // so we must construct tuple buffers from scratch instead of copying them. if (dst_buffer.on_device_shape().IsTuple()) { From b77b28d9db08a5f29988e7ca5e628df2b168d433 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 10:06:08 -0700 Subject: [PATCH 119/332] Compatibility compile fix PiperOrigin-RevId: 257217445 --- tensorflow/lite/delegates/gpu/gl/egl_context.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/delegates/gpu/gl/egl_context.cc b/tensorflow/lite/delegates/gpu/gl/egl_context.cc index 6ceaf36a928..146a1921e11 100644 --- a/tensorflow/lite/delegates/gpu/gl/egl_context.cc +++ b/tensorflow/lite/delegates/gpu/gl/egl_context.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/egl_context.h" +#include + #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/gl/gl_call.h" #include "tensorflow/lite/delegates/gpu/gl/gl_errors.h" @@ -54,7 +56,7 @@ Status CreateContext(EGLDisplay display, EGLContext shared_context, } bool HasExtension(EGLDisplay display, const char* name) { - return strstr(eglQueryString(display, EGL_EXTENSIONS), name); + return std::strstr(eglQueryString(display, EGL_EXTENSIONS), name); } } // namespace From 8b8b54b7849404a7a72ace64d6e0644ee06a838e Mon Sep 17 00:00:00 2001 From: Karim Nosir Date: Tue, 9 Jul 2019 10:19:38 -0700 Subject: [PATCH 120/332] Add MirrorPad op to MLIR TfLite converter PiperOrigin-RevId: 257220411 --- .../compiler/mlir/lite/flatbuffer_operator.cc | 7 +++ tensorflow/compiler/mlir/lite/ir/tfl_ops.td | 42 +++++++++++++++++ .../compiler/mlir/lite/tests/legalize-tf.mlir | 20 ++++++++ .../mlir/lite/transforms/legalize_patterns.td | 2 + .../mlir/tensorflow/ir/tf_generated_ops.td | 47 +++++++++++++++++++ 5 files changed, 118 insertions(+) diff --git a/tensorflow/compiler/mlir/lite/flatbuffer_operator.cc b/tensorflow/compiler/mlir/lite/flatbuffer_operator.cc index a6120c91eaf..5a480ae8439 100644 --- a/tensorflow/compiler/mlir/lite/flatbuffer_operator.cc +++ b/tensorflow/compiler/mlir/lite/flatbuffer_operator.cc @@ -42,6 +42,13 @@ static tflite::Padding ConvertTFL_PaddingAttrForOptionWriter( .Case("VALID", tflite::Padding_VALID); } +static tflite::MirrorPadMode ConvertTFL_MirrorPaddingAttrForOptionWriter( + llvm::StringRef str, flatbuffers::FlatBufferBuilder* builder) { + return llvm::StringSwitch(str) + .Case("REFLECT", tflite::MirrorPadMode_REFLECT) + .Case("SYMMETRIC", tflite::MirrorPadMode_SYMMETRIC); +} + static tflite::TensorType ConvertDerivedTypeAttrForOptionWriter( mlir::Type type, flatbuffers::FlatBufferBuilder* builder) { switch (type.getKind()) { diff --git a/tensorflow/compiler/mlir/lite/ir/tfl_ops.td b/tensorflow/compiler/mlir/lite/ir/tfl_ops.td index 0b89a85580e..0ced385ea2e 100644 --- a/tensorflow/compiler/mlir/lite/ir/tfl_ops.td +++ b/tensorflow/compiler/mlir/lite/ir/tfl_ops.td @@ -77,11 +77,17 @@ def TFL_AFAttr : StrEnumAttr< // These should match the padding enum in TFLite schema. def TFL_PAD_Same : StrEnumAttrCase<"SAME">; def TFL_PAD_Valid : StrEnumAttrCase<"VALID">; +def TFL_MIRRORPAD_Reflect : StrEnumAttrCase<"REFLECT">; +def TFL_MIRRORPAD_Symmetric : StrEnumAttrCase<"SYMMETRIC">; def TFL_PaddingAttr : StrEnumAttr<"Padding", "padding enum", [ TFL_PAD_Same, TFL_PAD_Valid ]>; +def TFL_MirrorPaddingAttr : StrEnumAttr<"Padding", "Mirror pad enum", [ + TFL_MIRRORPAD_Reflect, TFL_MIRRORPAD_Symmetric + ]>; + //===----------------------------------------------------------------------===// // Min-max range pair definitions. //===----------------------------------------------------------------------===// @@ -1959,6 +1965,42 @@ def TFL_StridedSliceOp: TFL_Op<"strided_slice", } +def TFL_MirrorPadOp: TFL_Op<"mirror_pad", [ + NoSideEffect, TFL_OperandHasRank<1, 2>]> { + let summary = "MirrorPad Operator. Pads a tensor with mirrored values."; + + let description = [{ + This operation pads a input with mirrored values according to the paddings + you specify. paddings is an integer tensor with shape [n, 2], + where n is the rank of input. + For each dimension D of input, paddings[D, 0] indicates how many values + to add before the contents of input in that dimension, + and paddings[D, 1] indicates how many values to add after the contents of + input in that dimension. + + Both paddings[D, 0] and paddings[D, 1] must be no greater than + input.dim_size(D) (or input.dim_size(D) - 1) + if copy_border is true (if false, respectively). + + The padded size of each dimension D of the output is: + + paddings(D, 0) + input.dim_size(D) + paddings(D, 1) + }]; + + let arguments = (ins + // TODO: add uint8 support when ready. + TensorOf<[F32, I32, I64]>:$input, + TensorOf<[I32, I64]>:$pad, + TFL_MirrorPaddingAttr:$mode + ); + + let results = (outs + TensorOf<[F32, I32, I64]>:$output + ); + + let hasOptions = 1; +} + //===----------------------------------------------------------------------===// // Quantization ops. //===----------------------------------------------------------------------===// diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir index 53dcc83feda..25ceede688e 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir @@ -829,3 +829,23 @@ func @slice1Tensor(%arg0: tensor<2x3x5xf32>, %arg1: tensor<3xi32>, %arg2: tensor // CHECK-LABEL: slice1Tensor // CHECK: "tfl.slice"(%arg0, %arg1, %arg2) : (tensor<2x3x5xf32>, tensor<3xi32>, tensor<3xi32>) -> tensor } + +func @mirror_pad(tensor<2x1x3xf32>, tensor<3x2xi32>) -> tensor { +^bb0(%arg0: tensor<2x1x3xf32>, %arg1: tensor<3x2xi32>): + %0 = "tf.MirrorPad"(%arg0, %arg1) { mode = "SYMMETRIC" }: (tensor<2x1x3xf32>, tensor<3x2xi32>) -> tensor + return %0#0 : tensor + + // CHECK-LABEL: mirror_pad + // CHECK: %0 = "tfl.mirror_pad"(%arg0, %arg1) {mode = "SYMMETRIC"} : (tensor<2x1x3xf32>, tensor<3x2xi32>) -> tensor + // CHECK: return %0 : tensor +} + +func @mirror_pad_reflect(tensor<2x1x3xf32>, tensor<3x2xi32>) -> tensor { +^bb0(%arg0: tensor<2x1x3xf32>, %arg1: tensor<3x2xi32>): + %0 = "tf.MirrorPad"(%arg0, %arg1) { mode = "REFLECT" }: (tensor<2x1x3xf32>, tensor<3x2xi32>) -> tensor + return %0#0 : tensor + + // CHECK-LABEL: mirror_pad_reflect + // CHECK: %0 = "tfl.mirror_pad"(%arg0, %arg1) {mode = "REFLECT"} : (tensor<2x1x3xf32>, tensor<3x2xi32>) -> tensor + // CHECK: return %0 : tensor +} diff --git a/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td b/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td index 0e7534bb513..bcc00d12b01 100644 --- a/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td +++ b/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td @@ -240,6 +240,8 @@ def : Pat<(TF_SpaceToBatchNDOp $input, $block_shape, $paddings), (TFL_SpaceToBat def : Pat<(TF_ResizeBilinearOp $images, $size, $align_corners, ConstBoolAttrFalse:$half_pixel_centers), (TFL_ResizeBilinearOp $images, $size, $align_corners)>; +def : Pat<(TF_MirrorPadOp $arg0, $arg1, $cst), (TFL_MirrorPadOp $arg0, $arg1, $cst)>; + def : Pat< (TF_StridedSliceOp $input, $begin, $end, $strides, $begin_mask, $end_mask, $ellipsis_mask, $new_axis_mask, $shrink_axis_mask), (TFL_StridedSliceOp $input, $begin, $end, $strides, diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index dda278a6cf0..876b3f47e6b 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -1348,6 +1348,53 @@ def TF_MinimumOp : TF_Op<"Minimum", [Broadcastable, NoSideEffect]>, TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<0>; } +def TF_MirrorPadOp : TF_Op<"MirrorPad", [NoSideEffect]> { + let summary = "Pads a tensor with mirrored values."; + + let description = [{ +This operation pads a `input` with mirrored values according to the `paddings` +you specify. `paddings` is an integer tensor with shape `[n, 2]`, where n is +the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates +how many values to add before the contents of `input` in that dimension, and +`paddings[D, 1]` indicates how many values to add after the contents of `input` +in that dimension. Both `paddings[D, 0]` and `paddings[D, 1]` must be no greater +than `input.dim_size(D)` (or `input.dim_size(D) - 1`) if `copy_border` is true +(if false, respectively). + +The padded size of each dimension D of the output is: + +`paddings(D, 0) + input.dim_size(D) + paddings(D, 1)` + +For example: + +``` +# 't' is [[1, 2, 3], [4, 5, 6]]. +# 'paddings' is [[1, 1]], [2, 2]]. +# 'mode' is SYMMETRIC. +# rank of 't' is 2. +pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2] + [2, 1, 1, 2, 3, 3, 2] + [5, 4, 4, 5, 6, 6, 5] + [5, 4, 4, 5, 6, 6, 5]] +``` + }]; + + let arguments = (ins + TF_Tensor:$input, + TF_I32OrI64Tensor:$paddings, + + TF_AnyStrAttrOf<["REFLECT", "SYMMETRIC"]>:$mode + ); + + let results = (outs + TF_Tensor:$output + ); + + TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<0>; + TF_DerivedOperandTypeAttr Tpaddings = TF_DerivedOperandTypeAttr<1>; +} + + def TF_MulOp : TF_Op<"Mul", [Broadcastable, Commutative, NoSideEffect]>, WithBroadcastableBinOpBuilder { let summary = "Returns x * y element-wise."; From 7d944239facdfc198911f5781bea4b13917441da Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Tue, 9 Jul 2019 10:32:05 -0700 Subject: [PATCH 121/332] Do not infer batch size for dataset/iterator/generator/sequence input. Those data type does not use batch size or already include batch information with themselves. Returning the default 32 as batch size is bit confusing. PiperOrigin-RevId: 257223120 --- tensorflow/python/keras/engine/training.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index bc64d5af38f..44b938d1235 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -61,6 +61,7 @@ from tensorflow.python.training.tracking import base as trackable from tensorflow.python.training.tracking import layer_utils as trackable_layer_utils from tensorflow.python.util import nest from tensorflow.python.util import serialization +from tensorflow.python.util import tf_inspect from tensorflow.python.util.tf_export import keras_export try: @@ -1703,7 +1704,13 @@ class Model(network.Network): if steps is None: batch_size = static_batch_size - if batch_size is None and steps is None: + if (batch_size is None + and steps is None + and not isinstance(x, (dataset_ops.DatasetV2, + iterator_ops.Iterator, + iterator_ops.IteratorV2, + data_utils.Sequence)) + and not tf_inspect.isgenerator(x)): # Backwards compatibility batch_size = 32 return batch_size From f72dc13764cc4b3f23d1595d8fde069cd4ea7804 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Tue, 9 Jul 2019 10:37:28 -0700 Subject: [PATCH 122/332] Disable tf_record_test on windows Fails with: AttributeError: Can't pickle local object 'TFRecordWriterCloseAndFlushTests.testFlush..childProcess' There is some weirdness with multiprocessing on windows PiperOrigin-RevId: 257224409 --- tensorflow/python/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index fcab7a7e802..a2ee9e07458 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -5297,6 +5297,7 @@ tf_py_test( ":lib", ":util", ], + tags = ["no_windows"], # b/137097422 ) cuda_py_test( From eebe2f7247ff8ece3864bb100f3030f0168d1120 Mon Sep 17 00:00:00 2001 From: Philip Pham Date: Tue, 9 Jul 2019 10:48:13 -0700 Subject: [PATCH 123/332] Support Tensorflow ops in Keras functional API for TPUs TPU rewriting adds extra metadata attributes to graph nodes that is not captured by the TensorflowOpLayer. In particular, ops and constants require the `_tpu_replicate`, which are usually added in the rewrite step. Reconstructing the constants and post-processing the ops with the control flow context covers these cases. PiperOrigin-RevId: 257226841 --- tensorflow/python/keras/BUILD | 2 ++ tensorflow/python/keras/engine/base_layer.py | 25 +++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index 3da6e3f4efa..fb4bb82d197 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -262,6 +262,8 @@ py_library( ":constraints", ":engine_utils", ":regularizers", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:constant_op", "//tensorflow/python/data", "//tensorflow/python/distribute:distribute_coordinator", "//tensorflow/python/distribute:distribute_lib", diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index c5bc2d2c219..a7b1721335c 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -37,6 +37,7 @@ from tensorflow.python.eager import context from tensorflow.python.eager import execute from tensorflow.python.eager import function from tensorflow.python.framework import auto_control_deps +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import func_graph from tensorflow.python.framework import ops @@ -2401,21 +2402,30 @@ class TensorFlowOpLayer(Layer): return self._defun_call(inputs) return self._make_op(inputs) + def _make_node_def(self, graph): + node_def = node_def_pb2.NodeDef() + node_def.CopyFrom(self.node_def) + node_def.name = graph.unique_name(node_def.name) + return node_def + def _make_op(self, inputs): inputs = nest.flatten(inputs) graph = inputs[0].graph + node_def = self._make_node_def(graph) with graph.as_default(): for index, constant in self.constants.items(): - constant = ops.convert_to_tensor(constant) + # Recreate constant in graph to add distribution context. + value = tensor_util.constant_value(constant) + if value is not None: + constant = constant_op.constant(value, name=node_def.input[index]) inputs.insert(index, constant) - - self.node_def.name = graph.unique_name(self.node_def.name) # Check for case where first input should be a list of Tensors. - if 'N' in self.node_def.attr: - num_tensors = self.node_def.attr['N'].i + if 'N' in node_def.attr: + num_tensors = node_def.attr['N'].i inputs = [inputs[:num_tensors]] + inputs[num_tensors:] - c_op = ops._create_c_op(graph, self.node_def, inputs, control_inputs=[]) + c_op = ops._create_c_op(graph, node_def, inputs, control_inputs=[]) op = graph._create_op_from_tf_operation(c_op) + op._control_flow_post_processing() # Record the gradient because custom-made ops don't go through the # code-gen'd eager call path @@ -2426,8 +2436,7 @@ class TensorFlowOpLayer(Layer): attrs.append(attr_name) attrs.append(op.get_attr(attr_name)) attrs = tuple(attrs) - execute.record_gradient(op_type, op.inputs, attrs, op.outputs, - op.name) + execute.record_gradient(op_type, op.inputs, attrs, op.outputs, op.name) if len(op.outputs) == 1: return op.outputs[0] From b05c0ee7541f7c7fe5db58de061cbc5697483195 Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Tue, 9 Jul 2019 10:55:31 -0700 Subject: [PATCH 124/332] Fix keras/backend_test for v2 control flow. PiperOrigin-RevId: 257228448 --- tensorflow/python/keras/backend.py | 4 ++-- tensorflow/python/keras/backend_test.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/keras/backend.py b/tensorflow/python/keras/backend.py index ce557b834d5..c7ebb4b2524 100644 --- a/tensorflow/python/keras/backend.py +++ b/tensorflow/python/keras/backend.py @@ -5429,9 +5429,9 @@ def ctc_label_dense_to_sparse(labels, label_lengths): num_batches_tns = array_ops.stack([label_shape[0]]) max_num_labels_tns = array_ops.stack([label_shape[1]]) - def range_less_than(_, current_input): + def range_less_than(old_input, current_input): return array_ops.expand_dims( - math_ops.range(label_shape[1]), 0) < array_ops.fill( + math_ops.range(array_ops.shape(old_input)[1]), 0) < array_ops.fill( max_num_labels_tns, current_input) init = math_ops.cast( diff --git a/tensorflow/python/keras/backend_test.py b/tensorflow/python/keras/backend_test.py index e3bc5467261..51b57d2d1ff 100644 --- a/tensorflow/python/keras/backend_test.py +++ b/tensorflow/python/keras/backend_test.py @@ -1661,6 +1661,7 @@ class BackendCrossEntropyLossesTest(test.TestCase): @test_util.run_all_in_graph_and_eager_modes +@test_util.with_control_flow_v2 class TestCTC(test.TestCase): def test_ctc_decode(self): From 47c5c54e3e51c8800d7b03101ae4ed7b7183c8d0 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Tue, 9 Jul 2019 11:15:07 -0700 Subject: [PATCH 125/332] Fix the related docs for MatchingFilesOp --- tensorflow/core/api_def/base_api/api_def_MatchingFiles.pbtxt | 2 +- tensorflow/go/op/wrappers.go | 2 +- tensorflow/python/training/input.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/api_def/base_api/api_def_MatchingFiles.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatchingFiles.pbtxt index 97fd39f6478..56027301fba 100644 --- a/tensorflow/core/api_def/base_api/api_def_MatchingFiles.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_MatchingFiles.pbtxt @@ -16,6 +16,6 @@ END description: < Date: Tue, 9 Jul 2019 11:25:04 -0700 Subject: [PATCH 126/332] Update tensorflow/core/kernels/scatter_functor.h Co-Authored-By: Penporn Koanantakool <38085909+penpornk@users.noreply.github.com> --- tensorflow/core/kernels/scatter_functor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/scatter_functor.h b/tensorflow/core/kernels/scatter_functor.h index 5051b508f06..729189456f9 100644 --- a/tensorflow/core/kernels/scatter_functor.h +++ b/tensorflow/core/kernels/scatter_functor.h @@ -207,7 +207,7 @@ struct ScatterFunctorBase { const Index N = static_cast(indices.size()); const Index limit = static_cast(params.dimension(0)); mutex mu_; - Index bad_index = -1 GUARDED_BY(mu_); + Index bad_index GUARDED_BY(mu_) = -1; auto ParallelScatter = [&](Index start, Index end) LOCKS_EXCLUDED(mu_){ for (Index i = start; i < end; i++) { // Grab the index and check its validity. Do this carefully, From ca57a9d5a88096cd68a1d7b59ec55cff373d62a4 Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Tue, 9 Jul 2019 11:05:15 -0700 Subject: [PATCH 127/332] Do not perform colocation checks for IdentityN since it just forwards its inputs. PiperOrigin-RevId: 257230757 --- tensorflow/core/kernels/BUILD | 4 +++- tensorflow/core/kernels/identity_n_op.cc | 6 ++++++ tensorflow/python/distribute/BUILD | 1 + tensorflow/python/distribute/step_fn_test.py | 2 ++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 35d6a81751f..571c6aea635 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -1097,7 +1097,9 @@ tf_kernel_library( tf_kernel_library( name = "identity_n_op", prefix = "identity_n_op", - deps = ARRAY_DEPS, + deps = ARRAY_DEPS + [ + "//tensorflow/core:core_cpu_internal", + ], ) tf_kernel_library( diff --git a/tensorflow/core/kernels/identity_n_op.cc b/tensorflow/core/kernels/identity_n_op.cc index 9746b1fab3e..746a29bf5aa 100644 --- a/tensorflow/core/kernels/identity_n_op.cc +++ b/tensorflow/core/kernels/identity_n_op.cc @@ -16,6 +16,7 @@ limitations under the License. // See docs in ../ops/array_ops.cc. #include "tensorflow/core/kernels/identity_n_op.h" +#include "tensorflow/core/common_runtime/input_colocation_exemption_registry.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" @@ -24,5 +25,10 @@ limitations under the License. namespace tensorflow { REGISTER_KERNEL_BUILDER(Name("IdentityN").Device(DEVICE_DEFAULT), IdentityNOp); +// Do not worry about colocating IdentityN op with its resource inputs since +// it just forwards it's inputs anyway. This is needed because we create +// IdentityN nodes to club "all" outputs of functional ops while lowering to +// make the original functional op fetchable. +REGISTER_INPUT_COLOCATION_EXEMPTION("IdentityN"); } // namespace tensorflow diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD index 28f3493ecc4..4f750277e44 100644 --- a/tensorflow/python/distribute/BUILD +++ b/tensorflow/python/distribute/BUILD @@ -963,6 +963,7 @@ distribute_py_test( deps = [ ":single_loss_example", "//tensorflow/contrib/tpu:tpu_lib", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:variables", "//tensorflow/python/distribute:combinations", "//tensorflow/python/distribute:strategy_combinations", diff --git a/tensorflow/python/distribute/step_fn_test.py b/tensorflow/python/distribute/step_fn_test.py index 3a1a7e6eca4..28e6ad28c77 100644 --- a/tensorflow/python/distribute/step_fn_test.py +++ b/tensorflow/python/distribute/step_fn_test.py @@ -25,9 +25,11 @@ from tensorflow.python.distribute import strategy_combinations from tensorflow.python.distribute.single_loss_example import single_loss_example from tensorflow.python.eager import context from tensorflow.python.eager import test +from tensorflow.python.framework import test_util from tensorflow.python.ops import variables +@test_util.with_control_flow_v2 class SingleLossStepTest(test.TestCase, parameterized.TestCase): @combinations.generate( From e43b2625111ec6cd56ae06f597acbfbd8be293e5 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Tue, 9 Jul 2019 11:36:01 -0700 Subject: [PATCH 128/332] Cherry-pick release notes for 1.12.3 release --- RELEASE.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index c2c50c590ba..f8abf9f4dce 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,10 @@ +# Release 1.12.3 + +## Bug Fixes and Other Changes + +* Updates `png_archive` dependency to 1.6.37 to not be affected by CVE-2019-7317, CVE-2018-13785, and CVE-2018-14048. +* Updates `sqlite` depenency to 3.28.0 to not be affected by CVE-2018-20506, CVE-2018-20346, and CVE-2018-20505. + # Release 1.12.2 ## Bug Fixes and Other Changes From 3928354651b991db5cc1b69c78b7efe66a394ce1 Mon Sep 17 00:00:00 2001 From: Christian Sigg Date: Tue, 9 Jul 2019 11:10:53 -0700 Subject: [PATCH 129/332] Make helper function global so it can be pickled. Fixes Windows tests after CL/257051579. PiperOrigin-RevId: 257232107 --- tensorflow/python/lib/io/tf_record_test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/lib/io/tf_record_test.py b/tensorflow/python/lib/io/tf_record_test.py index f620c0ec39e..c3000c09432 100644 --- a/tensorflow/python/lib/io/tf_record_test.py +++ b/tensorflow/python/lib/io/tf_record_test.py @@ -443,6 +443,10 @@ class TFRecordIteratorTest(TFCompressionTestCase): for _ in tf_record.tf_record_iterator(fn_truncated): pass +def ChildProcess(writer, rs): + for r in rs: + writer.write(r) + writer.flush() class TFRecordWriterCloseAndFlushTests(test.TestCase): """TFRecordWriter close and flush tests""" @@ -481,13 +485,8 @@ class TFRecordWriterCloseAndFlushTests(test.TestCase): """test Flush""" records = list(map(self._Record, range(self._num_records))) - def childProcess(writer, rs): - for r in rs: - writer.write(r) - writer.flush() - write_process = multiprocessing.Process( - target=childProcess, args=(self._writer, records)) + target=ChildProcess, args=(self._writer, records)) write_process.start() write_process.join() actual = list(tf_record.tf_record_iterator(self._fn, self._options)) From c1135834cff3594e4af897b99fbbde4d69f39a18 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Tue, 9 Jul 2019 11:37:59 -0700 Subject: [PATCH 130/332] Cherry-pick release notes for 1.14.0 release --- RELEASE.md | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index f8abf9f4dce..543b4f949b5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,137 @@ +# Release 1.14.0 + +## Major Features and Improvements + +* This is the first 1.x release containing the compat.v2 module. This module is required to allow libraries to publish code which works in both 1.x and 2.x. After this release, no backwards incompatible changes are allowed in the 2.0 Python API. +* Turn on MKL-DNN contraction kernels by default. MKL-DNN dynamically dispatches the best kernel implementation based on CPU vector architecture. To disable them, build with --define=tensorflow_mkldnn_contraction_kernel=0. + +## Behavioral changes + +* Set default loss reduction as `AUTO` for improving reliability of loss scaling with distribution strategy and custom training loops. `AUTO` indicates that the reduction option will be determined by the usage context. For almost all cases this defaults to `SUM_OVER_BATCH_SIZE`. When used in distribution strategy scope, outside of built-in training loops such as `tf.keras` `compile` and `fit`, we expect reduction value to be 'None' or 'SUM'. Using other values will raise an error. +* Wraps losses passed to the `compile` API (strings and v1 losses) which are not instances of v2 `Loss` class in `LossWrapper` class. => All losses will now use `SUM_OVER_BATCH_SIZE` reduction as default. +* Disable `run_eagerly` and distribution strategy if there are symbolic tensors added to the model using `add_metric` or `add_loss`. +* tf.linspace(start, stop, num) now always uses "stop" as last value (for num > 1) +* `ResourceVariable` and `Variable` no longer accepts `constraint` in the constructor, nor expose it as a @property. +* The behavior of tf.gather is now correct when axis=None and batch_dims<0. +* Only create a GCS directory object if the object does not already exist. +* In `map_vectorization` optimization, reduce the degree of parallelism in the vectorized map node. +* Bug fix: loss and gradients should now more reliably be correctly scaled w.r.t. the global batch size when using a tf.distribute.Strategy. +* Updating cosine similarity loss - removed the negate sign from cosine similarity. +* DType is no longer convertible to an int. Use dtype.as_datatype_enum instead of int(dtype) to get the same result. +* Changed default for gradient accumulation for TPU embeddings to true. +* Callbacks now log values in eager mode when a deferred build model is used. +* Transitive dependencies on :pooling_ops were removed. Some users may need to add explicit dependencies on :pooling_ops if they reference the operators from that library. + +## Bug Fixes and Other Changes +* Documentation +* Deprecations and Symbol renames. + * Remove unused StringViewVariantWrapper + * Delete unused Fingerprint64Map op registration + * SignatureDef util functions have been deprecated. + * Renamed tf.image functions to remove duplicate "image" where it is redundant. + * tf.keras.experimental.export renamed to tf.keras.experimental.export_saved_model + * Standardize the LayerNormalization API by replacing the args `norm_axis` and `params_axis` with `axis`. + * Tensor::UnsafeCopyFromInternal deprecated in favor Tensor::BitcastFrom +* Keras & Python API + * Add v2 module aliases for: + * tf.initializers => tf.keras.initializers + * tf.losses => tf.keras.losses & tf.metrics => tf.keras.metrics + * tf.optimizers => tf.keras.optimizers + * Add tf.keras.layers.AbstractRNNCell as the preferred implementation of RNN cell for TF v2. User can use it to implement RNN cell with custom behavior. + * Adding `clear_losses` API to be able to clear losses at the end of forward pass in a custom training loop in eager. + * Add support for passing list of lists to the `metrics` param in Keras `compile`. + * Added top-k to precision and recall to keras metrics. + * Adding public APIs for `cumsum` and `cumprod` keras backend functions. + * Fix: model.add_loss(symbolic_tensor) should work in ambient eager. + * Add name argument to tf.string_split and tf.strings_split + * Minor change to SavedModels exported from Keras using tf.keras.experimental.export. (SignatureDef key for evaluation mode is now "eval" instead of "test"). This will be reverted back to "test" in the near future. + * Updates binary cross entropy logic in Keras when input is probabilities. Instead of converting probabilities to logits, we are using the cross entropy formula for probabilities. + * Raw TensorFlow functions can now be used in conjunction with the Keras Functional API during model creation. This obviates the need for users to create Lambda layers in most cases when using the Functional API. Like Lambda layers, TensorFlow functions that result in Variable creation or assign ops are not supported. + * Keras training and validation curves are shown on the same plot. + * Introduce `dynamic` constructor argument in Layer and Model, which should be set to True when using imperative control flow in the `call` method. + * Removing of dtype in the constructor of initializers and partition_info in call. +* New ops and improved op functionality + * Add OpKernels for some stateless maps + * Add v2 APIs for AUCCurve and AUCSummationMethod enums. #tf-metrics-convergence + * Add tf.math.nextafter op. + * Add CompositeTensor base class. + * Add tf.linalg.tridiagonal_solve op. + * Add opkernel templates for common table operations. + * Added support for TFLite in TensorFlow 2.0. + * Adds summary trace API for collecting graph and profile information. + * Add batch_dims argument to tf.gather. + * Add support for `add_metric` in the graph function mode. + * Add C++ Gradient for BatchMatMulV2. + * Added tf.random.binomial + * Added gradient for SparseToDense op. + * Add legacy string flat hash map op kernels + * Add a ragged size op and register it to the op dispatcher + * Add broadcasting support to tf.matmul. + * Add ellipsis (...) support for tf.einsum() + * Added LinearOperator.adjoint and LinearOperator.H (alias). + * Added GPU implementation of tf.linalg.tridiagonal_solve. + * Added strings.byte_split + * Add RaggedTensor.placeholder() + * Add a new "result_type" parameter to tf.strings.split + * `add_update` can now be passed a zero-arg callable in order to support turning off the update when setting `trainable=False` on a Layer of a Model compiled with `run_eagerly=True`. + * Add variant wrapper for absl::string_view + * Add expand_composites argument to all nest.* methods. + * Add pfor converter for Squeeze. + * Bug fix for tf.tile gradient + * Expose CriticalSection in core as tf.CriticalSection. + * Update Fingerprint64Map to use aliases + * ResourceVariable support for gather_nd. + * ResourceVariable's gather op supports batch dimensions. + * Variadic reduce is supported on CPU + * Extend tf.function with basic support for CompositeTensors arguments (such as SparseTensor and RaggedTensor). + * Add templates and interfaces for creating lookup tables + * Post-training quantization tool supports quantizing weights shared by multiple operations. The models made with versions of this tool will use INT8 types for weights and will only be executable interpreters from this version onwards. + * Malformed gif images could result in an access out of bounds in the color palette of the frame. This has been fixed now + * image.resize now considers proper pixel centers and has new kernels (incl. anti-aliasing). +* Performance + * Turn on MKL-DNN contraction kernels by default. MKL-DNN dynamically dispatches the best kernel implementation based on CPU vector architecture. To disable them, build with --define=tensorflow_mkldnn_contraction_kernel=0. + * Support for multi-host ncclAllReduce in Distribution Strategy. + * Expose a flag that allows the number of threads to vary across Python benchmarks. +* TensorFlow 2.0 Development + * Add v2 sparse categorical crossentropy metric. + * Allow non-Tensors through v2 losses. + * Add UnifiedGRU as the new GRU implementation for tf2.0. Change the default recurrent activation function for GRU from 'hard_sigmoid' to 'sigmoid', and 'reset_after' to True in 2.0. Historically recurrent activation is 'hard_sigmoid' since it is fast than 'sigmoid'. With new unified backend between CPU and GPU mode, since the CuDNN kernel is using sigmoid, we change the default for CPU mode to sigmoid as well. With that, the default GRU will be compatible with both CPU and GPU kernel. This will enable user with GPU to use CuDNN kernel by default and get a 10x performance boost in training. Note that this is checkpoint breaking change. If user want to use their 1.x pre-trained checkpoint, please construct the layer with GRU(recurrent_activation='hard_sigmoid', reset_after=False) to fallback to 1.x behavior. + * TF 2.0 - Update metric name to always reflect what the user has given in compile. Affects following cases 1. When name is given as 'accuracy'/'crossentropy' 2. When an aliased function name is used eg. 'mse' 3. Removing the `weighted` prefix from weighted metric names. + * Begin adding Go wrapper for C Eager API + * image.resize in 2.0 now supports gradients for the new resize kernels. + * removed tf.string_split from v2 API + * Expose tf.contrib.proto.* ops in tf.io (they will exist in TF2) + * "Updates the TFLiteConverter API in 2.0. Changes from_concrete_function to from_concrete_functions." + * Enable tf.distribute.experimental.MultiWorkerMirroredStrategy working in eager mode. + * Support both binary and -1/1 label input in v2 hinge and squared hinge losses. +* TensorFlow Lite + * "Adds support for tflite_convert in 2.0." + * "Remove lite.OpHint, lite.experimental, and lite.constant from 2.0 API." +* tf.contrib + * Added Neural Turing Implementation as described in https://arxiv.org/abs/1807.08518. + * Remove tf.contrib.timeseries dependency on TF distributions. +* tf.data + * Add num_parallel_reads and passing in a Dataset containing filenames into TextLineDataset and FixedLengthRecordDataset + * Going forward we operate in TF 2.0, this change is part of the effort to slowly converting XYZDataset to DatasetV2 type which is the official version going to be used in TF 2.0 and motivated by some compatibility issue found, _BigtableXYZDataset (of type DatasetV2) does not implement the _as_variant_tensor() of DatasetV1, when moving contrib.bigtable to tensorflow_io. Converting into DatasetV2 removes the overheads to maintain V1 while we are moving into TF 2.0. + * Add dataset ops to the graph (or create kernels in Eager execution) during the python Dataset object creation instead doing it during Iterator creation time. + * Add support for TensorArrays to tf.data Dataset. + * Switching tf.data functions to use `defun`, providing an escape hatch to continue using the legacy `Defun`. +* Toolchains + * CUDNN_INSTALL_PATH, TENSORRT_INSTALL_PATH, NCCL_INSTALL_PATH, NCCL_HDR_PATH are deprecated. Use TF_CUDA_PATHS instead which supports a comma-separated list of base paths that are searched to find CUDA libraries and headers. + * TF code now resides in `tensorflow_core` and `tensorflow` is just a virtual pip package. No code changes are needed for projects using TensorFlow, the change is transparent +* XLA + * XLA HLO graphs can be inspected with interactive_graphviz tool now. +* Estimator + * Use tf.compat.v1.estimator.inputs instead of tf.estimator.inputs + * Replace contrib references with tf.estimator.experimental.* for apis in early_stopping.py + + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +1e100, 4d55397500, a6802739, abenmao, Adam Weiss, Ag Ramesh, Alan Du, Albin Joy, Alex, Aman Patel, Amit, Amit Kumar Jaiswal, Amit Srivastava, Andreas Eberle, Andy Craze, Anthony Platanios, Armen Poghosov, armenpoghosov, arp95, Arpit Shah, Ashwin Ramaswami, Aurelien Geron, AuréLien Geron, aweers, awesomealex1, Ayush Agrawal, Ben Barsdell, Bharat Raghunathan, Bhavani Subramanian, blairhan, BléNesi Attila, Brandon Carter, candy.dc, Chao Liu, chenchc, chie8842, Christian Hansen, Christian Sigg, Clayne Robison, crafet, csukuangfj, ctiijima, Dan Jarvis, Dan Lazewatsky, Daniel Ingram, Daniel Salvadori, Dave Airlie, David Norman, Dayananda V, Dayananda-V, delock, Denis Khalikov, Deven Desai, Dheeraj Rajaram Reddy, dmitrievanthony, Donovan Ong, Drew Szurko, Duncan Riach, Dustin Neighly, Edward Forgacs, EFanZh, Fei Hu, Felix Lemke, Filip Matzner, fo40225, frreiss, Gautam, gehring, Geoffrey Irving, Grzegorz George Pawelczak, Grzegorz Pawelczak, Gyoung-Yoon Ryoo, HanGuo97, Hanton Yang, Hari Shankar, hehongliang, Heungsub Lee, Hoeseong Kim, I-Hong Jhuo, Ilango R, Innovimax, Irene Dea, Jacky Ko, Jakub Lipinski, Jason Zaman, jcf94, Jeffrey Poznanovic, Jens Elofsson, Jeroen BéDorf, Jia Qingtong, Jiankang, Joe Q, Joe Quadrino, Joeran Beel, Jonas Rauber, Jonathan, Jonathan Kyl, Joppe Geluykens, Joseph Friedman, jtressle, jwu, K Yasaswi Sri Chandra Gandhi, K. Hodges, Kaixi Hou, Karl Lessard, Karl Weinmeister, Karthik Muthuraman, Kashif Rasul, KDR, Keno Fischer, Kevin Mader, kjopek, Koan-Sin Tan, kouml, ktaebum, Lakshay Tokas, Laurent Le Brun, Letian Kang, Li, Guizi, Loo Rong Jie, Lucas Hendren, Lukas Geiger, Luke Han, luxupu, Ma, Guokai, Mahmoud Abuzaina, Mandar Deshpande, manhyuk, Marco Gaido, Marek Drozdowski, Mark Collier, Mark Ryan, mars20, Mateusz Chudyk, Matt Conley, MattConley, mbhuiyan, mdfaijul, Melissa Grueter, Michael KäUfl, MickaëL Schoentgen, Miguel Morin, Mihail Salnikov, Mike Arpaia, Mike Holcomb, monklof, Moses Marin, Mshr-H, nammbash, Natalia Gimelshein, Nayana-Ibm, neargye, Neeraj Pradhan, Nehal J Wani, Nick, Niels Ole Salscheider, Niranjan Hasabnis, nlewycky, Nuka-137, Nutti, olicht, P Sudeepam, Palmer Lao, Pan Daoxin, Pariksheet Pinjari, Pavel Samolysov, PENGWA, Pooya Davoodi, R S Nikhil Krishna, Rohit Gupta, Roman Soldatow, rthadur, Ruizhe, Ryan Jiang, Samantha Andow, Sami Kama, Sana-Damani, Saurabh Deoras, sdamani, seanshpark, Sebastien Iooss, Serv-Inc, Shahzad Lone, Shashank Gupta, Shashi, shashvat, shashvatshahi1998, Siju, Siju Samuel, Snease-Abq, Spencer Schaber, sremedios, srinivasan.narayanamoorthy, Steve Lang, Steve Nesae, Sumesh Udayakumaran, Supriya Rao, Taylor Jakobson, Taylor Thornton, Ted Chang, ThisIsPIRI, Thomas Deegan, Thomas Hagebols, tianyapiaozi, Tim Zaman, tomguluson92, Tongxuan Liu, TungJerry, v1incent, Vagif, vcarpani, Vikram Tiwari, Vishwak Srinivasan, Vitor-Alves, wangsiyu, wateryzephyr, WeberXie, WeijieSun, Wen-Heng (Jack) Chung, wenxizhu, Will Battel, William D. Irons, wyzhao, Xin, Yasuhiro Matsumoto, ymodak, Yong Tang, Younes Khoudli, Yuan Lin, Yves-Noel Weweler, Zantares, zjjott, 卜居, 王振华 (Wang Zhenhua), 黄鑫 + # Release 1.12.3 ## Bug Fixes and Other Changes From 546308e322a6b95542ba9f3cbb14136128aaad1e Mon Sep 17 00:00:00 2001 From: Katherine Wu Date: Tue, 9 Jul 2019 11:29:51 -0700 Subject: [PATCH 131/332] Fix performance regression with convolution layers by lazily creating the convolution op attribute. PiperOrigin-RevId: 257236586 --- .../python/keras/layers/convolutional.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tensorflow/python/keras/layers/convolutional.py b/tensorflow/python/keras/layers/convolutional.py index 2175a96d865..dcece7fc754 100644 --- a/tensorflow/python/keras/layers/convolutional.py +++ b/tensorflow/python/keras/layers/convolutional.py @@ -176,6 +176,7 @@ class Conv(Layer): self.bias = None self.input_spec = InputSpec(ndim=self.rank + 2, axes={channel_axis: input_dim}) + self._convolution_op = None self.built = True def call(self, inputs): @@ -185,15 +186,16 @@ class Conv(Layer): op_padding = self.padding if not isinstance(op_padding, (list, tuple)): op_padding = op_padding.upper() - conv_op = nn_ops.Convolution( - inputs.shape, - filter_shape=self.kernel.shape, - dilation_rate=self.dilation_rate, - strides=self.strides, - padding=op_padding, - data_format=conv_utils.convert_data_format(self.data_format, - self.rank + 2)) - outputs = conv_op(inputs, self.kernel) + if self._convolution_op is None: + self._convolution_op = nn_ops.Convolution( + inputs.shape, + filter_shape=self.kernel.shape, + dilation_rate=self.dilation_rate, + strides=self.strides, + padding=op_padding, + data_format=conv_utils.convert_data_format(self.data_format, + self.rank + 2)) + outputs = self._convolution_op(inputs, self.kernel) if self.use_bias: if self.data_format == 'channels_first': From b34ed5bd1e0a8d04226ab4ceff5b8bae6ca7cba0 Mon Sep 17 00:00:00 2001 From: Ayush Dubey Date: Tue, 9 Jul 2019 11:31:35 -0700 Subject: [PATCH 132/332] Remove redundant test tags from collectives tests. Before this change, collective implementation tests would not run without config=cuda because they were tagged with tf_cuda_tests_tags. This change removes the tag, which is redundant because the tf_cc_tests_gpu target does the right thing. This change also expands test coverage by ensuring these tests run without config=cuda, i.e. on CPU. PiperOrigin-RevId: 257236942 --- tensorflow/core/BUILD | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 621bbcd3576..e28a115c25a 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -4336,7 +4336,7 @@ tf_cc_tests_gpu( "common_runtime/ring_reducer_test.cc", ], linkstatic = tf_kernel_tests_linkstatic(), - tags = tf_cuda_tests_tags(), + tags = ["no_cuda_on_cpu_tap"], deps = [ ":all_kernels", ":core", @@ -4365,7 +4365,7 @@ tf_cc_tests_gpu( "common_runtime/ring_gatherer_test.cc", ], linkstatic = tf_kernel_tests_linkstatic(), - tags = tf_cuda_tests_tags(), + tags = ["no_cuda_on_cpu_tap"], deps = [ ":all_kernels", ":core", @@ -4394,7 +4394,7 @@ tf_cc_tests_gpu( "common_runtime/hierarchical_tree_broadcaster_test.cc", ], linkstatic = tf_kernel_tests_linkstatic(), - tags = tf_cuda_tests_tags(), + tags = ["no_cuda_on_cpu_tap"], deps = [ ":all_kernels", ":core", From 7ece5ce95f73a50890e1bc4bac3cb90df9a86826 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Tue, 9 Jul 2019 11:50:54 -0700 Subject: [PATCH 133/332] Add hooks to allow lazyLoad TensorFlow public API. - Updates existing DeprecationWrapper with the ability to import modules only when they are referred. - Updates how TensorFlow generates public API. Wraps all generated TensorFlow __init__.py modules with this enhanced wrapper. To enable lazy-loading in the future, toggle _LAZY_LOADING flag in create_python_api.py. Once lazy loading is enabled, the wrapper will have the following behaviors: - dir() will always return module?s attributes. - __all__ will always return all public APIs. - __dict__ will be populated as attributes are being referred. - After wrapper instance is created, to add more attributes, use setattr(import does not explicitly call setattr) to make sure dir, __all__, __dict__ are updated. - import * will work as expected. Built and tested with pip package. PiperOrigin-RevId: 257240535 --- tensorflow/api_template.__init__.py | 34 +-- tensorflow/api_template_v1.__init__.py | 30 +-- tensorflow/compat_template.__init__.py | 12 +- tensorflow/compat_template_v1.__init__.py | 7 +- tensorflow/python/BUILD | 4 +- .../tools/api/generator/create_python_api.py | 105 +++++---- .../api/generator/create_python_api_test.py | 22 +- tensorflow/python/util/deprecation_wrapper.py | 133 +----------- .../python/util/deprecation_wrapper_test.py | 75 ------- tensorflow/python/util/module_wrapper.py | 205 ++++++++++++++++++ tensorflow/python/util/module_wrapper_test.py | 136 ++++++++++++ .../tools/api/tests/deprecation_test.py | 29 +-- tensorflow/tools/api/tests/module_test.py | 5 + .../tools/pip_package/build_pip_package.sh | 8 +- .../virtual_root_template_v1.__init__.py | 22 -- .../virtual_root_template_v2.__init__.py | 28 --- 16 files changed, 490 insertions(+), 365 deletions(-) delete mode 100644 tensorflow/python/util/deprecation_wrapper_test.py create mode 100644 tensorflow/python/util/module_wrapper.py create mode 100644 tensorflow/python/util/module_wrapper_test.py diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py index b75ec148ae8..8d5e43b672b 100644 --- a/tensorflow/api_template.__init__.py +++ b/tensorflow/api_template.__init__.py @@ -41,12 +41,15 @@ from tensorflow.python.tools import module_util as _module_util # API IMPORTS PLACEHOLDER +# WRAPPER_PLACEHOLDER + # Make sure directory containing top level submodules is in # the __path__ so that "from tensorflow.foo import bar" works. # We're using bitwise, but there's nothing special about that. -_API_MODULE = bitwise # pylint: disable=undefined-variable -_current_module = _sys.modules[__name__] +_API_MODULE = sys.modules[__name__].bitwise # pylint: disable=undefined-variable _tf_api_dir = _os.path.dirname(_os.path.dirname(_API_MODULE.__file__)) +_current_module = _sys.modules[__name__] + if not hasattr(_current_module, '__path__'): __path__ = [_tf_api_dir] elif _tf_api_dir not in __path__: @@ -57,6 +60,7 @@ try: from tensorboard.summary._tf import summary _current_module.__path__ = ( [_module_util.get_parent_dir(summary)] + _current_module.__path__) + setattr(_current_module, "summary", summary) except ImportError: _logging.warning( "Limited tf.summary API due to missing TensorBoard installation.") @@ -65,6 +69,7 @@ try: from tensorflow_estimator.python.estimator.api._v2 import estimator _current_module.__path__ = ( [_module_util.get_parent_dir(estimator)] + _current_module.__path__) + setattr(_current_module, "estimator", estimator) except ImportError: pass @@ -72,6 +77,7 @@ try: from tensorflow.python.keras.api._v2 import keras _current_module.__path__ = ( [_module_util.get_parent_dir(keras)] + _current_module.__path__) + setattr(_current_module, "keras", keras) except ImportError: pass @@ -122,25 +128,17 @@ if _running_from_pip_package(): # pylint: disable=undefined-variable try: del python - if '__all__' in vars(): - vars()['__all__'].remove('python') - del core - if '__all__' in vars(): - vars()['__all__'].remove('core') except NameError: - # Don't fail if these modules are not available. - # For e.g. this file will be originally placed under tensorflow/_api/v1 which - # does not have 'python', 'core' directories. Then, it will be copied - # to tensorflow/ which does have these two directories. pass -# Similarly for compiler. Do it separately to make sure we do this even if the -# others don't exist. +try: + del core +except NameError: + pass try: del compiler - if '__all__' in vars(): - vars()['__all__'].remove('compiler') except NameError: pass +# pylint: enable=undefined-variable # Add module aliases if hasattr(_current_module, 'keras'): @@ -148,6 +146,10 @@ if hasattr(_current_module, 'keras'): metrics = keras.metrics optimizers = keras.optimizers initializers = keras.initializers + setattr(_current_module, "losses", losses) + setattr(_current_module, "metrics", metrics) + setattr(_current_module, "optimizers", optimizers) + setattr(_current_module, "initializers", initializers) -compat.v2.compat.v1 = compat.v1 +_current_module.compat.v2.compat.v1 = _current_module.compat.v1 # pylint: enable=undefined-variable diff --git a/tensorflow/api_template_v1.__init__.py b/tensorflow/api_template_v1.__init__.py index 4fa92b07051..6d1c40a2428 100644 --- a/tensorflow/api_template_v1.__init__.py +++ b/tensorflow/api_template_v1.__init__.py @@ -30,10 +30,12 @@ from tensorflow.python.tools import module_util as _module_util # API IMPORTS PLACEHOLDER +# WRAPPER_PLACEHOLDER + # Make sure directory containing top level submodules is in # the __path__ so that "from tensorflow.foo import bar" works. # We're using bitwise, but there's nothing special about that. -_API_MODULE = bitwise # pylint: disable=undefined-variable +_API_MODULE = _sys.modules[__name__].bitwise # pylint: disable=undefined-variable _current_module = _sys.modules[__name__] _tf_api_dir = _os.path.dirname(_os.path.dirname(_API_MODULE.__file__)) if not hasattr(_current_module, '__path__'): @@ -46,6 +48,7 @@ try: from tensorflow_estimator.python.estimator.api._v1 import estimator _current_module.__path__ = ( [_module_util.get_parent_dir(estimator)] + _current_module.__path__) + setattr(_current_module, "estimator", estimator) except ImportError: pass @@ -53,6 +56,7 @@ try: from tensorflow.python.keras.api._v1 import keras _current_module.__path__ = ( [_module_util.get_parent_dir(keras)] + _current_module.__path__) + setattr(_current_module, "keras", keras) except ImportError: pass @@ -77,9 +81,8 @@ if '__all__' in vars(): from tensorflow.python.platform import flags # pylint: disable=g-import-not-at-top # The 'app' module will be imported as part of the placeholder section above. -app.flags = flags # pylint: disable=undefined-variable -if '__all__' in vars(): - vars()['__all__'].append('flags') +_current_module.app.flags = flags # pylint: disable=undefined-variable +setattr(_current_module, "flags", flags) # Load all plugin libraries from site-packages/tensorflow-plugins if we are # running under pip. @@ -122,25 +125,16 @@ if _running_from_pip_package(): # pylint: disable=undefined-variable try: del python - if '__all__' in vars(): - vars()['__all__'].remove('python') - del core - if '__all__' in vars(): - vars()['__all__'].remove('core') except NameError: - # Don't fail if these modules are not available. - # For e.g. this file will be originally placed under tensorflow/_api/v1 which - # does not have 'python', 'core' directories. Then, it will be copied - # to tensorflow/ which does have these two directories. pass -# Similarly for compiler. Do it separately to make sure we do this even if the -# others don't exist. +try: + del core +except NameError: + pass try: del compiler - if '__all__' in vars(): - vars()['__all__'].remove('compiler') except NameError: pass -compat.v2.compat.v1 = compat.v1 +_current_module.compat.v2.compat.v1 = _current_module.compat.v1 # pylint: enable=undefined-variable diff --git a/tensorflow/compat_template.__init__.py b/tensorflow/compat_template.__init__.py index ad2443a0c32..b830af58832 100644 --- a/tensorflow/compat_template.__init__.py +++ b/tensorflow/compat_template.__init__.py @@ -28,12 +28,16 @@ from tensorflow.python.tools import module_util as _module_util # API IMPORTS PLACEHOLDER +# WRAPPER_PLACEHOLDER + # Hook external TensorFlow modules. _current_module = _sys.modules[__name__] try: from tensorboard.summary._tf import summary _current_module.__path__ = ( [_module_util.get_parent_dir(summary)] + _current_module.__path__) + # Make sure we get the correct summary module with lazy loading + setattr(_current_module, "summary", summary) except ImportError: _logging.warning( "Limited tf.compat.v2.summary API due to missing TensorBoard " @@ -43,6 +47,7 @@ try: from tensorflow_estimator.python.estimator.api._v2 import estimator _current_module.__path__ = ( [_module_util.get_parent_dir(estimator)] + _current_module.__path__) + setattr(_current_module, "estimator", estimator) except ImportError: pass @@ -50,6 +55,7 @@ try: from tensorflow.python.keras.api._v2 import keras _current_module.__path__ = ( [_module_util.get_parent_dir(keras)] + _current_module.__path__) + setattr(_current_module, "keras", keras) except ImportError: pass @@ -61,11 +67,15 @@ except ImportError: # # This make this one symbol available directly. from tensorflow.python.compat.v2_compat import enable_v2_behavior # pylint: disable=g-import-not-at-top +setattr(_current_module, "enable_v2_behavior", enable_v2_behavior) # Add module aliases -_current_module = _sys.modules[__name__] if hasattr(_current_module, 'keras'): losses = keras.losses metrics = keras.metrics optimizers = keras.optimizers initializers = keras.initializers + setattr(_current_module, "losses", losses) + setattr(_current_module, "metrics", metrics) + setattr(_current_module, "optimizers", optimizers) + setattr(_current_module, "initializers", initializers) diff --git a/tensorflow/compat_template_v1.__init__.py b/tensorflow/compat_template_v1.__init__.py index 23c722edef7..48374b766b7 100644 --- a/tensorflow/compat_template_v1.__init__.py +++ b/tensorflow/compat_template_v1.__init__.py @@ -27,12 +27,15 @@ from tensorflow.python.tools import module_util as _module_util # API IMPORTS PLACEHOLDER +# WRAPPER_PLACEHOLDER + # Hook external TensorFlow modules. _current_module = _sys.modules[__name__] try: from tensorflow_estimator.python.estimator.api._v1 import estimator _current_module.__path__ = ( [_module_util.get_parent_dir(estimator)] + _current_module.__path__) + setattr(_current_module, "estimator", estimator) except ImportError: pass @@ -40,9 +43,11 @@ try: from tensorflow.python.keras.api._v1 import keras _current_module.__path__ = ( [_module_util.get_parent_dir(keras)] + _current_module.__path__) + setattr(_current_module, "keras", keras) except ImportError: pass from tensorflow.python.platform import flags # pylint: disable=g-import-not-at-top -app.flags = flags # pylint: disable=undefined-variable +_current_module.app.flags = flags # pylint: disable=undefined-variable +setattr(_current_module, "flags", flags) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index a2ee9e07458..915f3518873 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -4545,9 +4545,9 @@ tf_py_test( ) tf_py_test( - name = "deprecation_wrapper_test", + name = "module_wrapper_test", size = "small", - srcs = ["util/deprecation_wrapper_test.py"], + srcs = ["util/module_wrapper_test.py"], additional_deps = [ ":client_testlib", ":util", diff --git a/tensorflow/python/tools/api/generator/create_python_api.py b/tensorflow/python/tools/api/generator/create_python_api.py index 7dd3f97b79d..aeeec69cec8 100644 --- a/tensorflow/python/tools/api/generator/create_python_api.py +++ b/tensorflow/python/tools/api/generator/create_python_api.py @@ -48,15 +48,48 @@ _GENERATED_FILE_HEADER = """# This file is MACHINE GENERATED! Do not edit. from __future__ import print_function as _print_function +import sys as _sys + """ _GENERATED_FILE_FOOTER = '\n\ndel _print_function\n' _DEPRECATION_FOOTER = """ -import sys as _sys -from tensorflow.python.util import deprecation_wrapper as _deprecation_wrapper +from tensorflow.python.util import module_wrapper as _module_wrapper -if not isinstance(_sys.modules[__name__], _deprecation_wrapper.DeprecationWrapper): - _sys.modules[__name__] = _deprecation_wrapper.DeprecationWrapper( - _sys.modules[__name__], "%s") +if not isinstance(_sys.modules[__name__], _module_wrapper.TFModuleWrapper): + _sys.modules[__name__] = _module_wrapper.TFModuleWrapper( + _sys.modules[__name__], "%s", public_apis=_PUBLIC_APIS, deprecation=%s, + has_lite=%s) +""" +_MODULE_TEXT_TEMPLATE = """ +# Inform pytype that this module is dynamically populated (b/111239204). +_LAZY_LOADING = False + +_PUBLIC_APIS = { +%s +} + +if _LAZY_LOADING: + _HAS_DYNAMIC_ATTRIBUTES = True +else: + import importlib as _importlib + for symbol, symbol_loc_info in _PUBLIC_APIS.items(): + if symbol_loc_info[0]: + attr = getattr(_importlib.import_module(symbol_loc_info[0]), symbol_loc_info[1]) + else: + attr = _importlib.import_module(symbol_loc_info[1]) + setattr(_sys.modules[__name__], symbol, attr) + try: + del symbol + except NameError: + pass + try: + del symbol_loc_info + except NameError: + pass + try: + del attr + except NameError: + pass """ @@ -76,17 +109,7 @@ def format_import(source_module_name, source_name, dest_name): Returns: An import statement string. """ - if source_module_name: - if source_name == dest_name: - return 'from %s import %s' % (source_module_name, source_name) - else: - return 'from %s import %s as %s' % ( - source_module_name, source_name, dest_name) - else: - if source_name == dest_name: - return 'import %s' % source_name - else: - return 'import %s as %s' % (source_name, dest_name) + return " '%s': ('%s', '%s')," % (dest_name, source_module_name, source_name) def get_canonical_import(import_set): @@ -129,7 +152,6 @@ class _ModuleInitCodeBuilder(object): lambda: collections.defaultdict(set)) self._dest_import_to_id = collections.defaultdict(int) # Names that start with underscore in the root module. - self._underscore_names_in_root = [] self._api_version = api_version def _check_already_imported(self, symbol_id, api_name): @@ -166,9 +188,6 @@ class _ModuleInitCodeBuilder(object): symbol_id = -1 if not symbol else id(symbol) self._check_already_imported(symbol_id, full_api_name) - if not dest_module_name and dest_name.startswith('_'): - self._underscore_names_in_root.append(dest_name) - # The same symbol can be available in multiple modules. # We store all possible ways of importing this symbol and later pick just # one. @@ -197,11 +216,13 @@ class _ModuleInitCodeBuilder(object): submodule = module_split[submodule_index-1] parent_module += '.' + submodule if parent_module else submodule import_from = self._output_package - if submodule_index > 0: - import_from += '.' + '.'.join(module_split[:submodule_index]) + import_from += '.' + '.'.join(module_split[:submodule_index + 1]) self.add_import( - None, import_from, module_split[submodule_index], - parent_module, module_split[submodule_index]) + symbol=None, + source_module_name='', + source_name=import_from, + dest_module_name=parent_module, + dest_name=module_split[submodule_index]) def build(self): """Get a map from destination module to __init__.py code for that module. @@ -221,26 +242,20 @@ class _ModuleInitCodeBuilder(object): get_canonical_import(imports) for _, imports in dest_name_to_imports.items() ] - module_text_map[dest_module] = '\n'.join(sorted(imports_list)) + module_text_map[dest_module] = _MODULE_TEXT_TEMPLATE % '\n'.join( + sorted(imports_list)) - # Expose exported symbols with underscores in root module - # since we import from it using * import. - underscore_names_str = ', '.join( - '\'%s\'' % name for name in self._underscore_names_in_root) - # We will always generate a root __init__.py file to let us handle * - # imports consistently. Be sure to have a root __init__.py file listed in - # the script outputs. - module_text_map[''] = module_text_map.get('', '') + ''' -_names_with_underscore = [%s] -__all__ = [_s for _s in dir() if not _s.startswith('_')] -__all__.extend([_s for _s in _names_with_underscore]) -''' % underscore_names_str - - if self._api_version == 1: # Add 1.* deprecations. - for dest_module, _ in self._module_imports.items(): + for dest_module, _ in self._module_imports.items(): + deprecation = 'False' + has_lite = 'False' + if self._api_version == 1: # Add 1.* deprecations. if not dest_module.startswith(_COMPAT_MODULE_PREFIX): - footer_text_map[dest_module] = _DEPRECATION_FOOTER % ( - dest_module) + deprecation = 'True' + # Workaround to make sure not load lite from lite/__init__.py + if not dest_module and 'lite' in self._module_imports: + has_lite = 'True' + footer_text_map[dest_module] = _DEPRECATION_FOOTER % ( + dest_module, deprecation, has_lite) return module_text_map, footer_text_map @@ -519,7 +534,11 @@ def create_api_files(output_files, packages, root_init_template, output_dir, _GENERATED_FILE_HEADER % get_module_docstring( module, packages[0], api_name) + text + _GENERATED_FILE_FOOTER) if module in deprecation_footer_map: - contents += deprecation_footer_map[module] + if '# WRAPPER_PLACEHOLDER' in contents: + contents = contents.replace('# WRAPPER_PLACEHOLDER', + deprecation_footer_map[module]) + else: + contents += deprecation_footer_map[module] with open(module_name_to_file_path[module], 'w') as fp: fp.write(contents) diff --git a/tensorflow/python/tools/api/generator/create_python_api_test.py b/tensorflow/python/tools/api/generator/create_python_api_test.py index 6e0970ec80a..98afd9a241f 100644 --- a/tensorflow/python/tools/api/generator/create_python_api_test.py +++ b/tensorflow/python/tools/api/generator/create_python_api_test.py @@ -67,15 +67,16 @@ class CreatePythonApiTest(test.TestCase): output_package='tensorflow', api_name='tensorflow', api_version=1) - expected_import = ( - 'from tensorflow.python.test_module ' - 'import test_op as test_op1') + expected_import = ('\'test_op1\': ' + '(\'tensorflow.python.test_module\',' + ' \'test_op\')') self.assertTrue( expected_import in str(imports), msg='%s not in %s' % (expected_import, str(imports))) - expected_import = ('from tensorflow.python.test_module ' - 'import test_op') + expected_import = ('\'test_op\': ' + '(\'tensorflow.python.test_module\',' + ' \'test_op\')') self.assertTrue( expected_import in str(imports), msg='%s not in %s' % (expected_import, str(imports))) @@ -89,8 +90,10 @@ class CreatePythonApiTest(test.TestCase): output_package='tensorflow', api_name='tensorflow', api_version=2) - expected_import = ('from tensorflow.python.test_module ' - 'import TestClass') + expected_import = ( + '\'NewTestClass\':' + ' (\'tensorflow.python.test_module\',' + ' \'TestClass\')') self.assertTrue( 'TestClass' in str(imports), msg='%s not in %s' % (expected_import, str(imports))) @@ -101,8 +104,9 @@ class CreatePythonApiTest(test.TestCase): output_package='tensorflow', api_name='tensorflow', api_version=1) - expected = ('from tensorflow.python.test_module ' - 'import _TEST_CONSTANT') + expected = ('\'_TEST_CONSTANT\':' + ' (\'tensorflow.python.test_module\',' + ' \'_TEST_CONSTANT\')') self.assertTrue(expected in str(imports), msg='%s not in %s' % (expected, str(imports))) diff --git a/tensorflow/python/util/deprecation_wrapper.py b/tensorflow/python/util/deprecation_wrapper.py index 0bdaf1631da..2e0eee5ea32 100644 --- a/tensorflow/python/util/deprecation_wrapper.py +++ b/tensorflow/python/util/deprecation_wrapper.py @@ -12,138 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Provides wrapper for TensorFlow modules to support deprecation messages. +"""Compatibility wrapper for TensorFlow modules to support deprecation messages. -TODO(annarev): potentially merge with LazyLoader. +Please use module_wrapper instead. +TODO(yifeif): remove once no longer referred by estimator """ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import sys -import types +from tensorflow.python.util import module_wrapper -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import tf_decorator -from tensorflow.python.util import tf_inspect -from tensorflow.python.util import tf_stack -from tensorflow.tools.compatibility import all_renames_v2 - - -_PER_MODULE_WARNING_LIMIT = 1 - - -def get_rename_v2(name): - if name not in all_renames_v2.symbol_renames: - return None - return all_renames_v2.symbol_renames[name] - - -def _call_location(): - # We want to get stack frame 2 frames up from current frame, - # i.e. above _getattr__ and _call_location calls. - stack = tf_stack.extract_stack_file_and_line(max_length=3) - if not stack: # should never happen as we're in a function - return 'UNKNOWN' - frame = stack[0] - return '{}:{}'.format(frame.file, frame.line) - - -def contains_deprecation_decorator(decorators): - return any( - d.decorator_name == 'deprecated' for d in decorators) - - -def has_deprecation_decorator(symbol): - """Checks if given object has a deprecation decorator. - - We check if deprecation decorator is in decorators as well as - whether symbol is a class whose __init__ method has a deprecation - decorator. - Args: - symbol: Python object. - - Returns: - True if symbol has deprecation decorator. - """ - decorators, symbol = tf_decorator.unwrap(symbol) - if contains_deprecation_decorator(decorators): - return True - if tf_inspect.isfunction(symbol): - return False - if not tf_inspect.isclass(symbol): - return False - if not hasattr(symbol, '__init__'): - return False - init_decorators, _ = tf_decorator.unwrap(symbol.__init__) - return contains_deprecation_decorator(init_decorators) - - -class DeprecationWrapper(types.ModuleType): - """Wrapper for TensorFlow modules to support deprecation messages.""" - - def __init__(self, wrapped, module_name): # pylint: disable=super-on-old-class - super(DeprecationWrapper, self).__init__(wrapped.__name__) - self.__dict__.update(wrapped.__dict__) - # Prefix all local attributes with _dw_ so that we can - # handle them differently in attribute access methods. - self._dw_wrapped_module = wrapped - self._dw_module_name = module_name - # names we already checked for deprecation - self._dw_deprecated_checked = set() - self._dw_warning_count = 0 - - def __getattribute__(self, name): # pylint: disable=super-on-old-class - attr = super(DeprecationWrapper, self).__getattribute__(name) - if name.startswith('__') or name.startswith('_dw_'): - return attr - - if (self._dw_warning_count < _PER_MODULE_WARNING_LIMIT and - name not in self._dw_deprecated_checked): - - self._dw_deprecated_checked.add(name) - - if self._dw_module_name: - full_name = 'tf.%s.%s' % (self._dw_module_name, name) - else: - full_name = 'tf.%s' % name - rename = get_rename_v2(full_name) - if rename and not has_deprecation_decorator(attr): - call_location = _call_location() - # skip locations in Python source - if not call_location.startswith('<'): - logging.warning( - 'From %s: The name %s is deprecated. Please use %s instead.\n', - _call_location(), full_name, rename) - self._dw_warning_count += 1 - return attr - - def __setattr__(self, arg, val): # pylint: disable=super-on-old-class - if arg.startswith('_dw_'): - super(DeprecationWrapper, self).__setattr__(arg, val) - else: - setattr(self._dw_wrapped_module, arg, val) - self.__dict__[arg] = val - - def __dir__(self): - return dir(self._dw_wrapped_module) - - def __delattr__(self, name): # pylint: disable=super-on-old-class - if name.startswith('_dw_'): - super(DeprecationWrapper, self).__delattr__(name) - else: - delattr(self._dw_wrapped_module, name) - - def __repr__(self): - return self._dw_wrapped_module.__repr__() - - def __getstate__(self): - return self.__name__ - - def __setstate__(self, d): - # pylint: disable=protected-access - self.__init__( - sys.modules[d]._dw_wrapped_module, - sys.modules[d]._dw_module_name) - # pylint: enable=protected-access +# For backward compatibility for other pip packages that use this class. +DeprecationWrapper = module_wrapper.TFModuleWrapper diff --git a/tensorflow/python/util/deprecation_wrapper_test.py b/tensorflow/python/util/deprecation_wrapper_test.py deleted file mode 100644 index 84ff22c5937..00000000000 --- a/tensorflow/python/util/deprecation_wrapper_test.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorflow.python.util.deprecation_wrapper.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import logging -import types - -from tensorflow.python.platform import test -from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import deprecation_wrapper -from tensorflow.python.util import tf_inspect -from tensorflow.tools.compatibility import all_renames_v2 - -deprecation_wrapper._PER_MODULE_WARNING_LIMIT = 5 - - -class MockModule(types.ModuleType): - pass - - -class DeprecationWrapperTest(test.TestCase): - - def testWrapperIsAModule(self): - module = MockModule('test') - wrapped_module = deprecation_wrapper.DeprecationWrapper( - module, 'test') - self.assertTrue(tf_inspect.ismodule(wrapped_module)) - - @test.mock.patch.object(logging, 'warning', autospec=True) - def testDeprecationWarnings(self, mock_warning): - module = MockModule('test') - module.foo = 1 - module.bar = 2 - module.baz = 3 - all_renames_v2.symbol_renames['tf.test.bar'] = 'tf.bar2' - all_renames_v2.symbol_renames['tf.test.baz'] = 'tf.compat.v1.baz' - - wrapped_module = deprecation_wrapper.DeprecationWrapper( - module, 'test') - self.assertTrue(tf_inspect.ismodule(wrapped_module)) - - self.assertEqual(0, mock_warning.call_count) - bar = wrapped_module.bar - self.assertEqual(1, mock_warning.call_count) - foo = wrapped_module.foo - self.assertEqual(1, mock_warning.call_count) - baz = wrapped_module.baz - self.assertEqual(2, mock_warning.call_count) - baz = wrapped_module.baz - self.assertEqual(2, mock_warning.call_count) - - # Check that values stayed the same - self.assertEqual(module.foo, foo) - self.assertEqual(module.bar, bar) - self.assertEqual(module.baz, baz) - - -if __name__ == '__main__': - test.main() diff --git a/tensorflow/python/util/module_wrapper.py b/tensorflow/python/util/module_wrapper.py new file mode 100644 index 00000000000..aa232d58495 --- /dev/null +++ b/tensorflow/python/util/module_wrapper.py @@ -0,0 +1,205 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides wrapper for TensorFlow modules.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import importlib +import sys +import types + +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util import tf_decorator +from tensorflow.python.util import tf_inspect +from tensorflow.python.util import tf_stack +from tensorflow.tools.compatibility import all_renames_v2 + + +_PER_MODULE_WARNING_LIMIT = 1 + + +def get_rename_v2(name): + if name not in all_renames_v2.symbol_renames: + return None + return all_renames_v2.symbol_renames[name] + + +def _call_location(): + # We want to get stack frame 2 frames up from current frame, + # i.e. above _getattr__ and _call_location calls. + stack = tf_stack.extract_stack_file_and_line(max_length=3) + if not stack: # should never happen as we're in a function + return 'UNKNOWN' + frame = stack[0] + return '{}:{}'.format(frame.file, frame.line) + + +def contains_deprecation_decorator(decorators): + return any( + d.decorator_name == 'deprecated' for d in decorators) + + +def has_deprecation_decorator(symbol): + """Checks if given object has a deprecation decorator. + + We check if deprecation decorator is in decorators as well as + whether symbol is a class whose __init__ method has a deprecation + decorator. + Args: + symbol: Python object. + + Returns: + True if symbol has deprecation decorator. + """ + decorators, symbol = tf_decorator.unwrap(symbol) + if contains_deprecation_decorator(decorators): + return True + if tf_inspect.isfunction(symbol): + return False + if not tf_inspect.isclass(symbol): + return False + if not hasattr(symbol, '__init__'): + return False + init_decorators, _ = tf_decorator.unwrap(symbol.__init__) + return contains_deprecation_decorator(init_decorators) + + +class TFModuleWrapper(types.ModuleType): + """Wrapper for TF modules to support deprecation messages and lazyloading.""" + + def __init__( # pylint: disable=super-on-old-class + self, + wrapped, + module_name, + public_apis=None, + deprecation=True, + has_lite=False): # pylint: enable=super-on-old-class + super(TFModuleWrapper, self).__init__(wrapped.__name__) + self.__dict__.update(wrapped.__dict__) + # Prefix all local attributes with _tfmw_ so that we can + # handle them differently in attribute access methods. + self._tfmw_wrapped_module = wrapped + self._tfmw_module_name = module_name + self._tfmw_public_apis = public_apis + self._tfmw_print_deprecation_warnings = deprecation + self._tfmw_has_lite = has_lite + # Set __all__ so that import * work for lazy loaded modules + if self._tfmw_public_apis: + self._tfmw_wrapped_module.__all__ = list(self._tfmw_public_apis.keys()) + self.__all__ = list(self._tfmw_public_apis.keys()) + # names we already checked for deprecation + self._tfmw_deprecated_checked = set() + self._tfmw_warning_count = 0 + + def _tfmw_add_deprecation_warning(self, name, attr): + """Print deprecation warning for attr with given name if necessary.""" + if (self._tfmw_warning_count < _PER_MODULE_WARNING_LIMIT and + name not in self._tfmw_deprecated_checked): + + self._tfmw_deprecated_checked.add(name) + + if self._tfmw_module_name: + full_name = 'tf.%s.%s' % (self._tfmw_module_name, name) + else: + full_name = 'tf.%s' % name + rename = get_rename_v2(full_name) + if rename and not has_deprecation_decorator(attr): + call_location = _call_location() + # skip locations in Python source + if not call_location.startswith('<'): + logging.warning( + 'From %s: The name %s is deprecated. Please use %s instead.\n', + _call_location(), full_name, rename) + self._tfmw_warning_count += 1 + + def _tfmw_import_module(self, name): + symbol_loc_info = self._tfmw_public_apis[name] + if symbol_loc_info[0]: + module = importlib.import_module(symbol_loc_info[0]) + attr = getattr(module, symbol_loc_info[1]) + else: + attr = importlib.import_module(symbol_loc_info[1]) + setattr(self._tfmw_wrapped_module, name, attr) + self.__dict__[name] = attr + return attr + + def __getattribute__(self, name): # pylint: disable=super-on-old-class + # Workaround to make sure we do not import from tensorflow/lite/__init__.py + if name == 'lite': + if self._tfmw_has_lite: + attr = self._tfmw_import_module(name) + setattr(self._tfmw_wrapped_module, 'lite', attr) + return attr + + attr = super(TFModuleWrapper, self).__getattribute__(name) + if name.startswith('__') or name.startswith('_tfmw_'): + return attr + + if self._tfmw_print_deprecation_warnings: + self._tfmw_add_deprecation_warning(name, attr) + return attr + + def __getattr__(self, name): + try: + attr = getattr(self._tfmw_wrapped_module, name) + except AttributeError as e: + if not self._tfmw_public_apis: + raise e + if name not in self._tfmw_public_apis: + raise e + attr = self._tfmw_import_module(name) + + if self._tfmw_print_deprecation_warnings: + self._tfmw_add_deprecation_warning(name, attr) + return attr + + def __setattr__(self, arg, val): # pylint: disable=super-on-old-class + if not arg.startswith('_tfmw_'): + setattr(self._tfmw_wrapped_module, arg, val) + self.__dict__[arg] = val + if arg not in self.__all__ and arg != '__all__': + self.__all__.append(arg) + super(TFModuleWrapper, self).__setattr__(arg, val) + + def __dir__(self): + if self._tfmw_public_apis: + return list( + set(self._tfmw_public_apis.keys()).union( + set([ + attr for attr in dir(self._tfmw_wrapped_module) + if not attr.startswith('_') + ]))) + else: + return dir(self._tfmw_wrapped_module) + + def __delattr__(self, name): # pylint: disable=super-on-old-class + if name.startswith('_tfmw_'): + super(TFModuleWrapper, self).__delattr__(name) + else: + delattr(self._tfmw_wrapped_module, name) + + def __repr__(self): + return self._tfmw_wrapped_module.__repr__() + + def __getstate__(self): + return self.__name__ + + def __setstate__(self, d): + # pylint: disable=protected-access + self.__init__(sys.modules[d]._tfmw_wrapped_module, + sys.modules[d]._tfmw_module_name) + # pylint: enable=protected-access diff --git a/tensorflow/python/util/module_wrapper_test.py b/tensorflow/python/util/module_wrapper_test.py new file mode 100644 index 00000000000..582e98abdfa --- /dev/null +++ b/tensorflow/python/util/module_wrapper_test.py @@ -0,0 +1,136 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tensorflow.python.util.module_wrapper.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import logging +import types + +from tensorflow.python.platform import test +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util import module_wrapper +from tensorflow.python.util import tf_inspect +from tensorflow.tools.compatibility import all_renames_v2 + +module_wrapper._PER_MODULE_WARNING_LIMIT = 5 + + +class MockModule(types.ModuleType): + pass + + +class DeprecationWrapperTest(test.TestCase): + + def testWrapperIsAModule(self): + module = MockModule('test') + wrapped_module = module_wrapper.TFModuleWrapper(module, 'test') + self.assertTrue(tf_inspect.ismodule(wrapped_module)) + + @test.mock.patch.object(logging, 'warning', autospec=True) + def testDeprecationWarnings(self, mock_warning): + module = MockModule('test') + module.foo = 1 + module.bar = 2 + module.baz = 3 + all_renames_v2.symbol_renames['tf.test.bar'] = 'tf.bar2' + all_renames_v2.symbol_renames['tf.test.baz'] = 'tf.compat.v1.baz' + + wrapped_module = module_wrapper.TFModuleWrapper(module, 'test') + self.assertTrue(tf_inspect.ismodule(wrapped_module)) + + self.assertEqual(0, mock_warning.call_count) + bar = wrapped_module.bar + self.assertEqual(1, mock_warning.call_count) + foo = wrapped_module.foo + self.assertEqual(1, mock_warning.call_count) + baz = wrapped_module.baz # pylint: disable=unused-variable + self.assertEqual(2, mock_warning.call_count) + baz = wrapped_module.baz + self.assertEqual(2, mock_warning.call_count) + + # Check that values stayed the same + self.assertEqual(module.foo, foo) + self.assertEqual(module.bar, bar) + + +class LazyLoadingWrapperTest(test.TestCase): + + def testLazyLoad(self): + module = MockModule('test') + apis = {'cmd': ('', 'cmd'), 'ABCMeta': ('abc', 'ABCMeta')} + wrapped_module = module_wrapper.TFModuleWrapper( + module, 'test', public_apis=apis, deprecation=False) + import cmd as _cmd # pylint: disable=g-import-not-at-top + from abc import ABCMeta as _ABCMeta # pylint: disable=g-import-not-at-top, g-importing-member + self.assertEqual(wrapped_module.cmd, _cmd) + self.assertEqual(wrapped_module.ABCMeta, _ABCMeta) + + def testLazyLoadLocalOverride(self): + # Test that we can override and add fields to the wrapped module. + module = MockModule('test') + apis = {'cmd': ('', 'cmd')} + wrapped_module = module_wrapper.TFModuleWrapper( + module, 'test', public_apis=apis, deprecation=False) + import cmd as _cmd # pylint: disable=g-import-not-at-top + self.assertEqual(wrapped_module.cmd, _cmd) + setattr(wrapped_module, 'cmd', 1) + setattr(wrapped_module, 'cgi', 2) + self.assertEqual(wrapped_module.cmd, 1) # override + self.assertEqual(wrapped_module.cgi, 2) # add + + def testLazyLoadDict(self): + # Test that we can override and add fields to the wrapped module. + module = MockModule('test') + apis = {'cmd': ('', 'cmd')} + wrapped_module = module_wrapper.TFModuleWrapper( + module, 'test', public_apis=apis, deprecation=False) + import cmd as _cmd # pylint: disable=g-import-not-at-top + # At first cmd key does not exist in __dict__ + self.assertNotIn('cmd', wrapped_module.__dict__) + # After it is referred (lazyloaded), it gets added to __dict__ + wrapped_module.cmd # pylint: disable=pointless-statement + self.assertEqual(wrapped_module.__dict__['cmd'], _cmd) + # When we call setattr, it also gets added to __dict__ + setattr(wrapped_module, 'cmd2', _cmd) + self.assertEqual(wrapped_module.__dict__['cmd2'], _cmd) + + def testLazyLoadWildcardImport(self): + # Test that public APIs are in __all__. + module = MockModule('test') + module._should_not_be_public = 5 + apis = {'cmd': ('', 'cmd')} + wrapped_module = module_wrapper.TFModuleWrapper( + module, 'test', public_apis=apis, deprecation=False) + setattr(wrapped_module, 'hello', 1) + self.assertIn('hello', wrapped_module.__all__) + self.assertIn('cmd', wrapped_module.__all__) + self.assertNotIn('_should_not_be_public', wrapped_module.__all__) + + def testLazyLoadCorrectLiteModule(self): + # If set, always load lite module from public API list. + module = MockModule('test') + apis = {'lite': ('', 'cmd')} + module.lite = 5 + import cmd as _cmd # pylint: disable=g-import-not-at-top + wrapped_module = module_wrapper.TFModuleWrapper( + module, 'test', public_apis=apis, deprecation=False, has_lite=True) + self.assertEqual(wrapped_module.lite, _cmd) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/tools/api/tests/deprecation_test.py b/tensorflow/tools/api/tests/deprecation_test.py index 8f6748f5787..3a5cf0d043e 100644 --- a/tensorflow/tools/api/tests/deprecation_test.py +++ b/tensorflow/tools/api/tests/deprecation_test.py @@ -23,9 +23,9 @@ import tensorflow as tf from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.util import deprecation_wrapper +from tensorflow.python.util import module_wrapper -deprecation_wrapper._PER_MODULE_WARNING_LIMIT = 5 +module_wrapper._PER_MODULE_WARNING_LIMIT = 5 class DeprecationTest(test.TestCase): @@ -38,9 +38,8 @@ class DeprecationTest(test.TestCase): tf.tables_initializer() self.assertEqual(1, mock_warning.call_count) - self.assertRegexpMatches( - mock_warning.call_args[0][1], - "deprecation_test.py:") + self.assertRegexpMatches(mock_warning.call_args[0][1], + "module_wrapper.py:") self.assertRegexpMatches( mock_warning.call_args[0][2], r"tables_initializer") self.assertRegexpMatches( @@ -60,9 +59,8 @@ class DeprecationTest(test.TestCase): tf.ragged.RaggedTensorValue(value, row_splits) self.assertEqual(1, mock_warning.call_count) - self.assertRegexpMatches( - mock_warning.call_args[0][1], - "deprecation_test.py:") + self.assertRegexpMatches(mock_warning.call_args[0][1], + "module_wrapper.py:") self.assertRegexpMatches( mock_warning.call_args[0][2], r"ragged.RaggedTensorValue") self.assertRegexpMatches( @@ -84,9 +82,8 @@ class DeprecationTest(test.TestCase): tf.sparse_mask(array, mask_indices) self.assertEqual(1, mock_warning.call_count) - self.assertRegexpMatches( - mock_warning.call_args[0][1], - "deprecation_test.py:") + self.assertRegexpMatches(mock_warning.call_args[0][1], + "module_wrapper.py:") self.assertRegexpMatches( mock_warning.call_args[0][2], r"sparse_mask") self.assertRegexpMatches( @@ -103,9 +100,8 @@ class DeprecationTest(test.TestCase): tf.VarLenFeature(tf.dtypes.int32) self.assertEqual(1, mock_warning.call_count) - self.assertRegexpMatches( - mock_warning.call_args[0][1], - "deprecation_test.py:") + self.assertRegexpMatches(mock_warning.call_args[0][1], + "module_wrapper.py:") self.assertRegexpMatches( mock_warning.call_args[0][2], r"VarLenFeature") self.assertRegexpMatches( @@ -122,9 +118,8 @@ class DeprecationTest(test.TestCase): tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY # pylint: disable=pointless-statement self.assertEqual(1, mock_warning.call_count) - self.assertRegexpMatches( - mock_warning.call_args[0][1], - "deprecation_test.py:") + self.assertRegexpMatches(mock_warning.call_args[0][1], + "module_wrapper.py:") self.assertRegexpMatches( mock_warning.call_args[0][2], r"saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY") diff --git a/tensorflow/tools/api/tests/module_test.py b/tensorflow/tools/api/tests/module_test.py index 787df35ac30..257d558cda7 100644 --- a/tensorflow/tools/api/tests/module_test.py +++ b/tensorflow/tools/api/tests/module_test.py @@ -38,6 +38,11 @@ class ModuleTest(test.TestCase): def testDict(self): # Check that a few modules are in __dict__. + # pylint: disable=pointless-statement + tf.nn + tf.keras + tf.image + # pylint: enable=pointless-statement self.assertIn('nn', tf.__dict__) self.assertIn('keras', tf.__dict__) self.assertIn('image', tf.__dict__) diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh index 8c19ca010e9..5420769e25d 100755 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -178,15 +178,11 @@ function prepare_src() { # # import tensorflow as tf # - # which is not ok. We are removing the deprecation stuff by using sed and - # deleting the pattern that the wrapper uses (all lines between a line ending - # with _deprecation_wrapper -- the import line -- and a line containing - # _sys.modules[__name__] as the argument of a function -- the last line in - # the deprecation autogenerated pattern) + # which is not ok. We disable deprecation by using sed to toggle the flag # TODO(mihaimaruseac): When we move the API to root, remove this hack # Note: Can't do in place sed that works on all OS, so use a temp file instead sed \ - "/_deprecation_wrapper$/,/_sys.modules[__name__],/ d" \ + "s/deprecation=True/deprecation=False/g" \ "${TMPDIR}/tensorflow_core/__init__.py" > "${TMPDIR}/tensorflow_core/__init__.out" mv "${TMPDIR}/tensorflow_core/__init__.out" "${TMPDIR}/tensorflow_core/__init__.py" } diff --git a/tensorflow/virtual_root_template_v1.__init__.py b/tensorflow/virtual_root_template_v1.__init__.py index bb076759e60..785043a1a3f 100644 --- a/tensorflow/virtual_root_template_v1.__init__.py +++ b/tensorflow/virtual_root_template_v1.__init__.py @@ -98,28 +98,6 @@ for _m in _top_level_modules: # We still need all the names that are toplevel on tensorflow_core from tensorflow_core import * -# We also need to bring in keras if available in tensorflow_core -# Above import * doesn't import it as __all__ is updated before keras is hooked -try: - from tensorflow_core import keras -except ImportError as e: - pass - -# Similarly for estimator, but only if this file is not read via a -# import tensorflow_estimator (same reasoning as above when forwarding estimator -# separatedly from the rest of the top level modules) -if not _root_estimator: - try: - from tensorflow_core import estimator - except ImportError as e: - pass - -# And again for tensorboard (comes as summary) -try: - from tensorflow_core import summary -except ImportError as e: - pass - # In V1 API we need to print deprecation messages from tensorflow.python.util import deprecation_wrapper as _deprecation if not isinstance(_sys.modules[__name__], _deprecation.DeprecationWrapper): diff --git a/tensorflow/virtual_root_template_v2.__init__.py b/tensorflow/virtual_root_template_v2.__init__.py index bd212adf3d2..7d40733be7b 100644 --- a/tensorflow/virtual_root_template_v2.__init__.py +++ b/tensorflow/virtual_root_template_v2.__init__.py @@ -97,32 +97,4 @@ for _m in _top_level_modules: # We still need all the names that are toplevel on tensorflow_core from tensorflow_core import * -# We also need to bring in keras if available in tensorflow_core -# Above import * doesn't import it as __all__ is updated before keras is hooked -try: - from tensorflow_core import keras -except ImportError as e: - pass - -# Similarly for estimator, but only if this file is not read via a -# import tensorflow_estimator (same reasoning as above when forwarding estimator -# separatedly from the rest of the top level modules) -if not _root_estimator: - try: - from tensorflow_core import estimator - except ImportError as e: - pass - -# And again for tensorboard (comes as summary) -try: - from tensorflow_core import summary -except ImportError as e: - pass - -# Also import module aliases -try: - from tensorflow_core import losses, metrics, initializers, optimizers -except ImportError: - pass - # LINT.ThenChange(//tensorflow/virtual_root_template_v1.__init__.py.oss) From d37ebcee9f1b4f376fe280d9f5035a460c7689b8 Mon Sep 17 00:00:00 2001 From: Guangda Lai Date: Tue, 9 Jul 2019 11:52:46 -0700 Subject: [PATCH 134/332] Support FusedBatchNormV3 in generic layout optimizer. PiperOrigin-RevId: 257240886 --- .../generic_layout_optimizer_transposer.cc | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc index e9953f057aa..3d2b18d1326 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc @@ -1514,15 +1514,12 @@ Status UnaryGradTransposer::TransposeNode(TransposeContext* context, // Utils. bool IsDefaultLayoutSensitiveOp(const NodeDef& node) { - std::set default_layout_sensitive_ops = {"AvgPool", - "BiasAdd", - "Conv2D", - "DepthToSpace", - "FusedBatchNorm", - "FusedBatchNormV2", - "FusedConv2DBiasActivation", - "MaxPool", - "SpaceToDepth"}; + std::set default_layout_sensitive_ops = { + "AvgPool", "BiasAdd", + "Conv2D", "DepthToSpace", + "FusedBatchNorm", "FusedBatchNormV2", + "FusedBatchNormV3", "FusedConv2DBiasActivation", + "MaxPool", "SpaceToDepth"}; return default_layout_sensitive_ops.find(node.op()) != default_layout_sensitive_ops.end(); } From b1899424eae08c1f5b54b4e15b5dcfe0df585023 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 12:02:34 -0700 Subject: [PATCH 135/332] Delay the tf.gather using the batch_dims enabled C++ kernel. PiperOrigin-RevId: 257242802 --- tensorflow/python/ops/array_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 629ca6e9abf..041ae2d8526 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -3807,7 +3807,7 @@ def gather(params, A `Tensor`. Has the same type as `params`. """ del validate_indices - if compat.forward_compatible(2019, 7, 10): + if compat.forward_compatible(2019, 8, 10): if axis is None: axis = batch_dims if axis != 0: From 1059951b8e7445c9a8e09433fd22b885da35f3e3 Mon Sep 17 00:00:00 2001 From: Haoliang Zhang Date: Tue, 9 Jul 2019 12:08:22 -0700 Subject: [PATCH 136/332] Add dynamic_rnn into MLIR generated_examples test. PiperOrigin-RevId: 257243999 --- .../compiler/mlir/lite/tf_to_tfl_flatbuffer.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc index 759f93c94a3..647f1d75d7b 100644 --- a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc +++ b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc @@ -100,6 +100,16 @@ void AddTFToTFLConversionPasses(bool emit_builtin_tflite_ops, bool run_quantize, bool lower_tensor_list_ops, mlir::PassManager *pass_manager) { pass_manager->addPass(mlir::TFControlFlow::CreateRaiseTFControlFlowPass()); + + if (lower_tensor_list_ops) { + // Execute this pass before `CanonicalizerPass` in case some TensorList + // ops are constant folded into variant types. + // TODO(haoliang): Add this pass by default. + // TODO(b/137038401): Handle other ops that operate on variant types and + // improve error reporting. + pass_manager->addPass(mlir::TFL::CreateLowerStaticTensorListPass()); + } + // TODO(jpienaar): Revise post dialect constants. pass_manager->addPass(mlir::TF::CreateDecodeConstantPass()); // Canonicalization includes const folding, which is utilized here to optimize @@ -112,10 +122,6 @@ void AddTFToTFLConversionPasses(bool emit_builtin_tflite_ops, bool run_quantize, if (emit_builtin_tflite_ops) { // Prepare for TFLite dialect, rerun canonicalization, and then legalize to // the TFLite dialect. - // TODO(haoliang): Add this pass by default. - if (lower_tensor_list_ops) { - pass_manager->addPass(mlir::TFL::CreateLowerStaticTensorListPass()); - } pass_manager->addPass(mlir::TFL::CreatePrepareTFPass()); pass_manager->addPass(mlir::createCanonicalizerPass()); pass_manager->addPass(mlir::TFL::CreateLegalizeTFPass()); From 50c6158ee8de0bad7d818a0cdcbf0c55aa67cf4c Mon Sep 17 00:00:00 2001 From: Blake Hechtman Date: Tue, 9 Jul 2019 12:16:18 -0700 Subject: [PATCH 137/332] [XLA] Support fusion of trivial fusion nodes that only contain a parameter instruction. PiperOrigin-RevId: 257245425 --- .../compiler/xla/service/hlo_instructions.cc | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc index e2f4c30610a..754ccc1ff9f 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.cc +++ b/tensorflow/compiler/xla/service/hlo_instructions.cc @@ -1308,10 +1308,21 @@ void HloFusionInstruction::MergeFusionInstruction( unfused_instructions.push_back(fused_instruction); } } - CHECK(unfused_instructions.front() == cloned_fusion->fused_expression_root()); + + // If there are no unfused instructions, the fused computation must consist + // only of kParameter instructions. Make the operand of the corresponding + // parameter number the new root. + HloInstruction* unfused_root = + unfused_instructions.empty() + ? instruction_to_merge->mutable_operand( + instruction_to_merge->fused_instructions_computation() + ->root_instruction() + ->parameter_number()) + : unfused_instructions.front(); + CHECK(unfused_root == cloned_fusion->fused_expression_root() || + unfused_instructions.empty()); // Replace instruction_to_merge use of 'this' with unfused_root. - TF_CHECK_OK( - instruction_to_merge->ReplaceUseWith(this, unfused_instructions.front())); + TF_CHECK_OK(instruction_to_merge->ReplaceUseWith(this, unfused_root)); // Fuse 'unfused_instructions' into 'this'. for (auto& instruction : unfused_instructions) { FuseInstruction(instruction); @@ -1359,7 +1370,16 @@ void HloFusionInstruction::MergeFusionInstructionIntoMultiOutput( } } - HloInstruction* unfused_root = unfused_instructions.front(); + // If there are no unfused instructions, the fused computation must consist + // only of kParameter instructions. Make the operand of the corresponding + // parameter number the new root. + HloInstruction* unfused_root = + unfused_instructions.empty() + ? instruction_to_merge->mutable_operand( + instruction_to_merge->fused_instructions_computation() + ->root_instruction() + ->parameter_number()) + : unfused_instructions.front(); TF_CHECK_OK(instruction_to_merge->ReplaceAllUsesWith(unfused_root)); TF_CHECK_OK( @@ -1369,6 +1389,9 @@ void HloFusionInstruction::MergeFusionInstructionIntoMultiOutput( } // Fuse the root instruction and generate multiple outputs. + if (unfused_instructions.empty()) { + return; + } FuseInstructionIntoMultiOutput(unfused_root); TF_CHECK_OK(unfused_root->parent()->RemoveInstruction(unfused_root)); // The rest instructions are of normal fusing. From a1661765dd4cfdc45ed692b0de18ef3b6693ce20 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 13:07:55 -0700 Subject: [PATCH 138/332] Automated rollback of commit 0c9d4b5ea1ad7d948f006fcbd1398420a48904b9 PiperOrigin-RevId: 257255123 --- .../grappler/optimizers/constant_folding.cc | 231 ++++++------------ .../optimizers/constant_folding_test.cc | 105 ++------ .../core/grappler/optimizers/remapper.cc | 1 + 3 files changed, 92 insertions(+), 245 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index 8711d57dd24..2a593be9635 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -989,10 +989,11 @@ bool ConstantFolding::IsFoldable(const NodeDef& node, } } - // Don't fold nodes that have no outgoing edges except whitelisted nodes. - // Such nodes could be introduced by an earlier constant folding pass and are - // preserved in case users want to fetch their values; re-processing them - // would lead to an error of adding a duplicated node to graph. + // No need to (and don't) fold nodes that have no outgoing edges except + // whitelisted nodes. Such nodes could be introduced by an earlier constant + // folding pass and are preserved in case users want to fetch their values; + // re-processing them would lead to an error of adding a duplicated node + // to graph. const auto& outputs = node_map_->GetOutputs(node.name()); if (outputs.empty() && nodes_whitelist_.find(node.name()) == nodes_whitelist_.end()) { @@ -1028,7 +1029,6 @@ bool ConstantFolding::IsFoldable(const NodeDef& node, return false; } } - if (is_merge && !merge_has_constant_input) return false; // If we know the output shapes, make sure that the outputs are small enough // to materialize. @@ -1050,7 +1050,7 @@ bool ConstantFolding::IsFoldable(const NodeDef& node, } } - return true; + return !is_merge || merge_has_constant_input; } namespace { @@ -2827,162 +2827,83 @@ bool ConstantFolding::ConstantPushDown(GraphDef* optimized_graph, // / \ / \ // X Y C Y = leaves // - // where C is constant, X is non-constant, Y may be constant or non-constant, - // and '+' denotes an associative and commutative operator like addition or - // multiplication. This optimization pushes constants down in the tree to - // canonicalize it. Moreoever, in cases where the child node has a second - // constant input Y we will create a leaf node that can be folded, e.g. + // where C is constant and X is non-constant, and '+' denotes an + // associative and commutative operator like addition or multiplication. + // This optimization pushes constants down in the tree to canonicalize it. + // Moreoever, in cases where the child node has a second constant input Y + // we will create a leaf node that can be folded, e.g. // // Add(C1, Add(C2, X)) -> Add(X, Add(C1, C2)) -> Add(X, C1 + C2) // - // We also handle the non-commutative cases of subtraction and division - // by rotating the tree locally, e.g. - // Sub(C, Add(X, Y)) -> Sub(Sub(C, Y), X) - // Mul(C, Div(X, Y)) -> Mul(X, Div(C, Y)). - // - // Note: Don't touch BiasAdd since they can't handle vectors as their first + // TODO(rmlarsen): Handle non-associative/non-commutative operators like + // subtraction and division, as well as mixed subtraction/addition, + // division/multiplication. + // Don't touch BiasAdd since they can't handle vectors as their first // inputs. + if (has_fetch_ && (IsAdd(*node) || IsMul(*node)) && + NumNonControlInputs(*node) == 2) { + NodeDef* left_child = node_map_->GetNode(node->input(0)); + NodeDef* right_child = node_map_->GetNode(node->input(1)); + // One child must be constant, and the other the same op as the parent. + if (node->op() != left_child->op() && node->op() != right_child->op()) { + return false; + } + const bool left_child_is_constant = IsReallyConstant(*left_child); + const bool right_child_is_constant = IsReallyConstant(*right_child); + if (!left_child_is_constant && !right_child_is_constant) { + return false; + } + if (node->device() != left_child->device() || + node->device() != right_child->device()) { + return false; + } + NodeDef* op_child_node = left_child_is_constant ? right_child : left_child; + NodeDef* const_child_node = + left_child_is_constant ? left_child : right_child; + // Make sure that it is safe to change the value of the child node-> + if (op_child_node->input_size() < 2 || + nodes_to_preserve_.find(op_child_node->name()) != + nodes_to_preserve_.end() || + NumNonControlOutputs(*op_child_node, *node_map_) > 1) { + return false; + } - // Get parent op type. - const bool is_add = IsAdd(*node); - const bool is_mul = IsMul(*node); - const bool is_sub = IsSub(*node); - const bool is_div = IsDiv(*node); - const bool is_symmetric = is_add || is_mul; - if (!has_fetch_ || !(is_add || is_sub || is_mul || is_div) || - NumNonControlInputs(*node) != 2) { - return false; - } + // Identify the nodes to swap. + NodeDef* left_leaf = node_map_->GetNode(op_child_node->input(0)); + NodeDef* right_leaf = node_map_->GetNode(op_child_node->input(1)); + const bool left_leaf_is_constant = IsReallyConstant(*left_leaf); + const bool right_leaf_is_constant = IsReallyConstant(*right_leaf); + if (left_leaf_is_constant && right_leaf_is_constant) { + // Child is already foldable, leave it alone. + return false; + } + const int non_const_leaf_input = left_leaf_is_constant ? 1 : 0; + const int parent_const_input = left_child_is_constant ? 0 : 1; + const auto& child_output = node_map_->GetOutputs(op_child_node->name()); + if (child_output.find(const_child_node) != child_output.end()) { + // If there is a control edge from the child op to C, the transformation + // would create a cycle in the graph. We know that it must be a control + // edge. We can replace such a control edge with a control edge from A + // to C. + CHECK(MaybeRemoveControlInput(op_child_node->name(), const_child_node, + optimized_graph, node_map_.get())); + string other_leaf_input = left_leaf_is_constant ? op_child_node->input(0) + : op_child_node->input(1); + MaybeAddControlInput(other_leaf_input, const_child_node, optimized_graph, + node_map_.get()); + } - NodeDef* left_child = node_map_->GetNode(node->input(0)); - NodeDef* right_child = node_map_->GetNode(node->input(1)); - - const bool left_child_is_constant = IsReallyConstant(*left_child); - const bool right_child_is_constant = IsReallyConstant(*right_child); - if (!left_child_is_constant && !right_child_is_constant) { - return false; + // Swap the constant child with a non-constant leaf node. + node_map_->UpdateInput(node->name(), node->input(parent_const_input), + op_child_node->input(non_const_leaf_input)); + node_map_->UpdateInput(op_child_node->name(), + op_child_node->input(non_const_leaf_input), + node->input(parent_const_input)); + std::swap(*node->mutable_input(parent_const_input), + *op_child_node->mutable_input(non_const_leaf_input)); + return true; } - // Don't move nodes across devices. - if (node->device() != left_child->device() || - node->device() != right_child->device()) { - return false; - } - NodeDef* op_child = left_child_is_constant ? right_child : left_child; - NodeDef* const_child = left_child_is_constant ? left_child : right_child; - // Don't rewrite the tree if it might create cycles. - // TODO(rmlarsen): Add back handling of control dependency from op to C. - const auto& child_output = node_map_->GetOutputs(op_child->name()); - if (child_output.find(const_child) != child_output.end()) { - return false; - } - // Get child op type. - const bool is_child_add = IsAdd(*op_child); - const bool is_child_mul = IsMul(*op_child); - const bool is_child_sub = IsSub(*op_child); - const bool is_child_div = IsDiv(*op_child); - const bool is_add_sub = (is_add || is_sub) && (is_child_add || is_child_sub); - const bool is_mul_div = (is_mul || is_div) && (is_child_mul || is_child_div); - if (!is_add_sub && !is_mul_div) { - return false; - } - const bool is_child_symmetric = is_child_add || is_child_mul; - // Make sure that it is safe to change the value of the child node result. - if (op_child->input_size() < 2 || - nodes_to_preserve_.find(op_child->name()) != nodes_to_preserve_.end() || - NumNonControlOutputs(*op_child, *node_map_) > 1) { - return false; - } - // Do not rewrite integer expressions with subtraction or division. - if (!CheckAttrExists(*node, "T").ok()) return false; - DataType dtype = node->attr().at("T").type(); - if (!(is_symmetric && is_child_symmetric) && - !(DataTypeIsFloating(dtype) || DataTypeIsComplex(dtype))) { - return false; - } - - // Identify the nodes to swap. - NodeDef* left_leaf = node_map_->GetNode(op_child->input(0)); - NodeDef* right_leaf = node_map_->GetNode(op_child->input(1)); - const bool left_leaf_is_constant = IsReallyConstant(*left_leaf); - const bool right_leaf_is_constant = IsReallyConstant(*right_leaf); - if (left_leaf_is_constant && right_leaf_is_constant) { - // Child is already foldable, leave it alone. - return false; - } - // Don't move nodes across devices. - if (node->device() != left_leaf->device() || - node->device() != right_leaf->device()) { - return false; - } - // Get the node names corresponding to X, Y, and C. - const string input_x = - left_leaf_is_constant ? op_child->input(1) : op_child->input(0); - const string input_y = - input_x == op_child->input(0) ? op_child->input(1) : op_child->input(0); - const string input_c = - left_child_is_constant ? node->input(0) : node->input(1); - const string input_op = - left_child_is_constant ? node->input(1) : node->input(0); - - // Now we have identified the nodes to swap (non_const_leaf_input and - // const_child). - node_map_->UpdateInput(node->name(), input_c, input_x); - node_map_->AddOutput(input_c, op_child->name()); - if (input_x != input_y) { - node_map_->RemoveOutput(input_x, op_child->name()); - } - - if (is_symmetric && is_child_symmetric) { - // Easy case (only commutative ops). We always write this as one of - // + - // / \ - // X + - // / \ - // C Y - node->set_input(0, input_x); - node->set_input(1, input_op); - op_child->set_input(0, input_c); - op_child->set_input(1, input_y); - } else { - // More complicated case: When there are non-commutative operations like - // subtractions or divisions involved, we may have to rotate the tree - // and/or change op types. There are 6 non-trivial cases depending on - // the effective generalized "sign" of each of the three terms C, Y, and X. - // Here are the final trees we want to generate for those 6 cases: - // - // (CYX signs): ++- +-- -+- --+ +-+ -++ - // - - - - + + - // / \ / \ / \ / \ / \ / \ - // + X - X - X X + X - X - - // / \ / \ / \ / \ / \ / \ - // C Y C Y Y C Y C C Y Y C - // - NodeDef* non_const_leaf = left_leaf_is_constant ? right_leaf : left_leaf; - NodeDef* maybe_const_leaf = - non_const_leaf == right_leaf ? left_leaf : right_leaf; - - // First, let's determine the effective sign of each term in the original - // expression - auto is_leaf_negated = [&](const NodeDef* node) -> bool { - bool leaf_negated = !is_child_symmetric && (node == right_leaf); - bool child_negated = !is_symmetric && (op_child == right_child); - return leaf_negated != child_negated; - }; - const string symmetric_op = (is_add || is_sub) ? "Add" : "Mul"; - const string nonsymmetric_op = (is_add || is_sub) ? "Sub" : "Div"; - bool neg_c = !is_symmetric && (const_child == right_child); - bool neg_x = is_leaf_negated(non_const_leaf); - bool neg_y = is_leaf_negated(maybe_const_leaf); - // Rewrite the parent node. - node->set_op((neg_x || (neg_c && neg_y)) ? nonsymmetric_op : symmetric_op); - node->set_input(0, neg_x ? input_op : input_x); - node->set_input(1, neg_x ? input_x : input_op); - // Rewrite the child node. - op_child->set_op(neg_c != neg_y ? nonsymmetric_op : symmetric_op); - op_child->set_input(0, neg_c ? input_y : input_c); - op_child->set_input(1, neg_c ? input_c : input_y); - } - return true; + return false; } bool ConstantFolding::MulConvPushDown(GraphDef* optimized_graph, NodeDef* node, diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc index 3eba75c4214..3928fdff9ff 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc @@ -255,19 +255,20 @@ TEST_F(ConstantFoldingTest, SimpleFolding) { TEST_F(ConstantFoldingTest, AddTree) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - Output c1 = ops::Const(s.WithOpName("c1"), 1.0f, {1}); Output c2 = ops::Const(s.WithOpName("c2"), 2.0f, {2}); Output c3 = ops::Const(s.WithOpName("c3"), 3.0f, {2}); Output x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT, ops::Placeholder::Shape(TensorShape({2, 2}))); Output add_child = ops::Add(s.WithOpName("add_child"), c2, x); + Output c1 = ops::Const(s.WithOpName("c1").WithControlDependencies(add_child), + 1.0f, {1}); Output add_parent = ops::Add(s.WithOpName("add_parent"), c1, add_child); + Output y = ops::Placeholder(s.WithOpName("y"), DT_FLOAT, + ops::Placeholder::Shape(TensorShape({2, 2}))); Output c4 = ops::Const(s.WithOpName("c4"), 4.0f, {2}); Output c5 = ops::Const(s.WithOpName("c5"), 5.0f, {2}); Output c20 = ops::Const(s.WithOpName("c20"), 20.0f, {2}); - Output y = ops::Placeholder(s.WithOpName("y"), DT_FLOAT, - ops::Placeholder::Shape(TensorShape({2, 2}))); Output mul_child = ops::Mul(s.WithOpName("mul_child"), c4, y); Output mul_parent = ops::Mul(s.WithOpName("mul_parent"), c5, mul_child); Output addmul_child = ops::Add(s.WithOpName("addmul_child"), c4, x); @@ -297,16 +298,16 @@ TEST_F(ConstantFoldingTest, AddTree) { // / \ / \ // 5.0 y 4.0 5.0 - EXPECT_EQ(10, output.node_size()); + EXPECT_EQ(11, output.node_size()); for (const auto& node : output.node()) { if (node.name() == "add_child") { EXPECT_EQ("Const", node.op()); TensorProto t = node.attr().at("value").tensor(); - ASSERT_EQ(1, t.tensor_shape().dim_size()); + EXPECT_EQ(1, t.tensor_shape().dim_size()); EXPECT_EQ(2, t.tensor_shape().dim(0).size()); } else if (node.name() == "add_parent") { EXPECT_EQ("Add", node.op()); - ASSERT_EQ(2, node.input_size()); + EXPECT_EQ(2, node.input_size()); EXPECT_EQ("x", node.input(0)); EXPECT_EQ("add_child", node.input(1)); } else if (node.name() == "mul_child") { @@ -316,106 +317,30 @@ TEST_F(ConstantFoldingTest, AddTree) { EXPECT_EQ(2, t.tensor_shape().dim(0).size()); } else if (node.name() == "mul_parent") { EXPECT_EQ("Mul", node.op()); - ASSERT_EQ(2, node.input_size()); + EXPECT_EQ(2, node.input_size()); EXPECT_EQ("y", node.input(0)); EXPECT_EQ("mul_child", node.input(1)); } else if (node.name() == "addmul_child") { // Unchanged. EXPECT_EQ("Add", node.op()); - ASSERT_EQ(2, node.input_size()); + EXPECT_EQ(2, node.input_size()); EXPECT_EQ("c4", node.input(0)); EXPECT_EQ("x", node.input(1)); } } // Check that the result nodes have the expected value. - auto x_t = GenerateRandomTensor(TensorShape({2, 2})); - auto y_t = GenerateRandomTensor(TensorShape({2, 2})); - - std::vector fetch = {"add_parent", "mul_parent"}; - auto tensor_expected = - EvaluateNodes(item.graph, fetch, {{"x", x_t}, {"y", y_t}}); - ASSERT_EQ(fetch.size(), tensor_expected.size()); - fetch = {"add_parent", "mul_parent"}; - auto tensors = EvaluateNodes(output, fetch, {{"x", x_t}, {"y", y_t}}); - ASSERT_EQ(fetch.size(), tensors.size()); + std::vector fetch = {"c3", "c20"}; + auto tensor_expected = EvaluateNodes(item.graph, fetch); + EXPECT_EQ(fetch.size(), tensor_expected.size()); + fetch = {"add_child", "mul_child"}; + auto tensors = EvaluateNodes(output, fetch); + EXPECT_EQ(fetch.size(), tensors.size()); for (int i = 0; i < fetch.size(); i++) { test::ExpectTensorEqual(tensor_expected[i], tensors[i]); } } -TEST_F(ConstantFoldingTest, TreeCanonicalization) { - for (int is_add : {true, false}) { - for (int is_parent_commutative : {true, false}) { - for (int is_child_commutative : {true, false}) { - for (int is_left_child_const : {true, false}) { - for (int is_left_leaf_const : {true, false}) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - Output c2 = ops::Const(s.WithOpName("c2"), 2.0f, {2}); - Output c3 = ops::Const(s.WithOpName("c3"), 3.0f, {2}); - Output x = - ops::Placeholder(s.WithOpName("x"), DT_FLOAT, - ops::Placeholder::Shape(TensorShape({2, 2}))); - - auto get_op = [&](bool is_commutative, bool is_left_arg_cont, - const string& name, const Output& const_arg, - const Output non_const_arg) -> Output { - if (is_add) { - if (is_commutative) { - return ops::Add(s.WithOpName(name), - is_left_arg_cont ? const_arg : non_const_arg, - is_left_arg_cont ? non_const_arg : const_arg); - } else { - return ops::Sub(s.WithOpName(name), - is_left_arg_cont ? const_arg : non_const_arg, - is_left_arg_cont ? non_const_arg : const_arg); - } - } else { - if (is_commutative) { - return ops::Mul(s.WithOpName(name), - is_left_arg_cont ? const_arg : non_const_arg, - is_left_arg_cont ? non_const_arg : const_arg); - } else { - return ops::Div(s.WithOpName(name), - is_left_arg_cont ? const_arg : non_const_arg, - is_left_arg_cont ? non_const_arg : const_arg); - } - } - }; - - Output child = get_op(is_child_commutative, is_left_leaf_const, - "child", c2, x); - Output parent = get_op(is_parent_commutative, is_left_child_const, - "parent", c3, child); - GrapplerItem item; - item.fetch = {"parent"}; - TF_CHECK_OK(s.ToGraphDef(&item.graph)); - - ConstantFolding optimizer(/*cpu_device=*/nullptr); - GraphDef output; - Status status = - optimizer.Optimize(/*cluster=*/nullptr, item, &output); - TF_EXPECT_OK(status); - - // Check that the result nodes have the expected value. - auto x_t = GenerateRandomTensor(TensorShape({2, 2})); - std::vector fetch = {"parent"}; - auto tensor_expected = - EvaluateNodes(item.graph, fetch, {{"x", x_t}}); - ASSERT_EQ(fetch.size(), tensor_expected.size()); - fetch = {"parent"}; - auto tensors = EvaluateNodes(output, fetch, {{"x", x_t}}); - ASSERT_EQ(fetch.size(), tensors.size()); - for (int i = 0; i < fetch.size(); i++) { - test::ExpectTensorEqual(tensor_expected[i], tensors[i]); - } - } - } - } - } - } -} - TEST_F(ConstantFoldingTest, MulConvPushDownTest_Conv2D_ScalarConst) { for (string data_format : { "NHWC", diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc index 662dd08c523..3cfebdadcda 100644 --- a/tensorflow/core/grappler/optimizers/remapper.cc +++ b/tensorflow/core/grappler/optimizers/remapper.cc @@ -1634,6 +1634,7 @@ Status Remapper::Optimize(Cluster* cluster, const GrapplerItem& item, // Infer properties lazily in case they are not needed. if (!ctx.inferred_graph_properties && RequiresInferredShapes(ctx, i)) { const bool assume_valid_feeds = opt_level_ == RewriterConfig::AGGRESSIVE; + // TODO(rmlarsen): Get rid of tensor value copies. TF_RETURN_IF_ERROR(ctx.graph_properties.InferStatically( assume_valid_feeds, /*aggressive_shape_inference=*/false, From 62ce1e98bd05908a18aca1a2c9b9c0f5cc121647 Mon Sep 17 00:00:00 2001 From: Duncan Riach Date: Wed, 26 Jun 2019 21:20:43 -0700 Subject: [PATCH 139/332] Fix URL in docstring [skip ci] --- tensorflow/python/platform/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/python/platform/test.py b/tensorflow/python/platform/test.py index fdfdae093b9..a1669e6ad3a 100644 --- a/tensorflow/python/platform/test.py +++ b/tensorflow/python/platform/test.py @@ -15,6 +15,8 @@ """Testing. +See the [Testing](https://tensorflow.org/api_docs/python/tf/test) guide. + Note: `tf.compat.v1.test.mock` is an alias to the python `mock` or `unittest.mock` depending on the python version. """ From 8a6ba3c42bd878333aca2bd14474074ddfd13d13 Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Tue, 9 Jul 2019 13:10:54 -0700 Subject: [PATCH 140/332] Register numpy->TypeSpec converter. PiperOrigin-RevId: 257255699 --- tensorflow/python/framework/tensor_spec.py | 4 ++++ .../python/framework/tensor_spec_test.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/tensorflow/python/framework/tensor_spec.py b/tensorflow/python/framework/tensor_spec.py index 3ef8d443f1b..1e224e628c2 100644 --- a/tensorflow/python/framework/tensor_spec.py +++ b/tensorflow/python/framework/tensor_spec.py @@ -294,3 +294,7 @@ pywrap_tensorflow.RegisterType("TensorSpec", TensorSpec) type_spec.register_type_spec_from_value_converter( ops.Tensor, lambda tensor: TensorSpec(tensor.shape, tensor.dtype)) + +type_spec.register_type_spec_from_value_converter( + np.ndarray, + lambda array: TensorSpec(array.shape, array.dtype)) diff --git a/tensorflow/python/framework/tensor_spec_test.py b/tensorflow/python/framework/tensor_spec_test.py index 175aaebe67a..466f976ade6 100644 --- a/tensorflow/python/framework/tensor_spec_test.py +++ b/tensorflow/python/framework/tensor_spec_test.py @@ -24,9 +24,11 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import test_util +from tensorflow.python.framework import type_spec from tensorflow.python.ops import array_ops from tensorflow.python.platform import googletest @@ -146,6 +148,22 @@ class TensorSpecTest(test_util.TensorFlowTestCase): desc = tensor_spec.TensorSpec([1, 5], dtypes.float32, "test") self.assertEqual(pickle.loads(pickle.dumps(desc)), desc) + @test_util.deprecated_graph_mode_only + def testTypeSpecFromValue(self): + g = ops.Graph() + with g.as_default(): + v1 = np.array([1, 2, 3], np.int32) + t1 = constant_op.constant(v1) + + ops_before = g.get_operations() + + expected = tensor_spec.TensorSpec([3], dtypes.int32) + self.assertEqual(expected, type_spec.type_spec_from_value(v1)) + self.assertEqual(expected, type_spec.type_spec_from_value(t1)) + + # Check that creating TypeSpecs did not require building new Tensors. + self.assertLen(g.get_operations(), len(ops_before)) + class BoundedTensorSpecTest(test_util.TensorFlowTestCase): From f7154b226b1c01cf805cf0949801f4b6b6388f93 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Tue, 9 Jul 2019 13:20:13 -0700 Subject: [PATCH 141/332] [XLA:Python] Add an unsafe_buffer_pointer() to Python buffers. This is to enable users to experiment with calling their own custom kernels from Python, without necessarily going through the CustomCall APIs. As it says on the tin, it's unsafe and subtle to use correctly. PiperOrigin-RevId: 257257740 --- tensorflow/compiler/xla/python/BUILD | 1 + tensorflow/compiler/xla/python/xla.cc | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD index 5eed1aab07d..093152b975f 100644 --- a/tensorflow/compiler/xla/python/BUILD +++ b/tensorflow/compiler/xla/python/BUILD @@ -201,6 +201,7 @@ tf_pybind_extension( ":types", ":worker_thread", ":xrt", + "@com_google_absl//absl/base", "@com_google_absl//absl/hash", "@com_google_absl//absl/memory", "@com_google_absl//absl/synchronization", diff --git a/tensorflow/compiler/xla/python/xla.cc b/tensorflow/compiler/xla/python/xla.cc index 5dc102f785e..34ee5d7ea69 100644 --- a/tensorflow/compiler/xla/python/xla.cc +++ b/tensorflow/compiler/xla/python/xla.cc @@ -13,9 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include #include #include +#include "absl/base/casts.h" #include "absl/hash/hash.h" #include "absl/synchronization/mutex.h" #include "absl/types/optional.h" @@ -307,9 +309,22 @@ PYBIND11_MODULE(xla_extension, m) { .def("to_py", &PyLocalBuffer::ToPython) .def("shape", &PyLocalBuffer::on_host_shape) .def("device", &PyLocalBuffer::device_ordinal) - .def("is_deleted", [](const PyLocalBuffer& buffer) { - return buffer.DeviceBuffer() == nullptr; - }); + .def("is_deleted", + [](const PyLocalBuffer& buffer) { + return buffer.DeviceBuffer() == nullptr; + }) + .def("unsafe_buffer_pointer", + [](const PyLocalBuffer& buffer) -> StatusOr { + TF_ASSIGN_OR_RETURN(ShapedBuffer shaped_buffer, + buffer.AsShapedBuffer()); + if (shaped_buffer.on_device_shape().IsTuple()) { + return Unimplemented( + "unsafe_buffer_pointer is not implemented for tuple " + "buffers."); + } + return absl::bit_cast( + shaped_buffer.root_buffer().opaque()); + }); py::class_(m, "LocalExecutable") .def_static("Compile", &PyLocalExecutable::Compile, From f5ec649ad64321d09e9f06c376c4f511551ff3bd Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Tue, 9 Jul 2019 13:40:43 -0700 Subject: [PATCH 142/332] [XLA] Add a reduce-window test whose window only has base-dilation Also, delete window_util::IsInactiveWindowDimension which is unused. PiperOrigin-RevId: 257262250 --- .../compiler/xla/tests/reduce_window_test.cc | 20 +++++++++++++++++++ tensorflow/compiler/xla/window_util.cc | 6 ------ tensorflow/compiler/xla/window_util.h | 8 ++------ 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc index 531b76c1513..c5e1dbe7432 100644 --- a/tensorflow/compiler/xla/tests/reduce_window_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc @@ -1653,5 +1653,25 @@ ENTRY %reduce-window (parameter.0: f16[81,8], parameter.1: f16[]) -> f16[82,8] { EXPECT_TRUE(RunAndCompare(hlo_string, absl::nullopt)); } +XLA_TEST_F(ReduceWindowTextTest, R4OnlyDilation) { + const string hlo_string = R"( +HloModule R4OnlyDilation +mul { + lhs = f32[] parameter(0) + rhs = f32[] parameter(1) + ROOT mul = f32[] multiply(lhs, rhs) +} +ENTRY R4OnlyDilation { + operand = f32[2,2,2,2]{3,2,1,0} parameter(0) + constant = f32[] constant(1) + ROOT reduce-window = f32[3,3,3,3]{3,2,1,0} + reduce-window(operand, constant), + window={size=1x1x1x1 pad=0_0x0_0x0_0x0_0 lhs_dilate=2x2x2x2}, + to_apply=mul +} +)"; + EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{0.001})); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/window_util.cc b/tensorflow/compiler/xla/window_util.cc index f2e18311039..4f01325111c 100644 --- a/tensorflow/compiler/xla/window_util.cc +++ b/tensorflow/compiler/xla/window_util.cc @@ -198,12 +198,6 @@ bool HasDilation(const Window& window) { return HasBaseDilation(window) || HasWindowDilation(window); } -bool IsInactiveWindowDimension(const Window& window, int64 logical_dim) { - const WindowDimension& window_dim = window.dimensions(logical_dim); - return window_dim.size() == 1 && window_dim.stride() == 1 && - window_dim.padding_low() == 0 && window_dim.padding_high() == 0; -} - bool IsTrivialWindowDimension(const WindowDimension& window_dimension) { return window_dimension.size() == 1 && window_dimension.stride() == 1 && window_dimension.padding_low() == 0 && diff --git a/tensorflow/compiler/xla/window_util.h b/tensorflow/compiler/xla/window_util.h index e7099285c34..afb7d48f63f 100644 --- a/tensorflow/compiler/xla/window_util.h +++ b/tensorflow/compiler/xla/window_util.h @@ -58,12 +58,8 @@ bool HasDilation(const Window& window); bool HasWindowReversal(const Window& window); bool AllOrNoneReversed(const Window& window); -// Returns true if the given logical dimension is inactive in the sense that it -// has window bound 1, no striding and no padding. -bool IsInactiveWindowDimension(const Window& window, int64 logical_dim); - -// Returns true if the provided window dimension is trivial (inactive and has no -// dilation) +// Returns true if the provided window dimension is trivial in the sense that it +// has window bound 1, no striding, no padding and no dilation. bool IsTrivialWindowDimension(const WindowDimension& window_dimension); // Returns the new bound after dilation. From 83788ea45a7ea8cc37ad7e7b43f5bc474014e86c Mon Sep 17 00:00:00 2001 From: Haoliang Zhang Date: Tue, 9 Jul 2019 14:40:01 -0700 Subject: [PATCH 143/332] Refine TODO items related to lower-static-tensor-list pass. PiperOrigin-RevId: 257274127 --- tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc index 647f1d75d7b..8ae5bc2fe2f 100644 --- a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc +++ b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc @@ -104,9 +104,9 @@ void AddTFToTFLConversionPasses(bool emit_builtin_tflite_ops, bool run_quantize, if (lower_tensor_list_ops) { // Execute this pass before `CanonicalizerPass` in case some TensorList // ops are constant folded into variant types. + // TODO(b/137125056): Move this pass after `CanonicalizerPass` after we + // handle constant ops that produce `TensorList`. // TODO(haoliang): Add this pass by default. - // TODO(b/137038401): Handle other ops that operate on variant types and - // improve error reporting. pass_manager->addPass(mlir::TFL::CreateLowerStaticTensorListPass()); } From 77f6f8ccdeeca06a05b3513842f7e4e12febbe1b Mon Sep 17 00:00:00 2001 From: Juhyun Lee Date: Tue, 9 Jul 2019 14:50:05 -0700 Subject: [PATCH 144/332] TFLite GPU: Implement HARD_SWISH for MobileNet v3. PiperOrigin-RevId: 257276206 --- .../delegates/gpu/common/model_builder.cc | 20 ++ .../lite/delegates/gpu/common/operations.cc | 43 +-- .../lite/delegates/gpu/common/operations.h | 3 +- .../delegates/gpu/gl/kernels/elementwise.cc | 89 +++--- .../gpu/gl/kernels/elementwise_test.cc | 270 ++++++++++-------- .../lite/delegates/gpu/gl/kernels/registry.cc | 15 +- tensorflow/lite/delegates/gpu/metal/api.cc | 4 + .../lite/delegates/gpu/metal/kernels/BUILD | 13 + .../delegates/gpu/metal/kernels/hard_swish.cc | 47 +++ .../delegates/gpu/metal/kernels/hard_swish.h | 37 +++ 10 files changed, 344 insertions(+), 197 deletions(-) create mode 100644 tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.cc create mode 100644 tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.h diff --git a/tensorflow/lite/delegates/gpu/common/model_builder.cc b/tensorflow/lite/delegates/gpu/common/model_builder.cc index 253f792b0fd..f762a0fe0a3 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder.cc +++ b/tensorflow/lite/delegates/gpu/common/model_builder.cc @@ -816,6 +816,24 @@ class DepthwiseConvolutionOperationParser : public TFLiteOperationParser { } }; +class HardSwishOperationParser : public TFLiteOperationParser { + public: + Status IsSupported(const TfLiteContext* context, + const TfLiteNode* tflite_node, + const TfLiteRegistration*) final { + return CheckInputsOutputs(context, tflite_node, /*inputs=*/1, + /*outputs=*/1); + } + + Status Parse(const TfLiteNode*, const TfLiteRegistration*, + GraphFloat32* graph, ObjectReader* reader) final { + Node* node = graph->NewNode(); + node->operation.type = ToString(OperationType::HARD_SWISH); + RETURN_IF_ERROR(reader->AddInput(node, 0)); + return reader->AddOutputs(node); + } +}; + class ReshapeOperationParser : public TFLiteOperationParser { public: Status IsSupported(const TfLiteContext* context, @@ -2003,6 +2021,8 @@ std::unique_ptr NewOperationParser( return make_unique(OperationType::DIV); case kTfLiteBuiltinFullyConnected: return make_unique(); + case kTfLiteBuiltinHardSwish: + return make_unique(); case kTfLiteBuiltinLogistic: return make_unique(OperationType::SIGMOID); case kTfLiteBuiltinLog: diff --git a/tensorflow/lite/delegates/gpu/common/operations.cc b/tensorflow/lite/delegates/gpu/common/operations.cc index f7f9d1b7351..eb1f01804df 100644 --- a/tensorflow/lite/delegates/gpu/common/operations.cc +++ b/tensorflow/lite/delegates/gpu/common/operations.cc @@ -46,50 +46,58 @@ Padding2D& Padding2D::operator-(const Padding2D& value) { std::string ToString(enum OperationType op) { switch (op) { - case OperationType::UNKNOWN: - break; case OperationType::ABS: return "abs"; case OperationType::ADD: return "add"; case OperationType::APPLY_MASK: return "apply_mask"; - case OperationType::BATCH_TO_SPACE: - return "batch_to_space"; - case OperationType::POOLING_2D: - return "pooling_2d"; - case OperationType::MAX_UNPOOLING_2D: - return "max_unpooling"; case OperationType::BATCH_NORMALIZATION: return "batch_normalization"; + case OperationType::BATCH_TO_SPACE: + return "batch_to_space"; case OperationType::CONCAT: return "concat"; case OperationType::CONST: return "const"; case OperationType::CONVOLUTION_2D: return "convolution_2d"; + case OperationType::CONVOLUTION_TRANSPOSED: + return "convolution_transposed"; case OperationType::COS: return "cos"; case OperationType::DEPTHWISE_CONVOLUTION: return "depthwise_convolution"; case OperationType::DIV: return "div"; + case OperationType::FULLY_CONNECTED: + return "fully_connected"; + case OperationType::HARD_SWISH: + return "hard_swish"; case OperationType::LOG: return "log"; + case OperationType::LSTM: + return "lstm"; + case OperationType::MAX_UNPOOLING_2D: + return "max_unpooling"; case OperationType::MUL: return "mul"; + case OperationType::MULTIPLY_SCALAR: + return "multiply_scalar"; case OperationType::PAD: return "pad"; + case OperationType::POOLING_2D: + return "pooling_2d"; case OperationType::POW: return "pow"; case OperationType::PRELU: return "prelu"; case OperationType::RELU: return "relu"; - case OperationType::RESIZE: - return "resize"; case OperationType::RESHAPE: return "reshape"; + case OperationType::RESIZE: + return "resize"; case OperationType::RSQRT: return "rsqrt"; case OperationType::SIGMOID: @@ -110,18 +118,12 @@ std::string ToString(enum OperationType op) { return "squared_diff"; case OperationType::SUB: return "subtract"; - case OperationType::UPSAMPLE_2D: - return "upsample_2d"; - case OperationType::CONVOLUTION_TRANSPOSED: - return "convolution_transposed"; - case OperationType::MULTIPLY_SCALAR: - return "multiply_scalar"; - case OperationType::FULLY_CONNECTED: - return "fully_connected"; case OperationType::TANH: return "tanh"; - case OperationType::LSTM: - return "lstm"; + case OperationType::UPSAMPLE_2D: + return "upsample_2d"; + default: + break; } return "unknown_operation"; } @@ -140,6 +142,7 @@ OperationType OperationTypeFromString(const std::string& name) { {"cos", OperationType::COS}, {"depthwise_convolution", OperationType::DEPTHWISE_CONVOLUTION}, {"fully_connected", OperationType::FULLY_CONNECTED}, + {"hard_swish", OperationType::HARD_SWISH}, {"log", OperationType::LOG}, {"lstm", OperationType::LSTM}, {"max_unpooling", OperationType::MAX_UNPOOLING_2D}, diff --git a/tensorflow/lite/delegates/gpu/common/operations.h b/tensorflow/lite/delegates/gpu/common/operations.h index ef825376b31..5e564f6763c 100644 --- a/tensorflow/lite/delegates/gpu/common/operations.h +++ b/tensorflow/lite/delegates/gpu/common/operations.h @@ -46,14 +46,15 @@ enum class OperationType { DEPTHWISE_CONVOLUTION, DIV, FULLY_CONNECTED, + HARD_SWISH, LOG, LSTM, MAX_UNPOOLING_2D, MUL, MULTIPLY_SCALAR, + PAD, POOLING_2D, POW, - PAD, PRELU, RELU, RESHAPE, diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/elementwise.cc b/tensorflow/lite/delegates/gpu/gl/kernels/elementwise.cc index 37ee322ac8a..8ad2679e62e 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/elementwise.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/elementwise.cc @@ -34,60 +34,56 @@ class ElementwiseOneArgument : public NodeShader { GeneratedCode* generated_code) const final { std::string source; switch (operation_type_) { - case OperationType::ABS: { + case OperationType::ABS: source = "value_0 = abs(value_0);"; break; - } - case OperationType::SIN: { - source = "value_0 = sin(value_0);"; - break; - } - case OperationType::COS: { + case OperationType::COS: source = "value_0 = cos(value_0);"; break; - } - case OperationType::LOG: { + case OperationType::HARD_SWISH: + source = + "value_0 *= clamp(value_0 / 6.0 + vec4(0.5), vec4(0.0), " + "vec4(1.0));"; + break; + case OperationType::LOG: source = R"( - const float nan = normalize(vec4(0,0,0,0)).x; - value_0.x = value_0.x > 0.0 ? log(value_0.x) : nan; - value_0.y = value_0.y > 0.0 ? log(value_0.y) : nan; - value_0.z = value_0.z > 0.0 ? log(value_0.z) : nan; - value_0.w = value_0.w > 0.0 ? log(value_0.w) : nan; - )"; + const float nan = normalize(vec4(0, 0, 0, 0)).x; + value_0.x = value_0.x > 0.0 ? log(value_0.x) : nan; + value_0.y = value_0.y > 0.0 ? log(value_0.y) : nan; + value_0.z = value_0.z > 0.0 ? log(value_0.z) : nan; + value_0.w = value_0.w > 0.0 ? log(value_0.w) : nan; + )"; break; - } - case OperationType::SQRT: { + case OperationType::RSQRT: source = R"( - const float nan = normalize(vec4(0,0,0,0)).x; - value_0.x = value_0.x >= 0.0 ? sqrt(value_0.x) : nan; - value_0.y = value_0.y >= 0.0 ? sqrt(value_0.y) : nan; - value_0.z = value_0.z >= 0.0 ? sqrt(value_0.z) : nan; - value_0.w = value_0.w >= 0.0 ? sqrt(value_0.w) : nan; - )"; + const float nan = normalize(vec4(0, 0, 0, 0)).x; + value_0.x = value_0.x >= 0.0 ? 1.0 / sqrt(value_0.x) : nan; + value_0.y = value_0.y >= 0.0 ? 1.0 / sqrt(value_0.y) : nan; + value_0.z = value_0.z >= 0.0 ? 1.0 / sqrt(value_0.z) : nan; + value_0.w = value_0.w >= 0.0 ? 1.0 / sqrt(value_0.w) : nan; + )"; break; - } - case OperationType::RSQRT: { - source = R"( - const float nan = normalize(vec4(0,0,0,0)).x; - value_0.x = value_0.x >= 0.0 ? 1.0 / sqrt(value_0.x) : nan; - value_0.y = value_0.y >= 0.0 ? 1.0 / sqrt(value_0.y) : nan; - value_0.z = value_0.z >= 0.0 ? 1.0 / sqrt(value_0.z) : nan; - value_0.w = value_0.w >= 0.0 ? 1.0 / sqrt(value_0.w) : nan; - )"; - break; - } - case OperationType::SQUARE: { - source = "value_0 = value_0 * value_0;"; - break; - } - case OperationType::SIGMOID: { + case OperationType::SIGMOID: source = "value_0 = 1.0 / (1.0 + exp(-1.0 * value_0));"; break; - } - case OperationType::TANH: { + case OperationType::SIN: + source = "value_0 = sin(value_0);"; + break; + case OperationType::SQRT: + source = R"( + const float nan = normalize(vec4(0, 0, 0, 0)).x; + value_0.x = value_0.x >= 0.0 ? sqrt(value_0.x) : nan; + value_0.y = value_0.y >= 0.0 ? sqrt(value_0.y) : nan; + value_0.z = value_0.z >= 0.0 ? sqrt(value_0.z) : nan; + value_0.w = value_0.w >= 0.0 ? sqrt(value_0.w) : nan; + )"; + break; + case OperationType::SQUARE: + source = "value_0 = value_0 * value_0;"; + break; + case OperationType::TANH: source = "value_0 = tanh(value_0);"; break; - } default: return InvalidArgumentError("Incorrect elementwise operation type."); } @@ -183,19 +179,20 @@ std::unique_ptr NewElementwiseNodeShader( OperationType operation_type) { switch (operation_type) { case OperationType::ABS: - case OperationType::SIN: case OperationType::COS: case OperationType::LOG: - case OperationType::SQRT: + case OperationType::HARD_SWISH: case OperationType::RSQRT: - case OperationType::SQUARE: case OperationType::SIGMOID: + case OperationType::SIN: + case OperationType::SQRT: + case OperationType::SQUARE: case OperationType::TANH: return absl::make_unique(operation_type); - case OperationType::SUB: case OperationType::DIV: case OperationType::POW: case OperationType::SQUARED_DIFF: + case OperationType::SUB: return absl::make_unique(operation_type); default: return nullptr; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/elementwise_test.cc b/tensorflow/lite/delegates/gpu/gl/kernels/elementwise_test.cc index a0d088dbe48..6743664f7e2 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/elementwise_test.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/elementwise_test.cc @@ -28,139 +28,45 @@ namespace gpu { namespace gl { namespace { -class ElementwiseOneArgumentTest : public ::testing::Test { - public: - ElementwiseOneArgumentTest() = default; - ~ElementwiseOneArgumentTest() override = default; +TensorRef GetTensorRef(int ref, const BHWC& shape) { + TensorRef tensor_ref; + tensor_ref.type = DataType::FLOAT32; + tensor_ref.ref = ref; + tensor_ref.shape = shape; + return tensor_ref; +} - TensorRef GetTensorRef(int ref) { - TensorRef tensor_ref; - tensor_ref.type = DataType::FLOAT32; - tensor_ref.ref = ref; - tensor_ref.shape = BHWC(1, 2, 2, 1); - return tensor_ref; - } -}; - -TEST_F(ElementwiseOneArgumentTest, Abs) { +TEST(ElementwiseTest, Abs) { OperationType op_type = OperationType::ABS; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); ASSERT_TRUE(model.PopulateTensor(0, {0.0, -6.2, 2.0, 4.0})); ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); EXPECT_THAT(model.GetOutput(0), Pointwise(FloatNear(1e-6), {0.0, 6.2, 2.0, 4.0})); } -TEST_F(ElementwiseOneArgumentTest, Sin) { - OperationType op_type = OperationType::SIN; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); - ASSERT_TRUE(model.PopulateTensor(0, {0.0, 3.1415926, -3.1415926, 1.0})); - ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); - EXPECT_THAT(model.GetOutput(0), - Pointwise(FloatNear(1e-6), {0.0, 0.0, 0.0, 0.841471})); -} - -TEST_F(ElementwiseOneArgumentTest, Cos) { +TEST(ElementwiseTest, Cos) { OperationType op_type = OperationType::COS; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); ASSERT_TRUE(model.PopulateTensor(0, {0.0, 3.1415926, -3.1415926, 1})); ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); EXPECT_THAT(model.GetOutput(0), Pointwise(FloatNear(1e-6), {1.0, -1.0, -1.0, 0.540302})); } -TEST_F(ElementwiseOneArgumentTest, Log) { - OperationType op_type = OperationType::LOG; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); - ASSERT_TRUE(model.PopulateTensor(0, {1.0, 3.1415926, 1.0, 1.0})); - ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); - EXPECT_THAT(model.GetOutput(0), - Pointwise(FloatNear(1e-6), {0.0, 1.14473, 0.0, 0.0})); -} - -TEST_F(ElementwiseOneArgumentTest, Sqrt) { - OperationType op_type = OperationType::SQRT; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); - ASSERT_TRUE(model.PopulateTensor(0, {0.0, 1.0, 2.0, 4.0})); - ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); - EXPECT_THAT(model.GetOutput(0), - Pointwise(FloatNear(1e-6), {0.0, 1.0, 1.414213, 2.0})); -} - -TEST_F(ElementwiseOneArgumentTest, Rsqrt) { - OperationType op_type = OperationType::RSQRT; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); - ASSERT_TRUE(model.PopulateTensor(0, {1.0, 2.0, 4.0, 9.0})); - ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); - EXPECT_THAT(model.GetOutput(0), - Pointwise(FloatNear(1e-6), {1.0, 0.707106, 0.5, 0.333333})); -} - -TEST_F(ElementwiseOneArgumentTest, Square) { - OperationType op_type = OperationType::SQUARE; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); - ASSERT_TRUE(model.PopulateTensor(0, {1.0, 2.0, 0.5, -3.0})); - ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); - EXPECT_THAT(model.GetOutput(0), - Pointwise(FloatNear(1e-6), {1.0, 4.0, 0.25, 9.0})); -} - -TEST_F(ElementwiseOneArgumentTest, Sigmoid) { - OperationType op_type = OperationType::SIGMOID; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); - ASSERT_TRUE(model.PopulateTensor(0, {0.0, -6.0, 2.0, 4.0})); - ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); - EXPECT_THAT(model.GetOutput(0), - Pointwise(FloatNear(1e-6), {0.5, 0.002473, 0.880797, 0.982014})); -} - -TEST_F(ElementwiseOneArgumentTest, Tanh) { - OperationType op_type = OperationType::TANH; - SingleOpModel model({ToString(op_type), {}}, {GetTensorRef(0)}, - {GetTensorRef(1)}); - ASSERT_TRUE(model.PopulateTensor(0, {0.0, -6.0, 2.0, 4.0})); - ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); - EXPECT_THAT(model.GetOutput(0), - Pointwise(FloatNear(1e-6), {0.0, -0.999987, 0.964027, 0.999329})); -} - -class ElementwiseTwoArgumentsTest : public ::testing::Test { - public: - ElementwiseTwoArgumentsTest() = default; - ~ElementwiseTwoArgumentsTest() override = default; - - TensorRef GetTensorRef(int ref) { - TensorRef tensor_ref; - tensor_ref.type = DataType::FLOAT32; - tensor_ref.ref = ref; - tensor_ref.shape = BHWC(1, 2, 2, 1); - return tensor_ref; - } -}; - -TEST_F(ElementwiseTwoArgumentsTest, Sub) { - OperationType op_type = OperationType::SUB; - SingleOpModel model({ToString(op_type), {}}, - {GetTensorRef(0), GetTensorRef(1)}, {GetTensorRef(2)}); - ASSERT_TRUE(model.PopulateTensor(0, {0.0, -6.2, 2.0, 4.0})); - ASSERT_TRUE(model.PopulateTensor(1, {1.0, 2.0, 3.0, 4.0})); - ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); - EXPECT_THAT(model.GetOutput(0), - Pointwise(FloatNear(1e-6), {-1.0, -8.2, -1.0, 0.0})); -} - -TEST_F(ElementwiseTwoArgumentsTest, Div) { +TEST(ElementwiseTest, Div) { OperationType op_type = OperationType::DIV; - SingleOpModel model({ToString(op_type), {}}, - {GetTensorRef(0), GetTensorRef(1)}, {GetTensorRef(2)}); + const BHWC shape(1, 2, 2, 1); + SingleOpModel model( + {/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape), GetTensorRef(1, shape)}, + /*outputs=*/{GetTensorRef(2, shape)}); ASSERT_TRUE(model.PopulateTensor(0, {0.0, -6.2, 2.0, 4.0})); ASSERT_TRUE(model.PopulateTensor(1, {1.0, 2.0, -0.5, 4.0})); ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); @@ -168,10 +74,39 @@ TEST_F(ElementwiseTwoArgumentsTest, Div) { Pointwise(FloatNear(1e-6), {0.0, -3.1, -4.0, 1.0})); } -TEST_F(ElementwiseTwoArgumentsTest, Pow) { +TEST(ElementwiseTest, HardSwish) { + OperationType op_type = OperationType::HARD_SWISH; + const BHWC shape(1, 1, 1, 7); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); + ASSERT_TRUE( + model.PopulateTensor(0, {-4.5f, -3.0f, -1.5f, 0.0f, 1.5f, 3.0f, 4.5f})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6f), + {0.0f, 0.0f, -0.375f, 0.0f, 1.125f, 3.f, 4.5f})); +} + +TEST(ElementwiseTest, Log) { + OperationType op_type = OperationType::LOG; + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); + ASSERT_TRUE(model.PopulateTensor(0, {1.0, 3.1415926, 1.0, 1.0})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6), {0.0, 1.14473, 0.0, 0.0})); +} + +TEST(ElementwiseTest, Pow) { OperationType op_type = OperationType::POW; - SingleOpModel model({ToString(op_type), {}}, - {GetTensorRef(0), GetTensorRef(1)}, {GetTensorRef(2)}); + const BHWC shape(1, 2, 2, 1); + SingleOpModel model( + {/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape), GetTensorRef(1, shape)}, + /*outputs=*/{GetTensorRef(2, shape)}); ASSERT_TRUE(model.PopulateTensor(0, {0.0, 1.0, 2.0, 4.0})); ASSERT_TRUE(model.PopulateTensor(1, {1.0, 2.0, 3.0, 4.0})); ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); @@ -179,10 +114,73 @@ TEST_F(ElementwiseTwoArgumentsTest, Pow) { Pointwise(FloatNear(1e-6), {0.0, 1.0, 8.0, 256.0})); } -TEST_F(ElementwiseTwoArgumentsTest, SquaredDiff) { +TEST(ElementwiseTest, Rsqrt) { + OperationType op_type = OperationType::RSQRT; + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); + ASSERT_TRUE(model.PopulateTensor(0, {1.0, 2.0, 4.0, 9.0})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6), {1.0, 0.707106, 0.5, 0.333333})); +} + +TEST(ElementwiseTest, Sigmoid) { + OperationType op_type = OperationType::SIGMOID; + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); + ASSERT_TRUE(model.PopulateTensor(0, {0.0, -6.0, 2.0, 4.0})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6), {0.5, 0.002473, 0.880797, 0.982014})); +} + +TEST(ElementwiseTest, Sin) { + OperationType op_type = OperationType::SIN; + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); + ASSERT_TRUE(model.PopulateTensor(0, {0.0, 3.1415926, -3.1415926, 1.0})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6), {0.0, 0.0, 0.0, 0.841471})); +} + +TEST(ElementwiseTest, Sqrt) { + OperationType op_type = OperationType::SQRT; + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); + ASSERT_TRUE(model.PopulateTensor(0, {0.0, 1.0, 2.0, 4.0})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6), {0.0, 1.0, 1.414213, 2.0})); +} + +TEST(ElementwiseTest, Square) { + OperationType op_type = OperationType::SQUARE; + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); + ASSERT_TRUE(model.PopulateTensor(0, {1.0, 2.0, 0.5, -3.0})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6), {1.0, 4.0, 0.25, 9.0})); +} + +TEST(ElementwiseTest, SquaredDiff) { OperationType op_type = OperationType::SQUARED_DIFF; - SingleOpModel model({ToString(op_type), {}}, - {GetTensorRef(0), GetTensorRef(1)}, {GetTensorRef(2)}); + const BHWC shape(1, 2, 2, 1); + SingleOpModel model( + {/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape), GetTensorRef(1, shape)}, + /*outputs=*/{GetTensorRef(2, shape)}); ASSERT_TRUE(model.PopulateTensor(0, {0.0, 2.0, 2.0, 4.0})); ASSERT_TRUE(model.PopulateTensor(1, {1.0, 1.0, 5.0, 4.0})); ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); @@ -190,6 +188,32 @@ TEST_F(ElementwiseTwoArgumentsTest, SquaredDiff) { Pointwise(FloatNear(1e-6), {1.0, 1.0, 9.0, 0.0})); } +TEST(ElementwiseTest, Sub) { + OperationType op_type = OperationType::SUB; + const BHWC shape(1, 2, 2, 1); + SingleOpModel model( + {/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape), GetTensorRef(1, shape)}, + /*outputs=*/{GetTensorRef(2, shape)}); + ASSERT_TRUE(model.PopulateTensor(0, {0.0, -6.2, 2.0, 4.0})); + ASSERT_TRUE(model.PopulateTensor(1, {1.0, 2.0, 3.0, 4.0})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6), {-1.0, -8.2, -1.0, 0.0})); +} + +TEST(ElementwiseTest, Tanh) { + OperationType op_type = OperationType::TANH; + const BHWC shape(1, 2, 2, 1); + SingleOpModel model({/*type=*/ToString(op_type), /*attributes=*/{}}, + /*inputs=*/{GetTensorRef(0, shape)}, + /*outputs=*/{GetTensorRef(1, shape)}); + ASSERT_TRUE(model.PopulateTensor(0, {0.0, -6.0, 2.0, 4.0})); + ASSERT_OK(model.Invoke(*NewElementwiseNodeShader(op_type))); + EXPECT_THAT(model.GetOutput(0), + Pointwise(FloatNear(1e-6), {0.0, -0.999987, 0.964027, 0.999329})); +} + } // namespace } // namespace gl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc b/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc index 2201d0018dd..7c93ebd1caf 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc @@ -60,10 +60,10 @@ class Registry : public NodeShader { using Type = OperationType; using NewShaderFunc = std::function()>; - auto insert_op = [&](Type type, NewShaderFunc func) { + const auto insert_op = [&](Type type, NewShaderFunc func) { shaders_[ToString(type)].push_back(func()); }; - auto insert_elementwise_op = [&](Type operation_type) { + const auto insert_elementwise_op = [&](Type operation_type) { shaders_[ToString(operation_type)].push_back( NewElementwiseNodeShader(operation_type)); }; @@ -82,26 +82,27 @@ class Registry : public NodeShader { insert_op(Type::MULTIPLY_SCALAR, NewMultiplyScalarNodeShader); insert_op(Type::PAD, NewPadNodeShader); insert_op(Type::POOLING_2D, NewPoolingNodeShader); + insert_op(Type::PRELU, NewPReLUNodeShader); insert_op(Type::RELU, NewReLUNodeShader); insert_op(Type::RESHAPE, NewReshapeNodeShader); - insert_op(Type::PRELU, NewPReLUNodeShader); insert_op(Type::SLICE, NewSliceNodeShader); insert_op(Type::SOFT_MAX, NewSoftMaxNodeShader); insert_op(Type::UPSAMPLE_2D, NewUpsamplingNodeShader); insert_elementwise_op(Type::ABS); insert_elementwise_op(Type::COS); + insert_elementwise_op(Type::DIV); + insert_elementwise_op(Type::HARD_SWISH); insert_elementwise_op(Type::LOG); + insert_elementwise_op(Type::POW); insert_elementwise_op(Type::RSQRT); insert_elementwise_op(Type::SIGMOID); insert_elementwise_op(Type::SIN); insert_elementwise_op(Type::SQRT); insert_elementwise_op(Type::SQUARE); - insert_elementwise_op(Type::TANH); - insert_elementwise_op(Type::SUB); - insert_elementwise_op(Type::DIV); - insert_elementwise_op(Type::POW); insert_elementwise_op(Type::SQUARED_DIFF); + insert_elementwise_op(Type::SUB); + insert_elementwise_op(Type::TANH); #ifndef TFLITE_GPU_BINARY_RELEASE insert_op(Type::MAX_UNPOOLING_2D, NewMaxUnpoolingNodeShader); diff --git a/tensorflow/lite/delegates/gpu/metal/api.cc b/tensorflow/lite/delegates/gpu/metal/api.cc index 3588cd97169..856939eb9b2 100644 --- a/tensorflow/lite/delegates/gpu/metal/api.cc +++ b/tensorflow/lite/delegates/gpu/metal/api.cc @@ -30,6 +30,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/metal/kernels/depthwise_conv.h" #include "tensorflow/lite/delegates/gpu/metal/kernels/elementwise.h" #include "tensorflow/lite/delegates/gpu/metal/kernels/fully_connected.h" +#include "tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.h" #include "tensorflow/lite/delegates/gpu/metal/kernels/max_unpooling.h" #include "tensorflow/lite/delegates/gpu/metal/kernels/mul.h" #include "tensorflow/lite/delegates/gpu/metal/kernels/padding.h" @@ -172,6 +173,9 @@ Status Compile(const GraphFloat32& graph, const RuntimeOptions& options, node->operation.attributes), options); break; + case OperationType::HARD_SWISH: + tasks = HardSwish(node_id, inputs[0], outputs[0], options); + break; case OperationType::MAX_UNPOOLING_2D: tasks = MaxUnpooling(node_id, inputs[0], inputs[1], outputs[0], absl::any_cast( diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/BUILD b/tensorflow/lite/delegates/gpu/metal/kernels/BUILD index 48ff3632a02..c1b57bd4fc0 100644 --- a/tensorflow/lite/delegates/gpu/metal/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/metal/kernels/BUILD @@ -12,6 +12,7 @@ cc_library( ":depthwise_conv", ":elementwise", ":fully_connected", + ":hard_swish", ":max_unpooling", ":mul", ":padding", @@ -122,6 +123,18 @@ cc_library( ], ) +cc_library( + name = "hard_swish", + srcs = ["hard_swish.cc"], + hdrs = ["hard_swish.h"], + deps = [ + "//tensorflow/lite/delegates/gpu/common:model", + "//tensorflow/lite/delegates/gpu/common:types", + "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", + "//tensorflow/lite/delegates/gpu/metal:runtime_options", + ], +) + cc_library( name = "max_unpooling", srcs = ["max_unpooling.cc"], diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.cc b/tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.cc new file mode 100644 index 00000000000..fbf2be92627 --- /dev/null +++ b/tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.cc @@ -0,0 +1,47 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.h" + +#include +#include + +#include "tensorflow/lite/delegates/gpu/common/model.h" +#include "tensorflow/lite/delegates/gpu/metal/compute_task_descriptor.h" +#include "tensorflow/lite/delegates/gpu/metal/runtime_options.h" + +namespace tflite { +namespace gpu { +namespace metal { + +std::vector HardSwish(int id, ValueId input_id, + ValueId output_id, + const RuntimeOptions& options) { + auto desc = std::make_shared(); + desc->id = id; + desc->is_linkable = true; + desc->shader_source = R"( + FLT4 linkable$0(FLT4 value, int linear_index, uint3 gid) { + return value * clamp(value / 6.0f + FLT4(0.5f), FLT4(0.0f), FLT4(1.0f)); + } + )"; + desc->input_buffers = {{input_id}}; + desc->output_buffer = {output_id}; + return {desc}; +} + +} // namespace metal +} // namespace gpu +} // namespace tflite diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.h b/tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.h new file mode 100644 index 00000000000..fa040ebcb97 --- /dev/null +++ b/tensorflow/lite/delegates/gpu/metal/kernels/hard_swish.h @@ -0,0 +1,37 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_DELEGATES_GPU_METAL_KERNELS_HARD_SWISH_H_ +#define TENSORFLOW_LITE_DELEGATES_GPU_METAL_KERNELS_HARD_SWISH_H_ + +#include + +#include "tensorflow/lite/delegates/gpu/common/model.h" +#include "tensorflow/lite/delegates/gpu/metal/compute_task_descriptor.h" +#include "tensorflow/lite/delegates/gpu/metal/runtime_options.h" + +namespace tflite { +namespace gpu { +namespace metal { + +std::vector HardSwish(int id, ValueId input_id, + ValueId output_id, + const RuntimeOptions& options); + +} // namespace metal +} // namespace gpu +} // namespace tflite + +#endif // TENSORFLOW_LITE_DELEGATES_GPU_METAL_KERNELS_HARD_SWISH_H_ From ac6a943173973f55ee27f97c8eba6bbfceb4a733 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Tue, 9 Jul 2019 14:52:23 -0700 Subject: [PATCH 145/332] Disable docs generation test. PiperOrigin-RevId: 257276657 --- tensorflow/tools/docs/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD index bb3757099c3..767c918a145 100644 --- a/tensorflow/tools/docs/BUILD +++ b/tensorflow/tools/docs/BUILD @@ -140,6 +140,7 @@ py_test( "nomsan", "notsan", "optonly", + "notap", ], deps = [ ":generate2_lib", From 449c70062f323dfef41a0ea1c180ff5fdbb7dbe0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 15:05:43 -0700 Subject: [PATCH 146/332] NFC: Remove unnecessary CHECK-EMPTY lines. These don't test anything useful in these tests. PiperOrigin-RevId: 257279580 --- .../tests/graphdef2mlir/graph-device-retval.pbtxt | 3 +-- .../tensorflow/tests/graphdef2mlir/graph-func-attr.pbtxt | 6 ++---- .../tests/graphdef2mlir/graph-function-defs.pbtxt | 6 ++---- .../tests/graphdef2mlir/graph-function-static-output.pbtxt | 6 ++---- .../tests/graphdef2mlir/graph-functional-while-loop.pbtxt | 6 ++---- .../tensorflow/tests/graphdef2mlir/graph-gradient-def.pbtxt | 6 ++---- .../mlir/tensorflow/tests/graphdef2mlir/graph-library.pbtxt | 6 ++---- 7 files changed, 13 insertions(+), 26 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-device-retval.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-device-retval.pbtxt index dcbf299119d..fcd0e62ab63 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-device-retval.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-device-retval.pbtxt @@ -91,8 +91,7 @@ versions { # CHECK-NEXT: %0:2 = "_tf.PartitionedCall"() {Tin = [], Tout = ["tfdtype$DT_INT32"], config = "", config_proto = "", device = "", executor_type = "", f = @foo0, name = "PartitionedCall"} : () -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: return # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @foo0() -> tensor +# CHECK: func @foo0() -> tensor # CHECK-NEXT: attributes {tf.experimental_ints_on_device = true} { # CHECK-NEXT: %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "Const", value = dense<5> : tensor} : () -> (tensor, !_tf.control) # CHECK-NEXT: %1:2 = "_tf.Identity"(%0#0) {T = "tfdtype$DT_INT32", device = "", name = "Identity"} : (tensor) -> (tensor, !_tf.control) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-func-attr.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-func-attr.pbtxt index ce1c9926b5c..e8b9ce86ddb 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-func-attr.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-func-attr.pbtxt @@ -157,13 +157,11 @@ versions { # CHECK-NEXT: %1:2 = "_tf.Case"(%0#0) {Tin = [], Tout = ["tfdtype$DT_FLOAT"], branches = [@foo0, @bar0], device = "", name = "Case", output_shapes = []} : (tensor) -> (tensor<*xf32>, !_tf.control) # CHECK-NEXT: return # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @foo0() -> tensor<10xf32> { +# CHECK: func @foo0() -> tensor<10xf32> { # CHECK-NEXT: %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_FLOAT", name = "const_1", value = dense<1.000000e+00> : tensor<10xf32>} : () -> (tensor<10xf32>, !_tf.control) # CHECK-NEXT: return %0#0 : tensor<10xf32> # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @bar0() -> tensor<10xf32> { +# CHECK: func @bar0() -> tensor<10xf32> { # CHECK-NEXT: %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_FLOAT", name = "const_2", value = dense<2.000000e+00> : tensor<10xf32>} : () -> (tensor<10xf32>, !_tf.control) # CHECK-NEXT: return %0#0 : tensor<10xf32> # CHECK-NEXT: } diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt index c5c42955e58..fe91ebe41e9 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt @@ -526,14 +526,12 @@ versions { # CHECK-NEXT: %18:2 = "_tf.Identity"(%17#0, %6) {T = "tfdtype$DT_INT32", device = "", name = "output_1_shard_0"} : (tensor<*xi32>, !_tf.control) -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: return # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @cond_false0(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { +# CHECK: func @cond_false0(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { # CHECK-NEXT: %0:2 = "_tf.Identity"(%arg0) {T = "tfdtype$DT_INT32", device = "", name = "Identity"} : (tensor<*xi32>) -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: %1:2 = "_tf.Identity"(%arg1) {T = "tfdtype$DT_INT32", device = "", name = "Identity_1"} : (tensor<*xi32>) -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: return %1#0, %0#0 : tensor<*xi32>, tensor<*xi32> # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @cond_true0(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { +# CHECK: func @cond_true0(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>) { # CHECK-NEXT: %0:2 = "_tf.Identity"(%arg0) {T = "tfdtype$DT_INT32", device = "", name = "Identity"} : (tensor<*xi32>) -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: %1:2 = "_tf.Identity"(%arg1) {T = "tfdtype$DT_INT32", device = "", name = "Identity_1"} : (tensor<*xi32>) -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: return %0#0, %1#0 : tensor<*xi32>, tensor<*xi32> diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-static-output.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-static-output.pbtxt index dc4e74cd028..41107cfbff4 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-static-output.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-static-output.pbtxt @@ -145,12 +145,10 @@ versions { #CHECK-NEXT: %2:2 = "_tf.If"(%0#0, %1#0) {Tcond = "tfdtype$DT_BOOL", Tin = ["tfdtype$DT_INT32"], Tout = ["tfdtype$DT_INT32"], device = "", else_branch = @get_zeros0, name = "If", output_shapes = [], then_branch = @identity0} : (tensor<*xi1>, tensor<*xi32>) -> (tensor<*xi32>, !_tf.control) #CHECK-NEXT: return #CHECK-NEXT: } -#CHECK-EMPTY: -#CHECK-NEXT: func @get_zeros0(%arg0: tensor<*xi32>) -> tensor<2xi32> { +#CHECK: func @get_zeros0(%arg0: tensor<*xi32>) -> tensor<2xi32> { #CHECK-NEXT: %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "const", value = dense<[1, 2]> : tensor<2xi32>} : () -> (tensor<2xi32>, !_tf.control) #CHECK-NEXT: return %0#0 : tensor<2xi32> #CHECK-NEXT: } -#CHECK-EMPTY: -#CHECK-NEXT: func @identity0(%arg0: tensor<*xi32>) -> tensor<*xi32> { +#CHECK: func @identity0(%arg0: tensor<*xi32>) -> tensor<*xi32> { #CHECK-NEXT: return %arg0 : tensor<*xi32> #CHECK-NEXT: } diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-functional-while-loop.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-functional-while-loop.pbtxt index 34e688a5605..456bf4951bd 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-functional-while-loop.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-functional-while-loop.pbtxt @@ -303,16 +303,14 @@ versions { # CHECK-NEXT: %3:4 = "_tf.While"(%1#0, %2#0, %0#0) {T = ["tfdtype$DT_INT32", "tfdtype$DT_INT32", "tfdtype$DT_INT32"], _lower_using_switch_merge = true, body = @while_body_60, cond = @while_cond_50, device = "", name = "while", output_shapes = ["tfshape$", "tfshape$", "tfshape$"], parallel_iterations = 10 : i64} : (tensor, tensor, tensor) -> (tensor, tensor, tensor, !_tf.control) # CHECK-NEXT: return %3#2 : tensor # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @while_body_60(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>) { +# CHECK: func @while_body_60(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor<*xi32>) -> (tensor<*xi32>, tensor<*xi32>, tensor<*xi32>) { # CHECK-NEXT: %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "Add/y", value = dense<1> : tensor} : () -> (tensor, !_tf.control) # CHECK-NEXT: %1:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "add_1/y", value = dense<1> : tensor} : () -> (tensor, !_tf.control) # CHECK-NEXT: %2:2 = "_tf.Add"(%arg2, %0#0) {T = "tfdtype$DT_INT32", device = "", name = "Add"} : (tensor<*xi32>, tensor) -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: %3:2 = "_tf.Add"(%arg0, %1#0) {T = "tfdtype$DT_INT32", device = "", name = "add_1"} : (tensor<*xi32>, tensor) -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: return %3#0, %arg1, %2#0 : tensor<*xi32>, tensor<*xi32>, tensor<*xi32> # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @while_cond_50(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor<*xi32>) -> tensor<*xi1> { +# CHECK: func @while_cond_50(%arg0: tensor<*xi32>, %arg1: tensor<*xi32>, %arg2: tensor<*xi32>) -> tensor<*xi1> { # CHECK-NEXT: %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "Less/y", value = dense<10> : tensor} : () -> (tensor, !_tf.control) # CHECK-NEXT: %1:2 = "_tf.Less"(%arg2, %0#0) {T = "tfdtype$DT_INT32", device = "", name = "Less"} : (tensor<*xi32>, tensor) -> (tensor<*xi1>, !_tf.control) # CHECK-NEXT: return %1#0 : tensor<*xi1> diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-gradient-def.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-gradient-def.pbtxt index 512ffd12eef..c1045bf19af 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-gradient-def.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-gradient-def.pbtxt @@ -279,14 +279,12 @@ versions { # CHECK-NEXT: %5:2 = "_tf.SymbolicGradient"(%0#0, %4#0) {Tin = ["tfdtype$DT_FLOAT", "tfdtype$DT_FLOAT"], Tout = ["tfdtype$DT_FLOAT"], device = "", f = @foo0, f._disable_call_shape_inference = true, name = "gradients/foo_grad/SymbolicGradient"} : (tensor, tensor<*xf32>) -> (tensor, !_tf.control) # CHECK-NEXT: return # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @foo_grad0(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> +# CHECK: func @foo_grad0(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> # CHECK-NEXT: attributes {tf._disable_call_shape_inference = true} { # CHECK-NEXT: %0:2 = "_tf.Mul"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT", device = "", name = "mul_0"} : (tensor<*xf32>, tensor<*xf32>) -> (tensor<*xf32>, !_tf.control) # CHECK-NEXT: return %0#0 : tensor<*xf32> # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @foo0(%arg0: tensor<*xf32>) -> tensor<*xf32> +# CHECK: func @foo0(%arg0: tensor<*xf32>) -> tensor<*xf32> # CHECK-NEXT: attributes {tf._disable_call_shape_inference = true, tf.gradient = @foo_grad0} { # CHECK-NEXT: %0:2 = "_tf.Exp"(%arg0) {T = "tfdtype$DT_FLOAT", device = "", name = "Exp"} : (tensor<*xf32>) -> (tensor<*xf32>, !_tf.control) # CHECK-NEXT: %1:2 = "_tf.Neg"(%arg0) {T = "tfdtype$DT_FLOAT", device = "", name = "Neg"} : (tensor<*xf32>) -> (tensor<*xf32>, !_tf.control) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-library.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-library.pbtxt index 80697690e8a..83ca4466869 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-library.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-library.pbtxt @@ -41,12 +41,10 @@ versions { # CHECK-NEXT: %1 = "_tf.bar0"() {device = "", name = "unnamed1"} : () -> !_tf.control # CHECK-NEXT: return # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @foo0() { +# CHECK: func @foo0() { # CHECK-NEXT: %0 = "_tf.bar0"() {device = "", name = "unnamed"} : () -> !_tf.control # CHECK-NEXT: return # CHECK-NEXT: } -# CHECK-EMPTY: -# CHECK-NEXT: func @bar0() { +# CHECK: func @bar0() { # CHECK-NEXT: return # CHECK-NEXT: } From 3b753303b18ffd3431f95ae8497b3f2e6808442e Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Tue, 9 Jul 2019 15:22:45 -0700 Subject: [PATCH 147/332] Disable stackhandler_test where it doesn't work PiperOrigin-RevId: 257283229 --- tensorflow/python/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 915f3518873..1ae2c9acb56 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -270,6 +270,7 @@ tf_py_test( tags = [ "no_windows", "nomac", + "notap", # TODO(b/137133525): enable after this is fixed. ], ) From 1ef629438e9a1ebc11adc9e4043abd1f130e4f25 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 15:37:56 -0700 Subject: [PATCH 148/332] Add sparkfun_edge platform support for micro_vision example. PiperOrigin-RevId: 257286154 --- .../micro_vision/apollo3evb/image_provider.cc | 5 - .../himax_driver/platform_Sparkfun_Edge.h | 1 + .../sparkfun_edge/image_provider.cc | 199 ++++++++++++++++++ 3 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 tensorflow/lite/experimental/micro/examples/micro_vision/sparkfun_edge/image_provider.cc diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/apollo3evb/image_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_vision/apollo3evb/image_provider.cc index 7189efca9b2..2015aecf2b5 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_vision/apollo3evb/image_provider.cc +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/apollo3evb/image_provider.cc @@ -75,11 +75,6 @@ static hm01b0_cfg_t s_HM01B0Cfg = { pfnGpioIsr : NULL, }; -static constexpr int kDebugRowLenElements = 16; -// Each byte takes two characters plus a space, and the offset takes an -// additional 8 characters plus a space. -static constexpr int kDebugLineLenBytes = kDebugRowLenElements * 3 + 9; - bool g_is_camera_initialized = false; void boost_mode_enable(tflite::ErrorReporter* error_reporter, bool bEnable) { diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/platform_Sparkfun_Edge.h b/tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/platform_Sparkfun_Edge.h index 4f13c8befbe..3a580914987 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/platform_Sparkfun_Edge.h +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/platform_Sparkfun_Edge.h @@ -35,6 +35,7 @@ extern "C" { #define HM01B0_PIN_INT 4 #define HM01B0_PIN_SCL 8 #define HM01B0_PIN_SDA 9 +#define HM01B0_PIN_DVDD_EN 10 // Define AP3B's CTIMER and output pin for HM01B0 MCLK generation #define HM01B0_MCLK_GENERATOR_MOD 0 diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/sparkfun_edge/image_provider.cc b/tensorflow/lite/experimental/micro/examples/micro_vision/sparkfun_edge/image_provider.cc new file mode 100644 index 00000000000..f9e5da29e29 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/sparkfun_edge/image_provider.cc @@ -0,0 +1,199 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_vision/image_provider.h" + +#include "tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/HM01B0.h" +#include "tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/HM01B0_RAW8_QVGA_8bits_lsb_5fps.h" +#include "tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/HM01B0_debug.h" +#include "tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/HM01B0_optimized.h" +#include "tensorflow/lite/experimental/micro/examples/micro_vision/himax_driver/platform_Sparkfun_Edge.h" + +// These are headers from Ambiq's Apollo3 SDK. +#include "am_bsp.h" // NOLINT +#include "am_mcu_apollo.h" // NOLINT +#include "am_util.h" // NOLINT + +// #define DEMO_HM01B0_FRAMEBUFFER_DUMP_ENABLE + +// Enabling logging increases power consumption by preventing low power mode +// from being enabled. +#define ENABLE_LOGGING + +namespace { + +//***************************************************************************** +// +// HM01B0 Configuration +// +//***************************************************************************** +static hm01b0_cfg_t s_HM01B0Cfg = { + // i2c settings + ui16SlvAddr : HM01B0_DEFAULT_ADDRESS, + eIOMMode : HM01B0_IOM_MODE, + ui32IOMModule : HM01B0_IOM_MODULE, + sIOMCfg : { + eInterfaceMode : HM01B0_IOM_MODE, + ui32ClockFreq : HM01B0_I2C_CLOCK_FREQ, + }, + pIOMHandle : NULL, + + // MCLK settings + ui32CTimerModule : HM01B0_MCLK_GENERATOR_MOD, + ui32CTimerSegment : HM01B0_MCLK_GENERATOR_SEG, + ui32CTimerOutputPin : HM01B0_PIN_MCLK, + + // data interface + ui8PinSCL : HM01B0_PIN_SCL, + ui8PinSDA : HM01B0_PIN_SDA, + ui8PinD0 : HM01B0_PIN_D0, + ui8PinD1 : HM01B0_PIN_D1, + ui8PinD2 : HM01B0_PIN_D2, + ui8PinD3 : HM01B0_PIN_D3, + ui8PinD4 : HM01B0_PIN_D4, + ui8PinD5 : HM01B0_PIN_D5, + ui8PinD6 : HM01B0_PIN_D6, + ui8PinD7 : HM01B0_PIN_D7, + ui8PinVSYNC : HM01B0_PIN_VSYNC, + ui8PinHSYNC : HM01B0_PIN_HSYNC, + ui8PinPCLK : HM01B0_PIN_PCLK, + + ui8PinTrig : HM01B0_PIN_TRIG, + ui8PinInt : HM01B0_PIN_INT, + pfnGpioIsr : NULL, +}; + +bool g_is_camera_initialized = false; + +void boost_mode_enable(tflite::ErrorReporter* error_reporter, bool bEnable) { + am_hal_burst_avail_e eBurstModeAvailable; + am_hal_burst_mode_e eBurstMode; + + // Check that the Burst Feature is available. + if (AM_HAL_STATUS_SUCCESS == + am_hal_burst_mode_initialize(&eBurstModeAvailable)) { + if (AM_HAL_BURST_AVAIL == eBurstModeAvailable) { + error_reporter->Report("Apollo3 Burst Mode is Available\n"); + } else { + error_reporter->Report("Apollo3 Burst Mode is Not Available\n"); + return; + } + } else { + error_reporter->Report("Failed to Initialize for Burst Mode operation\n"); + } + + // Make sure we are in "Normal" mode. + if (AM_HAL_STATUS_SUCCESS == am_hal_burst_mode_disable(&eBurstMode)) { + if (AM_HAL_NORMAL_MODE == eBurstMode) { + error_reporter->Report("Apollo3 operating in Normal Mode (48MHz)\n"); + } + } else { + error_reporter->Report("Failed to Disable Burst Mode operation\n"); + } + + // Put the MCU into "Burst" mode. + if (bEnable) { + if (AM_HAL_STATUS_SUCCESS == am_hal_burst_mode_enable(&eBurstMode)) { + if (AM_HAL_BURST_MODE == eBurstMode) { + error_reporter->Report("Apollo3 operating in Burst Mode (96MHz)\n"); + } + } else { + error_reporter->Report("Failed to Enable Burst Mode operation\n"); + } + } +} + +} // namespace + +TfLiteStatus InitCamera(tflite::ErrorReporter* error_reporter) { + // Enable the ITM print interface. + am_bsp_itm_printf_enable(); + + error_reporter->Report("Initializing HM01B0...\n"); + + am_hal_clkgen_control(AM_HAL_CLKGEN_CONTROL_SYSCLK_MAX, 0); + + // Set the default cache configuration + am_hal_cachectrl_config(&am_hal_cachectrl_defaults); + am_hal_cachectrl_enable(); + + // Configure the board for low power operation. This breaks logging by + // turning off the itm and uart interfaces. +#ifndef ENABLE_LOGGING + am_bsp_low_power_init(); +#endif + + // Enable interrupts so we can receive messages from the boot host. + am_hal_interrupt_master_enable(); + + boost_mode_enable(error_reporter, true); + + // Turn on the 1.8V regulator for DVDD on the camera. + am_hal_gpio_pinconfig(HM01B0_PIN_DVDD_EN, g_AM_HAL_GPIO_OUTPUT_12); + am_hal_gpio_output_set(HM01B0_PIN_DVDD_EN); + + hm01b0_power_up(&s_HM01B0Cfg); + + // TODO(njeff): check the delay time to just fit the spec. + am_util_delay_ms(1); + + hm01b0_mclk_enable(&s_HM01B0Cfg); + + // TODO(njeff): check the delay time to just fit the spec. + am_util_delay_ms(1); + + hm01b0_init_if(&s_HM01B0Cfg); + + hm01b0_init_system(&s_HM01B0Cfg, (hm_script_t*)sHM01B0InitScript, + sizeof(sHM01B0InitScript) / sizeof(hm_script_t)); + + // Put camera into streaming mode - this makes it so that the camera + // constantly captures images. It is still OK to read and image since the + // camera uses a double-buffered input. This means there is always one valid + // image to read while the other buffer fills. Streaming mode allows the + // camera to perform auto exposure constantly. + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_RED); + uint32_t error_code = + hm01b0_set_mode(&s_HM01B0Cfg, HM01B0_REG_MODE_SELECT_STREAMING, 0); + if (error_code == HM01B0_ERR_OK) { + am_hal_gpio_output_set(AM_BSP_GPIO_LED_RED); + + return kTfLiteError; + } + + return kTfLiteOk; +} + +// Capture single frame. Frame pointer passed in to reduce memory usage. This +// allows the input tensor to be used instead of requiring an extra copy. +TfLiteStatus GetImage(tflite::ErrorReporter* error_reporter, int frame_width, + int frame_height, int channels, uint8_t* frame) { + if (!g_is_camera_initialized) { + TfLiteStatus init_status = InitCamera(error_reporter); + if (init_status != kTfLiteOk) { + return init_status; + } + g_is_camera_initialized = true; + } + + hm01b0_blocking_read_oneframe_scaled(frame, frame_width, frame_height, + channels); + +#ifdef DEMO_HM01B0_FRAMEBUFFER_DUMP_ENABLE + hm01b0_framebuffer_dump(frame, frame_width * frame_height * channels); +#endif + + return kTfLiteOk; +} From 8d9b34c4cde3c8773e57118f58d00f35df6db6de Mon Sep 17 00:00:00 2001 From: Nupur Garg Date: Tue, 9 Jul 2019 15:39:18 -0700 Subject: [PATCH 149/332] Propagate node debug information. PiperOrigin-RevId: 257286387 --- tensorflow/lite/python/convert.py | 77 ++++++++++++------ tensorflow/lite/python/convert_test.py | 3 + tensorflow/lite/python/lite.py | 78 ++++++++++--------- tensorflow/lite/python/wrap_toco.py | 12 ++- tensorflow/lite/toco/python/BUILD | 2 + tensorflow/lite/toco/python/toco.i | 11 ++- .../lite/toco/python/toco_from_protos.py | 38 ++++++++- .../lite/toco/python/toco_python_api.cc | 72 ++++++++++++++--- tensorflow/lite/toco/python/toco_python_api.h | 9 ++- 9 files changed, 216 insertions(+), 86 deletions(-) diff --git a/tensorflow/lite/python/convert.py b/tensorflow/lite/python/convert.py index 3dfd112cef8..d849e9d0adc 100644 --- a/tensorflow/lite/python/convert.py +++ b/tensorflow/lite/python/convert.py @@ -93,7 +93,11 @@ class ConverterError(Exception): pass -def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str): +def toco_convert_protos(model_flags_str, + toco_flags_str, + input_data_str, + debug_info_str="", + enable_mlir_converter=False): """Convert `input_data_str` according to model and toco parameters. Unless you know what you are doing consider using @@ -105,6 +109,10 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str): toco_flags_str: Serialized proto describing conversion properties, see `toco/toco_flags.proto`. input_data_str: Input data in serialized form (e.g. a graphdef is common) + debug_info_str: Serialized `GraphDebugInfo` proto describing logging + information. (default "") + enable_mlir_converter: Enables the MLIR converter instead of the TOCO + converter. (default False) Returns: Converted model in serialized form (e.g. a TFLITE model is common). Raises: @@ -118,10 +126,12 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str): if not _toco_from_proto_bin: try: model_str = wrap_toco.wrapped_toco_convert(model_flags_str, - toco_flags_str, input_data_str) + toco_flags_str, input_data_str, + debug_info_str, + enable_mlir_converter) return model_str except Exception as e: - raise ConverterError("TOCO failed: %s" % e) + raise ConverterError(str(e)) # Windows and TemporaryFile are not that useful together, # since you cannot have two readers/writers. So we have to @@ -132,16 +142,17 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str): # Build all input files with _tempfile.NamedTemporaryFile(delete=False) as fp_toco, \ _tempfile.NamedTemporaryFile(delete=False) as fp_model, \ - _tempfile.NamedTemporaryFile(delete=False) as fp_input: + _tempfile.NamedTemporaryFile(delete=False) as fp_input, \ + _tempfile.NamedTemporaryFile(delete=False) as fp_debug: toco_filename = fp_toco.name input_filename = fp_input.name model_filename = fp_model.name + debug_filename = fp_debug.name + fp_model.write(model_flags_str) fp_toco.write(toco_flags_str) fp_input.write(input_data_str) - fp_model.flush() - fp_toco.flush() - fp_input.flush() + fp_debug.write(debug_info_str) # Reserve an output file with _tempfile.NamedTemporaryFile(delete=False) as fp: @@ -149,9 +160,15 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str): # Run cmd = [ - _toco_from_proto_bin, model_filename, toco_filename, input_filename, - output_filename + _toco_from_proto_bin, + model_filename, + toco_filename, + input_filename, + output_filename, + "--debug_proto_file={}".format(debug_filename), ] + if enable_mlir_converter: + cmd.append("--enable_mlir_converter") cmdline = " ".join(cmd) is_windows = _platform.system() == "Windows" proc = _subprocess.Popen( @@ -168,8 +185,7 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str): else: stdout = _try_convert_to_unicode(stdout) stderr = _try_convert_to_unicode(stderr) - raise ConverterError( - "TOCO failed. See console for info.\n%s\n%s\n" % (stdout, stderr)) + raise ConverterError("See console for info.\n%s\n%s\n" % (stdout, stderr)) finally: # Must manually cleanup files. for filename in [ @@ -211,9 +227,9 @@ def build_toco_convert_protos(input_tensors, output_tensors: List of output tensors (only .name is used from this). inference_type: Target data type of real-number arrays in the output file. Must be `{tf.float32, tf.uint8}`. (default tf.float32) + Must be `{tf.float32, tf.uint8}`. (default `inference_type`) inference_input_type: Target data type of real-number input arrays. Allows for a different type for input arrays in the case of quantization. - Must be `{tf.float32, tf.uint8}`. (default `inference_type`) input_format: Type of data to read Currently must be `{TENSORFLOW_GRAPHDEF}`. (default TENSORFLOW_GRAPHDEF) input_shapes: Input array shape. It needs to be a list of the same length @@ -266,7 +282,7 @@ def build_toco_convert_protos(input_tensors, Returns: model_flags, toco_flags, debug_info: three protocol buffers describing the - conversion process and debug information. + conversion process and debug information. Raises: ValueError: @@ -330,7 +346,7 @@ def build_toco_convert_protos(input_tensors, def toco_convert_graph_def(input_data, input_arrays_with_shape, output_arrays, - *args, **kwargs): + enable_mlir_converter, *args, **kwargs): """"Convert a model using TOCO. This function is used to convert GraphDefs that cannot be loaded into @@ -347,6 +363,8 @@ def toco_convert_graph_def(input_data, input_arrays_with_shape, output_arrays, output_arrays: List of output tensors to freeze graph with. Use only when graph cannot be loaded into TensorFlow and when `output_tensors` is None. (default None) + enable_mlir_converter: Enables the MLIR converter instead of the TOCO + converter. *args: See `build_toco_convert_protos`, **kwargs: See `build_toco_convert_protos`. @@ -375,14 +393,16 @@ def toco_convert_graph_def(input_data, input_arrays_with_shape, output_arrays, for name in output_arrays: model_flags.output_arrays.append(name) - data = toco_convert_protos(model_flags.SerializeToString(), - toco_flags.SerializeToString(), - input_data.SerializeToString()) + data = toco_convert_protos( + model_flags.SerializeToString(), + toco_flags.SerializeToString(), + input_data.SerializeToString(), + enable_mlir_converter=enable_mlir_converter) return data -def toco_convert_impl(input_data, input_tensors, output_tensors, *args, - **kwargs): +def toco_convert_impl(input_data, input_tensors, output_tensors, + enable_mlir_converter, *args, **kwargs): """"Convert a model using TOCO. Typically this function is used to convert from TensorFlow GraphDef to TFLite. @@ -394,6 +414,8 @@ def toco_convert_impl(input_data, input_tensors, output_tensors, *args, input_tensors: List of input tensors. Type and shape are computed using `foo.shape` and `foo.dtype`. output_tensors: List of output tensors (only .name is used from this). + enable_mlir_converter: Enables the MLIR converter instead of the TOCO + converter. *args: See `build_toco_convert_protos`, **kwargs: See `build_toco_convert_protos`. @@ -404,11 +426,15 @@ def toco_convert_impl(input_data, input_tensors, output_tensors, *args, Raises: Defined in `build_toco_convert_protos`. """ - model_flags, toco_flags, _ = build_toco_convert_protos( + model_flags, toco_flags, debug_info = build_toco_convert_protos( input_tensors, output_tensors, *args, **kwargs) - data = toco_convert_protos(model_flags.SerializeToString(), - toco_flags.SerializeToString(), - input_data.SerializeToString()) + debug_info_str = debug_info.SerializeToString() if debug_info else "" + data = toco_convert_protos( + model_flags.SerializeToString(), + toco_flags.SerializeToString(), + input_data.SerializeToString(), + debug_info_str=debug_info_str, + enable_mlir_converter=enable_mlir_converter) return data @@ -437,5 +463,6 @@ def toco_convert(input_data, input_tensors, output_tensors, *args, **kwargs): Raises: Defined in `build_toco_convert_protos`. """ - return toco_convert_impl(input_data, input_tensors, output_tensors, *args, - **kwargs) + enable_mlir_converter = kwargs.get("enable_mlir_converter", False) + return toco_convert_impl(input_data, input_tensors, output_tensors, + enable_mlir_converter, *args, **kwargs) diff --git a/tensorflow/lite/python/convert_test.py b/tensorflow/lite/python/convert_test.py index 693f41cf082..382c351f7a7 100644 --- a/tensorflow/lite/python/convert_test.py +++ b/tensorflow/lite/python/convert_test.py @@ -90,6 +90,7 @@ class ConvertTest(test_util.TensorFlowTestCase): tflite_model = convert.toco_convert_graph_def( sess.graph_def, [("input", [1, 16, 16, 3])], ["add"], + enable_mlir_converter=False, inference_type=lite_constants.FLOAT) self.assertTrue(tflite_model) @@ -126,6 +127,7 @@ class ConvertTest(test_util.TensorFlowTestCase): sess.graph_def, input_arrays_map, output_arrays, + enable_mlir_converter=False, inference_type=lite_constants.QUANTIZED_UINT8, quantized_input_stats=[(0., 1.), (0., 1.)]) self.assertTrue(tflite_model) @@ -171,6 +173,7 @@ class ConvertTest(test_util.TensorFlowTestCase): sess.graph_def, input_arrays_map, output_arrays, + enable_mlir_converter=False, inference_type=lite_constants.QUANTIZED_UINT8) self.assertEqual( "std_dev and mean must be defined when inference_input_type is " diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py index fe3922a8654..87646a572c8 100644 --- a/tensorflow/lite/python/lite.py +++ b/tensorflow/lite/python/lite.py @@ -233,6 +233,25 @@ class TFLiteConverterBase(object): self.representative_dataset.input_gen, inference_input_type, inference_output_type, allow_float) + def _get_base_converter_args(self): + """Returns the base converter args. + + Returns: + {key str: val} + """ + float16_quantize = self._is_float16_quantize() + args = { + "input_format": constants.TENSORFLOW_GRAPHDEF, + "allow_custom_ops": self.allow_custom_ops, + "post_training_quantize": (self._is_int8_weight_only_quantize() or + float16_quantize), + "quantize_to_float16": float16_quantize, + "debug_info": self._debug_info, + "target_ops": self._target_ops, + "enable_mlir_converter": self.experimental_enable_mlir_converter, + } + return args + @_tf_export("lite.TFLiteConverter", v1=[]) class TFLiteConverterV2(TFLiteConverterBase): @@ -251,6 +270,8 @@ class TFLiteConverterV2(TFLiteConverterBase): representative_dataset: A representative dataset that can be used to generate input and output samples for the model. The converter can use the dataset to evaluate different optimizations. + experimental_enable_mlir_converter: Experimental flag, subject to change. + Enables the MLIR converter instead of the TOCO converter. Example usage: @@ -287,6 +308,7 @@ class TFLiteConverterV2(TFLiteConverterBase): self.allow_custom_ops = False self.target_spec = TargetSpec() self._debug_info = None + self.experimental_enable_mlir_converter = False @classmethod def from_concrete_functions(cls, funcs): @@ -414,23 +436,7 @@ class TFLiteConverterV2(TFLiteConverterBase): self._validate_representative_dataset() self._debug_info = _get_debug_info( _build_debug_info_func(self._funcs[0].graph), graph_def) - - float16_quantize = self._is_float16_quantize() - - converter_kwargs = { - "input_format": - constants.TENSORFLOW_GRAPHDEF, - "allow_custom_ops": - self.allow_custom_ops, - "post_training_quantize": - self._is_int8_weight_only_quantize() or float16_quantize, - "quantize_to_float16": - float16_quantize, - "target_ops": - self.target_spec.supported_ops, - "debug_info": - self._debug_info - } + converter_kwargs = self._get_base_converter_args() # Converts model. result = _toco_convert_impl( @@ -522,6 +528,8 @@ class TFLiteConverter(TFLiteConverterBase): representative_dataset: A representative dataset that can be used to generate input and output samples for the model. The converter can use the dataset to evaluate different optimizations. + experimental_enable_mlir_converter: Experimental flag, subject to change. + Enables the MLIR converter instead of the TOCO converter. Example usage: @@ -597,6 +605,7 @@ class TFLiteConverter(TFLiteConverterBase): self.target_spec = TargetSpec() self._debug_info_func = experimental_debug_info_func self._debug_info = None + self.experimental_enable_mlir_converter = False # Attributes are used by models that cannot be loaded into TensorFlow. if not self._has_valid_tensors(): @@ -939,31 +948,11 @@ class TFLiteConverter(TFLiteConverterBase): "Provide an inference_input_type and inference_output_type of type " "tf.float32.") - float16_quantize = self._is_float16_quantize() - if not post_training_optimize and self.inference_output_type is not None: raise ValueError( "inference_output_type is currently not supported if optimizations " "are not enabled.") - converter_kwargs = { - "inference_type": self.inference_type, - "inference_input_type": toco_inference_input_type, - "input_format": constants.TENSORFLOW_GRAPHDEF, - "output_format": self.output_format, - "quantized_input_stats": quantized_stats, - "default_ranges_stats": self.default_ranges_stats, - "drop_control_dependency": self.drop_control_dependency, - "reorder_across_fake_quant": self.reorder_across_fake_quant, - "change_concat_input_ranges": self.change_concat_input_ranges, - "allow_custom_ops": self.allow_custom_ops, - "post_training_quantize": weight_only_quantize or float16_quantize, - "quantize_to_float16": float16_quantize, - "target_ops": self._target_ops, - "dump_graphviz_dir": self.dump_graphviz_dir, - "dump_graphviz_video": self.dump_graphviz_video - } - optimized_graph = self._graph_def if self.inference_type != constants.QUANTIZED_UINT8: try: @@ -977,13 +966,26 @@ class TFLiteConverter(TFLiteConverterBase): self._debug_info = _get_debug_info(self._debug_info_func, optimized_graph) + converter_kwargs = self._get_base_converter_args() + converter_kwargs.update({ + "inference_type": self.inference_type, + "inference_input_type": toco_inference_input_type, + "output_format": self.output_format, + "quantized_input_stats": quantized_stats, + "default_ranges_stats": self.default_ranges_stats, + "drop_control_dependency": self.drop_control_dependency, + "reorder_across_fake_quant": self.reorder_across_fake_quant, + "change_concat_input_ranges": self.change_concat_input_ranges, + "dump_graphviz_dir": self.dump_graphviz_dir, + "dump_graphviz_video": self.dump_graphviz_video + }) + # Converts model. if self._has_valid_tensors(): result = _toco_convert_impl( input_data=optimized_graph, input_tensors=self._input_tensors, output_tensors=self._output_tensors, - debug_info=self._debug_info, **converter_kwargs) else: result = _toco_convert_graph_def( diff --git a/tensorflow/lite/python/wrap_toco.py b/tensorflow/lite/python/wrap_toco.py index 7b514804b37..aa17e2ff192 100644 --- a/tensorflow/lite/python/wrap_toco.py +++ b/tensorflow/lite/python/wrap_toco.py @@ -29,10 +29,16 @@ _toco_python = LazyLoader( del LazyLoader -def wrapped_toco_convert(model_flags_str, toco_flags_str, input_data_str): +def wrapped_toco_convert(model_flags_str, toco_flags_str, input_data_str, + debug_info_str, enable_mlir_converter): """Wraps TocoConvert with lazy loader.""" - return _toco_python.TocoConvert(model_flags_str, toco_flags_str, - input_data_str) + return _toco_python.TocoConvert( + model_flags_str, + toco_flags_str, + input_data_str, + False, # extended_return + debug_info_str, + enable_mlir_converter) def wrapped_get_potentially_supported_ops(): diff --git a/tensorflow/lite/toco/python/BUILD b/tensorflow/lite/toco/python/BUILD index c2b7d7d9668..e11e9cf1578 100644 --- a/tensorflow/lite/toco/python/BUILD +++ b/tensorflow/lite/toco/python/BUILD @@ -22,9 +22,11 @@ cc_library( name = "toco_python_api", srcs = ["toco_python_api.cc"], hdrs = ["toco_python_api.h"], + features = ["no_layering_check"], deps = [ "//third_party/python_runtime:headers", "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", "//tensorflow/lite/python/interpreter_wrapper:python_utils", "//tensorflow/lite/toco:model_flags_proto_cc", "//tensorflow/lite/toco:toco_flags_proto_cc", diff --git a/tensorflow/lite/toco/python/toco.i b/tensorflow/lite/toco/python/toco.i index de10fca99e8..3aa9ce6553f 100644 --- a/tensorflow/lite/toco/python/toco.i +++ b/tensorflow/lite/toco/python/toco.i @@ -26,13 +26,18 @@ namespace toco { // parameters (see relevant .protos for more information). Returns a string // representing the contents of the converted model. When extended_return // flag is set to true returns a dictionary that contains string representation -// of the converted model and some statitics like arithmetic ops count. +// of the converted model and some statistics like arithmetic ops count. +// `debug_info_str` contains the `GraphDebugInfo` proto. When +// `enable_mlir_converter` is True, the MLIR converter is used instead of the +// TOCO converter. PyObject* TocoConvert(PyObject* model_flags_proto_txt_raw, PyObject* toco_flags_proto_txt_raw, PyObject* input_contents_txt_raw, - bool extended_return = false); + bool extended_return = false, + PyObject* debug_info_txt_raw = nullptr, + bool enable_mlir_converter = false); // Returns a list of names of all ops potentially supported by tflite. PyObject* TocoGetPotentiallySupportedOps(); -} // namespace toco \ No newline at end of file +} // namespace toco diff --git a/tensorflow/lite/toco/python/toco_from_protos.py b/tensorflow/lite/toco/python/toco_from_protos.py index 152dd241eab..0566cb8ba60 100644 --- a/tensorflow/lite/toco/python/toco_from_protos.py +++ b/tensorflow/lite/toco/python/toco_from_protos.py @@ -26,11 +26,30 @@ FLAGS = None def execute(unused_args): - model_str = open(FLAGS.model_proto_file, "rb").read() - toco_str = open(FLAGS.toco_proto_file, "rb").read() - input_str = open(FLAGS.model_input_file, "rb").read() + """Runs the converter.""" + with open(FLAGS.model_proto_file, "rb") as model_file: + model_str = model_file.read() - output_str = tensorflow_wrap_toco.TocoConvert(model_str, toco_str, input_str) + with open(FLAGS.toco_proto_file, "rb") as toco_file: + toco_str = toco_file.read() + + with open(FLAGS.model_input_file, "rb") as input_file: + input_str = input_file.read() + + debug_info_str = "" + if FLAGS.debug_proto_file: + with open(FLAGS.debug_proto_file, "rb") as debug_info_file: + debug_info_str = debug_info_file.read() + + enable_mlir_converter = FLAGS.enable_mlir_converter + + output_str = tensorflow_wrap_toco.TocoConvert( + model_str, + toco_str, + input_str, + False, # extended_return + debug_info_str, + enable_mlir_converter) open(FLAGS.model_output_file, "wb").write(output_str) sys.exit(0) @@ -53,6 +72,17 @@ def main(): "model_output_file", type=str, help="Result of applying TOCO conversion is written here.") + parser.add_argument( + "--debug_proto_file", + type=str, + default="", + help=("File containing serialized `GraphDebugInfo` proto that describes " + "logging information.")) + parser.add_argument( + "--enable_mlir_converter", + action="store_true", + help=("Boolean indiciating whether to enable the MLIR converter instead " + "of TOCO converter. (default False)")) FLAGS, unparsed = parser.parse_known_args() diff --git a/tensorflow/lite/toco/python/toco_python_api.cc b/tensorflow/lite/toco/python/toco_python_api.cc index 22557a34cc5..de714b28e3a 100644 --- a/tensorflow/lite/toco/python/toco_python_api.cc +++ b/tensorflow/lite/toco/python/toco_python_api.cc @@ -12,6 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/lite/toco/python/toco_python_api.h" + #include #include #include @@ -20,20 +22,27 @@ limitations under the License. #include "tensorflow/lite/python/interpreter_wrapper/python_utils.h" #include "tensorflow/lite/toco/import_tensorflow.h" #include "tensorflow/lite/toco/model_flags.pb.h" -#include "tensorflow/lite/toco/python/toco_python_api.h" #include "tensorflow/lite/toco/toco_flags.pb.h" #include "tensorflow/lite/toco/toco_graphviz_dump_options.h" #include "tensorflow/lite/toco/toco_port.h" #include "tensorflow/lite/toco/toco_tooling.h" #include "tensorflow/lite/toco/toco_types.h" +#if defined(PLATFORM_GOOGLE) +#include "tensorflow/compiler/mlir/lite/python/graphdef_to_tfl_flatbuffer.h" +#else +#include "tensorflow/core/protobuf/graph_debug_info.pb.h" +#endif + namespace toco { // NOTE(aselle): We are using raw PyObject's here because we want to make // sure we input and output bytes rather than unicode strings for Python3. PyObject* TocoConvert(PyObject* model_flags_proto_txt_raw, PyObject* toco_flags_proto_txt_raw, - PyObject* input_contents_txt_raw, bool extended_return) { + PyObject* input_contents_txt_raw, bool extended_return, + PyObject* debug_info_txt_raw, + bool enable_mlir_converter) { // Use Python C API to validate and convert arguments. In py3 (bytes), // in py2 (str). auto ConvertArg = [&](PyObject* obj, bool* error) { @@ -70,12 +79,35 @@ PyObject* TocoConvert(PyObject* model_flags_proto_txt_raw, // Use TOCO to produce new outputs. toco::ModelFlags model_flags; if (!model_flags.ParseFromString(model_flags_proto_txt)) { - PyErr_SetString(PyExc_ValueError, "Model proto failed to parse."); + PyErr_SetString(PyExc_ValueError, + "Failed to convert Model to Python String."); return nullptr; } toco::TocoFlags toco_flags; if (!toco_flags.ParseFromString(toco_flags_proto_txt)) { - PyErr_SetString(PyExc_ValueError, "Toco proto failed to parse."); + PyErr_SetString(PyExc_ValueError, + "Failed to convert Toco to Python String."); + return nullptr; + } + + tensorflow::GraphDebugInfo debug_info; + if (debug_info_txt_raw) { + std::string debug_info_txt = ConvertArg(debug_info_txt_raw, &error); + if (error) { + PyErr_SetString(PyExc_ValueError, "Input DebugInfo is invalid."); + return nullptr; + } + if (!debug_info.ParseFromString(debug_info_txt)) { + PyErr_SetString(PyExc_ValueError, + "Failed to convert DebugInfo to Python String."); + return nullptr; + } + } + + tensorflow::GraphDef graph_def; + if (!graph_def.ParseFromString(input_contents_txt)) { + PyErr_SetString(PyExc_ValueError, + "Failed to convert GraphDef to Python String."); return nullptr; } @@ -87,18 +119,36 @@ PyObject* TocoConvert(PyObject* model_flags_proto_txt_raw, dump_options.dump_graphviz_video = toco_flags.dump_graphviz_include_video(); } - // Convert model. - std::unique_ptr model = - toco::Import(toco_flags, model_flags, input_contents_txt); - toco::Transform(toco_flags, model.get()); string output_file_contents_txt; - auto status = Export(toco_flags, *model, toco_flags.allow_custom_ops(), - &output_file_contents_txt); + tensorflow::Status status; + std::unique_ptr model; + + // Convert model. + if (enable_mlir_converter) { +#if defined(PLATFORM_GOOGLE) + status = tensorflow::ConvertGraphDefToTFLiteFlatBuffer( + model_flags, toco_flags, debug_info, graph_def, + &output_file_contents_txt); +#else + // TODO(b/124314620): Remove this condition. + PyErr_SetString(PyExc_Exception, + "This flag is not supported by this version of the " + "TFLite converter. This functionality is being " + "actively worked on."); + return nullptr; +#endif + } else { + model = toco::Import(toco_flags, model_flags, input_contents_txt); + toco::Transform(toco_flags, model.get()); + status = Export(toco_flags, *model, toco_flags.allow_custom_ops(), + &output_file_contents_txt); + } + if (!status.ok()) { PyErr_SetString(PyExc_Exception, status.error_message().c_str()); return nullptr; } - if (extended_return) { + if (extended_return && !enable_mlir_converter) { PyObject* dict = PyDict_New(); PyDict_SetItemString( dict, "flatbuffer", diff --git a/tensorflow/lite/toco/python/toco_python_api.h b/tensorflow/lite/toco/python/toco_python_api.h index 20390c32a5e..add7bf9f4ed 100644 --- a/tensorflow/lite/toco/python/toco_python_api.h +++ b/tensorflow/lite/toco/python/toco_python_api.h @@ -25,11 +25,16 @@ namespace toco { // parameters (see relevant .protos for more information). Returns a string // representing the contents of the converted model. When extended_return // flag is set to true returns a dictionary that contains string representation -// of the converted model and some statitics like arithmetic ops count. +// of the converted model and some statistics like arithmetic ops count. +// `debug_info_str` contains the `GraphDebugInfo` proto. When +// `enable_mlir_converter` is True, the MLIR converter is used instead of the +// TOCO converter. PyObject* TocoConvert(PyObject* model_flags_proto_txt_raw, PyObject* toco_flags_proto_txt_raw, PyObject* input_contents_txt_raw, - bool extended_return = false); + bool extended_return = false, + PyObject* debug_info_txt_raw = nullptr, + bool enable_mlir_converter = false); // Returns a list of names of all ops potentially supported by tflite. PyObject* TocoGetPotentiallySupportedOps(); From 06eea697f10cd0004b6d68dda49a74bd3a7870f6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 16:00:13 -0700 Subject: [PATCH 150/332] Replace TraceMe for EagerCopyToDevice with TraceMe recording loops calling it. PiperOrigin-RevId: 257290089 --- .../core/common_runtime/eager/execute.cc | 70 ++++++++++--------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index a206d4068e8..785a30750b4 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -214,6 +214,8 @@ Status ValidateInputTypeAndPlacement( EagerContext* ctx, EagerOperation* op, const core::RefCountPtr& kernel, RunMetadata* run_metadata) { + profiler::TraceMe activity("ValidateInputTypeAndPlacement", + profiler::TraceMeLevel::kInfo); if (kernel->num_inputs() != op->Inputs().size()) { return errors::InvalidArgument("expected ", kernel->num_inputs(), " inputs, got ", op->Inputs().size()); @@ -487,6 +489,8 @@ Status EagerLocalExecute(EagerOperation* op, std::unordered_map input_resource_variable_dtypes_and_shapes; if (is_multi_device_function) { + profiler::TraceMe activity("EagerCopyToDeviceAndAddCacheKey", + profiler::TraceMeLevel::kInfo); input_dev_ptrs.reserve(op->Inputs().size()); // All inputs need to be on local devices. // TODO(b/122851476): This is a limitation of the current code base (but @@ -807,34 +811,38 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, eager::Operation* remote_op = request->add_queue()->mutable_operation(); - for (int i = 0; i < op->Inputs().size(); i++) { - tensorflow::TensorHandle* input = op->Inputs()[i]; - tensorflow::Device* input_device = input->device(); - if (op->Device() != input_device && - // If the expected and actual devices are on the same task, don't - // explicitly copy, and instead depend on the copy to happen locally - // when the op is executed on the device. - !ctx->OnSameTask(op->Device(), input_device)) { - tensorflow::Device* remote_cpu_device; - TF_RETURN_IF_ERROR( - ctx->CPUDeviceOnTask(op->Device(), &remote_cpu_device)); - // TODO(b/110044833): It's possible the same tensor gets copied to the - // remote device repeatedly. - // Always copy to the remote CPU so that the actual device can be - // correctly determined after the kernel is selected/instantiated, since - // the op might have its inputs on host memory. - TensorHandle* handle = nullptr; - TF_RETURN_IF_ERROR( - MaybeCopyInputToExpectedDevice(op, op->Device(), i, remote_cpu_device, - /* run_metadata= */ nullptr, &handle)); - op->UpdateInput(i, handle); - input = handle; - input_device = remote_cpu_device; - // Unref handle since it has a ref as an input now - handle->Unref(); - } + { + profiler::TraceMe activity("CopyInputToExpectedDevice", + profiler::TraceMeLevel::kInfo); + for (int i = 0; i < op->Inputs().size(); i++) { + tensorflow::TensorHandle* input = op->Inputs()[i]; + tensorflow::Device* input_device = input->device(); + if (op->Device() != input_device && + // If the expected and actual devices are on the same task, don't + // explicitly copy, and instead depend on the copy to happen locally + // when the op is executed on the device. + !ctx->OnSameTask(op->Device(), input_device)) { + tensorflow::Device* remote_cpu_device; + TF_RETURN_IF_ERROR( + ctx->CPUDeviceOnTask(op->Device(), &remote_cpu_device)); + // TODO(b/110044833): It's possible the same tensor gets copied to the + // remote device repeatedly. + // Always copy to the remote CPU so that the actual device can be + // correctly determined after the kernel is selected/instantiated, since + // the op might have its inputs on host memory. + TensorHandle* handle = nullptr; + TF_RETURN_IF_ERROR(MaybeCopyInputToExpectedDevice( + op, op->Device(), i, remote_cpu_device, + /* run_metadata= */ nullptr, &handle)); + op->UpdateInput(i, handle); + input = handle; + input_device = remote_cpu_device; + // Unref handle since it has a ref as an input now + handle->Unref(); + } - TF_RETURN_IF_ERROR(AddRemoteInput(remote_op, input, input_device)); + TF_RETURN_IF_ERROR(AddRemoteInput(remote_op, input, input_device)); + } } PrepareRemoteOp(remote_op, op); @@ -1259,8 +1267,7 @@ Status ExecuteSend(EagerContext* ctx, Device* device, TensorHandle* h, } else { eager::EagerClient* eager_client; uint64 context_id = ctx->GetContextId(); - TF_RETURN_IF_ERROR( - ctx->GetClient(device, &eager_client)); + TF_RETURN_IF_ERROR(ctx->GetClient(device, &eager_client)); std::unique_ptr request(new eager::EnqueueRequest); eager::EnqueueResponse response; @@ -1328,8 +1335,7 @@ Status ExecuteRecv(EagerContext* ctx, Device* device, DataType dtype, } else { eager::EagerClient* eager_client; uint64 context_id = ctx->GetContextId(); - TF_RETURN_IF_ERROR( - ctx->GetClient(device, &eager_client)); + TF_RETURN_IF_ERROR(ctx->GetClient(device, &eager_client)); std::unique_ptr request(new eager::EnqueueRequest); eager::EnqueueResponse response; @@ -1385,8 +1391,6 @@ string GetUniqueWireID() { Status EagerCopyToDevice(TensorHandle* h, EagerContext* ctx, Device* device, bool mirror, TensorHandle** result) { - profiler::TraceMe activity("EagerCopyToDevice", - profiler::TraceMeLevel::kInfo); Device* send_device = h->DeviceOrHostCPU(ctx); bool sender_is_local = ctx->IsLocal(send_device); From d128386daeb8e3226ebbf51b6529da0093bb0b9e Mon Sep 17 00:00:00 2001 From: Ayush Dubey Date: Tue, 9 Jul 2019 16:04:01 -0700 Subject: [PATCH 151/332] Enable scoped allocator optimizer to work well when an output is consumed by ops in different scopes. Previously, if the same output was input to different ops, and the ops belonged to different scopes, then ScopedAllocatorOptimizer would fail. The failure was actually silent, i.e. CheckExistingScopedAllocator was buggy, and so the optimizer would actually assign the same output to 2 different scope ids. This caused undefined behavior during execution. This change allows using the same output in different scoped allocator groups. To do this, we insert an identity node between the input and the op for each time it is repeated after the first group. This ensures that the other op groups use the identity as its input, and hence each output in the graph is assigned at most one scope id. We also change CheckExistingScopedAllocator to actually (sanity) check that an output has not been previously assigned a scope id. PiperOrigin-RevId: 257291024 --- .../optimizers/scoped_allocator_optimizer.cc | 117 ++++++++++------ .../scoped_allocator_optimizer_test.cc | 127 ++++++++++++------ 2 files changed, 168 insertions(+), 76 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc index 13fb883217a..29bc154eb0e 100644 --- a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc @@ -40,6 +40,9 @@ namespace tensorflow { namespace grappler { namespace { + +const char kScopedAllocatorAttrName[] = "_scoped_allocator"; + // Node names often have some kind of name_scope prefix, with slashes, // and a _nn numeric suffix. Returns true if the main part of the node_name // matches op_name, i.e. it looks from the name like this node is @@ -161,27 +164,51 @@ Status RemoveEdge(const string& input_edge_name, const string& from_node_name, return Status::OK(); } -// If `input` to `op` is an Exit node, rewrite the graph to insert an identity -// op between `input` and `op`. Return the input to op in `new_input`. -// `edge_name` gives the name of the edge from `input` to `op`, and -// `edge_position` is the output position of this edge on `input`. +// In certain cases, we would like to insert an identity op between `input` and +// `op` to ensure correctness. We currently do this in 2 cases: when `input` is +// Exit node, or when `input` is already marked for allocation with another +// scoped allocator op. // -// (loop) is rewritten to (loop) +// If `input` is an Exit node, we add an identity to avoid the case when Exit +// has inputs from different frames. +// +// If `input` has kScopedAllocatorAttrName attribute, this means that it was +// previously marked for allocation with a different scope id. Since there can +// be only one scope id per output, we insert an identity between the input and +// op. This will ensure that the identity becomes the new input to op, and this +// identity can be marked with a new scope id different from `input`. +// +// If the graph is rewritten, this function will perform the following change: +// +// input input // | | -// Exit Exit -// | | -// (collective op) Identity +// op Identity // | -// (collective op) -// This avoids the case when Exit has inputs from different frames. -Status MaybeRewriteExitNode(ScopedAllocatorOptimizer* sa_opti, - int64 invocation_count, GraphDef* graph, - NodeMap* node_map, const DataType& dtype, - NodeDef* input, const string& edge_name, - int edge_position, NodeDef* op, - NodeDef** new_input) { - if (!IsExit(*input)) { +// op +// +// This function returns the input to op in `new_input`, and the output index +// from input to op in `new_output_index`. +// `edge_name` gives the name of the edge from `input` to `op`, and +// `output_index` is the output index of this edge on `input`. +Status MaybeRewriteInput(ScopedAllocatorOptimizer* sa_opti, + int64 invocation_count, GraphDef* graph, + NodeMap* node_map, const DataType& dtype, + NodeDef* input, const string& edge_name, + int output_index, NodeDef* op, NodeDef** new_input, + int* new_output_index) { + bool rewrite = false; + if (IsExit(*input)) { + rewrite = true; + } else { + AttrSlice input_attrs = AttrSlice(*input); + std::vector scopes; + Status sa_status = + GetNodeAttr(input_attrs, kScopedAllocatorAttrName, &scopes); + rewrite = sa_status.ok(); + } + if (!rewrite) { *new_input = input; + *new_output_index = output_index; return Status::OK(); } @@ -197,15 +224,21 @@ Status MaybeRewriteExitNode(ScopedAllocatorOptimizer* sa_opti, NodeDefBuilder identity_builder(identity_name, "Identity"); identity_builder.Device(op->device()); identity_builder.Attr("T", dtype); - // Connect `input` to `identity`. - identity_builder.Input(NodeDefBuilder::NodeOut(input->name(), 0, dtype)); + // Connect output at `output_index` from `input` to `identity`. + identity_builder.Input( + NodeDefBuilder::NodeOut(input->name(), output_index, dtype)); LOG_WARNING_AND_RETURN_IF_ERROR(identity_builder.Finalize(identity)); node_map->AddNode(identity_name, identity); node_map->AddOutput(input->name(), identity_name); - // Connect `identity` to `op`. + // Connect `identity` to `op`. Both output index at `identity` and input + // index at `op` are 0. node_map->AddOutput(identity_name, op->name()); - *op->mutable_input(edge_position) = strings::StrCat(identity_name, ":", 0); + *op->mutable_input(0) = strings::StrCat(identity_name, ":", 0); *new_input = identity; + *new_output_index = 0; + VLOG(1) << "Rewrite input " << edge_name << " op " << op->name() + << " old output index " << output_index << " with identity " + << identity_name << " new output index 0"; return Status::OK(); } @@ -219,7 +252,7 @@ Status GetInputs(ScopedAllocatorOptimizer* sa_opti, int64 invocation_count, VLOG(1) << "Getinputs"; for (NodeDef* n : ops) { NodeDef* inode = nullptr; - int position = 0; + int output_index = 0; VLOG(2) << "for node " << n->name(); for (const auto& input_name : n->input()) { if (!IsControlInput(input_name)) { @@ -227,16 +260,18 @@ Status GetInputs(ScopedAllocatorOptimizer* sa_opti, int64 invocation_count, return errors::Internal("Found more than one input for node ", n->name()); } - ParseNodeName(input_name, &position); + ParseNodeName(input_name, &output_index); inode = node_map->GetNode(input_name); if (inode == nullptr) { return errors::Internal("Did not find node ", input_name); } - VLOG(2) << "inode " << inode->DebugString(); - LOG_WARNING_AND_RETURN_IF_ERROR(MaybeRewriteExitNode( + VLOG(2) << "inode " << inode->DebugString() << " output_index " + << output_index; + LOG_WARNING_AND_RETURN_IF_ERROR(MaybeRewriteInput( sa_opti, invocation_count, graph, node_map, dtype, inode, - input_name, position, n, &inode)); - VLOG(2) << "inode after rewrite " << inode->DebugString(); + input_name, output_index, n, &inode, &output_index)); + VLOG(2) << "inode after rewrite " << inode->DebugString() + << " output_index " << output_index; } } AttrSlice inode_attrs = AttrSlice(*inode); @@ -247,7 +282,7 @@ Status GetInputs(ScopedAllocatorOptimizer* sa_opti, int64 invocation_count, return errors::Internal("ScopedAllocatorOptimizer expected input type ", dtype, " but found ", inode_dtype); } - inputs->emplace_back(inode, position, n); + inputs->emplace_back(inode, output_index, n); } return Status::OK(); } @@ -274,19 +309,24 @@ class UnaryElementwiseRewriter : public ScopedAllocatorOptimizer::Rewriter { ~UnaryElementwiseRewriter() override {} // Return non-OK if any input is already committed to a ScopedAllocator. + // + // We insert an identity to ensure that inputs are not committed to different + // scope ids in `MaybeRewriteInput`, so this function is basically a sanity + // check. Status CheckExistingScopedAllocator(const std::vector& inputs) { for (const InputDesc& nd : inputs) { VLOG(2) << "get attrs for " << nd.from_node_def->name(); AttrSlice n_attrs = AttrSlice(*nd.from_node_def); - int sa_id; - Status ss = GetNodeAttr(n_attrs, "sa_id", &sa_id); + std::vector scope_ids; + Status ss = GetNodeAttr(n_attrs, kScopedAllocatorAttrName, &scope_ids); if (ss.ok()) { - LOG(INFO) << "Abandoning PARewriter because input " - << nd.from_node_def->name() << " is already assigned " - << "to ScopedAllocator " << sa_id; + LOG(INFO) << "Abandoning ScopedAllocatorOptimizer because input " + << nd.from_node_def->name() << " output " << scope_ids[0] + << " is already assigned to scope_id " << scope_ids[1]; return errors::Internal( - "Abandoning PARewriter because input ", nd.from_node_def->name(), - " is already assigned to ScopedAllocator ", sa_id); + "Abandoning ScopedAllocatorOptimizer because input ", + nd.from_node_def->name(), " output ", scope_ids[0], " is already ", + "assigned to scope_id ", scope_ids[1]); } } return Status::OK(); @@ -397,7 +437,7 @@ class UnaryElementwiseRewriter : public ScopedAllocatorOptimizer::Rewriter { nd.from_node_def->add_input(strings::StrCat("^", sa_name)); // This attribute says: allocate output_slot from // ScopedAllocator instance sa_id + 1 + i. - ScopedAllocatorOptimizer::ExtendNodeAttr("_scoped_allocator", + ScopedAllocatorOptimizer::ExtendNodeAttr(kScopedAllocatorAttrName, {nd.output_slot, sa_id + 1 + i}, nd.from_node_def); node_map->AddOutput(sa_name, nd.from_node_def->name()); @@ -535,7 +575,7 @@ class UnaryElementwiseRewriter : public ScopedAllocatorOptimizer::Rewriter { // maintained by NodeMap in the loop. std::set output_nodes = node_map->GetOutputs(old_op->name()); VLOG(3) << "old_op " << old_op->name() << " had " << output_nodes.size() - << " outputs. Moving them to the PASplit node."; + << " outputs. Moving them to the ScopedAllocatorSplit node."; if (VLOG_IS_ON(2)) { for (NodeDef* n : output_nodes) { VLOG(3) << " output: " << n->name(); @@ -908,7 +948,8 @@ Status ScopedAllocatorOptimizer::ProcessGraphDef( VLOG(1) << "Processing " << op_name << " set size " << it.second.size(); Rewriter* rewriter = GetRewriter(op_name); if (!rewriter) { - LOG(ERROR) << "Failed to find PARewriter for op_name " << op_name; + LOG(ERROR) << "Failed to find Rewriter in ScopedAllocatorOptimizer " + << "for op_name " << op_name; continue; } rewriter->SetGraphProperties(graph_properties); diff --git a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer_test.cc b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer_test.cc index a6b9257cd25..5fd8a12e7e1 100644 --- a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer_test.cc @@ -50,7 +50,7 @@ class ScopedAllocatorOptimizerTest : public ::testing::Test { std::vector EvaluateNodes(const GraphDef& graph, const std::vector& fetch) { SessionOptions options; - std::unique_ptr session(NewSession(options)); + std::unique_ptr session(NewSession(options)); TF_CHECK_OK(session->Create(graph)); RunOptions run_options; std::vector output_tensors; @@ -78,7 +78,7 @@ class ScopedAllocatorOptimizerTest : public ::testing::Test { r1 r2 */ void BuildAbsGraph(GraphDef* graph_def, bool forward) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Scope s = Scope::NewRootScope(); s = s.WithDevice("/job:localhost/replica:0/task:0/device:CPU:0"); Output a = @@ -104,6 +104,55 @@ class ScopedAllocatorOptimizerTest : public ::testing::Test { TF_CHECK_OK(s.ToGraphDef(graph_def)); } + // Constructs the following graph. + // + // We have 2 different name scopes in this graph. s3, a3, a4, r3, and r4 are + // all under "sub" scope. All other nodes are in the root scope. + // + // The intention is to test that ScopedAllocatorOptimizer works well with a + // graph that has multiple name scopes. In particular, it should work when a + // node (in this case s2) is an input to two nodes in different name scopes + // (a2 and sub/a3) which may be scope allocated. + /* + a b c a b + \ / \ / \ / + s1 s2------ sub/s3 + | | | | + a1 a2 sub/a4 sub/a3 + | | | | + r1 r2 sub/r4 sub/r3 + */ + void BuildGraphWithMultipleScopes(GraphDef* graph_def) { + Scope root_scope = Scope::NewRootScope(); + root_scope = + root_scope.WithDevice("/job:localhost/replica:0/task:0/device:CPU:0"); + + Output a = ops::Const(root_scope.WithOpName("a"), + {1.0, 0.0, 0.0, -1.0}, {2, 2}); + Output b = ops::Const(root_scope.WithOpName("b"), + {1.0, -2.0, 3.0, 4.0}, {2, 2}); + Output c = ops::Const(root_scope.WithOpName("c"), + {-5.0, -2.0, 0.0, -2.0}, {2, 2}); + + // Root scope ops. + Output s1 = ops::Add(root_scope.WithOpName("s1"), a, b); + Output s2 = ops::Add(root_scope.WithOpName("s2"), b, c); + Output a1 = ops::Abs(root_scope.WithOpName("a1"), s1); + Output a2 = ops::Abs(root_scope.WithOpName("a2"), s2); + Output r1 = ops::Reshape(root_scope.WithOpName("r1"), a1, {1, 4}); + Output r2 = ops::Reshape(root_scope.WithOpName("r2"), a2, {4, 1}); + + // Sub scope ops. + Scope sub_scope = root_scope.NewSubScope("sub"); + Output s3 = ops::Add(sub_scope.WithOpName("s3"), a, b); + Output a3 = ops::Abs(sub_scope.WithOpName("a3"), s3); + Output a4 = ops::Abs(sub_scope.WithOpName("a4"), s2); + Output r3 = ops::Reshape(sub_scope.WithOpName("r3"), a3, {1, 4}); + Output r4 = ops::Reshape(sub_scope.WithOpName("r4"), a4, {4, 1}); + + TF_CHECK_OK(root_scope.ToGraphDef(graph_def)); + } + void SetShapes(GraphDef* graph_def) { TensorShapeProto shape_proto; shape_proto.add_dim()->set_size(2); @@ -116,12 +165,11 @@ class ScopedAllocatorOptimizerTest : public ::testing::Test { } } - // Constructs a graph by calling BuildAbsGraph, then executes it and returns - // r1, r2, and scoped_allocator_1_2_Abs:0. - void BuildAndExecuteAbsGraph(bool forward, std::vector* outputs) { - GrapplerItem item; - BuildAbsGraph(&item.graph, forward); - + // Invokes ScopedAllocatorOptimizer on `graph_def`, then executes it and + // returns the outputs specifed by `output_names` in `outputs`. + void ExecuteGraph(const GraphDef& graph_def, + const std::vector& output_names, + std::vector* outputs) { // Turn off all optimization except the ScopedAllocatorOptimizer // to avoid anything that would alter the expected graph input/output, // e.g. by constant folding away all calculations. @@ -136,36 +184,22 @@ class ScopedAllocatorOptimizerTest : public ::testing::Test { rwcfg->clear_optimizers(); (*rwcfg->add_optimizers()) = "scoped_allocator"; rwcfg->mutable_scoped_allocator_opts()->add_enable_op("Abs"); - std::unique_ptr session(CreateSession(item.graph, config)); + std::unique_ptr session(CreateSession(graph_def, config)); - // Request two targets: one fetch output and one non-fetched output. - std::vector output_names = {"r1:0", "r2:0"}; - if (!forward) { - output_names.push_back("scoped_allocator_1_2_Abs:0"); - } std::vector> inputs; std::vector target_nodes = {}; Status s = session->Run(inputs, output_names, target_nodes, outputs); TF_ASSERT_OK(s); - ASSERT_EQ(outputs->size(), forward ? 2 : 3); + ASSERT_EQ(outputs->size(), output_names.size()); } - // Validates that output[0] matches expected0 and outputs[1] matches - // expected1. + // Validates that outputs match expected. void ValidateValues(const std::vector& outputs, - const std::vector& expected0, - const std::vector& expected1) { - for (int oi = 0; oi < outputs.size(); ++oi) { - if (oi == 0) { - ASSERT_EQ(expected0.size(), outputs[oi].NumElements()); - for (int i = 0; i < expected0.size(); ++i) { - EXPECT_EQ(expected0[i], outputs[oi].flat()(i)); - } - } else if (oi == 1) { - ASSERT_EQ(expected1.size(), outputs[oi].NumElements()); - for (int i = 0; i < expected1.size(); ++i) { - EXPECT_EQ(expected1[i], outputs[oi].flat()(i)); - } + const std::vector>& expected) { + for (int i = 0; i < expected.size(); ++i) { + EXPECT_EQ(expected[i].size(), outputs[i].NumElements()); + for (int j = 0; j < expected[i].size(); ++j) { + EXPECT_EQ(expected[i][j], outputs[i].flat()(j)); } } } @@ -230,13 +264,29 @@ TEST_F(ScopedAllocatorOptimizerTest, UnaryRewriteOnly) { TEST_F(ScopedAllocatorOptimizerTest, UnaryExecute) { // Builds the same graph as UnaryRewriteOnly but also executes it and // validates the output. + GraphDef graph_def; + BuildAbsGraph(&graph_def, /*forward=*/false); + SetShapes(&graph_def); std::vector outputs; - BuildAndExecuteAbsGraph(false, &outputs); + ExecuteGraph(graph_def, + /*output_names=*/{"r1:0", "r2:0", "scoped_allocator_1_2_Abs:0"}, + &outputs); // a + b == 2, -2, 3, 3 // b + c == -4, -4, 3, 2 - std::vector expected_r1({2, 2, 3, 3}); - std::vector expected_r2({4, 4, 3, 2}); - ValidateValues(outputs, expected_r1, expected_r2); + ValidateValues(outputs, /*expected=*/{{2, 2, 3, 3}, {4, 4, 3, 2}}); +} + +TEST_F(ScopedAllocatorOptimizerTest, MultipleScopes) { + GraphDef graph_def; + BuildGraphWithMultipleScopes(&graph_def); + SetShapes(&graph_def); + std::vector outputs; + ExecuteGraph(graph_def, + /*output_names=*/{"r1:0", "r2:0", "sub/r3:0", "sub/r4:0"}, + &outputs); + ValidateValues( + outputs, + /*expected=*/{{2, 2, 3, 3}, {4, 4, 3, 2}, {2, 2, 3, 3}, {4, 4, 3, 2}}); } // Tests static ScopedAllocatorOptimizer::ExtendNodeAttr. @@ -264,13 +314,14 @@ TEST_F(ScopedAllocatorOptimizerTest, Extend) { TEST_F(ScopedAllocatorOptimizerTest, ForwardInputToOutput) { // Test that kernels that forward the input to output using `set_output` work // well with scoped allocator optimization. + GraphDef graph_def; + BuildAbsGraph(&graph_def, /*forward=*/true); + SetShapes(&graph_def); std::vector outputs; - BuildAndExecuteAbsGraph(true, &outputs); + ExecuteGraph(graph_def, /*output_names=*/{"r1:0", "r2:0"}, &outputs); // a + b == 2, -2, 3, 3 // b + c == -4, -4, 3, 2 - std::vector expected_r1({2, 2, 3, 3}); - std::vector expected_r2({4, 4, 3, 2}); - ValidateValues(outputs, expected_r1, expected_r2); + ValidateValues(outputs, /*expected=*/{{2, 2, 3, 3}, {4, 4, 3, 2}}); } } // namespace From 14365469a2b38624141c03f1aa7842d672c0af50 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 16:07:42 -0700 Subject: [PATCH 152/332] Added function for generating floating points to include more numbers. Floating point values generated can now be generated by selecting exponents from a uniform distribution approximately between the floating point type's minimum and maximum exponents. PiperOrigin-RevId: 257291678 --- tensorflow/compiler/xla/tests/test_utils.h | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tensorflow/compiler/xla/tests/test_utils.h b/tensorflow/compiler/xla/tests/test_utils.h index b3c8a739058..f2570ecff37 100644 --- a/tensorflow/compiler/xla/tests/test_utils.h +++ b/tensorflow/compiler/xla/tests/test_utils.h @@ -54,6 +54,29 @@ class PseudorandomGenerator { std::mt19937 generator_; }; +// Populates a floating point literal with random floating points sampled from a +// uniform-log distribution spanning approximately the entire range of the +// representable floating point. +template +void PopulateWithRandomFullRangeFloatingPointData(Literal* literal, + std::minstd_rand0* engine) { + // Generates floating points with a log-uniform distribution. This causes the + // exponent of the floating point to have a uniform distribution. + int min_exp, max_exp; + if (std::is_same()) { + min_exp = std::numeric_limits::min_exponent; + max_exp = std::numeric_limits::max_exponent; + } else { + min_exp = std::numeric_limits::min_exponent; + max_exp = std::numeric_limits::max_exponent; + } + std::uniform_real_distribution generator(min_exp - 1, max_exp - 1); + for (FloatT& value : literal->data()) { + float sign = ((*engine)() % 2 == 0) ? 1 : -1; + value = static_cast(pow(2, generator(*engine)) * sign); + } +} + // Generates fake data in a literal of the given shape, or returns an error // status if the element type is currently unhandled for fake data // generation. See below for documentation of pseudo_random. From 5c2d400e90b5fd0a40a9f67f8d026db768913a18 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 16:17:55 -0700 Subject: [PATCH 153/332] NFC: Rename Function to FuncOp. PiperOrigin-RevId: 257293379 --- .../mlir/lite/flatbuffer_translate.cc | 21 +++++++++---------- .../compiler/mlir/lite/mlir_tflite_runner.cc | 2 +- .../mlir/lite/tf_to_tfl_flatbuffer.cc | 2 +- .../transforms/lower_static_tensor_list.cc | 12 +++++------ .../mlir/lite/transforms/post_quantize.cc | 2 +- .../mlir/lite/utils/quantization_driver.cc | 4 ++-- .../mlir/lite/utils/quantization_utils.h | 2 +- .../functional_control_flow_to_cfg.cc | 2 +- .../tensorflow/translate/export_graphdef.cc | 20 +++++++++--------- .../tensorflow/translate/import_graphdef.cc | 4 ++-- .../translate/translate_tf_dialect_op.cc | 2 +- 11 files changed, 36 insertions(+), 37 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc index 23d15663fd4..ba4c88f865a 100644 --- a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc +++ b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc @@ -77,7 +77,7 @@ using llvm::Twine; using mlir::Block; using mlir::Dialect; using mlir::ElementsAttr; -using mlir::Function; +using mlir::FuncOp; using mlir::MLIRContext; using mlir::Module; using mlir::NoneType; @@ -247,14 +247,14 @@ static bool IsValidTFLiteMlirModule(Module module) { MLIRContext* context = module.getContext(); // Verify that module has a function named main. - Function main_fn = module.getNamedFunction("main"); + FuncOp main_fn = module.getNamedFunction("main"); if (!main_fn) { return emitError(UnknownLoc::get(context), "should have a function named 'main'"), false; } - for (auto fn : module.getOps()) { + for (auto fn : module.getOps()) { if (fn.getBlocks().size() != 1) { return fn.emitError("should have exactly one basic block"), false; } @@ -391,11 +391,11 @@ class Translator { Operation* inst, const std::vector& operands, const std::vector& results); - Optional> BuildSubGraph(Function fn); + Optional> BuildSubGraph(FuncOp fn); // Uses the tf.entry_function attribute (if set) to initialize the op to name // mapping. - void InitializeNamesFromAttribute(Function fn); + void InitializeNamesFromAttribute(FuncOp fn); // Returns a unique name for `op`. std::string UniqueName(mlir::Operation* op); @@ -781,7 +781,7 @@ Optional> Translator::BuildOperator( llvm::None; } -void Translator::InitializeNamesFromAttribute(Function fn) { +void Translator::InitializeNamesFromAttribute(FuncOp fn) { auto dict_attr = fn.getAttrOfType("tf.entry_function"); if (!dict_attr) return; @@ -823,8 +823,7 @@ void Translator::InitializeNamesFromAttribute(Function fn) { } } -Optional> Translator::BuildSubGraph( - Function fn) { +Optional> Translator::BuildSubGraph(FuncOp fn) { InitializeNamesFromAttribute(fn); std::vector> tensors; llvm::DenseMap tensor_index_map; @@ -941,14 +940,14 @@ Optional Translator::TranslateInternal() { // Create a list of functions in the module with main function being the // first function in the list. This is required as the first subgraph in the // model is entry point for the model. - std::vector functions; + std::vector functions; functions.reserve(std::distance(module_.begin(), module_.end())); int subgraph_idx = 0; - Function main_fn = module_.getNamedFunction("main"); + FuncOp main_fn = module_.getNamedFunction("main"); subgraph_index_map_[main_fn.getName().str()] = subgraph_idx++; functions.push_back(main_fn); - for (auto fn : module_.getOps()) { + for (auto fn : module_.getOps()) { if (fn == main_fn) continue; subgraph_index_map_[fn.getName().str()] = subgraph_idx++; diff --git a/tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc b/tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc index d65e17e0683..c89e7602cc9 100644 --- a/tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc +++ b/tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc @@ -98,7 +98,7 @@ int main(int argc, char** argv) { if (!module) return 1; // TODO(jpienaar): Expand to support inputs. - mlir::Function main = module->getNamedFunction("main"); + mlir::FuncOp main = module->getNamedFunction("main"); QCHECK(main) << "No 'main' function specified."; if (main.getType().getNumInputs() != 0) LOG(QFATAL) << "NYI: Only nullary functions supported."; diff --git a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc index 8ae5bc2fe2f..02e753e13d9 100644 --- a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc +++ b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc @@ -88,7 +88,7 @@ StatusOr LoadFromGraphdefOrMlirSource( } bool ShouldRunQuantizePasses(mlir::Module m) { - if (mlir::Function main_fn = m.getNamedFunction("main")) { + if (mlir::FuncOp main_fn = m.getNamedFunction("main")) { return main_fn.getAttrOfType("tf.quantize") != mlir::Attribute(); } diff --git a/tensorflow/compiler/mlir/lite/transforms/lower_static_tensor_list.cc b/tensorflow/compiler/mlir/lite/transforms/lower_static_tensor_list.cc index 266aeb3de0d..27175b05660 100644 --- a/tensorflow/compiler/mlir/lite/transforms/lower_static_tensor_list.cc +++ b/tensorflow/compiler/mlir/lite/transforms/lower_static_tensor_list.cc @@ -61,7 +61,7 @@ namespace { class TensorListPatternRewriter : public PatternRewriter { public: - explicit TensorListPatternRewriter(Function fn) + explicit TensorListPatternRewriter(FuncOp fn) : PatternRewriter(fn.getBody()) {} Operation *createOperation(const OperationState &state) override { @@ -77,7 +77,7 @@ struct LowerStaticTensorListPass void runOnModule() override; // Apply type and op changes within a function. - LogicalResult RewriteFunction(Function func, + LogicalResult RewriteFunction(FuncOp func, TensorListPatternRewriter *rewriter); // Changes the function type of `cond_func` and `body_func`, and the result @@ -276,8 +276,8 @@ LogicalResult LowerStaticTensorListPass::UpdateWhileFunctionType( auto *context = &getContext(); auto module = getModule(); - Function cond_func = module.getNamedFunction(while_op->getCond()); - Function body_func = module.getNamedFunction(while_op->getBody()); + FuncOp cond_func = module.getNamedFunction(while_op->getCond()); + FuncOp body_func = module.getNamedFunction(while_op->getBody()); if (cond_func) { // Change `cond_func`'s argument types to `unranked_argument_types`. @@ -327,7 +327,7 @@ LogicalResult LowerStaticTensorListPass::UpdateWhileFunctionType( } LogicalResult LowerStaticTensorListPass::RewriteFunction( - Function func, TensorListPatternRewriter *rewriter) { + FuncOp func, TensorListPatternRewriter *rewriter) { auto *context = &getContext(); for (Block &block : func) { @@ -388,7 +388,7 @@ void LowerStaticTensorListPass::runOnModule() { // have a potential issue when one function taking a `DT_VARIANT` is processed // before the function that produces the `DT_VARIANT`. We need to carefully // order the functions to be processed. - std::vector funcs_in_module; + std::vector funcs_in_module; for (auto func : getModule().getOps()) { // Always place the main function to be the first in the list. if (func.getName() == "main") { diff --git a/tensorflow/compiler/mlir/lite/transforms/post_quantize.cc b/tensorflow/compiler/mlir/lite/transforms/post_quantize.cc index f0f1d3f822c..94c19d27adc 100644 --- a/tensorflow/compiler/mlir/lite/transforms/post_quantize.cc +++ b/tensorflow/compiler/mlir/lite/transforms/post_quantize.cc @@ -48,7 +48,7 @@ class PostQuantizePass : public FunctionPass { bool emit_quant_adaptor_ops_; }; -void RemoveQuantizationAdaptorOps(Function func) { +void RemoveQuantizationAdaptorOps(FuncOp func) { mlir::OpBuilder builder(func.getBody()); auto& bb = func.getBlocks().front(); auto* terminator = bb.getTerminator(); diff --git a/tensorflow/compiler/mlir/lite/utils/quantization_driver.cc b/tensorflow/compiler/mlir/lite/utils/quantization_driver.cc index 7ceb0f5c86e..fa99cf38c8e 100644 --- a/tensorflow/compiler/mlir/lite/utils/quantization_driver.cc +++ b/tensorflow/compiler/mlir/lite/utils/quantization_driver.cc @@ -121,7 +121,7 @@ struct RequantizeState { // class QuantizationDriver { public: - explicit QuantizationDriver(Function fn) : builder_(fn.getBody()) {} + explicit QuantizationDriver(FuncOp fn) : builder_(fn.getBody()) {} // The entry point of the quantization parameters propagation. void Run(); @@ -706,7 +706,7 @@ void QuantizationDriver::Run() { } } -void ApplyQuantizationParamsPropagation(mlir::Function func) { +void ApplyQuantizationParamsPropagation(mlir::FuncOp func) { QuantizationDriver(func).Run(); } diff --git a/tensorflow/compiler/mlir/lite/utils/quantization_utils.h b/tensorflow/compiler/mlir/lite/utils/quantization_utils.h index 33fe39cb05c..a7b9179e5b4 100644 --- a/tensorflow/compiler/mlir/lite/utils/quantization_utils.h +++ b/tensorflow/compiler/mlir/lite/utils/quantization_utils.h @@ -116,7 +116,7 @@ quant::QuantizedType GetUniformQuantizedTypeForBias( // quantization parameters are stored as adjacent quantize and dequantize ops // and the propagation results are materialized by inserting pairs of quantize // and dequantize ops to this function. -void ApplyQuantizationParamsPropagation(mlir::Function func); +void ApplyQuantizationParamsPropagation(mlir::FuncOp func); } // end namespace TFL } // end namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc b/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc index a585d6de9f1..103e1442863 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc @@ -70,7 +70,7 @@ static Value* LowerCondition(Location loc, Value* value, OpBuilder* builder) { // that is compatible for tensor cast. // static Operation* CallFn(Location loc, - const std::function& get_arg, Function fn, + const std::function& get_arg, FuncOp fn, OpBuilder* builder) { FunctionType fn_type = fn.getType(); llvm::SmallVector operands; diff --git a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc index 196837b8e3e..dc3108c3f77 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc @@ -92,19 +92,19 @@ class Exporter { std::unique_ptr* graph, FunctionLibraryDefinition* flib_def); - // Converts a given Function to a FunctionDef and adds it to the function + // Converts a given FuncOp to a FunctionDef and adds it to the function // definition library static Status ConvertLibFunction(const ExporterConfigs& configs, const Dialect* tf_dialect, - mlir::Function function, + mlir::FuncOp function, FunctionDefLibrary* flib); - // Converts the given CFG Function to a Graph. The arguments and returns of + // Converts the given FuncOp to a Graph. The arguments and returns of // function are added to the graph with special op names kArgOp and kRetOp. // Later on, this graph can be converted a function definition and added to // another graph. static StatusOr> Convert( const ExporterConfigs& configs, const Dialect* tf_dialect, - mlir::Function function, FunctionDefLibrary* flib); + mlir::FuncOp function, FunctionDefLibrary* flib); private: explicit Exporter(Graph* graph, const Dialect* tf_dialect) @@ -376,11 +376,11 @@ Status Exporter::AddNextIterationNode(mlir::Operation* inst) { StatusOr> Exporter::Convert(const ExporterConfigs& confs, const Dialect* tf_dialect, - mlir::Function function, + mlir::FuncOp function, FunctionDefLibrary* flib) { if (function.getBlocks().size() != 1) { return errors::FailedPrecondition( - "Input Function must have only one basic block!"); + "Input FuncOp must have only one basic block!"); } mlir::Block& block = function.front(); @@ -433,7 +433,7 @@ StatusOr> Exporter::Convert(const ExporterConfigs& confs, mlir::Type type = arg->getType(); if (!type.isa()) { return errors::InvalidArgument( - "Functions arguments must have tensor types. Found ", + "FuncOps arguments must have tensor types. Found ", mlir::debugString(type), " in function ", function.getName().str()); } @@ -483,7 +483,7 @@ StatusOr> Exporter::Convert(const ExporterConfigs& confs, Status Exporter::ConvertLibFunction(const ExporterConfigs& configs, const Dialect* tf_dialect, - mlir::Function function, + mlir::FuncOp function, FunctionDefLibrary* flib) { // First look for the function in the current function library. If found, // nothing needs to be done. @@ -536,10 +536,10 @@ Status Exporter::Convert(mlir::Module module, const ExporterConfigs& configs, FunctionLibraryDefinition* flib_def) { mlir::Identifier entry_func_id = mlir::Identifier::get("main", module.getContext()); - absl::optional entry_func; + absl::optional entry_func; FunctionDefLibrary flib; auto tf_dialect = module.getContext()->getRegisteredDialect("tf"); - for (auto function : module.getOps()) { + for (auto function : module.getOps()) { if (function.isExternal()) return errors::FailedPrecondition("External functions not supported"); diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc index c0553de183e..e592862f6af 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc @@ -1151,8 +1151,8 @@ Status Importer::Convert(llvm::StringRef func_name, const absl::InlinedVector& ret_nodes, llvm::ArrayRef attrs) { // TODO(b/122040776): Uses debug info for FunctionDef. - auto function = mlir::Function::create(mlir::UnknownLoc::get(context_), - func_name, func_type, attrs); + auto function = mlir::FuncOp::create(mlir::UnknownLoc::get(context_), + func_name, func_type, attrs); module_.push_back(function); builder_ = absl::make_unique(function.getBody()); diff --git a/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc b/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc index 932bc7b28c2..aa53ed9481f 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc @@ -23,7 +23,7 @@ limitations under the License. namespace mlir { static mlir::Operation* ExtractOnlyOp(mlir::Module module) { - mlir::Function fn = module.getNamedFunction("main"); + mlir::FuncOp fn = module.getNamedFunction("main"); if (!fn) return nullptr; if (fn.getBlocks().size() != 1) return nullptr; From 466a59b50448dd521e6561d54f97b5af41122a9b Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Tue, 9 Jul 2019 10:17:27 -0700 Subject: [PATCH 154/332] Tests for TextLineDatasetOp --- .../core/kernels/data/dataset_test_base.cc | 82 +++ .../core/kernels/data/dataset_test_base.h | 26 + .../kernels/data/text_line_dataset_op_test.cc | 625 ++++++++++++++++++ 3 files changed, 733 insertions(+) create mode 100644 tensorflow/core/kernels/data/text_line_dataset_op_test.cc diff --git a/tensorflow/core/kernels/data/dataset_test_base.cc b/tensorflow/core/kernels/data/dataset_test_base.cc index 274b99cee3c..61424bb3458 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.cc +++ b/tensorflow/core/kernels/data/dataset_test_base.cc @@ -22,6 +22,88 @@ limitations under the License. namespace tensorflow { namespace data { +constexpr char kNewLine[] = "\n"; + +string CompressionName(CompressionType compression_type) { + switch (compression_type) { + case ZLIB: + return "ZLIB"; + case GZIP: + return "GZIP"; + case RAW: + return "RAW"; + case UNCOMPRESSED: + return ""; + default: + LOG(WARNING) << "Undefined compression type: " << compression_type; + return "UNDEFINED"; + } +} + +io::ZlibCompressionOptions GetZlibCompressionOptions( + CompressionType compression_type) { + switch (compression_type) { + case ZLIB: + return io::ZlibCompressionOptions::DEFAULT(); + case GZIP: + return io::ZlibCompressionOptions::GZIP(); + case RAW: + return io::ZlibCompressionOptions::RAW(); + default: + LOG(WARNING) << "Undefined zlib compression type: " << compression_type; + return io::ZlibCompressionOptions::DEFAULT(); + } +} + +Status CreateFile(const string& filename, const std::vector& text, + int input_buffer_size, int output_buffer_size, + const CompressionType& compression_type) { + Env* env = Env::Default(); + std::unique_ptr file_writer; + TF_RETURN_IF_ERROR(env->NewWritableFile(filename, &file_writer)); + if (compression_type == UNCOMPRESSED) { + for (const string& line : text) { + TF_RETURN_IF_ERROR(file_writer->Append(line)); + TF_RETURN_IF_ERROR(file_writer->Append(kNewLine)); + } + } else if (compression_type == ZLIB || compression_type == GZIP) { + auto zlib_compression_options = GetZlibCompressionOptions(compression_type); + io::ZlibOutputBuffer out(file_writer.get(), input_buffer_size, + output_buffer_size, zlib_compression_options); + TF_RETURN_IF_ERROR(out.Init()); + for (const string& line : text) { + TF_RETURN_IF_ERROR(out.Append(line)); + TF_RETURN_IF_ERROR(out.Append(kNewLine)); + } + TF_RETURN_IF_ERROR(out.Close()); + } else { + return tensorflow::errors::InvalidArgument("Unsupported compression_type: ", + compression_type); + } + + TF_RETURN_IF_ERROR(file_writer->Flush()); + TF_RETURN_IF_ERROR(file_writer->Close()); + + return Status::OK(); +} + +Status CreateMultiTextFiles(const std::vector& filenames, + const std::vector& multi_texts, + int input_buffer_size, int output_buffer_size, + const CompressionType& compression_type) { + int lines_per_file = multi_texts.size() / filenames.size(); + auto first = multi_texts.begin(); + for (int i = 0; i < filenames.size(); ++i) { + auto last = + i == filenames.size() - 1 ? multi_texts.end() : first + lines_per_file; + std::vector text(first, last); + TF_RETURN_IF_ERROR(CreateFile(filenames[i], text, input_buffer_size, + output_buffer_size, compression_type)); + first += lines_per_file; + } + return Status::OK(); +} + template Status IsEqual(const Tensor& t1, const Tensor& t2) { if (t1.dtype() != t2.dtype()) { diff --git a/tensorflow/core/kernels/data/dataset_test_base.h b/tensorflow/core/kernels/data/dataset_test_base.h index edd12667724..c35336c51ef 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.h +++ b/tensorflow/core/kernels/data/dataset_test_base.h @@ -38,9 +38,35 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/ptr_util.h" +#include "tensorflow/core/lib/io/zlib_compression_options.h" +#include "tensorflow/core/lib/io/zlib_outputbuffer.h" +#include "tensorflow/core/platform/env.h" + namespace tensorflow { namespace data { +enum CompressionType : int { ZLIB = 0, GZIP = 1, RAW = 2, UNCOMPRESSED = 3 }; + +// Gets the compression name. +string CompressionName(CompressionType compression_type); + +// Gets the specified zlib compression options according to the compression +// type. Note that `CompressionType : UNCOMPRESSED` is not supported because +// `ZlibCompressionOptions` does not have the option for uncompressed data. +io::ZlibCompressionOptions GetZlibCompressionOptions( + CompressionType compression_type); + +// Saves the input text into the file with the specified compression. +Status CreateFile(const string& filename, const std::vector& text, + int input_buffer_size, int output_buffer_size, + const CompressionType& compression_type); + +// Saves the input texts into multiple files with the specified compression. +Status CreateMultiTextFiles(const std::vector& filenames, + const std::vector& multi_texts, + int input_buffer_size, int output_buffer_size, + const CompressionType& compression_type); + // Helpful functions to test Dataset op kernels. class DatasetOpsTestBase : public ::testing::Test { public: diff --git a/tensorflow/core/kernels/data/text_line_dataset_op_test.cc b/tensorflow/core/kernels/data/text_line_dataset_op_test.cc new file mode 100644 index 00000000000..112b8536ba6 --- /dev/null +++ b/tensorflow/core/kernels/data/text_line_dataset_op_test.cc @@ -0,0 +1,625 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/kernels/data/text_line_dataset_op.h" + +#include "tensorflow/core/kernels/data/dataset_test_base.h" + +namespace tensorflow { +namespace data { +namespace { + +constexpr char kNodeName[] = "text_line_dataset"; +constexpr char kIteratorPrefix[] = "Iterator"; + +class TextLineDatasetOpTest : public DatasetOpsTestBase { + protected: + // Create a new `TextLineDataset` op kernel. + Status CreateTextLineDatasetOpKernel( + std::unique_ptr* text_line_dataset_op_kernel) { + NodeDef node_def = test::function::NDef( + kNodeName, name_utils::OpName(TextLineDatasetOp::kDatasetType), + {TextLineDatasetOp::kFileNames, TextLineDatasetOp::kCompressionType, + TextLineDatasetOp::kBufferSize}, + {}); + TF_RETURN_IF_ERROR(CreateOpKernel(node_def, text_line_dataset_op_kernel)); + return Status::OK(); + } + + // Create a new `TextLineDataset` op kernel context + Status CreateTextLineDatasetContext( + OpKernel* const op_kernel, + gtl::InlinedVector* const inputs, + std::unique_ptr* context) { + TF_RETURN_IF_ERROR(CheckOpKernelInput(*op_kernel, *inputs)); + TF_RETURN_IF_ERROR(CreateOpKernelContext(op_kernel, inputs, context)); + return Status::OK(); + } +}; + +struct TestCase { + std::vector filenames; + CompressionType compression_type; + int64 buffer_size; + std::vector expected_outputs; + DataTypeVector expected_output_dtypes; + std::vector expected_output_shapes; + int64 expected_cardinality; + std::vector breakpoints; +}; + +std::vector ToStringVector(const std::vector& str_tensors) { + std::vector str_vec; + for (const Tensor& tensor : str_tensors) { + for (int i = 0; i < tensor.NumElements(); ++i) { + str_vec.push_back(tensor.flat()(i)); + } + } + return str_vec; +} + +// Test case 1: multiple text files with ZLIB compression. +TestCase TestCase1() { + return { + /*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_1"), + absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_2")}, + /*compression_type*/ ZLIB, + /*buffer_size*/ 10, + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"hello world"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"11223334455"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"abcd, EFgH"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {" "}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), {"$%^&*()"})}, + /*expected_output_dtypes*/ {DT_STRING}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ kUnknownCardinality, + /*breakpoints*/ {0, 2, 6}}; +} + +// Test case 2: multiple text files with GZIP compression. +TestCase TestCase2() { + return {/*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_GZIP_1"), + absl::StrCat(testing::TmpDir(), "/text_line_GZIP_2")}, + /*compression_type*/ GZIP, + /*buffer_size*/ 10, + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"hello world"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"11223334455"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"abcd, EFgH"})}, + /*expected_output_dtypes*/ {DT_STRING}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ kUnknownCardinality, + /*breakpoints*/ {0, 2, 6}}; +} + +// Test case 3: a single text file without compression. +TestCase TestCase3() { + return {/*filenames*/ { + absl::StrCat(testing::TmpDir(), "/text_line_UNCOMPRESSED")}, + /*compression_type*/ UNCOMPRESSED, + /*buffer_size*/ 10, + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"hello world"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"11223334455"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"abcd, EFgH"})}, + /*expected_output_dtypes*/ {DT_STRING}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ kUnknownCardinality, + /*breakpoints*/ {0, 2, 6}}; +} + +class ParameterizedTextLineDatasetOpTest + : public TextLineDatasetOpTest, + public ::testing::WithParamInterface {}; + +TEST_P(ParameterizedTextLineDatasetOpTest, GetNext) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + bool end_of_sequence = false; + std::vector out_tensors; + while (!end_of_sequence) { + std::vector next; + TF_EXPECT_OK( + iterator->GetNext(iterator_ctx.get(), &next, &end_of_sequence)); + out_tensors.insert(out_tensors.end(), next.begin(), next.end()); + } + + TF_EXPECT_OK(ExpectEqual(out_tensors, test_case.expected_outputs, + /*compare_order*/ true)); +} + +TEST_F(TextLineDatasetOpTest, DatasetNodeName) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = TestCase1(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + EXPECT_EQ(text_line_dataset->node_name(), kNodeName); +} + +TEST_F(TextLineDatasetOpTest, DatasetTypeString) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = TestCase1(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + EXPECT_EQ(text_line_dataset->type_string(), + name_utils::OpName(TextLineDatasetOp::kDatasetType)); +} + +TEST_P(ParameterizedTextLineDatasetOpTest, DatasetOutputDtypes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + TF_EXPECT_OK(VerifyTypesMatch(text_line_dataset->output_dtypes(), + test_case.expected_output_dtypes)); +} + +TEST_P(ParameterizedTextLineDatasetOpTest, DatasetOutputShapes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + TF_EXPECT_OK(VerifyShapesCompatible(text_line_dataset->output_shapes(), + test_case.expected_output_shapes)); +} + +TEST_P(ParameterizedTextLineDatasetOpTest, Cardinality) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + EXPECT_EQ(text_line_dataset->Cardinality(), test_case.expected_cardinality); +} + +TEST_P(ParameterizedTextLineDatasetOpTest, DatasetSave) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr serialization_context; + TF_ASSERT_OK(CreateSerializationContext(&serialization_context)); + VariantTensorData data; + VariantTensorDataWriter writer(&data); + TF_ASSERT_OK(text_line_dataset->Save(serialization_context.get(), &writer)); + TF_ASSERT_OK(writer.Flush()); +} + +TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputDtypes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + + TF_EXPECT_OK(VerifyTypesMatch(iterator->output_dtypes(), + test_case.expected_output_dtypes)); +} + +TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputShapes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + + TF_EXPECT_OK(VerifyShapesCompatible(iterator->output_shapes(), + test_case.expected_output_shapes)); +} + +TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputPrefix) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + + EXPECT_EQ(iterator->prefix(), + name_utils::IteratorPrefix(TextLineDatasetOp::kDatasetType, + kIteratorPrefix)); +} + +TEST_P(ParameterizedTextLineDatasetOpTest, Roundtrip) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateTextLineDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + + std::unique_ptr serialization_ctx; + TF_ASSERT_OK(CreateSerializationContext(&serialization_ctx)); + + bool end_of_sequence = false; + std::vector out_tensors; + int cur_iteration = 0; + const std::vector& breakpoints = test_case.breakpoints; + for (int breakpoint : breakpoints) { + VariantTensorData data; + VariantTensorDataWriter writer(&data); + TF_EXPECT_OK(iterator->Save(serialization_ctx.get(), &writer)); + TF_EXPECT_OK(writer.Flush()); + VariantTensorDataReader reader(&data); + TF_EXPECT_OK(RestoreIterator(iterator_ctx.get(), &reader, kIteratorPrefix, + *text_line_dataset, &iterator)); + + while (cur_iteration <= breakpoint) { + std::vector next; + TF_EXPECT_OK( + iterator->GetNext(iterator_ctx.get(), &next, &end_of_sequence)); + out_tensors.insert(out_tensors.end(), next.begin(), next.end()); + cur_iteration++; + } + } + + TF_EXPECT_OK(ExpectEqual(out_tensors, test_case.expected_outputs, + /*compare_order*/ true)); +} + +INSTANTIATE_TEST_SUITE_P(TextLineDatasetOpTest, + ParameterizedTextLineDatasetOpTest, + ::testing::ValuesIn(std::vector( + {TestCase1(), TestCase2(), TestCase3()}))); + +} // namespace +} // namespace data +} // namespace tensorflow From a7440c393a7b700fee1e3d16d9b7ce91a8471766 Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Tue, 9 Jul 2019 16:20:43 -0700 Subject: [PATCH 155/332] [XLA] Extract a common visitor for rewriting instructions. PiperOrigin-RevId: 257293820 --- tensorflow/compiler/xla/service/BUILD | 1 + .../xla/service/algebraic_simplifier.cc | 42 ++------------- .../xla/service/batchnorm_expander.cc | 37 +------------- .../service/bfloat16_conversion_folding.cc | 1 + .../xla/service/bfloat16_normalization.cc | 1 + .../compiler/xla/service/call_inliner.cc | 1 + .../service/dfs_hlo_visitor_with_default.h | 51 +++++++++++++++++-- .../service/gpu/cudnn_batchnorm_rewriter.cc | 2 + .../compiler/xla/service/gpu/gemm_rewriter.cc | 23 ++------- .../compiler/xla/service/hlo_computation.h | 1 - .../compiler/xla/service/hlo_verifier.cc | 1 + .../xla/service/logical_buffer_analysis.h | 1 + .../compiler/xla/tools/hlo_extractor.cc | 1 + 13 files changed, 65 insertions(+), 98 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 2f74a1378cb..880ea14fed5 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -644,6 +644,7 @@ cc_library( hdrs = ["call_inliner.h"], deps = [ ":call_graph", + ":hlo", ":hlo_dce", ":hlo_pass", "//tensorflow/compiler/xla:statusor", diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 10030ce6491..53a2a57617c 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -168,13 +168,8 @@ bool IsUnstridedSlice(const HloInstruction* hlo) { // algebraic expressions to simplified forms. Note: This only supports // simplifications that simply look at the operands of an instruction. For the // more general case a worklist based approach would be needed. -class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { +class AlgebraicSimplifierVisitor : public DfsHloRewriteVisitor { public: - // Default visitor action is to do nothing and return OK. - Status DefaultAction(HloInstruction* /*hlo_instruction*/) override { - return Status::OK(); - } - Status HandleAdd(HloInstruction* add) override; Status HandleAnd(HloInstruction* logical_and) override; @@ -250,9 +245,6 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { Status HandleMap(HloInstruction* map) override; - // Returns whether algebraic simplification has occurred. - const bool changed() const { return changed_; } - // Runs the visitor on a computation. static bool Run(HloComputation* computation, const AlgebraicSimplifierOptions& options, @@ -350,35 +342,6 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { StatusOr TryToSinkBroadcastAfterOpWithUniqueNonScalarOperand( HloInstruction* broadcast); - // Replaces the existing HLO instruction old_instruction, with - // new_instruction, and marks the optimizer status as changed. - // Returns the Status representing the result of the replace operation. - Status ReplaceWithNewInstruction( - HloInstruction* old_instruction, - std::unique_ptr new_instruction) { - VLOG(3) << "Replacing instruction:"; - VLOG(3) << " old: " << old_instruction->ToString(); - VLOG(3) << " new: " << new_instruction->ToString(); - TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction( - old_instruction, std::move(new_instruction))); - changed_ = true; - return Status::OK(); - } - - // Replaces the existing HLO instruction old_instruction, with - // new_instruction, and marks the optimizer status as changed. - // Returns the Status representing the result of the replace operation. - Status ReplaceInstruction(HloInstruction* old_instruction, - HloInstruction* new_instruction) { - VLOG(3) << "Replacing instruction:"; - VLOG(3) << " old: " << old_instruction->ToString(); - VLOG(3) << " new: " << new_instruction->ToString(); - TF_RETURN_IF_ERROR( - computation_->ReplaceInstruction(old_instruction, new_instruction)); - changed_ = true; - return Status::OK(); - } - StatusOr OptimizeDotOfConcat(HloInstruction* dot); StatusOr OptimizeDotOfConcatHelper( const HloInstruction& dot, HloInstruction* lhs, int64 lhs_contracting_dim, @@ -445,7 +408,7 @@ bool AlgebraicSimplifierVisitor::Run(HloComputation* computation, AlgebraicSimplifier* simplifier) { AlgebraicSimplifierVisitor visitor(computation, options, simplifier); TF_CHECK_OK(computation->Accept(&visitor)); - return visitor.changed_; + return visitor.changed_ || visitor.changed(); } bool AlgebraicSimplifierVisitor::SameShape(const HloInstruction* lhs, @@ -1723,6 +1686,7 @@ AlgebraicSimplifierVisitor::OptimizeDotOfReorderContractingDims( } Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) { + CHECK(computation_ == dot->parent()); HloInstruction *lhs, *rhs; CHECK(Match(dot, m::Dot(m::Op(&lhs), m::Op(&rhs)))); if (options_.is_layout_sensitive()) { diff --git a/tensorflow/compiler/xla/service/batchnorm_expander.cc b/tensorflow/compiler/xla/service/batchnorm_expander.cc index d14e803be6a..131b50efc9c 100644 --- a/tensorflow/compiler/xla/service/batchnorm_expander.cc +++ b/tensorflow/compiler/xla/service/batchnorm_expander.cc @@ -46,13 +46,8 @@ using absl::optional; // BatchNormExpanderVisitor traverses the HLO computation and rewrites BatchNorm // operations into smaller operations. -class BatchNormExpanderVisitor : public DfsHloVisitorWithDefault { +class BatchNormExpanderVisitor : public DfsHloRewriteVisitor { public: - // Default visitor action is to do nothing and return OK. - Status DefaultAction(HloInstruction* /*hlo_instruction*/) override { - return Status::OK(); - } - Status HandleBatchNormTraining(HloInstruction* batch_norm) override; Status HandleBatchNormInference(HloInstruction* batch_norm) override; @@ -63,9 +58,6 @@ class BatchNormExpanderVisitor : public DfsHloVisitorWithDefault { static bool Run(HloComputation* computation, bool rewrite_training_op, bool rewrite_inference_op, bool rewrite_grad_op); - // Returns whether any batch norm ops were rewritten. - const bool changed() const { return changed_; } - ~BatchNormExpanderVisitor() override = default; private: @@ -133,28 +125,6 @@ class BatchNormExpanderVisitor : public DfsHloVisitorWithDefault { elements_per_feature_u32); } - // Replaces the existing HLO instruction old_instruction, with - // new_instruction, and marks the optimizer status as changed. - // Returns the Status representing the result of the replace operation. - Status ReplaceWithNewInstruction( - HloInstruction* old_instruction, - std::unique_ptr new_instruction) { - TF_RETURN_IF_ERROR(computation_->ReplaceWithNewInstruction( - old_instruction, std::move(new_instruction))); - changed_ = true; - return Status::OK(); - } - - // Replaces the existing HLO instruction old_instruction, with - // new_instruction, and marks the optimizer status as changed. - // Returns the Status representing the result of the replace operation. - Status ReplaceInstruction(HloInstruction* old_instruction, - HloInstruction* new_instruction) { - TF_RETURN_IF_ERROR( - computation_->ReplaceInstruction(old_instruction, new_instruction)); - changed_ = true; - return Status::OK(); - } // Current HloComputation instance the BatchNormExpander is // traversing. HloComputation* computation_; @@ -162,9 +132,6 @@ class BatchNormExpanderVisitor : public DfsHloVisitorWithDefault { bool rewrite_training_op_; bool rewrite_inference_op_; bool rewrite_grad_op_; - - // Whether rewrite has occurred. - bool changed_ = false; }; } // namespace @@ -179,7 +146,7 @@ bool BatchNormExpanderVisitor::Run(HloComputation* computation, /*rewrite_inference_op=*/rewrite_inference_op, /*rewrite_grad_op=*/rewrite_grad_op); TF_CHECK_OK(computation->Accept(&visitor)); - return visitor.changed_; + return visitor.changed(); } Status BatchNormExpanderVisitor::HandleBatchNormTraining( diff --git a/tensorflow/compiler/xla/service/bfloat16_conversion_folding.cc b/tensorflow/compiler/xla/service/bfloat16_conversion_folding.cc index 430172b474c..23d2a9225a8 100644 --- a/tensorflow/compiler/xla/service/bfloat16_conversion_folding.cc +++ b/tensorflow/compiler/xla/service/bfloat16_conversion_folding.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/bfloat16_conversion_folding.h" #include "absl/types/span.h" +#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" diff --git a/tensorflow/compiler/xla/service/bfloat16_normalization.cc b/tensorflow/compiler/xla/service/bfloat16_normalization.cc index 85e1113bf77..f1ab34d6141 100644 --- a/tensorflow/compiler/xla/service/bfloat16_normalization.cc +++ b/tensorflow/compiler/xla/service/bfloat16_normalization.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/bfloat16_normalization.h" #include "absl/types/span.h" +#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" diff --git a/tensorflow/compiler/xla/service/call_inliner.cc b/tensorflow/compiler/xla/service/call_inliner.cc index 1d421404440..062110af867 100644 --- a/tensorflow/compiler/xla/service/call_inliner.cc +++ b/tensorflow/compiler/xla/service/call_inliner.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "tensorflow/compiler/xla/service/call_graph.h" +#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo_dce.h" #include "tensorflow/core/lib/core/errors.h" diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h index 7f900d9fc55..fcb68a200d9 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h @@ -20,6 +20,8 @@ limitations under the License. #include "absl/types/span.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/service/dfs_hlo_visitor.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" @@ -29,9 +31,6 @@ limitations under the License. namespace xla { -class HloComputation; -class HloInstruction; - // DfsHloVisitor with default action based on the HloInstruction being visited. // Users should not use this class directly, but use the type aliases // DfsHloVisitorWithDefault/ConstDfsHloVisitorWithDefault instead. @@ -246,6 +245,52 @@ using DfsHloVisitorWithDefault = DfsHloVisitorWithDefaultBase; using ConstDfsHloVisitorWithDefault = DfsHloVisitorWithDefaultBase; +// A common base class for visitors performing rewriting operation. +// +// Subclasses call ReplaceWithNewInstruction and ReplaceInstruction while +// visiting. +class DfsHloRewriteVisitor : public DfsHloVisitorWithDefault { + public: + // Default visitor action is to do nothing and return OK. + Status DefaultAction(HloInstruction* /*hlo_instruction*/) override { + return Status::OK(); + } + + bool changed() const { return changed_; } + + protected: + // Replaces the existing HLO instruction old_instruction, with + // new_instruction, and marks the optimizer status as changed. + // Returns the Status representing the result of the replace operation. + Status ReplaceWithNewInstruction( + HloInstruction* old_instruction, + std::unique_ptr new_instruction) { + VLOG(3) << "Replacing instruction:"; + VLOG(3) << " old: " << old_instruction->ToString(); + VLOG(3) << " new: " << new_instruction->ToString(); + TF_RETURN_IF_ERROR(old_instruction->parent()->ReplaceWithNewInstruction( + old_instruction, std::move(new_instruction))); + changed_ = true; + return Status::OK(); + } + + // Replaces the existing HLO instruction old_instruction, with + // new_instruction, and marks the optimizer status as changed. + // Returns the Status representing the result of the replace operation. + Status ReplaceInstruction(HloInstruction* old_instruction, + HloInstruction* new_instruction) { + VLOG(3) << "Replacing instruction:"; + VLOG(3) << " old: " << old_instruction->ToString(); + VLOG(3) << " new: " << new_instruction->ToString(); + TF_RETURN_IF_ERROR(old_instruction->parent()->ReplaceInstruction( + old_instruction, new_instruction)); + changed_ = true; + return Status::OK(); + } + + bool changed_ = false; +}; + // (Const)FunctionVisitor lets you transform an // std::function into a (Const)DfsHloVisitor. // diff --git a/tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.cc b/tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.cc index 2cceb0422d0..4d61f09a7a9 100644 --- a/tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.cc +++ b/tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.cc @@ -14,8 +14,10 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.h" + #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h" namespace xla { diff --git a/tensorflow/compiler/xla/service/gpu/gemm_rewriter.cc b/tensorflow/compiler/xla/service/gpu/gemm_rewriter.cc index 36098cbfb72..df7ee3cdc69 100644 --- a/tensorflow/compiler/xla/service/gpu/gemm_rewriter.cc +++ b/tensorflow/compiler/xla/service/gpu/gemm_rewriter.cc @@ -61,7 +61,7 @@ static complex128 GetScalarConstantAsComplex(const Literal &literal) { // and provided C has no other users). // We then guide the buffer assignment to alias the buffer of the custom call // and C. -class GemmRewriterVisitor : public DfsHloVisitorWithDefault { +class GemmRewriterVisitor : public DfsHloRewriteVisitor { public: Status HandleDot(HloInstruction *instr) override { if (IsMatrixMultiplication(*instr)) { @@ -107,9 +107,7 @@ class GemmRewriterVisitor : public DfsHloVisitorWithDefault { config.set_alpha_real(new_alpha.real()); config.set_alpha_imag(new_alpha.imag()); TF_RETURN_IF_ERROR(existing_gemm->set_backend_config(config)); - TF_RETURN_IF_ERROR( - instr->parent()->ReplaceInstruction(instr, existing_gemm)); - changed_ = true; + TF_RETURN_IF_ERROR(ReplaceInstruction(instr, existing_gemm)); } } return Status::OK(); @@ -141,27 +139,12 @@ class GemmRewriterVisitor : public DfsHloVisitorWithDefault { } return Status::OK(); } - - Status DefaultAction(HloInstruction *) override { return Status::OK(); } - - bool IsChanged() { return changed_; } - - private: - Status ReplaceWithNewInstruction( - HloInstruction *instr, std::unique_ptr replacement) { - TF_RETURN_IF_ERROR(instr->parent()->ReplaceWithNewInstruction( - instr, std::move(replacement))); - changed_ = true; - return Status::OK(); - } - - bool changed_ = false; }; static StatusOr RunOnComputation(HloComputation *computation) { GemmRewriterVisitor visitor; TF_RETURN_IF_ERROR(computation->Accept(&visitor)); - return visitor.IsChanged(); + return visitor.changed(); } StatusOr GemmRewriter::Run(HloModule *module) { diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h index fc4aaedde15..316c3514aeb 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.h +++ b/tensorflow/compiler/xla/service/hlo_computation.h @@ -30,7 +30,6 @@ limitations under the License. #include "tensorflow/compiler/xla/iterator_util.h" #include "tensorflow/compiler/xla/map_util.h" #include "tensorflow/compiler/xla/service/dfs_hlo_visitor.h" -#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_clone_context.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 1de9a66adcb..fa2631bc364 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -20,6 +20,7 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "absl/strings/str_join.h" #include "tensorflow/compiler/xla/primitive_util.h" +#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo_casting_utils.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.h b/tensorflow/compiler/xla/service/logical_buffer_analysis.h index 276a157a15a..5f774bb25a6 100644 --- a/tensorflow/compiler/xla/service/logical_buffer_analysis.h +++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_LOGICAL_BUFFER_ANALYSIS_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_LOGICAL_BUFFER_ANALYSIS_H_ +#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_module.h" #include "tensorflow/compiler/xla/service/logical_buffer.h" diff --git a/tensorflow/compiler/xla/tools/hlo_extractor.cc b/tensorflow/compiler/xla/tools/hlo_extractor.cc index f3ce5f99b0c..5d681f61ff6 100644 --- a/tensorflow/compiler/xla/tools/hlo_extractor.cc +++ b/tensorflow/compiler/xla/tools/hlo_extractor.cc @@ -21,6 +21,7 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/memory/memory.h" +#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo_clone_context.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_verifier.h" From 794b5d4cf6be2cd4e26ac141d3591b1e331fe3ca Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Tue, 9 Jul 2019 16:24:26 -0700 Subject: [PATCH 156/332] Introduce a new class RemoteMgr to maintain remote state for eager. This change includes: 1. Move RemoteTensorHandleMap into this class in order to allow eager/execute.cc add TensorHandle into the map. 2. Move NextId() function from Context to RemoteMgr and only allow master generating op_id. 3. Create two functions to serialize and deserialize RemoteTensorHandle. PiperOrigin-RevId: 257294505 --- tensorflow/c/eager/BUILD | 1 + tensorflow/c/eager/c_api.cc | 5 +- tensorflow/core/common_runtime/eager/BUILD | 7 +- .../core/common_runtime/eager/context.cc | 15 ++-- .../core/common_runtime/eager/context.h | 31 +++++-- .../core/common_runtime/eager/execute.cc | 24 ++--- .../core/distributed_runtime/eager/BUILD | 20 +++++ .../eager/eager_service_impl.cc | 20 +++-- .../eager/eager_service_impl.h | 54 ----------- .../eager/eager_service_impl_test.cc | 4 +- .../distributed_runtime/eager/remote_mgr.cc | 89 +++++++++++++++++++ .../distributed_runtime/eager/remote_mgr.h | 85 ++++++++++++++++++ 12 files changed, 259 insertions(+), 96 deletions(-) create mode 100644 tensorflow/core/distributed_runtime/eager/remote_mgr.cc create mode 100644 tensorflow/core/distributed_runtime/eager/remote_mgr.h diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD index 516b8256cec..d0b2f215975 100644 --- a/tensorflow/c/eager/BUILD +++ b/tensorflow/c/eager/BUILD @@ -64,6 +64,7 @@ tf_cuda_library( }) + [ "@com_google_absl//absl/memory", "//tensorflow/core/common_runtime/eager:eager_operation", + "//tensorflow/core/distributed_runtime/eager:remote_mgr", "//tensorflow/core/distributed_runtime/eager:eager_client", "//tensorflow/core/distributed_runtime/rpc/eager:grpc_eager_client", "//tensorflow/core/distributed_runtime/rpc:grpc_channel", diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 71a217c447d..22c1f219f38 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -54,6 +54,7 @@ limitations under the License. #include "tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h" #include "tensorflow/core/distributed_runtime/server_lib.h" #include "tensorflow/core/distributed_runtime/worker_env.h" +#include "tensorflow/core/distributed_runtime/eager/remote_mgr.h" #endif // !IS_MOBILE_PLATFORM #include "tensorflow/core/framework/node_def_util.h" #include "tensorflow/core/framework/rendezvous.h" @@ -260,12 +261,14 @@ tensorflow::Status UpdateTFE_ContextWithServerDef( TF_RETURN_IF_ERROR(r->Initialize(worker_session.get())); auto* device_mgr = grpc_server->worker_env()->device_mgr; + auto remote_mgr = + absl::make_unique(/*is_master=*/true); return ctx->context->InitializeRemoteMaster( std::move(server), grpc_server->worker_env(), worker_session, std::move(remote_eager_workers), std::move(remote_device_mgr), remote_workers, context_id, r, device_mgr, keep_alive_secs, - worker_session->cluster_flr.get()); + worker_session->cluster_flr.get(), std::move(remote_mgr)); #undef LOG_AND_RETURN_IF_ERROR } #endif // !IS_MOBILE_PLATFORM diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD index 5d5c93130dc..70c79b1dc8f 100644 --- a/tensorflow/core/common_runtime/eager/BUILD +++ b/tensorflow/core/common_runtime/eager/BUILD @@ -48,6 +48,7 @@ tf_cuda_library( deps = [ ":eager_executor", ":kernel_and_device", + "//tensorflow/core/distributed_runtime/eager:remote_tensor_handle", "//tensorflow/core/distributed_runtime:rendezvous_mgr_interface", "//tensorflow/core/distributed_runtime:worker_env", ] + select({ @@ -253,6 +254,7 @@ cc_library( "//tensorflow/core:android_tensorflow_lib_lite", ], "//conditions:default": [ + "//tensorflow/core/distributed_runtime/eager:remote_mgr", "//tensorflow/core:core_cpu_lib", "//tensorflow/core:framework", "//tensorflow/core:framework_internal", @@ -335,6 +337,9 @@ filegroup( "*.cc", "*.h", ], - exclude = ["*_test.cc"], + exclude = [ + "*_test.cc", + "remote_*", + ], ), ) diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc index 429523aaaca..ce739ae6413 100644 --- a/tensorflow/core/common_runtime/eager/context.cc +++ b/tensorflow/core/common_runtime/eager/context.cc @@ -641,7 +641,9 @@ Status EagerContext::InitializeRemoteMaster( std::unique_ptr remote_device_manager, const std::vector& remote_contexts, uint64 context_id, Rendezvous* r, DeviceMgr* local_device_mgr, int keep_alive_secs, - DistributedFunctionLibraryRuntime* cluster_flr) { + DistributedFunctionLibraryRuntime* cluster_flr, + std::unique_ptr> + remote_mgr) { mutex_lock l(remote_state_mu_); if (!remote_contexts_.empty()) { @@ -673,6 +675,7 @@ Status EagerContext::InitializeRemoteMaster( } server_ = std::move(server); + remote_mgr_ = std::move(remote_mgr); worker_env_ = worker_env; worker_session_ = worker_session; remote_eager_workers_ = std::move(remote_eager_workers); @@ -749,7 +752,9 @@ Status EagerContext::InitializeRemoteWorker( std::unique_ptr remote_eager_workers, const DeviceMgr* remote_device_mgr, const std::vector& remote_contexts, uint64 context_id, - std::function rendezvous_creator) { + std::function rendezvous_creator, + std::unique_ptr> + remote_mgr) { mutex_lock l(remote_state_mu_); if (remote_device_manager_ != nullptr || server_ != nullptr || @@ -764,6 +769,7 @@ Status EagerContext::InitializeRemoteWorker( rendezvous_creator_ = std::move(rendezvous_creator); remote_eager_workers_ = std::move(remote_eager_workers); + remote_mgr_ = std::move(remote_mgr); remote_unowned_device_manager_ = remote_device_mgr; InitDeviceMapAndAsync(); @@ -773,11 +779,6 @@ Status EagerContext::InitializeRemoteWorker( return Status::OK(); } - -tensorflow::uint64 EagerContext::NextId() { - tensorflow::mutex_lock l(next_id_mutex_); - return next_id_++; -} #endif // !IS_MOBILE_PLATFORM } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h index 3d77c96d69b..a4fe6ad2a22 100644 --- a/tensorflow/core/common_runtime/eager/context.h +++ b/tensorflow/core/common_runtime/eager/context.h @@ -41,6 +41,7 @@ limitations under the License. #include "tensorflow/core/util/device_name_utils.h" #if !defined(IS_MOBILE_PLATFORM) #include "tensorflow/core/distributed_runtime/eager/eager_client.h" +#include "tensorflow/core/distributed_runtime/eager/remote_tensor_handle.h" #include "tensorflow/core/distributed_runtime/rendezvous_mgr_interface.h" #include "tensorflow/core/distributed_runtime/server_lib.h" #include "tensorflow/core/distributed_runtime/worker_cache.h" @@ -64,6 +65,13 @@ limitations under the License. namespace tensorflow { +namespace eager { +// We need this forward declaration because we have circular dependency: +// Context -> RemoteMgr -> TensorHandle -> Context. +// TODO(fishx): Remove this once we remove Context dependency in TensorHandle. +class RemoteMgr; +} // namespace eager + // LINT.IfChange // Note: Keep in sync with exported copy of enum in eager/c_api.h. enum ContextDevicePlacementPolicy { @@ -280,7 +288,9 @@ class EagerContext : public core::RefCounted { std::unique_ptr remote_device_manager, const std::vector& remote_contexts, uint64 context_id, Rendezvous* r, DeviceMgr* local_device_mgr, int keep_alive_secs, - DistributedFunctionLibraryRuntime* cluster_flr); + DistributedFunctionLibraryRuntime* cluster_flr, + std::unique_ptr> + remote_mgr); // Similar with InitializeRemoteMaster but this context will not kill remote // contexts in shutdown. @@ -288,20 +298,25 @@ class EagerContext : public core::RefCounted { std::unique_ptr remote_eager_workers, const DeviceMgr* remote_device_mgr, const std::vector& remote_contexts, uint64 context_id, - std::function rendezvous_creator); + std::function rendezvous_creator, + std::unique_ptr> + remote_mgr); Status StoreCollectiveOpsServer( std::unique_ptr server, DeviceMgr* device_mgr, CollectiveExecutorMgrInterface* rpc_collective_executor_mgr); + // TODO(fishx): Remove the custom deleter once we remove forward declaration. + const std::unique_ptr>& + RemoteMgr() { + return remote_mgr_; + } + // If true, then tensors should be shipped across processes via the // EagerService.SendTensor RPC. If false, _Send/_Recv ops should be used // instead (which in-turn use WorkerService.RecvTensor RPCs). bool UseSendTensorRPC() { return use_send_tensor_rpc_; } - - // Helper function to create monotonically increasing ids unique to this - // context. - uint64 NextId(); #endif // IS_MOBILE_PLATFORM bool PinSmallOpsToCPU() { return pin_small_ops_to_cpu_; } @@ -432,8 +447,8 @@ class EagerContext : public core::RefCounted { condition_variable keep_alive_thread_cv_; bool shutting_down_ GUARDED_BY(keep_alive_thread_shutdown_mu_) = false; - mutex next_id_mutex_; - uint64 next_id_ GUARDED_BY(next_id_mutex_) = 1; + std::unique_ptr> + remote_mgr_; #endif // IS_MOBILE_PLATFORM bool use_send_tensor_rpc_; diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index 785a30750b4..430e11ecc15 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -43,6 +43,7 @@ limitations under the License. #include "tensorflow/core/profiler/lib/traceme.h" #include "tensorflow/core/util/device_name_utils.h" #if !defined(IS_MOBILE_PLATFORM) +#include "tensorflow/core/distributed_runtime/eager/remote_mgr.h" #include "tensorflow/core/distributed_runtime/eager/eager_client.h" #include "tensorflow/core/distributed_runtime/eager/remote_execute_node.h" #endif // IS_MOBILE_PLATFORM @@ -710,7 +711,7 @@ Status EagerRemoteSendTensor(EagerContext* ctx, TensorHandle* h, eager::SendTensorResponse response; request.set_context_id(context_id); - request.set_op_id(ctx->NextId()); + request.set_op_id(ctx->RemoteMgr()->NextOpId()); request.set_device_name(recv_device->name()); // AsProtoTensorContent doesn't work when the tensor is on the GPU, hence @@ -749,19 +750,6 @@ Status EagerRemoteSendTensor(EagerContext* ctx, TensorHandle* h, return status; } -Status AddRemoteInput(eager::Operation* remote_op, TensorHandle* input, - Device* input_device) { - tensorflow::int64 op_id; - int32 output_num; - TF_RETURN_IF_ERROR(input->RemoteAddress(input_device, &op_id, &output_num)); - - auto* remote_op_input = remote_op->add_inputs(); - remote_op_input->set_op_id(op_id); - remote_op_input->set_output_num(output_num); - - return Status::OK(); -} - Status EnqueueAndWait(eager::EagerClient* eager_client, const std::unique_ptr& request, eager::EnqueueResponse* response) { @@ -780,7 +768,7 @@ Status EnqueueAndWait(eager::EagerClient* eager_client, void PrepareRemoteOp(eager::Operation* remote_op, EagerOperation* op) { EagerContext* ctx = op->EagerContext(); - remote_op->set_id(ctx->NextId()); + remote_op->set_id(ctx->RemoteMgr()->NextOpId()); remote_op->set_name(op->Name()); op->Attrs().FillAttrValueMap(remote_op->mutable_attrs()); @@ -841,7 +829,8 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, handle->Unref(); } - TF_RETURN_IF_ERROR(AddRemoteInput(remote_op, input, input_device)); + TF_RETURN_IF_ERROR(ctx->RemoteMgr()->SerializeRemoteTensorHandle( + input, remote_op->add_inputs(), input_device)); } } @@ -1275,7 +1264,8 @@ Status ExecuteSend(EagerContext* ctx, Device* device, TensorHandle* h, request->set_context_id(context_id); auto* remote_op = request->add_queue()->mutable_operation(); - TF_RETURN_IF_ERROR(AddRemoteInput(remote_op, h, h->device())); + TF_RETURN_IF_ERROR(ctx->RemoteMgr()->SerializeRemoteTensorHandle( + h, remote_op->add_inputs(), h->device())); PrepareRemoteOp(remote_op, &op); diff --git a/tensorflow/core/distributed_runtime/eager/BUILD b/tensorflow/core/distributed_runtime/eager/BUILD index 22d5c50bd4b..aa7adb88b0e 100644 --- a/tensorflow/core/distributed_runtime/eager/BUILD +++ b/tensorflow/core/distributed_runtime/eager/BUILD @@ -62,6 +62,7 @@ cc_library( "eager_service_impl.h", ], deps = [ + ":remote_mgr", ":remote_tensor_handle", "//tensorflow:grpc++", "//tensorflow/c:c_api_internal", @@ -92,6 +93,7 @@ tf_cc_test( srcs = ["eager_service_impl_test.cc"], deps = [ ":eager_service_impl", + ":remote_mgr", "//tensorflow/c:c_api", "//tensorflow/c:c_api_internal", "//tensorflow/core:eager_service_proto_cc", @@ -109,6 +111,24 @@ tf_cc_test( ], ) +cc_library( + name = "remote_mgr", + srcs = [ + "remote_mgr.cc", + ], + hdrs = [ + "remote_mgr.h", + ], + visibility = ["//tensorflow:internal"], + deps = [ + ":remote_tensor_handle", + "//tensorflow/core:eager_service_proto_cc", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/common_runtime/eager:tensor_handle", + ], +) + cc_library( name = "remote_tensor_handle_data", srcs = ["remote_tensor_handle_data.cc"], diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc index 6db300bc944..a63386f70ba 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/core/common_runtime/eager/execute.h" #include "tensorflow/core/common_runtime/function.h" #include "tensorflow/core/common_runtime/process_util.h" +#include "tensorflow/core/distributed_runtime/eager/remote_mgr.h" #include "tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h" #include "tensorflow/core/distributed_runtime/server_lib.h" #include "tensorflow/core/distributed_runtime/session_mgr.h" @@ -142,9 +143,12 @@ Status EagerServiceImpl::CreateContext(const CreateContextRequest* request, return s; } + auto remote_mgr = + absl::make_unique(/*is_master=*/false); s = ctx->InitializeRemoteWorker( std::move(remote_eager_workers), worker_session->remote_device_mgr(), - remote_workers, request->context_id(), std::move(rendezvous_creator)); + remote_workers, request->context_id(), std::move(rendezvous_creator), + std::move(remote_mgr)); if (!s.ok()) { delete ctx; return s; @@ -209,9 +213,9 @@ Status EagerServiceImpl::ExecuteOp(const Operation& operation, profiler::TraceMeLevel::kVerbose); for (const auto& remote_handle : operation.inputs()) { tensorflow::TensorHandle* handle; - TF_RETURN_IF_ERROR(server_context->GetTensorHandle( - RemoteTensorHandleInternal(remote_handle), &handle)); - + TF_RETURN_IF_ERROR( + server_context->Context()->RemoteMgr()->DeserializeRemoteTensorHandle( + remote_handle, &handle)); op->AddInput(handle); } } @@ -228,7 +232,8 @@ Status EagerServiceImpl::ExecuteOp(const Operation& operation, tensorflow::gtl::InlinedVector retvals; TF_RETURN_IF_ERROR(EagerExecute(op.get(), &retvals, &num_retvals)); - server_context->AddOperationOutputs(retvals, operation.id()); + server_context->Context()->RemoteMgr()->AddOperationOutputs(retvals, + operation.id()); for (auto* handle : retvals) { TF_RETURN_IF_ERROR(TensorHandleShape(handle, queue_response->add_shape())); @@ -253,7 +258,7 @@ Status EagerServiceImpl::Enqueue(const EnqueueRequest* request, if (item.has_operation()) { TF_RETURN_IF_ERROR(ExecuteOp(item.operation(), context, queue_response)); } else { - TF_RETURN_IF_ERROR(context->DeleteTensorHandle( + TF_RETURN_IF_ERROR(context->Context()->RemoteMgr()->DeleteTensorHandle( RemoteTensorHandleInternal(item.handle_to_decref()))); } } @@ -341,7 +346,8 @@ Status EagerServiceImpl::SendTensor(const SendTensorRequest* request, tensor_handle->Unref(); } - context->AddOperationOutputs(tensors, request->op_id()); + context->Context()->RemoteMgr()->AddOperationOutputs(tensors, + request->op_id()); return Status::OK(); } diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.h b/tensorflow/core/distributed_runtime/eager/eager_service_impl.h index a33c678e789..b64c0ffb28c 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.h +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.h @@ -16,7 +16,6 @@ limitations under the License. #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_EAGER_EAGER_SERVICE_IMPL_H_ #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_EAGER_EAGER_SERVICE_IMPL_H_ -#include #include "tensorflow/core/common_runtime/eager/context.h" #include "tensorflow/core/common_runtime/eager/tensor_handle.h" @@ -112,56 +111,12 @@ class EagerServiceImpl { RecordAccess(); } ~ServerContext() { - for (const auto& entry : tensors_) { - entry.second->Unref(); - } ctx_->Unref(); } tensorflow::EagerContext* Context() const { return ctx_; } - void AddOperationOutputs( - const gtl::ArraySlice& handles, - int64 operation_id) { - mutex_lock l(tensors_mu_); - for (int i = 0; i < handles.size(); i++) { - // TODO(nareshmodi): Correctly handle operation_id not being unique. - tensors_.emplace(RemoteTensorHandleInternal(operation_id, i), - handles[i]); - } - } - - Status GetTensorHandle(const RemoteTensorHandleInternal& remote_handle, - tensorflow::TensorHandle** handle) { - mutex_lock l(tensors_mu_); - auto iter = tensors_.find(remote_handle); - if (iter == tensors_.end()) { - return errors::InvalidArgument( - "Unable to find the relevant tensor remote_handle: Op ID: ", - remote_handle.op_id, ", Output num: ", remote_handle.output_num); - } - - *handle = iter->second; - - return Status::OK(); - } - - Status DeleteTensorHandle(const RemoteTensorHandleInternal& remote_handle) { - mutex_lock l(tensors_mu_); - auto iter = tensors_.find(remote_handle); - if (iter == tensors_.end()) { - return errors::InvalidArgument( - "Unable to find the relevant tensor remote_handle: Op ID: ", - remote_handle.op_id, ", Output num: ", remote_handle.output_num); - } - - iter->second->Unref(); - tensors_.erase(iter); - - return Status::OK(); - } - void RecordAccess() { mutex_lock l(last_accessed_mu_); last_accessed_micros_ = env_->env->NowMicros(); @@ -175,18 +130,9 @@ class EagerServiceImpl { } private: - using RemoteTensorHandleMap = - gtl::FlatMap; - // The context for this execution. tensorflow::EagerContext* ctx_; - // The state related to the context for this execution. - mutex tensors_mu_; - RemoteTensorHandleMap tensors_ GUARDED_BY(tensors_mu_); - const WorkerEnv* const env_; // Not owned. mutex last_accessed_mu_; diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc index c3b3cb42a51..39384c127ed 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/c/c_api_internal.h" #include "tensorflow/core/common_runtime/eager/tensor_handle.h" +#include "tensorflow/core/distributed_runtime/eager/remote_mgr.h" #include "tensorflow/core/distributed_runtime/rpc/rpc_rendezvous_mgr.h" #include "tensorflow/core/distributed_runtime/session_mgr.h" #include "tensorflow/core/distributed_runtime/test_utils.h" @@ -48,7 +49,8 @@ class TestEagerServiceImpl : public EagerServiceImpl { TF_RETURN_IF_ERROR(GetServerContext(context_id, &context)); core::ScopedUnref context_unref(context); - return context->GetTensorHandle(remote_handle, handle); + return context->Context()->RemoteMgr()->GetTensorHandle(remote_handle, + handle); } }; diff --git a/tensorflow/core/distributed_runtime/eager/remote_mgr.cc b/tensorflow/core/distributed_runtime/eager/remote_mgr.cc new file mode 100644 index 00000000000..a7e00272029 --- /dev/null +++ b/tensorflow/core/distributed_runtime/eager/remote_mgr.cc @@ -0,0 +1,89 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/distributed_runtime/eager/remote_mgr.h" + +#include "tensorflow/core/distributed_runtime/eager/remote_tensor_handle.h" +#include "tensorflow/core/lib/core/errors.h" + +namespace tensorflow { +namespace eager { + +void RemoteMgr::AddOperationOutputs( + const gtl::ArraySlice handles, + int64 operation_id) { + mutex_lock l(remote_tensor_handle_mu_); + for (int i = 0; i < handles.size(); i++) { + // TODO(nareshmodi): Correctly handle operation_id not being unique. + remote_tensor_handle_map_.emplace( + RemoteTensorHandleInternal(operation_id, i), handles[i]); + } +} + +Status RemoteMgr::GetTensorHandle( + const RemoteTensorHandleInternal& remote_handle, + tensorflow::TensorHandle** handle) { + tf_shared_lock l(remote_tensor_handle_mu_); + auto iter = remote_tensor_handle_map_.find(remote_handle); + if (iter == remote_tensor_handle_map_.end()) { + return errors::InvalidArgument( + "Unable to find the relevant tensor remote_handle: Op ID: ", + remote_handle.op_id, ", Output num: ", remote_handle.output_num); + } + + *handle = iter->second; + + return Status::OK(); +} + +Status RemoteMgr::DeleteTensorHandle( + const RemoteTensorHandleInternal& remote_handle) { + mutex_lock l(remote_tensor_handle_mu_); + auto iter = remote_tensor_handle_map_.find(remote_handle); + if (iter == remote_tensor_handle_map_.end()) { + return errors::InvalidArgument( + "Unable to find the relevant tensor remote_handle: Op ID: ", + remote_handle.op_id, ", Output num: ", remote_handle.output_num); + } + + iter->second->Unref(); + remote_tensor_handle_map_.erase(iter); + + return Status::OK(); +} + +Status RemoteMgr::SerializeRemoteTensorHandle(TensorHandle* in, + RemoteTensorHandle* out, + Device* device) { + // TODO(fishx): support serializing local tensor handle. + int64 op_id; + int32 output_num; + TF_RETURN_IF_ERROR(in->RemoteAddress(device, &op_id, &output_num)); + out->Clear(); + out->set_op_id(op_id); + out->set_output_num(output_num); + return Status::OK(); +} + +Status RemoteMgr::DeserializeRemoteTensorHandle(const RemoteTensorHandle& in, + TensorHandle** out) { + // TODO(fishx): support the case when the remote tensor handle does not exist + // in the map. + TF_RETURN_IF_ERROR(GetTensorHandle(RemoteTensorHandleInternal(in), out)); + return Status::OK(); +} + +} // namespace eager +} // namespace tensorflow diff --git a/tensorflow/core/distributed_runtime/eager/remote_mgr.h b/tensorflow/core/distributed_runtime/eager/remote_mgr.h new file mode 100644 index 00000000000..7b4a9bfa84f --- /dev/null +++ b/tensorflow/core/distributed_runtime/eager/remote_mgr.h @@ -0,0 +1,85 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_EAGER_REMOTE_MGR_H_ +#define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_EAGER_REMOTE_MGR_H_ + +#include + +#include "tensorflow/core/common_runtime/eager/tensor_handle.h" +#include "tensorflow/core/distributed_runtime/eager/remote_tensor_handle.h" +#include "tensorflow/core/platform/mutex.h" + +namespace tensorflow { +namespace eager { + +// This class manages the states required to setup an eager cluster. +// TODO(fishx): Move remote state from context to this class. +class RemoteMgr { + public: + explicit RemoteMgr(bool is_master) : is_master_(is_master) {} + + ~RemoteMgr() { + for (const auto& entry : remote_tensor_handle_map_) { + entry.second->Unref(); + } + } + + bool IsMaster() { return is_master_; } + + void AddOperationOutputs( + const gtl::ArraySlice handles, + int64 operation_id); + + Status GetTensorHandle(const RemoteTensorHandleInternal& remote_handle, + tensorflow::TensorHandle** handle); + + Status DeleteTensorHandle(const RemoteTensorHandleInternal& remote_handle); + + // Helper function to create monotonically increasing ids unique to this + // context. + uint64 NextOpId() { + DCHECK(is_master_); + mutex_lock l(next_id_mutex_); + return next_op_id_++; + } + + Status SerializeRemoteTensorHandle(TensorHandle* in, RemoteTensorHandle* out, + Device* device); + + Status DeserializeRemoteTensorHandle(const RemoteTensorHandle& in, + TensorHandle** out); + + private: + bool is_master_; + + using RemoteTensorHandleMap = + gtl::FlatMap; + mutex remote_tensor_handle_mu_; + // This map maintains the TensorHandles that is required by remote worker + // in the cluster. + RemoteTensorHandleMap remote_tensor_handle_map_ + GUARDED_BY(remote_tensor_handle_mu_); + + mutex next_id_mutex_; + uint64 next_op_id_ GUARDED_BY(next_id_mutex_) = 1; +}; + +} // namespace eager +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_EAGER_REMOTE_MGR_H_ From 7a6aa853e16b68a221a78cafb92fc157bc1caf48 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 16:33:51 -0700 Subject: [PATCH 157/332] Add support for integer sampled textures PiperOrigin-RevId: 257296221 --- .../gpu/gl/compiler/object_accessor.cc | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.cc b/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.cc index bd11be6be62..fff15b455c1 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.cc @@ -360,9 +360,37 @@ struct TextureSamplerTypeGetter { return (*this)(uint2()); } - std::string operator()(const uint2&) const { return "sampler2D"; } + std::string operator()(const uint2&) const { + switch (type) { + case DataType::FLOAT16: + case DataType::FLOAT32: + return "sampler2D"; + case DataType::INT32: + case DataType::INT16: + return "isampler2D"; + case DataType::UINT32: + case DataType::UINT16: + return "usampler2D"; + default: + return "unknown_sampler2D"; + } + } - std::string operator()(const uint3&) const { return "sampler2DArray"; } + std::string operator()(const uint3&) const { + switch (type) { + case DataType::FLOAT16: + case DataType::FLOAT32: + return "sampler2DArray"; + case DataType::INT32: + case DataType::INT16: + return "isampler2DArray"; + case DataType::UINT32: + case DataType::UINT16: + return "usampler2DArray"; + default: + return "unknown_sampler2DArray"; + } + } DataType type; }; From a46248283239879ca0d3c8c19efc9208dc4382c3 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Tue, 9 Jul 2019 17:03:19 -0700 Subject: [PATCH 158/332] IM --- tensorflow/core/kernels/data/BUILD | 19 + .../core/kernels/data/dataset_test_base.h | 11 +- .../fixed_length_record_dataset_op_test.cc | 656 ++++++++++++++++++ 3 files changed, 685 insertions(+), 1 deletion(-) create mode 100644 tensorflow/core/kernels/data/fixed_length_record_dataset_op_test.cc diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD index 250d3221db3..a6dacab6893 100644 --- a/tensorflow/core/kernels/data/BUILD +++ b/tensorflow/core/kernels/data/BUILD @@ -1017,6 +1017,25 @@ tf_cc_test( ], ) +tf_cc_test( + name = "fixed_length_record_dataset_op_test", + size = "small", + srcs = ["fixed_length_record_dataset_op_test.cc"], + deps = [ + ":dataset_test_base", + ":dataset_utils", + ":iterator_ops", + ":reader_dataset_ops", + "//tensorflow/core:core_cpu_internal", + "//tensorflow/core:dataset_ops_op_lib", + "//tensorflow/core:framework", + "//tensorflow/core:lib_internal", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + tf_kernel_library( name = "iterator_ops", srcs = ["iterator_ops.cc"], diff --git a/tensorflow/core/kernels/data/dataset_test_base.h b/tensorflow/core/kernels/data/dataset_test_base.h index c35336c51ef..e014c9d3390 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.h +++ b/tensorflow/core/kernels/data/dataset_test_base.h @@ -61,12 +61,21 @@ Status CreateFile(const string& filename, const std::vector& text, int input_buffer_size, int output_buffer_size, const CompressionType& compression_type); -// Saves the input texts into multiple files with the specified compression. +// Saves the input texts into multiple text files with the specified +// compression. Status CreateMultiTextFiles(const std::vector& filenames, const std::vector& multi_texts, int input_buffer_size, int output_buffer_size, const CompressionType& compression_type); +// Saves the input contents into multiple fixed length record files with the +// specified compression. +Status CreateMultiFixedLengthRecordFiles( + const std::vector& filenames, const string& header, + const string& footer, const std::vector& contents, + int input_buffer_size, int output_buffer_size, + const CompressionType& compression_type); + // Helpful functions to test Dataset op kernels. class DatasetOpsTestBase : public ::testing::Test { public: diff --git a/tensorflow/core/kernels/data/fixed_length_record_dataset_op_test.cc b/tensorflow/core/kernels/data/fixed_length_record_dataset_op_test.cc new file mode 100644 index 00000000000..073b506863c --- /dev/null +++ b/tensorflow/core/kernels/data/fixed_length_record_dataset_op_test.cc @@ -0,0 +1,656 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/kernels/data/text_line_dataset_op.h" + +#include "tensorflow/core/kernels/data/dataset_test_base.h" + +namespace tensorflow { +namespace data { +namespace { + +constexpr char kNodeName[] = "fixed_length_record_dataset"; +constexpr char kIteratorPrefix[] = "Iterator"; + +class FixedLengthRecordDatasetOpTest : public DatasetOpsTestBase { + protected: + // Create a new `TextLineDataset` op kernel. + Status CreateFixedLengthRecordDatasetOpKernel( + std::unique_ptr* text_line_dataset_op_kernel) { + NodeDef node_def = test::function::NDef( + kNodeName, "FixedLengthRecordDatasetV2", + {"filenames", "header_bytes", "record_bytes", "footer_bytes", + "buffer_size", "compression_type"}, + {}); + TF_RETURN_IF_ERROR(CreateOpKernel(node_def, text_line_dataset_op_kernel)); + return Status::OK(); + } + + // Create a new `FixedLengthRecordDataset` op kernel context + Status CreateFixedLengthRecordDatasetContext( + OpKernel* const op_kernel, + gtl::InlinedVector* const inputs, + std::unique_ptr* context) { + TF_RETURN_IF_ERROR(CheckOpKernelInput(*op_kernel, *inputs)); + TF_RETURN_IF_ERROR(CreateOpKernelContext(op_kernel, inputs, context)); + return Status::OK(); + } +}; + +struct TestCase { + std::vector filenames; + int64 header_bytes; + int64 record_bytes; + int64 footer_bytes; + int64 buffer_size; + CompressionType compression_type; + string header; + string footer; + std::vector expected_outputs; + DataTypeVector expected_output_dtypes; + std::vector expected_output_shapes; + int64 expected_cardinality; + std::vector breakpoints; +}; + +std::vector ToStringVector(const std::vector& str_tensors) { + std::vector str_vec; + for (const Tensor& tensor : str_tensors) { + for (int i = 0; i < tensor.NumElements(); ++i) { + str_vec.push_back(tensor.flat()(i)); + } + } + return str_vec; +} + +// Test case 1: multiple text files with ZLIB compression. +TestCase TestCase1() { + return { + /*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_1"), + absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_2")}, + /*header_bytes*/ 5, + /*record_bytes*/ 3, + /*footer_bytes*/ 2, + /*buffer_size*/ 10, + /*compression_type*/ ZLIB, + /*header*/ "HHHHH", + /*footer*/ "FF", + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"111"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"222"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"333"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"444"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), {"555"})}, + /*expected_output_dtypes*/ {DT_STRING}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ kUnknownCardinality, + /*breakpoints*/ {0, 2, 6}}; +} + +// Test case 2: multiple text files with GZIP compression. +TestCase TestCase2() { + return {/*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_GZIP_1"), + absl::StrCat(testing::TmpDir(), "/text_line_GZIP_2")}, + /*header_bytes*/ 5, + /*record_bytes*/ 3, + /*footer_bytes*/ 2, + /*buffer_size*/ 10, + /*compression_type*/ GZIP, + /*header*/ "HHHHH", + /*footer*/ "FF", + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"aaa"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"bbb"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"ccc"})}, + /*expected_output_dtypes*/ {DT_STRING}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ kUnknownCardinality, + /*breakpoints*/ {0, 2, 6}}; +} + +// Test case 3: a single text file without compression. +TestCase TestCase3() { + return {/*filenames*/ { + absl::StrCat(testing::TmpDir(), "/text_line_UNCOMPRESSED")}, + /*header_bytes*/ 5, + /*record_bytes*/ 3, + /*footer_bytes*/ 2, + /*buffer_size*/ 10, + /*compression_type*/ UNCOMPRESSED, + /*header*/ "HHHHH", + /*footer*/ "FF", + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"aa1"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"b2b"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"3cc"})}, + /*expected_output_dtypes*/ {DT_STRING}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ kUnknownCardinality, + /*breakpoints*/ {0, 2, 6}}; +} + +class ParameterizedFixedLengthRecordDatasetOpTest + : public FixedLengthRecordDatasetOpTest, + public ::testing::WithParamInterface {}; + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, GetNext) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + bool end_of_sequence = false; + std::vector out_tensors; + while (!end_of_sequence) { + std::vector next; + TF_EXPECT_OK( + iterator->GetNext(iterator_ctx.get(), &next, &end_of_sequence)); + out_tensors.insert(out_tensors.end(), next.begin(), next.end()); + } + + TF_EXPECT_OK(ExpectEqual(out_tensors, test_case.expected_outputs, + /*compare_order*/ true)); +} + +TEST_F(FixedLengthRecordDatasetOpTest, DatasetNodeName) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = TestCase1(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + EXPECT_EQ(text_line_dataset->node_name(), kNodeName); +} + +TEST_F(FixedLengthRecordDatasetOpTest, DatasetTypeString) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = TestCase1(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + EXPECT_EQ(text_line_dataset->type_string(), + name_utils::OpName(FixedLengthRecordDatasetOp::kDatasetType)); +} + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, DatasetOutputDtypes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + TF_EXPECT_OK(VerifyTypesMatch(text_line_dataset->output_dtypes(), + test_case.expected_output_dtypes)); +} + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, DatasetOutputShapes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + TF_EXPECT_OK(VerifyShapesCompatible(text_line_dataset->output_shapes(), + test_case.expected_output_shapes)); +} + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, Cardinality) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + EXPECT_EQ(text_line_dataset->Cardinality(), test_case.expected_cardinality); +} + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, DatasetSave) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr serialization_context; + TF_ASSERT_OK(CreateSerializationContext(&serialization_context)); + VariantTensorData data; + VariantTensorDataWriter writer(&data); + TF_ASSERT_OK(text_line_dataset->Save(serialization_context.get(), &writer)); + TF_ASSERT_OK(writer.Flush()); +} + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, IteratorOutputDtypes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + + TF_EXPECT_OK(VerifyTypesMatch(iterator->output_dtypes(), + test_case.expected_output_dtypes)); +} + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, IteratorOutputShapes) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + + TF_EXPECT_OK(VerifyShapesCompatible(iterator->output_shapes(), + test_case.expected_output_shapes)); +} + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, IteratorOutputPrefix) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + + EXPECT_EQ(iterator->prefix(), + name_utils::IteratorPrefix(FixedLengthRecordDatasetOp::kDatasetType, + kIteratorPrefix)); +} + +TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, Roundtrip) { + int thread_num = 2, cpu_num = 2; + TestCase test_case = GetParam(); + TF_ASSERT_OK(InitThreadPool(thread_num)); + TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); + + std::vector multi_texts = ToStringVector(test_case.expected_outputs); + TF_ASSERT_OK(CreateMultiTextFiles( + test_case.filenames, multi_texts, test_case.buffer_size, + test_case.buffer_size, test_case.compression_type)); + + std::unique_ptr text_line_dataset_kernel; + TF_ASSERT_OK( + CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); + + int64 num_files = test_case.filenames.size(); + Tensor filenames = + CreateTensor(TensorShape({num_files}), test_case.filenames); + Tensor compression_type = CreateTensor( + TensorShape({}), {CompressionName(test_case.compression_type)}); + Tensor buffer_size = + CreateTensor(TensorShape({}), {test_case.buffer_size}); + gtl::InlinedVector inputs{TensorValue(&filenames), + TensorValue(&compression_type), + TensorValue(&buffer_size)}; + std::unique_ptr text_line_dataset_context; + TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( + text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); + + DatasetBase* text_line_dataset; + TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), + text_line_dataset_context.get(), + &text_line_dataset)); + core::ScopedUnref scoped_unref(text_line_dataset); + + std::unique_ptr iterator_ctx; + TF_ASSERT_OK( + CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); + std::unique_ptr iterator; + TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), + kIteratorPrefix, &iterator)); + + std::unique_ptr serialization_ctx; + TF_ASSERT_OK(CreateSerializationContext(&serialization_ctx)); + + bool end_of_sequence = false; + std::vector out_tensors; + int cur_iteration = 0; + const std::vector& breakpoints = test_case.breakpoints; + for (int breakpoint : breakpoints) { + VariantTensorData data; + VariantTensorDataWriter writer(&data); + TF_EXPECT_OK(iterator->Save(serialization_ctx.get(), &writer)); + TF_EXPECT_OK(writer.Flush()); + VariantTensorDataReader reader(&data); + TF_EXPECT_OK(RestoreIterator(iterator_ctx.get(), &reader, kIteratorPrefix, + *text_line_dataset, &iterator)); + + while (cur_iteration <= breakpoint) { + std::vector next; + TF_EXPECT_OK( + iterator->GetNext(iterator_ctx.get(), &next, &end_of_sequence)); + out_tensors.insert(out_tensors.end(), next.begin(), next.end()); + cur_iteration++; + } + } + + TF_EXPECT_OK(ExpectEqual(out_tensors, test_case.expected_outputs, + /*compare_order*/ true)); +} + +INSTANTIATE_TEST_SUITE_P(FixedLengthRecordDatasetOpTest, + ParameterizedFixedLengthRecordDatasetOpTest, + ::testing::ValuesIn(std::vector( + {TestCase1(), TestCase2(), TestCase3()}))); + +} // namespace +} // namespace data +} // namespace tensorflow From e70b1dd8820b36203cead190d415e161e6dc0faa Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 16:47:51 -0700 Subject: [PATCH 159/332] [XLA] Modify operation semantics description for XLA Gather. NFC. PiperOrigin-RevId: 257298693 --- .../compiler/xla/g3doc/operation_semantics.md | 103 ++++++++++-------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md index f886a06be8b..8b425bd40a4 100644 --- a/tensorflow/compiler/xla/g3doc/operation_semantics.md +++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md @@ -1367,12 +1367,12 @@ For a more intuitive description, see the "Informal Description" section below. : : : detailed description. : | `offset_dims` | `ArraySlice` | The set of dimensions in the | : : : output shape that offset into : -: : : a array sliced from operand. : +: : : an array sliced from operand. : | `slice_sizes` | `ArraySlice` | `slice_sizes[i]` is the | : : : bounds for the slice on : : : : dimension `i`. : | `collapsed_slice_dims` | `ArraySlice` | The set of dimensions in each | -: : : \: slice that are collapsed : +: : : slice that are collapsed : : : : away. These dimensions must : : : : have size 1. : | `start_index_map` | `ArraySlice` | A map that describes how to | @@ -1383,8 +1383,11 @@ For a more intuitive description, see the "Informal Description" section below. For convenience, we label dimensions in the output array not in `offset_dims` as `batch_dims`. -The output is an array of rank `batch_dims.size` + `operand.rank` - -`collapsed_slice_dims`.size. +The output is an array of rank `batch_dims.size` + `offset_dims.size`. + +The `operand.rank` must equal the sume of `offset_dims.size` and +`collapsed_slice_dims`. Also, `slice_sizes.size` has to be equal to +`operand.rank`. If `index_vector_dim` is equal to `start_indices.rank` we implicitly consider `start_indices` to have a trailing `1` dimension (i.e. if `start_indices` was of @@ -1405,61 +1408,65 @@ accounting for `collapsed_slice_dims` (i.e. we pick `adjusted_slice_sizes`[`k`] where `adjusted_slice_sizes` is `slice_sizes` with the bounds at indices `collapsed_slice_dims` removed). -Formally, the operand index `In` corresponding to an output index `Out` is -computed as follows: +Formally, the operand index `In` corresponding to a given output index `Out` is +calculated as follows: -1. Let `G` = { `Out`[`k`] for `k` in `batch_dims` }. Use `G` to slice out -vector `S` such that `S`[`i`] = `start_indices`[Combine(`G`, `i`)] where -Combine(A, b) inserts b at position `index_vector_dim` into A. Note that -this is well defined even if `G` is empty -- if `G` is empty then `S` = -`start_indices`. +1. Let `G` = { `Out`[`k`] for `k` in `batch_dims` }. Use `G` to slice out a + vector `S` such that `S`[`i`] = `start_indices`[Combine(`G`, `i`)] where + Combine(A, b) inserts b at position `index_vector_dim` into A. Note that + this is well defined even if `G` is empty -- if `G` is empty then `S` = + `start_indices`. -2. Create a starting index, `S``in`, into `operand` using `S` by -scattering `S` using `start_index_map`. More precisely: -1. `S``in`[`start_index_map`[`k`]] = `S`[`k`] if `k` < -`start_index_map.size`. -2. `S``in`[`_`] = `0` otherwise. +2. Create a starting index, `S``in`, into `operand` using `S` by + scattering `S` using `start_index_map`. More precisely: -3. Create an index `O``in` into `operand` by scattering the indices -at the offset dimensions in `Out` according to the `collapsed_slice_dims` -set. More precisely: -1. `O``in`[`expand_offset_dims`(`k`)] = -`Out`[`offset_dims`[`k`]] if `k` < `offset_dims.size` -(`expand_offset_dims` is defined below). -2. `O``in`[`_`] = `0` otherwise. -4. `In` is `O``in` + `S``in` where + is element-wise -addition. + 1. `S``in`[`start_index_map`[`k`]] = `S`[`k`] if `k` < + `start_index_map.size`. -`expand_offset_dims` is the monotonic function with domain [`0`, `offset.size`) -and range [`0`, `operand.rank`) \ `collapsed_slice_dims`. So if, e.g., + 2. `S``in`[`_`] = `0` otherwise. + +3. Create an index `O``in` into `operand` by scattering the indices + at the offset dimensions in `Out` according to the `collapsed_slice_dims` + set. More precisely: + + 1. `O``in`[`remapped_offset_dims`(`k`)] = + `Out`[`offset_dims`[`k`]] if `k` < `offset_dims.size` + (`remapped_offset_dims` is defined below). + + 2. `O``in`[`_`] = `0` otherwise. + +4. `In` is `O``in` + `S``in` where + is element-wise + addition. + +`remapped_offset_dims` is a monotonic function with domain [`0`, `offset.size`) +and range [`0`, `operand.rank`) \ `collapsed_slice_dims`. So if, e.g., `offset.size` is `4`, `operand.rank` is `6` and `collapsed_slice_dims` is {`0`, -`2`} then `expand_offset_dims` is {`0`→`1`, `1`→`3`, `2`→`4`, `3`→`5`}. +`2`} then `remapped_offset_dims` is {`0`→`1`, `1`→`3`, `2`→`4`, `3`→`5`}. ### Informal Description and Examples Informally, every index `Out` in the output array corresponds to an element `E` in the operand array, computed as follows: -- We use the batch dimensions in `Out` to look up a starting index from -`start_indices`. +- We use the batch dimensions in `Out` to look up a starting index from + `start_indices`. -- We use `start_index_map` to map the starting index (which may have size less -than operand.rank) to a "full" starting index into operand. +- We use `start_index_map` to map the starting index (whose size may be less + than operand.rank) to a "full" starting index into the `operand`. -- We dynamic-slice out a slice with size `slice_sizes` using the full starting -index. +- We dynamic-slice out a slice with size `slice_sizes` using the full starting + index. -- We reshape the slice by collapsing the `collapsed_slice_dims` dimensions. -Since all collapsed slice dimensions have to have bound 1 this reshape is -always legal. +- We reshape the slice by collapsing the `collapsed_slice_dims` dimensions. + Since all collapsed slice dimensions must have a bound of 1, this reshape is + always legal. -- We use the offset dimensions in `Out` to index into this slice to get the -input element, `E`, corresponding to output index `Out`. +- We use the offset dimensions in `Out` to index into this slice to get the + input element, `E`, corresponding to output index `Out`. -`index_vector_dim` is set to `start_indices.rank` - `1` in all of the -examples that follow. More interesting values for `index_vector_dim` does not -change the operation fundamentally, but makes the visual representation more -cumbersome. +`index_vector_dim` is set to `start_indices.rank` - `1` in all of the examples +that follow. More interesting values for `index_vector_dim` do not change the +operation fundamentally, but make the visual representation more cumbersome. To get an intuition on how all of the above fits together, let's look at an example that gathers 5 slices of shape `[8,6]` from a `[16,11]` array. The @@ -1526,12 +1533,12 @@ As a final example, we use (2) and (3) to implement `tf.gather_nd`: `G``0` and `G``1` are used to slice out a starting index from the gather indices array as usual, except the starting index has only one -element, `X`. Similarly, there is only one output offset index with the value -`O``0`. However, before being used as indices into the input array, +element, `X`. Similarly, there is only one output offset index with the value +`O``0`. However, before being used as indices into the input array, these are expanded in accordance to "Gather Index Mapping" (`start_index_map` in -the formal description) and "Offset Mapping" (`expand_offset_dims` in the formal -description) into [`X`,`0`] and [`0`,`O``0`] respectively, adding up -to [`X`,`O``0`]. In other words, the output index +the formal description) and "Offset Mapping" (`remapped_offset_dims` in the +formal description) into [`X`,`0`] and [`0`,`O``0`] respectively, +adding up to [`X`,`O``0`]. In other words, the output index [`G``0`,`G``1`,`O``0`] maps to the input index [`GatherIndices`[`G``0`,`G``1`,`0`],`X`] which gives us the semantics for `tf.gather_nd`. From 82c46d74d9ece4f6b5832f27a7ae580d95e1a312 Mon Sep 17 00:00:00 2001 From: Anudhyan Boral Date: Tue, 9 Jul 2019 16:49:48 -0700 Subject: [PATCH 160/332] Add functools32 to TensorFlow dependencies. This library is a backport for Python 2 of the Python 3 functools standard library. We want to use functools.lru_cache for caching opt_einsum results. One other option is adding backports/functools_lru_cache. But functools also has other components which might be useful. PiperOrigin-RevId: 257299040 --- tensorflow/opensource_only.files | 1 + tensorflow/python/BUILD | 1 + tensorflow/tools/pip_package/BUILD | 1 + tensorflow/tools/pip_package/setup.py | 2 ++ tensorflow/workspace.bzl | 11 +++++++++++ third_party/functools32.BUILD | 18 ++++++++++++++++++ 6 files changed, 34 insertions(+) create mode 100644 third_party/functools32.BUILD diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files index e15b2eda7b3..6d0e94e9569 100644 --- a/tensorflow/opensource_only.files +++ b/tensorflow/opensource_only.files @@ -212,6 +212,7 @@ tensorflow/third_party/nccl/BUILD tensorflow/third_party/boringssl/BUILD tensorflow/third_party/mpi/.gitignore tensorflow/third_party/mpi/BUILD +tensorflow/third_party/functools32.BUILD tensorflow/third_party/tensorrt/LICENSE tensorflow/third_party/tensorrt/BUILD tensorflow/third_party/tensorrt/build_defs.bzl.tpl diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 1ae2c9acb56..5f38820fd87 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3555,6 +3555,7 @@ py_library( ":math_ops", ":platform", "//tensorflow/compiler/tf2xla/ops:gen_xla_ops", + "@functools32_archive//:functools32", "@opt_einsum_archive//:opt_einsum", ], ) diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 9d32e54eada..85fc4faaa74 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -171,6 +171,7 @@ filegroup( "@farmhash_archive//:COPYING", "@fft2d//:fft2d/readme2d.txt", "@flatbuffers//:LICENSE.txt", + "@functools32_archive//:LICENSE", "@gast_archive//:PKG-INFO", "@gemmlowp//:LICENSE", "@gif_archive//:COPYING", diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 2b70f64c093..103c8724edb 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -88,6 +88,8 @@ else: REQUIRED_PACKAGES.append('wheel') # mock comes with unittest.mock for python3, need to install for python2 REQUIRED_PACKAGES.append('mock >= 2.0.0') + # functools comes with python3, need to install the backport for python2 + REQUIRED_PACKAGES.append('functools32 >= 3.2.3') # tf-nightly should depend on tb-nightly if 'tf_nightly' in project_name: diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 1322496c0e4..b6a1808640c 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -316,6 +316,17 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ], ) + tf_http_archive( + name = "functools32_archive", + build_file = clean_dep("//third_party:functools32.BUILD"), + sha256 = "f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d", + strip_prefix = "functools32-3.2.3-2", + urls = [ + "http://mirror.tensorflow.org/pypi.python.org/packages/c5/60/6ac26ad05857c601308d8fb9e87fa36d0ebf889423f47c3502ef034365db/functools32-3.2.3-2.tar.gz", + "https://pypi.python.org/packages/c5/60/6ac26ad05857c601308d8fb9e87fa36d0ebf889423f47c3502ef034365db/functools32-3.2.3-2.tar.gz", + ], + ) + tf_http_archive( name = "gast_archive", build_file = clean_dep("//third_party:gast.BUILD"), diff --git a/third_party/functools32.BUILD b/third_party/functools32.BUILD new file mode 100644 index 00000000000..32dccf3b72d --- /dev/null +++ b/third_party/functools32.BUILD @@ -0,0 +1,18 @@ +# Description: +# functools32 provides a backport of the functools module for Python 2. + +licenses(["notice"]) # Python 2.0 + +exports_files(["LICENSE"]) + +py_library( + name = "functools32", + srcs = [ + "functools32/__init__.py", + "functools32/_dummy_thread32.py", + "functools32/functools32.py", + "functools32/reprlib32.py", + ], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) From 6370774652ea87e42b3f662ba2a96b5a502016f8 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Tue, 9 Jul 2019 17:28:31 -0700 Subject: [PATCH 161/332] Update MLIR rev & build file. PiperOrigin-RevId: 257305941 --- third_party/mlir/BUILD | 20 ++++++++++++-------- third_party/mlir/mlir_configure.bzl | 4 ++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/third_party/mlir/BUILD b/third_party/mlir/BUILD index 0645b51eb7e..ca8a310d123 100644 --- a/third_party/mlir/BUILD +++ b/third_party/mlir/BUILD @@ -808,6 +808,7 @@ cc_library( ":AffineOps", ":Analysis", ":IR", + ":StandardDialectRegistration", ":StandardOps", ":Support", "@llvm//:support", @@ -823,6 +824,7 @@ cc_library( "lib/Transforms/DmaGeneration.cpp", "lib/Transforms/LoopFusion.cpp", "lib/Transforms/LoopInvariantCodeMotion.cpp", + "lib/Transforms/LoopParametricTiling.cpp", "lib/Transforms/LoopTiling.cpp", "lib/Transforms/LoopUnroll.cpp", "lib/Transforms/LoopUnrollAndJam.cpp", @@ -857,18 +859,19 @@ cc_library( ) cc_library( - name = "AffineToGPU", + name = "LoopsToGPU", srcs = [ - "lib/Conversion/AffineToGPU/AffineToGPU.cpp", + "lib/Conversion/LoopsToGPU/LoopsToGPU.cpp", ], hdrs = [ - "include/mlir/Conversion/AffineToGPU/AffineToGPU.h", + "include/mlir/Conversion/LoopsToGPU/LoopsToGPU.h", ], includes = ["include"], deps = [ ":AffineOps", ":GPUDialect", ":IR", + ":Linalg", ":StandardOps", ":Support", ":TransformUtils", @@ -878,17 +881,18 @@ cc_library( ) cc_library( - name = "AffineToGPUPass", + name = "LoopsToGPUPass", srcs = [ - "lib/Conversion/AffineToGPU/AffineToGPUPass.cpp", + "lib/Conversion/LoopsToGPU/LoopsToGPUPass.cpp", ], hdrs = [ - "include/mlir/Conversion/AffineToGPU/AffineToGPUPass.h", + "include/mlir/Conversion/LoopsToGPU/LoopsToGPUPass.h", ], includes = ["include"], deps = [ ":AffineOps", - ":AffineToGPU", + ":Linalg", + ":LoopsToGPU", ":Pass", "@llvm//:support", ], @@ -1186,13 +1190,13 @@ cc_binary( name = "mlir-opt", deps = [ ":AffineDialectRegistration", - ":AffineToGPUPass", ":Analysis", ":FxpMathOps", ":FxpMathOpsDialectRegistration", ":GPUDialectRegistration", ":IR", ":LinalgDialectRegistration", + ":LoopsToGPUPass", ":MlirOptLib", ":MlirOptMain", ":QuantOps", diff --git a/third_party/mlir/mlir_configure.bzl b/third_party/mlir/mlir_configure.bzl index 614cfd59158..f0855be3c0a 100644 --- a/third_party/mlir/mlir_configure.bzl +++ b/third_party/mlir/mlir_configure.bzl @@ -1,7 +1,7 @@ """Repository rule to setup the external MLIR repository.""" -_MLIR_REV = "35500c0d6c8fee4802d9cdedcac6cafc8900fe01" -_MLIR_SHA256 = "a8102a4ac1d40f6c24fd68bbefd317fccbc371416d2ce39139338496ad5c478d" +_MLIR_REV = "510870483fa9f897db59c59a983c6774fb522197" +_MLIR_SHA256 = "eda40ec8cfb9de60f6ab7165aa722e8b82d7c83bb6d256ac176e37d3bed5d412" def _mlir_autoconf_impl(repository_ctx): """Implementation of the mlir_configure repository rule.""" From 21c1ffe9f38f0ad47bd3d19c5f6a5d46d66ac1b1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jul 2019 17:57:24 -0700 Subject: [PATCH 162/332] Update ModuleOp::create(...) to take a Location instead of a context. This allows for giving a Module a more interesting location than 'Unknown'. PiperOrigin-RevId: 257310117 --- .../compiler/mlir/tensorflow/translate/import_graphdef.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc index e592862f6af..774d18c7911 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc @@ -1291,7 +1291,8 @@ StatusOr Importer::Convert( mlir::MLIRContext* context, const Graph& graph, const GraphDebugInfo& debug_info, const FunctionLibraryDefinition& flib_def, const NodeSpecs& specs) { - mlir::OwningModuleRef module = mlir::Module::create(context); + mlir::OwningModuleRef module = + mlir::Module::create(mlir::UnknownLoc::get(context)); std::unordered_map tf_name_to_mlir_name; Importer importer(flib_def, debug_info, specs, module.get(), &tf_name_to_mlir_name); From e737182ec257b55d575fde3a2c87ea5f5f2b0f7a Mon Sep 17 00:00:00 2001 From: Reed Wanderman-Milne Date: Tue, 9 Jul 2019 18:00:10 -0700 Subject: [PATCH 163/332] Automated rollback of commit 1ec90f6a7b8b4aa6a8ed9a596c0803406b4b7460 PiperOrigin-RevId: 257310508 --- tensorflow/python/framework/ops.py | 42 +++++ tensorflow/python/keras/BUILD | 11 -- tensorflow/python/keras/engine/base_layer.py | 13 +- .../python/keras/engine/base_layer_utils.py | 28 ++++ .../python/keras/engine/casting_utils.py | 74 --------- .../experimental/autocast_variable.py | 40 +++-- .../experimental/autocast_variable_test.py | 153 +++++++++++------- .../experimental/keras_test.py | 44 +---- 8 files changed, 192 insertions(+), 213 deletions(-) delete mode 100644 tensorflow/python/keras/engine/casting_utils.py diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index d0884de2a6e..390115764d7 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -4898,6 +4898,48 @@ class Graph(object): def _global_distribute_strategy_scope(self, distribute_strategy_scope): self._thread_local.distribute_strategy_scope = (distribute_strategy_scope) + @property + def _auto_cast_variable_read_dtype(self): + """The dtype that instances of `AutoCastVariable` will be casted to. + + This is None if `AutoCastVariables` should not be casted. + + See `AutoCastVariable` for more information. + + Returns: + The dtype that instances of `AutoCastVariable` will be casted to. + """ + if not hasattr(self._thread_local, "_auto_cast_variable_read_dtype"): + self._thread_local._auto_cast_variable_read_dtype = None # pylint: disable=protected-access + return self._thread_local._auto_cast_variable_read_dtype # pylint: disable=protected-access + + @_auto_cast_variable_read_dtype.setter + def _auto_cast_variable_read_dtype(self, _auto_cast_variable_read_dtype): + self._thread_local._auto_cast_variable_read_dtype = ( # pylint: disable=protected-access + _auto_cast_variable_read_dtype) + + @tf_contextlib.contextmanager + def _enable_auto_casting_variables(self, dtype): + """Context manager to automatically cast AutoCastVariables. + + If an AutoCastVariable `var` is used under this context manager, it will be + casted to `dtype` before being used. + + See `AutoCastVariable` for more information. + + Args: + dtype: The dtype that AutoCastVariables should be casted to. + + Yields: + Nothing. + """ + prev_read_dtype = self._auto_cast_variable_read_dtype + try: + self._auto_cast_variable_read_dtype = dtype + yield + finally: + self._auto_cast_variable_read_dtype = prev_read_dtype + def _mutation_lock(self): """Returns a lock to guard code that creates & mutates ops. diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index fb4bb82d197..c0c4ef10841 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -157,16 +157,6 @@ py_library( ], ) -py_library( - name = "casting_utils", - srcs = ["engine/casting_utils.py"], - srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:util", - "//tensorflow/python/keras/mixed_precision/experimental:autocast_variable", - ], -) - py_library( name = "engine", srcs = [ @@ -258,7 +248,6 @@ py_library( deps = [ ":backend", ":base_layer_utils", - ":casting_utils", ":constraints", ":engine_utils", ":regularizers", diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index a7b1721335c..04c9a387e24 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -48,7 +48,6 @@ from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers from tensorflow.python.keras.engine import base_layer_utils -from tensorflow.python.keras.engine import casting_utils from tensorflow.python.keras.engine import input_spec from tensorflow.python.keras.engine import node as node_module from tensorflow.python.keras.mixed_precision.experimental import autocast_variable @@ -698,10 +697,7 @@ class Layer(module.Module): if not self.dynamic: try: - # We do not directly pass self.weights, because we do not want - # to include weights of any layers in self.layers. - with casting_utils.autocast_context_manager( - self._trainable_weights + self._non_trainable_weights, + with base_layer_utils.autocast_context_manager( input_list, self._mixed_precision_policy.should_cast_variables): # Add auto_control_deps in V2 when they are not already added by @@ -757,11 +753,8 @@ class Layer(module.Module): # Eager execution on data tensors. with backend.name_scope(self._name_scope()): self._maybe_build(inputs) - # We do not directly pass self.weights, because we do not want - # to include weights of any layers in self.layers. - with casting_utils.autocast_context_manager( - self._trainable_weights + self._non_trainable_weights, input_list, - self._mixed_precision_policy.should_cast_variables): + with base_layer_utils.autocast_context_manager( + input_list, self._mixed_precision_policy.should_cast_variables): outputs = self.call(inputs, *args, **kwargs) self._handle_activity_regularization(inputs, outputs) self._set_mask_metadata(inputs, outputs, input_masks) diff --git a/tensorflow/python/keras/engine/base_layer_utils.py b/tensorflow/python/keras/engine/base_layer_utils.py index bfa9d6ed988..14e2cabf39b 100644 --- a/tensorflow/python/keras/engine/base_layer_utils.py +++ b/tensorflow/python/keras/engine/base_layer_utils.py @@ -438,6 +438,34 @@ def training_arg_passed_to_call(argspec, args, kwargs): return 'training' in full_args and full_args['training'] is not None +def _get_var_read_dtype(input_list, should_cast): + """Gets the dtype that AutoCastVariables should be read in.""" + if should_cast and input_list and input_list[0].dtype.is_floating: + return input_list[0].dtype.base_dtype + else: + return None + + +def autocast_context_manager(input_list, should_cast): + """Returns a context manager to autocast AutoCastVariables. + + Under this context manager, if `should_cast` is True, AutoCastVariables will + be casted. If `should_cast` is False, AutoCastVariables will not be casted, + which can be used to disable autocasting if nested under another + call to `autocast_context_manager`. + + Args: + input_list: The inputs to the layer with the AutoCastVariables. + should_cast: Whether AutoCastVariables should be casted. + + Returns: + A context manager to automatically cast AutoCastVariables. + """ + var_read_dtype = _get_var_read_dtype(input_list, should_cast) + return ops.get_default_graph()._enable_auto_casting_variables( # pylint: disable=protected-access + var_read_dtype) + + def is_subclassed(layer): """Returns True if the object is a subclassed layer or subclassed model.""" return (layer.__module__.find('keras.engine') == -1 and diff --git a/tensorflow/python/keras/engine/casting_utils.py b/tensorflow/python/keras/engine/casting_utils.py deleted file mode 100644 index 2b3f85d96f4..00000000000 --- a/tensorflow/python/keras/engine/casting_utils.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains private utilities related to casting.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from tensorflow.python.keras.mixed_precision.experimental import autocast_variable -from tensorflow.python.util import tf_contextlib - - -def _get_var_read_dtype(input_list): - """Gets the dtype that AutoCastVariables should be read in.""" - try: - # TODO(reedwm): Is choosing the first input the right choice? - is_floating = input_list and input_list[0].dtype.is_floating - except AttributeError: - is_floating = False - if is_floating: - return input_list[0].dtype.base_dtype - else: - return None - - -@tf_contextlib.contextmanager -def autocast_context_manager(layer_weights, input_list, should_cast): - """A context manager to autocast a layer's AutoCastVariables. - - Under this context manager, if `should_cast` is True, the AutoCastVariables in - `layer_weights` will be casted to the dtype of the first input in - `input_list`, if the first input is a floating-point dtype. If `should_cast` - is False, this context manager is a no-op. - - Args: - layer_weights: A list of weights of a layer. AutoCastVariables in this list - will be casted if `should_cast` is True. Non-AutoCastVariables are - ignored. - input_list: The inputs to the layer with the AutoCastVariables. - should_cast: Whether AutoCastVariables should be casted. - - Yields: - Nothing. - """ - if not should_cast: - yield - return - - var_read_dtype = _get_var_read_dtype(input_list) - if var_read_dtype is None: - yield - return - - autocast_vars = [var for var in layer_weights - if isinstance(var, autocast_variable.AutoCastVariable)] - old_read_dtypes = [var._read_dtype for var in autocast_vars] # pylint: disable=protected-access - for var in autocast_vars: - var._read_dtype = var_read_dtype # pylint: disable=protected-access - try: - yield - finally: - for var, old_read_dtype in zip(autocast_vars, old_read_dtypes): - var._read_dtype = old_read_dtype # pylint: disable=protected-access diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py index e968594ef08..1ab46e89773 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py @@ -30,9 +30,8 @@ class AutoCastVariable(trackable.Trackable): This class wraps a floating-point tf.Variable. It emulates the variable interface and delegates to the wrapped variable, but it additionally will cast - the wrapped variable to `auto_cast_variable._read_dtype`. `_read_dtype` - defaults to the wrapped Variable's dtype, meaning the casts are a no-op, but - `_read_dtype` can be set to a different value, + the wrapped variable under a `Graph._enable_variable_auto_cast(dtype)` context + manager. For example: @@ -40,15 +39,14 @@ class AutoCastVariable(trackable.Trackable): v = tf.Variable(1.0, dtype=tf.float32) v = AutoCastVariable(v) print(tf.identity(v).dtype) # tf.float32 - v._read_dtype = tf.float16 - print(tf.identity(v).dtype) # tf.float16, as v will cast itself to float16 - print(v.dtype) # tf.float16, as v.dtype also changes + with ops.get_default_graph()._enable_variable_auto_cast(tf.float16): + print(tf.identity(v).dtype) # tf.float16, as v will cast itself to float16 + print(v.dtype) # tf.float16, as v.dtype also changes under the ctx manager. ``` The purpose of this class is to allow Keras layers to create variables in float32, and automatically cast them to float16 or bfloat16 when the layer is - called. Keras layers will set `_read_dtype` to the appropriate dtype when - called, then set it back to None when the call returns. + called. """ def __init__(self, variable): @@ -68,11 +66,6 @@ class AutoCastVariable(trackable.Trackable): 'type: %s' % variable.dtype.name) self._variable = variable - # The dtype this variable will be read in. This is public to other internal - # classes, but not externally. It can be accessed externally via the `dtype` - # property. - self._read_dtype = self._variable.dtype - # Delegate to the underlying variable for checkpointing. self._gather_saveables_for_checkpoint = ( self._variable._gather_saveables_for_checkpoint) # pylint: disable=protected-access @@ -81,10 +74,21 @@ class AutoCastVariable(trackable.Trackable): def name(self): return self._variable.name + def _should_cast(self): + """Returns True if this variable should be casted when accessed.""" + g = ops.get_default_graph() + # pylint:disable=protected-access + return (g._auto_cast_variable_read_dtype is not None and + self.true_dtype != g._auto_cast_variable_read_dtype) + # pylint:enable=protected-access + @property def dtype(self): """The dtype this variable will be casted to when read.""" - return self._read_dtype + if self._should_cast(): + return ops.get_default_graph()._auto_cast_variable_read_dtype # pylint:disable=protected-access + else: + return self._variable.dtype @property def true_dtype(self): @@ -93,6 +97,8 @@ class AutoCastVariable(trackable.Trackable): def value(self): val = self._variable.value() + if not self._should_cast(): + return val # We colocate_with(None) to ignore the existing device constraints, so that # the cast is always done on the variable's device with ops.colocate_with(None, ignore_existing=True): @@ -101,11 +107,15 @@ class AutoCastVariable(trackable.Trackable): def read_value(self): val = self._variable.read_value() + if not self._should_cast(): + return val return math_ops.cast(val, self.dtype) def sparse_read(self, indices, name=None): """Reads the value of this variable sparsely, using `gather`.""" val = self._variable.sparse_read(indices, name=name) + if not self._should_cast(): + return val return math_ops.cast(val, self.dtype) def assign(self, value, use_locking=None, name=None, read_value=True): @@ -128,7 +138,7 @@ class AutoCastVariable(trackable.Trackable): def _dense_var_to_tensor(self, dtype=None, name=None, as_ref=False): """Converts this variable to a tensor.""" - if self.dtype == self.true_dtype: + if not self._should_cast(): return ops.internal_convert_to_tensor(self._variable, dtype, name, as_ref) # TODO(reedwm): Support as_ref? diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py index 850691c86b4..da5b660c36d 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py @@ -25,6 +25,7 @@ import numpy as np from tensorflow.python.distribute import mirrored_strategy from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras.mixed_precision.experimental import autocast_variable @@ -79,20 +80,47 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): x = get_autocast_var(x, distribute) self.evaluate(x.initializer) - # _read_dtype is same dtype as variable + # outside of auto cast scope. self.assertEqual(x.dtype, dtypes.float32) - self.assertEqual(x.true_dtype, dtypes.float32) self.assertEqual(x.value().dtype, dtypes.float32) self.assertEqual(x.read_value().dtype, dtypes.float32) self.assertEqual(array_ops.identity(x).dtype, dtypes.float32) - # Setting _read_dtype to a different dtype - x._read_dtype = dtypes.float16 - self.assertEqual(x.dtype, dtypes.float16) - self.assertEqual(x.true_dtype, dtypes.float32) - self.assertEqual(x.value().dtype, dtypes.float16) - self.assertEqual(x.read_value().dtype, dtypes.float16) - self.assertEqual(array_ops.identity(x).dtype, dtypes.float16) + # within auto cast scope of different dtype + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float16): + self.assertEqual(x.dtype, dtypes.float16) + self.assertEqual(x.value().dtype, dtypes.float16) + self.assertEqual(x.read_value().dtype, dtypes.float16) + self.assertEqual(array_ops.identity(x).dtype, dtypes.float16) + + # within auto cast scope of same dtype + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float32): + self.assertEqual(x.dtype, dtypes.float32) + self.assertEqual(x.value().dtype, dtypes.float32) + self.assertEqual(x.read_value().dtype, dtypes.float32) + self.assertEqual(array_ops.identity(x).dtype, dtypes.float32) + + @parameterized.named_parameters(*TESTCASES) + def test_read_nested_scopes(self, distribute): + with get_distribute_scope(distribute): + x = get_var(1., dtypes.float32) + x = get_autocast_var(x, distribute) + self.evaluate(x.initializer) + + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float16): + self.assertEqual(x.dtype, dtypes.float16) + self.assertEqual(x.read_value().dtype, dtypes.float16) + + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float32): + self.assertEqual(x.dtype, dtypes.float32) + self.assertEqual(x.read_value().dtype, dtypes.float32) + + self.assertEqual(x.dtype, dtypes.float16) + self.assertEqual(x.read_value().dtype, dtypes.float16) @parameterized.named_parameters(*TESTCASES) def test_operator_overloads(self, distribute): @@ -100,43 +128,43 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): for read_dtype in (dtypes.float32, dtypes.float16): x = get_var(7., dtypes.float32) x = get_autocast_var(x, distribute) - x._read_dtype = read_dtype - self.evaluate(x.initializer) - self.assertAlmostEqual(8, self.evaluate(x + 1)) - self.assertAlmostEqual(10, self.evaluate(3 + x)) - self.assertAlmostEqual(14, self.evaluate(x + x)) - self.assertAlmostEqual(5, self.evaluate(x - 2)) - self.assertAlmostEqual(6, self.evaluate(13 - x)) - self.assertAlmostEqual(0, self.evaluate(x - x)) - self.assertAlmostEqual(14, self.evaluate(x * 2)) - self.assertAlmostEqual(21, self.evaluate(3 * x)) - self.assertAlmostEqual(49, self.evaluate(x * x)) - self.assertAlmostEqual(3.5, self.evaluate(x / 2)) - self.assertAlmostEqual(1.5, self.evaluate(10.5 / x)) - self.assertAlmostEqual(3, self.evaluate(x // 2)) - self.assertAlmostEqual(2, self.evaluate(15 // x)) - if read_dtype == dtypes.float32: - # The "mod" operator does not support float16 - self.assertAlmostEqual(1, self.evaluate(x % 2)) - self.assertAlmostEqual(2, self.evaluate(16 % x)) - self.assertTrue(self.evaluate(x < 12)) - self.assertTrue(self.evaluate(x <= 12)) - self.assertFalse(self.evaluate(x > 12)) - self.assertFalse(self.evaluate(x >= 12)) - self.assertFalse(self.evaluate(12 < x)) - self.assertFalse(self.evaluate(12 <= x)) - self.assertTrue(self.evaluate(12 > x)) - self.assertTrue(self.evaluate(12 >= x)) - self.assertAlmostEqual(343, self.evaluate(pow(x, 3)), places=4) - self.assertAlmostEqual(128, self.evaluate(pow(2, x)), places=4) - self.assertAlmostEqual(-7, self.evaluate(-x)) - self.assertAlmostEqual(7, self.evaluate(abs(x))) + with ops.get_default_graph()._enable_auto_casting_variables( + read_dtype): + self.evaluate(x.initializer) + self.assertAlmostEqual(8, self.evaluate(x + 1)) + self.assertAlmostEqual(10, self.evaluate(3 + x)) + self.assertAlmostEqual(14, self.evaluate(x + x)) + self.assertAlmostEqual(5, self.evaluate(x - 2)) + self.assertAlmostEqual(6, self.evaluate(13 - x)) + self.assertAlmostEqual(0, self.evaluate(x - x)) + self.assertAlmostEqual(14, self.evaluate(x * 2)) + self.assertAlmostEqual(21, self.evaluate(3 * x)) + self.assertAlmostEqual(49, self.evaluate(x * x)) + self.assertAlmostEqual(3.5, self.evaluate(x / 2)) + self.assertAlmostEqual(1.5, self.evaluate(10.5 / x)) + self.assertAlmostEqual(3, self.evaluate(x // 2)) + self.assertAlmostEqual(2, self.evaluate(15 // x)) + if read_dtype == dtypes.float32: + # The "mod" operator does not support float16 + self.assertAlmostEqual(1, self.evaluate(x % 2)) + self.assertAlmostEqual(2, self.evaluate(16 % x)) + self.assertTrue(self.evaluate(x < 12)) + self.assertTrue(self.evaluate(x <= 12)) + self.assertFalse(self.evaluate(x > 12)) + self.assertFalse(self.evaluate(x >= 12)) + self.assertFalse(self.evaluate(12 < x)) + self.assertFalse(self.evaluate(12 <= x)) + self.assertTrue(self.evaluate(12 > x)) + self.assertTrue(self.evaluate(12 >= x)) + self.assertAlmostEqual(343, self.evaluate(pow(x, 3)), places=4) + self.assertAlmostEqual(128, self.evaluate(pow(2, x)), places=4) + self.assertAlmostEqual(-7, self.evaluate(-x)) + self.assertAlmostEqual(7, self.evaluate(abs(x))) - x = get_var([7, 8, 9], dtypes.float32) - x = get_autocast_var(x, distribute) - x._read_dtype = read_dtype - self.evaluate(x.initializer) - self.assertEqual(self.evaluate(x[1]), 8) + x = get_var([7, 8, 9], dtypes.float32) + x = get_autocast_var(x, distribute) + self.evaluate(x.initializer) + self.assertEqual(self.evaluate(x[1]), 8) @parameterized.named_parameters(*TESTCASES) def test_assign(self, distribute): @@ -145,6 +173,7 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): x = get_autocast_var(x, distribute) self.evaluate(x.initializer) + # outside of auto cast scope. v1 = constant_op.constant(3.14, dtype=dtypes.float32) v2 = constant_op.constant(3.14, dtype=dtypes.float16) @@ -176,37 +205,37 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): run_and_check() # reset x self.evaluate(x.assign(0.)) - x._read_dtype = dtypes.float16 - # assign still expect float32 value even _read_dtype is float16 - run_and_check() + # within auto cast scope. + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float16): + # assign still expect float32 value even if in float16 scope + run_and_check() @parameterized.named_parameters(*TESTCASES) def test_assign_stays_in_true_dtype(self, distribute): with get_distribute_scope(distribute): x = get_var(1., dtypes.float32) x = get_autocast_var(x, distribute) - x._read_dtype = dtypes.float16 self.evaluate(x.initializer) # small_val is a value such that 1.0 + small_val == 1.0 in fp16, but not # in fp32 small_val = np.finfo('float16').eps / 2 small_tensor = constant_op.constant(small_val, dtype=dtypes.float32) - # Variable should be increased, despite it appearing to be the same - # float16 value. - self.assertEqual(1. + small_val, - self.evaluate(x.assign(1. + small_tensor))) - self.assertEqual(1., self.evaluate(x.value())) - - x._read_dtype = dtypes.float32 + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float16): + # Variable should be increased, despite it appearing to be the same + # float16 value. + self.assertEqual(1. + small_val, + self.evaluate(x.assign(1. + small_tensor))) + self.assertEqual(1., self.evaluate(x.value())) self.assertEqual(1. + small_val, self.evaluate(x.value())) - x._read_dtype = dtypes.float16 self.evaluate(x.assign(1.)) - self.assertEqual(1. + small_val, - self.evaluate(x.assign_add(small_tensor))) - self.assertEqual(1., self.evaluate(x.value())) - - x._read_dtype = dtypes.float32 + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float16): + self.assertEqual(1. + small_val, + self.evaluate(x.assign_add(small_tensor))) + self.assertEqual(1., self.evaluate(x.value())) self.assertEqual(1. + small_val, self.evaluate(x.value())) @parameterized.named_parameters(*TESTCASES) diff --git a/tensorflow/python/keras/mixed_precision/experimental/keras_test.py b/tensorflow/python/keras/mixed_precision/experimental/keras_test.py index 247282c9aea..cb40ad43f9a 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/keras_test.py +++ b/tensorflow/python/keras/mixed_precision/experimental/keras_test.py @@ -27,7 +27,6 @@ from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import mirrored_strategy from tensorflow.python.eager import backprop from tensorflow.python.eager import context -from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import test_util @@ -125,14 +124,6 @@ class AddLayerWithoutAutoCast(AddLayer): return self._add(inputs, math_ops.cast(self.v, inputs.dtype)) -class AddLayerWithFunction(AddLayer): - """Same as AddLayer, but _add is decorated with a tf.function.""" - - @def_function.function - def _add(self, x, y): - return super(AddLayerWithFunction, self)._add(x, y) - - class IdentityRegularizer(regularizers.Regularizer): def __call__(self, x): @@ -190,30 +181,6 @@ class KerasLayerTest(keras_parameterized.TestCase): self.evaluate(variables.global_variables_initializer()) self.assertEqual(self.evaluate(y), 2.) - @parameterized.named_parameters(*TESTCASES) - @test_util.run_in_graph_and_eager_modes - def test_layer_calling_tf_function(self, strategy_fn): - x = constant_op.constant([1.], dtype=dtypes.float16) - with strategy_fn().scope(): - with policy.policy_scope('infer_float32_vars'): - layer = AddLayerWithFunction(assert_type=dtypes.float16) - y = layer(x) - self.assertEqual(layer.v.dtype, dtypes.float32) - self.assertEqual(y.dtype, dtypes.float16) - self.evaluate(variables.global_variables_initializer()) - self.assertEqual(self.evaluate(y), 2.) - - @parameterized.named_parameters(*TESTCASES) - @test_util.run_in_graph_and_eager_modes - def test_variable_not_casted_for_int_inputs(self, strategy_fn): - x = constant_op.constant([[1]], dtype=dtypes.int32) - with strategy_fn().scope(): - with policy.policy_scope('infer_float32_vars'): - layer = layers.Embedding(input_dim=10, output_dim=32) - y = layer(x) - self.assertEqual(layer.embeddings.dtype, dtypes.float32) - self.assertEqual(y.dtype, dtypes.float32) - @parameterized.named_parameters(*TESTCASES) @test_util.run_in_graph_and_eager_modes def test_layer_regularizer_runs_in_float32(self, strategy_fn): @@ -347,22 +314,17 @@ class KerasModelTest(keras_parameterized.TestCase): 'testcase_name': 'nocloning', 'strategy_fn': create_mirrored_strategy, 'cloning': False - }, { - 'testcase_name': 'function', - 'strategy_fn': create_mirrored_strategy, - 'layer_with_tf_function': True }) def test_model(self, strategy_fn, use_operator=False, use_regularizer=False, - cloning=True, layer_with_tf_function=False): + cloning=True): if not self._is_strategy_supported(strategy_fn): return regularizer = IdentityRegularizer() if use_regularizer else None - layer_class = AddLayerWithFunction if layer_with_tf_function else AddLayer with strategy_fn().scope(): with policy.policy_scope('infer_float32_vars'): x = layers.Input(shape=(1,), batch_size=2, dtype=dtypes.float16) - layer = layer_class(assert_type=dtypes.float16, - use_operator=use_operator, regularizer=regularizer) + layer = AddLayer(assert_type=dtypes.float16, use_operator=use_operator, + regularizer=regularizer) y = layer(x) y = math_ops.cast(y, dtypes.float32) model = models.Model(inputs=x, outputs=y) From b20c5f3438fb4f9bb55c3caad551a0d6354a4f53 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Tue, 9 Jul 2019 18:03:08 -0700 Subject: [PATCH 164/332] Remove use of tuple points to analysis in buffer assignment. The train to dataflow analysis is coming. PiperOrigin-RevId: 257311047 --- .../compiler/xla/service/buffer_assignment.h | 5 ----- tensorflow/compiler/xla/service/cpu/BUILD | 2 +- .../xla/service/cpu/cpu_executable.cc | 17 ++++++++-------- .../compiler/xla/service/cpu/cpu_executable.h | 8 ++++---- tensorflow/compiler/xla/service/gpu/BUILD | 2 +- .../xla/service/gpu/gpu_executable.cc | 20 +++++++++---------- .../compiler/xla/service/gpu/gpu_executable.h | 8 ++++---- tensorflow/compiler/xla/service/hlo_value.cc | 8 ++++++++ tensorflow/compiler/xla/service/hlo_value.h | 5 +++++ 9 files changed, 42 insertions(+), 33 deletions(-) diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h index bfc4dced907..602b6b1b4fe 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment.h +++ b/tensorflow/compiler/xla/service/buffer_assignment.h @@ -440,11 +440,6 @@ class BufferAssignment { bool HaveDisjointSlices(const HloInstruction* hlo_a, const HloInstruction* hlo_b) const; - // Returns the underlying points-to analysis used for this assignment. - const TuplePointsToAnalysis& points_to_analysis() const { - return liveness_->points_to_analysis(); - } - const HloDataflowAnalysis& dataflow_analysis() const { return alias_analysis_->dataflow_analysis(); } diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index 3e3af30d3aa..41a2d7f34bc 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -247,10 +247,10 @@ cc_library( "//tensorflow/compiler/xla/service:computation_layout", "//tensorflow/compiler/xla/service:executable", "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/compiler/xla/service:hlo_dataflow_analysis", "//tensorflow/compiler/xla/service:hlo_execution_profile", "//tensorflow/compiler/xla/service:logical_buffer", "//tensorflow/compiler/xla/service:shaped_buffer", - "//tensorflow/compiler/xla/service:tuple_points_to_analysis", "//tensorflow/core:lib", "//tensorflow/core:stream_executor_no_cuda", "//tensorflow/core/profiler/lib:traceme", diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc index b10c4006f30..476579883f3 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/cpu_executable.h" #include + #include #include #include @@ -224,19 +225,19 @@ StatusOr CpuExecutable::CreateResultShapedBuffer( // caller. TF_RETURN_IF_ERROR(result_buffer.buffers().ForEachMutableElementWithStatus( [&](const ShapeIndex& index, se::DeviceMemoryBase* device_memory) { - const auto& sources = this->GetRootPointsToSet().element(index); + const auto& sources = this->GetRootValueSet().element(index); // The points to set is unambiguous so the set should be a // singleton. - CHECK_EQ(1, sources.size()); - const LogicalBuffer* buffer_source = sources[0]; - HloInstruction* src = buffer_source->instruction(); + CHECK_EQ(1, sources.values().size()); + const HloValue* value_source = sources.values()[0]; + HloInstruction* src = value_source->instruction(); // The source for this result buffer can be a nested buffer such as // a tuple element. The source instruction should have a // non-parameter buffer assigned. TF_ASSIGN_OR_RETURN( const BufferAllocation::Slice slice, - this->assignment_->GetUniqueSlice(src, buffer_source->index())); + this->assignment_->GetUniqueSlice(src, value_source->index())); const BufferAllocation::Index buffer_index = slice.index(); se::OwningDeviceMemory& buffer = buffers[buffer_index]; if (!slice.allocation()->is_entry_computation_parameter()) { @@ -293,7 +294,7 @@ StatusOr CpuExecutable::ExecuteAsyncOnStreamImpl( const ServiceExecutableRunOptions* run_options, absl::Span arguments, HloExecutionProfile* hlo_execution_profile) { - if (GetRootPointsToSet().IsAmbiguous()) { + if (GetRootValueSet().IsAmbiguous()) { return Unimplemented("Points-to set of root instruction is ambiguous"); } @@ -371,8 +372,8 @@ StatusOr CpuExecutable::ExecuteAsyncOnStreamImpl( return ShapeUtil::ByteSizeOf(shape, sizeof(void*)); } -const PointsToSet& CpuExecutable::GetRootPointsToSet() const { - return assignment_->points_to_analysis().GetPointsToSet( +const InstructionValueSet& CpuExecutable::GetRootValueSet() const { + return assignment_->dataflow_analysis().GetInstructionValueSet( module().entry_computation()->root_instruction()); } diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.h b/tensorflow/compiler/xla/service/cpu/cpu_executable.h index 735a20749b9..169acdeffd4 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.h @@ -26,11 +26,11 @@ limitations under the License. #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h" #include "tensorflow/compiler/xla/service/executable.h" +#include "tensorflow/compiler/xla/service/hlo_dataflow_analysis.h" #include "tensorflow/compiler/xla/service/hlo_execution_profile.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_module.h" #include "tensorflow/compiler/xla/service/shaped_buffer.h" -#include "tensorflow/compiler/xla/service/tuple_points_to_analysis.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/platform/macros.h" @@ -129,9 +129,9 @@ class CpuExecutable : public Executable { const ServiceExecutableRunOptions* run_options, absl::Span buffers); - // Returns the points-to set of the root instruction of the entry - // computation. Uses points-to analysis from buffer assignment. - const PointsToSet& GetRootPointsToSet() const; + // Returns the instruction value set of the root instruction of the entry + // computation. Uses dataflow analysis from buffer assignment. + const InstructionValueSet& GetRootValueSet() const; // The JIT containing compiled modules. const std::unique_ptr jit_; diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 90fe70d3d35..476d4fdeb2e 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -477,11 +477,11 @@ cc_library( "//tensorflow/compiler/xla/service:executable", "//tensorflow/compiler/xla/service:hlo", "//tensorflow/compiler/xla/service:hlo_casting_utils", + "//tensorflow/compiler/xla/service:hlo_dataflow_analysis", "//tensorflow/compiler/xla/service:hlo_execution_profile", "//tensorflow/compiler/xla/service:logical_buffer", "//tensorflow/compiler/xla/service:shaped_buffer", "//tensorflow/compiler/xla/service:transfer_manager", - "//tensorflow/compiler/xla/service:tuple_points_to_analysis", "//tensorflow/compiler/xla/service/llvm_ir:buffer_assignment_util", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc index 8a65e11f809..ce559c2c1e7 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc @@ -256,7 +256,7 @@ StatusOr GpuExecutable::Execute( HloExecutionProfile* hlo_execution_profile, bool block_host_until_done) { se::DeviceMemoryAllocator* memory_allocator = run_options->allocator(); - if (GetRootPointsToSet().IsAmbiguous()) { + if (GetRootValueSet().IsAmbiguous()) { return Unimplemented("Points-to set of root instruction is ambiguous"); } @@ -327,20 +327,20 @@ StatusOr GpuExecutable::Execute( TF_RETURN_IF_ERROR(shaped_buffer.buffers().ForEachMutableElementWithStatus( [&buffer_allocations, &buffers_in_result, this]( const ShapeIndex& index, se::DeviceMemoryBase* device_memory) { - const auto& sources = this->GetRootPointsToSet().element(index); + const auto& sources = this->GetRootValueSet().element(index); // The points-to set is unambiguous so the set should be a // singleton. That is, we know exactly which instruction // produced the array at this element. - CHECK_EQ(1, sources.size()); - auto src_hlo = sources[0]->instruction(); + CHECK_EQ(1, sources.values().size()); + auto src_hlo = sources.values()[0]->instruction(); - VLOG(4) << "Looking at: " << sources[0]; + VLOG(4) << "Looking at: " << sources.values()[0]; // The source instruction should have a non-parameter buffer // assigned. - TF_ASSIGN_OR_RETURN( - const BufferAllocation::Slice slice, - this->assignment_->GetUniqueSlice(src_hlo, sources[0]->index())); + TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice slice, + this->assignment_->GetUniqueSlice( + src_hlo, sources.values()[0]->index())); se::DeviceMemoryBase src_base = buffer_allocations->GetDeviceAddress(slice.index()); @@ -398,8 +398,8 @@ StatusOr GpuExecutable::ExecuteAsyncOnStream( return Execute(run_options, arguments, nullptr, block_host_until_done); } -const PointsToSet& GpuExecutable::GetRootPointsToSet() const { - return assignment_->points_to_analysis().GetPointsToSet( +const InstructionValueSet& GpuExecutable::GetRootValueSet() const { + return assignment_->dataflow_analysis().GetInstructionValueSet( module().entry_computation()->root_instruction()); } diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.h b/tensorflow/compiler/xla/service/gpu/gpu_executable.h index 45ed345bbf6..fd1a17c0a92 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.h +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.h @@ -29,10 +29,10 @@ limitations under the License. #include "tensorflow/compiler/xla/service/gpu/stream_assignment.h" #include "tensorflow/compiler/xla/service/gpu/thunk.h" #include "tensorflow/compiler/xla/service/gpu/thunk_schedule.h" +#include "tensorflow/compiler/xla/service/hlo_dataflow_analysis.h" #include "tensorflow/compiler/xla/service/hlo_execution_profile.h" #include "tensorflow/compiler/xla/service/hlo_module.h" #include "tensorflow/compiler/xla/service/shaped_buffer.h" -#include "tensorflow/compiler/xla/service/tuple_points_to_analysis.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/platform/macros.h" @@ -109,9 +109,9 @@ class GpuExecutable : public Executable { bool block_host_until_done, HloExecutionProfile* hlo_execution_profile); - // Returns the points-to set of the root instruction of the entry - // computation. Uses points-to analysis from buffer assignment. - const PointsToSet& GetRootPointsToSet() const; + // Returns the value set of the root instruction of the entry + // computation. Uses dataflow analysis from buffer assignment. + const InstructionValueSet& GetRootValueSet() const; using BufferAllocToDeviceMemoryMap = absl::flat_hash_map; diff --git a/tensorflow/compiler/xla/service/hlo_value.cc b/tensorflow/compiler/xla/service/hlo_value.cc index 18ab401bc89..78c48e036d6 100644 --- a/tensorflow/compiler/xla/service/hlo_value.cc +++ b/tensorflow/compiler/xla/service/hlo_value.cc @@ -257,6 +257,14 @@ std::ostream& operator<<(std::ostream& out, const HloValueSet& value_set) { return out; } +bool InstructionValueSet::IsAmbiguous() const { + bool ambiguous = false; + for (auto& iter : *this) { + ambiguous |= iter.second.values().size() > 1; + } + return ambiguous; +} + bool InstructionValueSet::AssignUnionOf( absl::Span inputs) { CHECK_GT(inputs.size(), 0); diff --git a/tensorflow/compiler/xla/service/hlo_value.h b/tensorflow/compiler/xla/service/hlo_value.h index 1f01b0bb365..a1150ae299d 100644 --- a/tensorflow/compiler/xla/service/hlo_value.h +++ b/tensorflow/compiler/xla/service/hlo_value.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_VALUE_H_ #include + #include #include @@ -245,6 +246,10 @@ class InstructionValueSet : public ShapeTree { // this value set changed. bool AssignUnionOf(absl::Span inputs); + // Returns true if any value sets for any subshape element is not a + // singleton. + bool IsAmbiguous() const; + string ToString() const; }; From 2a883b2bfb7f8f1c15a3564f2a441ea03102678f Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Tue, 9 Jul 2019 18:24:42 -0700 Subject: [PATCH 165/332] Fix tf_executor.NextIteration.Sink print to include control operands PiperOrigin-RevId: 257313708 --- tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc | 2 +- tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops.mlir | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc index a32bd6ceeb7..29d73a71ad9 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc @@ -694,7 +694,7 @@ void Print(NextIterationSinkOp next_iteration, OpAsmPrinter *p) { *p << next_iteration.getOperationName() << " ["; p->printOperand(next_iteration.getOperand(0)); *p << "] "; - p->printOperand(next_iteration.getOperand(1)); + p->printOperands(llvm::drop_begin(next_iteration.getOperands(), 1)); *p << " : " << next_iteration.getOperand(1)->getType(); p->printOptionalAttrDict(next_iteration.getAttrs()); } diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops.mlir index 9772386c029..510aaccb26a 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf_executor_ops.mlir @@ -313,7 +313,7 @@ func @nextiteration_control(%arg0: tensor<*xf32>, %arg1: tensor) -> tensor<* %3:3 = tf_executor.NextIteration.Source : tensor<*xf32> tf_executor.NextIteration.Sink [%3#1] %3#0, %1#2 : tensor<*xf32> // CHECK: %3:3 = tf_executor.NextIteration.Source : tensor<*xf32> -// CHECK: tf_executor.NextIteration.Sink [%3#1] %3#0 : tensor<*xf32> +// CHECK: tf_executor.NextIteration.Sink [%3#1] %3#0, %1#2 : tensor<*xf32> tf_executor.fetch %3#0 : tensor<*xf32> } return %0 : tensor<*xf32> From 0a09474bfe8b750d1748f5f105847ec2fefbae3d Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Tue, 9 Jul 2019 18:26:41 -0700 Subject: [PATCH 166/332] Enable def_function_test on GPU internally + disable the one failing test. PiperOrigin-RevId: 257313896 --- tensorflow/python/eager/BUILD | 2 +- tensorflow/python/eager/def_function_test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 16b1663f71b..8cefd5a01c8 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -640,7 +640,7 @@ tf_py_test( ], ) -tf_py_test( +cuda_py_test( name = "def_function_test", srcs = ["def_function_test.py"], additional_deps = [ diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py index ccad29baf4d..46f45395272 100644 --- a/tensorflow/python/eager/def_function_test.py +++ b/tensorflow/python/eager/def_function_test.py @@ -578,8 +578,9 @@ class DefFunctionTest(test.TestCase): v_holder[1].assign(11.) self.assertAllClose([14., 15.], wrapper(constant_op.constant(2.))) + # TODO(b/137148281): reenable @test_util.run_gpu_only - def testDeviceAnnotationRespected(self): + def DISABLED_testDeviceAnnotationRespected(self): a = [] @def_function.function() From 07fc1591f2e7e0394cf4a3f132043a05065b9595 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Tue, 9 Jul 2019 19:12:12 -0700 Subject: [PATCH 167/332] Formula fix, added tests. --- tensorflow/python/ops/image_grad.py | 10 +++- tensorflow/python/ops/image_grad_test.py | 65 +++++++++++++++++++++--- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index 9eb671d2481..db4b0fc30b5 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -280,6 +280,9 @@ def _RGBToHSVGrad(op, grad): green_smallest * _CustomReciprocal(blues - greens)) dh_dr = dh_dr_1 + dh_dr_2 + dh_dr_3 + dh_dr_4 + dh_dr_5 + # Converting from degrees to [0,1] scale as specified in + # https://www.tensorflow.org/api_docs/python/tf/image/rgb_to_hsv + dh_dr = dh_dr / 360 # for green, dh_dg -> dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 # dh_dg_1 -> @@ -318,6 +321,9 @@ def _RGBToHSVGrad(op, grad): red_smallest * -1 * _CustomReciprocal(blues - reds)) dh_dg = dh_dg_1 + dh_dg_2 + dh_dg_3 + dh_dg_4 + dh_dg_5 + # Converting from degrees to [0,1] scale as specified in + # https://www.tensorflow.org/api_docs/python/tf/image/rgb_to_hsv + dh_dg = dh_dg / 360 # for blue, dh_db -> dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 # dh_db_1 -> @@ -356,6 +362,9 @@ def _RGBToHSVGrad(op, grad): red_smallest * _CustomReciprocal(greens - reds)) dh_db = dh_db_1 + dh_db_2 + dh_db_3 + dh_db_4 + dh_db_5 + # Converting from degrees to [0,1] scale as specified in + # https://www.tensorflow.org/api_docs/python/tf/image/rgb_to_hsv + dh_db = dh_db / 360 # Gradients wrt to inputs dv_drgb = array_ops.stack([grad[..., 2] * dv_dr, @@ -369,5 +378,4 @@ def _RGBToHSVGrad(op, grad): grad[..., 0] * dh_db], axis=-1) gradient_input = math_ops.add(math_ops.add(dv_drgb, ds_drgb), dh_drgb) - return gradient_input diff --git a/tensorflow/python/ops/image_grad_test.py b/tensorflow/python/ops/image_grad_test.py index b4326e07eb5..a91de9c354c 100644 --- a/tensorflow/python/ops/image_grad_test.py +++ b/tensorflow/python/ops/image_grad_test.py @@ -29,7 +29,8 @@ from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import image_ops from tensorflow.python.ops import gen_image_ops from tensorflow.python.platform import test - +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import array_ops @test_util.for_all_test_methods(test_util.disable_xla, 'align_corners=False not supported by XLA') @@ -477,19 +478,71 @@ class RGBToHSVOpTest(test.TestCase): hsv_out = self.evaluate(hsv_out) self.assertEqual(out_shape, list(hsv_out.shape)) - def testRGBToHSVGrad(self): - in_shape = [2, 20, 30, 3] + def testRGBToHSVGradSimpleCase(self): + def f(x): return gen_image_ops.rgb_to_hsv(x) - x = np.random.rand(2, 20, 30, 3).astype(np.float32) - rgb_input_tensor = constant_op.constant(x, shape=in_shape) + # Building a simple input tensor to avoid any discontinuity + x = np.array( + [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]).astype(np.float32) + rgb_input_tensor = constant_op.constant(x, shape=x.shape) + # Computing Analytical and Numerical gradients of f(x) analytical, numerical = gradient_checker_v2.compute_gradient( f, [rgb_input_tensor]) + self.assertAllClose(numerical, analytical, atol=1e-4) + + def testRGBToHSVGradRandomCase(self): + + def f(x): + return gen_image_ops.rgb_to_hsv(x) + np.random.seed(0) + # Building a simple input tensor to avoid any discontinuity + x = np.random.rand(1, 5, 5, 3).astype(np.float32) + rgb_input_tensor = constant_op.constant(x, shape=x.shape) + # Computing Analytical and Numerical gradients of f(x) self.assertLess( - gradient_checker_v2.max_error(analytical, numerical), 1e-2) + gradient_checker_v2.max_error(*gradient_checker_v2.compute_gradient( + f, [rgb_input_tensor])), 1e-4) + def testRGBToHSVGradSpecialCaseRGreatest(self): + # This test tests a specific subset of the input space + # with a dummy function implemented with native TF operations. + in_shape = [2, 10, 20, 3] + def f(x): + return gen_image_ops.rgb_to_hsv(x) + def f_dummy(x): + # This dummy function is a implementation of RGB to HSV using + # primitive TF functions for one particular case when R>G>B. + r = x[..., 0] + g = x[..., 1] + b = x[..., 2] + # Since MAX = r and MIN = b, we get the following h,s,v values. + v = r + s = 1 - math_ops.div_no_nan(b, r) + h = 60 * math_ops.div_no_nan(g - b, r - b) + h = h/360 + return array_ops.stack([h, s, v], axis=-1) + + # Building a custom input tensor where R>G>B + x_reds = np.ones((in_shape[0], in_shape[1], in_shape[2])).astype(np.float32) + x_greens = 0.5 * np.ones( + (in_shape[0], in_shape[1], in_shape[2])).astype(np.float32) + x_blues = 0.2 * np.ones( + (in_shape[0], in_shape[1], in_shape[2])).astype(np.float32) + x = np.stack([x_reds, x_greens, x_blues], axis=-1) + rgb_input_tensor = constant_op.constant(x, shape=in_shape) + + # Computing Analytical and Numerical gradients of f(x) + analytical, numerical = gradient_checker_v2.compute_gradient( + f, [rgb_input_tensor]) + # Computing Analytical and Numerical gradients of f_dummy(x) + analytical_dummy, numerical_dummy = gradient_checker_v2.compute_gradient( + f_dummy, [rgb_input_tensor]) + self.assertAllClose(numerical, analytical, atol=1e-4) + self.assertAllClose(analytical_dummy, analytical, atol=1e-4) + self.assertAllClose(numerical_dummy, numerical, atol=1e-4) if __name__ == "__main__": test.main() From 032bdc6d83486bcea38b47bb5a661dce67338bc0 Mon Sep 17 00:00:00 2001 From: Karthik Muthuraman Date: Tue, 9 Jul 2019 19:20:15 -0700 Subject: [PATCH 168/332] Fixed docstring for helper function. --- tensorflow/python/ops/image_grad.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/ops/image_grad.py b/tensorflow/python/ops/image_grad.py index db4b0fc30b5..a74abb00f5a 100644 --- a/tensorflow/python/ops/image_grad.py +++ b/tensorflow/python/ops/image_grad.py @@ -157,13 +157,12 @@ def _CropAndResizeGrad(op, grad): def _CustomReciprocal(x): """ - Performs reciprocal with an eps added to the input. This is to avoid - inversion errors or divide by zeros or NaNs. - Inputs: - x -> input tensor to be reciprocat-ed - my_eps -> custom machine precision epsilon + Wrapper function around `math_ops.div_no_nan()` to perform a "safe" + reciprocal incase the input is zero. Avoids divide by zero and NaNs. + Input: + x -> input tensor to be reciprocat-ed. Returns: - x_reciprocal -> reciprocal of x added with my_eps + x_reciprocal -> reciprocal of x without NaNs. """ return math_ops.div_no_nan(1.0, x) From d904d313400597259df5942662c1a6f94dbb7b8b Mon Sep 17 00:00:00 2001 From: Reed Wanderman-Milne Date: Tue, 9 Jul 2019 20:59:24 -0700 Subject: [PATCH 169/332] Implement AutoCastVariable's __repr__ method. PiperOrigin-RevId: 257328567 --- .../keras/mixed_precision/experimental/BUILD | 1 + .../experimental/autocast_variable.py | 16 +++++++++ .../experimental/autocast_variable_test.py | 36 +++++++++++++++++-- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/mixed_precision/experimental/BUILD b/tensorflow/python/keras/mixed_precision/experimental/BUILD index cf08fd257d8..982ebf0c0ea 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/BUILD +++ b/tensorflow/python/keras/mixed_precision/experimental/BUILD @@ -76,6 +76,7 @@ py_library( "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", "//tensorflow/python/distribute:values", + "//tensorflow/python/eager:context", ], ) diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py index 1ab46e89773..59a0e08cba1 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.distribute import values as distribute_values +from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops @@ -158,6 +159,18 @@ class AutoCastVariable(trackable.Trackable): """Pass resource_variable_ops.is_resource_variable check.""" pass + def __repr__(self): + if context.executing_eagerly() and not self._in_graph_mode: + repr_str = ("') + return repr_str.format( + v=self, np_repr=ops.numpy_text(self.read_value(), is_repr=True)) + else: + repr_str = ("') + return repr_str.format(v=self) + # Operator overloads: # Note we only overload operators that support floating-point types, as # non-float variables cannot be wrapped with an AutoCastVariable. @@ -236,3 +249,6 @@ class AutoCastDistributedVariable(AutoCastVariable, raise ValueError('variable must be of type DistributedValues, ' 'but got: %s' % variable) super(AutoCastDistributedVariable, self).__init__(variable) + + def __repr__(self): + return distribute_values.DistributedVariable.__repr__(self) diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py index da5b660c36d..60cb1ca0ee9 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py @@ -23,12 +23,12 @@ from absl.testing import parameterized import numpy as np from tensorflow.python.distribute import mirrored_strategy +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.keras.mixed_precision.experimental import autocast_variable - from tensorflow.python.ops import array_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -66,8 +66,8 @@ def get_autocast_var(var, distribute): return autocast_variable.AutoCastVariable(var) -def get_var(val, dtype): - return variables.VariableV1(val, use_resource=True, dtype=dtype) +def get_var(val, dtype, name=None): + return variables.VariableV1(val, use_resource=True, dtype=dtype, name=name) @test_util.run_all_in_graph_and_eager_modes @@ -273,6 +273,36 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): x = get_var(1., dtypes.float32) get_autocast_var(x, distribute) + def test_repr(self): + # We do not test with DistributionStrategy because we do not want to rely on + # the exact __repr__ output of a DistributedVariable. + x = get_var(1., dtypes.float32, name='x') + x = get_autocast_var(x, distribute=False) + if context.executing_eagerly(): + self.assertStartsWith( + repr(x), + "" + ) + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float16): + self.assertEqual( + repr(x), + "" + ) + if __name__ == '__main__': test.main() From 39f01017eeec6d3e718a16b41ea81ea6d8fb67c1 Mon Sep 17 00:00:00 2001 From: Yanan Cao Date: Tue, 9 Jul 2019 22:30:38 -0700 Subject: [PATCH 170/332] Refactor handling of various pass-through, resource source/user ops into helper functions for readability. [NFC] PiperOrigin-RevId: 257337225 --- tensorflow/compiler/tf2xla/resource_util.cc | 317 +++++++++++--------- 1 file changed, 182 insertions(+), 135 deletions(-) diff --git a/tensorflow/compiler/tf2xla/resource_util.cc b/tensorflow/compiler/tf2xla/resource_util.cc index 5d72943279f..f323dd57070 100644 --- a/tensorflow/compiler/tf2xla/resource_util.cc +++ b/tensorflow/compiler/tf2xla/resource_util.cc @@ -42,63 +42,199 @@ const char kRetvalOp[] = "_Retval"; const int kMaxCallDepth = 100; +Status AnalyzeResourceUsage( + const Graph* graph, const absl::optional& function_name, + const int call_depth, const absl::flat_hash_set& resource_arg_indices, + FunctionLibraryRuntime* lib_runtime, + absl::flat_hash_map>* + source_to_path); + bool IsControlFlowV1Node(const Node* n) { return (n->IsEnter() || n->IsExit() || n->IsSwitch() || n->IsMerge() || n->IsNextIteration()); } -// Given an output edge, find the corresponding input edge if given edge is -// coming from a pass-through node. Otherwise, return nullptr. -StatusOr WalkBackPassThroughEdge(const Edge* e) { - const Node* n = e->src(); - - if (n->IsIdentity()) { - const Edge* ret; - TF_RETURN_IF_ERROR(n->input_edge(0, &ret)); - return ret; - } - - if (n->type_string() == kIdentityNOp) { - const Edge* ret; - TF_RETURN_IF_ERROR(n->input_edge(e->src_output(), &ret)); - return ret; - } - - // Reaching here means e is not coming from a pass through node, return empty - // vector to indicate we can no longer trace back. - return nullptr; -} - // TODO(ycao): Add this as Tensorflow Node method. -StatusOr> OutputEdgesByIndex(const Node* n, +StatusOr> OutputEdgesByIndex(const Node& n, int idx) { absl::InlinedVector res; - if (idx >= n->num_outputs()) { + if (idx >= n.num_outputs()) { return errors::InvalidArgument("Invalid out_edge index: ", idx, ", Node ", - n->name(), " only has ", n->num_outputs(), + n.name(), " only has ", n.num_outputs(), " outputs."); } - for (const Edge* o : n->out_edges()) { + for (const Edge* o : n.out_edges()) { if (o->src_output() == idx) res.emplace_back(o); } return res; } -bool IsStackOrTensorArraySource(const Node* n) { - const XlaResourceOpInfo* op_info = GetResourceOpInfoForOp(n->type_string()); +bool IsStackOrTensorArraySource(const Node& n) { + const XlaResourceOpInfo* op_info = GetResourceOpInfoForOp(n.type_string()); if (!op_info) return false; if (op_info->resource_kind() != XlaResourceKind::kStack && op_info->resource_kind() != XlaResourceKind::kTensorArray) return false; - return n->num_outputs() > 0 && n->output_type(0) == DataType::DT_RESOURCE; + return n.num_outputs() > 0 && n.output_type(0) == DataType::DT_RESOURCE; +} + +void PropagateFromStackOrTensorArraySourceOp( + const Node& n, const absl::optional& function_name, + absl::flat_hash_map* + user_to_source) { + ResourceUsageAnalysis::NodeInfo src_node_info(function_name, n.name(), + n.type_string()); + for (const Edge* o : n.out_edges()) { + if (o->IsControlEdge()) continue; + if (o->dst()->input_type(o->dst_input()) != DataType::DT_RESOURCE) { + continue; + } + (*user_to_source)[o] = src_node_info; + } +} + +Status PropagateFromArgOp( + const Node& n, const absl::optional& function_name, + const absl::flat_hash_set& resource_arg_indices, + absl::flat_hash_map* + user_to_source) { + TF_RET_CHECK(n.type_string() == kArgOp); + + int index; + TF_RETURN_IF_ERROR(GetNodeAttr(n.attrs(), "index", &index)); + if (!resource_arg_indices.contains(index)) return Status::OK(); + + TF_RET_CHECK(function_name.has_value()) + << "ResourceUsageAnalysis does not support analyzing _Arg nodes " + "carrying Stack/TensorArray resource in given graph unless they " + "are in function calls."; + + const ResourceUsageAnalysis::NodeInfo src_node_info(function_name, n.name(), + n.type_string()); + + for (const Edge* o : n.out_edges()) { + if (o->IsControlEdge()) continue; + if (o->dst()->input_type(o->dst_input()) != DataType::DT_RESOURCE) { + continue; + } + (*user_to_source)[o] = src_node_info; + } + + return Status::OK(); +} + +Status PropagateThroughCallOp( + const Node& n, const absl::optional& function_name, + const int call_depth, FunctionLibraryRuntime* lib_runtime, + absl::flat_hash_map* + user_to_source, + absl::flat_hash_map>* + source_to_path) { + if (call_depth > kMaxCallDepth) { + return errors::InvalidArgument( + "Function call stack in given graph is too deep, last function ", + "name is: ", function_name.value()); + } + // resource_arg_indices_for_call contains all indices of the input + // arguments that carry Stack/TensorArray resource handles. + absl::flat_hash_set resource_arg_indices_for_call; + for (const Edge* e : n.in_edges()) { + if (!user_to_source->contains(e)) continue; + resource_arg_indices_for_call.emplace(e->dst_input()); + } + + absl::string_view called_function_name = n.type_string(); + FunctionLibraryRuntime::Handle handle; + TF_RETURN_IF_ERROR(InstantiateFunctionCall(n.def(), lib_runtime, &handle)); + auto release_handle_on_return = gtl::MakeCleanup( + [&] { TF_CHECK_OK(lib_runtime->ReleaseHandle(handle)); }); + const FunctionBody* fbody = lib_runtime->GetFunctionBody(handle); + + // Recursively analyze called function for resource sources and users. + absl::flat_hash_map> + called_function_source_to_path; + TF_RETURN_IF_ERROR(AnalyzeResourceUsage( + fbody->graph, absl::optional(called_function_name), + call_depth + 1, resource_arg_indices_for_call, lib_runtime, + &called_function_source_to_path)); + + std::unordered_map node_name_index = + fbody->graph->BuildNodeNameIndex(); + + for (auto it : called_function_source_to_path) { + ResourceUsageAnalysis::NodeInfo src_node_info = it.first; + + // If source is an _Arg, then the true source is actually corresponding + // edge that feeds into function call node with the same index. + if (src_node_info.op_ == kArgOp) { + const Node* arg_src = node_name_index[src_node_info.node_name_]; + int index; + TF_RETURN_IF_ERROR(GetNodeAttr(arg_src->attrs(), "index", &index)); + + const Edge* e; + TF_RETURN_IF_ERROR(n.input_edge(index, &e)); + const Node* true_src = e->src(); + src_node_info.function_name_ = function_name; + src_node_info.node_name_ = true_src->name(); + src_node_info.op_ = true_src->type_string(); + } + + for (const auto& dst_node_info : it.second) { + // If user is an _Retval, then the true user is actually corresponding + // edge of that _Retval. + if (dst_node_info.op_ == kRetvalOp) { + const Node* ret_user = node_name_index[dst_node_info.node_name_]; + int index; + TF_RETURN_IF_ERROR(GetNodeAttr(ret_user->attrs(), "index", &index)); + + absl::InlinedVector outs; + TF_ASSIGN_OR_RETURN(outs, OutputEdgesByIndex(n, index)); + for (const Edge* o : outs) (*user_to_source)[o] = src_node_info; + } else { + (*source_to_path)[src_node_info].emplace(dst_node_info); + } + } + } + + return Status::OK(); +} + +// Analyzes pass through values for Identity and IdentityN ops. +Status PropagateThroughIdentityOp( + const Node& n, + absl::flat_hash_map* + user_to_source) { + TF_RET_CHECK(n.IsIdentity() || n.type_string() == kIdentityNOp); + if (n.IsIdentity()) { + for (const Edge* o : n.out_edges()) { + if (o->IsControlEdge()) continue; + const Edge* in; + TF_RETURN_IF_ERROR(n.input_edge(0, &in)); + if (!user_to_source->contains(in)) continue; + user_to_source->emplace(std::make_pair(o, (*user_to_source)[in])); + } + } else { + for (const Edge* o : n.out_edges()) { + if (o->IsControlEdge()) continue; + const Edge* in; + TF_RETURN_IF_ERROR(n.input_edge(o->src_output(), &in)); + if (!user_to_source->contains(in)) continue; + user_to_source->emplace(std::make_pair(o, (*user_to_source)[in])); + } + } + + return Status::OK(); } Status AnalyzeResourceUsage( - const Graph* graph, FunctionLibraryRuntime* lib_runtime, - const absl::optional& function_name, const int call_depth, - const absl::flat_hash_set& resource_arg_indices, + const Graph* graph, const absl::optional& function_name, + const int call_depth, const absl::flat_hash_set& resource_arg_indices, + FunctionLibraryRuntime* lib_runtime, absl::flat_hash_map>* source_to_path) { @@ -127,120 +263,30 @@ Status AnalyzeResourceUsage( } // Record a resource source edge. - if (IsStackOrTensorArraySource(n)) { - ResourceUsageAnalysis::NodeInfo src_node_info(function_name, n->name(), - n->type_string()); - for (const Edge* o : n->out_edges()) { - if (o->IsControlEdge()) continue; - if (o->dst()->input_type(o->dst_input()) != DataType::DT_RESOURCE) { - continue; - } - user_to_source[o] = src_node_info; - } + if (IsStackOrTensorArraySource(*n)) { + PropagateFromStackOrTensorArraySourceOp(*n, function_name, + &user_to_source); continue; } // Arguments that are listed in resource_arg_indices are also considered as // resource sources. if (n->IsArg()) { - int index; - TF_RETURN_IF_ERROR(GetNodeAttr(n->attrs(), "index", &index)); - if (!resource_arg_indices.contains(index)) continue; - - TF_RET_CHECK(function_name.has_value()) - << "ResourceUsageAnalysis does not support analyzing _Arg nodes " - "carrying Stack/TensorArray resource in given graph unless they " - "are in function calls."; - - const ResourceUsageAnalysis::NodeInfo src_node_info( - function_name, n->name(), n->type_string()); - - for (const Edge* o : n->out_edges()) { - if (o->IsControlEdge()) continue; - if (o->dst()->input_type(o->dst_input()) != DataType::DT_RESOURCE) { - continue; - } - user_to_source[o] = src_node_info; - } + TF_RETURN_IF_ERROR(PropagateFromArgOp( + *n, function_name, resource_arg_indices, &user_to_source)); continue; } + // Recursively analyze function call ops. if (IsFunctionCall(*lib_runtime->GetFunctionLibraryDefinition(), *n)) { - if (call_depth > kMaxCallDepth) { - return errors::InvalidArgument( - "Function call stack in given graph is too deep, last function ", - "name is: ", function_name.value()); - } - // resource_arg_indices_for_call contains all indices of the input - // arguments that carry Stack/TensorArray resource handles. - absl::flat_hash_set resource_arg_indices_for_call; - for (const Edge* e : n->in_edges()) { - if (!user_to_source.contains(e)) continue; - resource_arg_indices_for_call.emplace(e->dst_input()); - } - - absl::string_view called_function_name = n->type_string(); - FunctionLibraryRuntime::Handle handle; - TF_RETURN_IF_ERROR( - InstantiateFunctionCall(n->def(), lib_runtime, &handle)); - auto release_handle_on_return = gtl::MakeCleanup( - [&] { TF_CHECK_OK(lib_runtime->ReleaseHandle(handle)); }); - const FunctionBody* fbody = lib_runtime->GetFunctionBody(handle); - - // Recursively analyze called function for resource sources and users. - absl::flat_hash_map> - called_function_source_to_path; - TF_RETURN_IF_ERROR(AnalyzeResourceUsage( - fbody->graph, lib_runtime, - absl::optional(called_function_name), call_depth + 1, - resource_arg_indices_for_call, &called_function_source_to_path)); - - std::unordered_map node_name_index = - fbody->graph->BuildNodeNameIndex(); - - for (auto it : called_function_source_to_path) { - ResourceUsageAnalysis::NodeInfo src_node_info = it.first; - - // If source is an _Arg, then the true source is actually corresponding - // edge that feeds into function call node with the same index. - if (src_node_info.op_ == kArgOp) { - const Node* arg_src = node_name_index[src_node_info.node_name_]; - int index; - TF_RETURN_IF_ERROR(GetNodeAttr(arg_src->attrs(), "index", &index)); - - const Edge* e; - TF_RETURN_IF_ERROR(n->input_edge(index, &e)); - const Node* true_src = e->src(); - src_node_info.function_name_ = function_name; - src_node_info.node_name_ = true_src->name(); - src_node_info.op_ = true_src->type_string(); - } - - for (const auto& dst_node_info : it.second) { - // If user is an _Retval, then the true user is actually corresponding - // edge of that _Retval. - if (dst_node_info.op_ == kRetvalOp) { - const Node* ret_user = node_name_index[dst_node_info.node_name_]; - int index; - TF_RETURN_IF_ERROR(GetNodeAttr(ret_user->attrs(), "index", &index)); - - absl::InlinedVector outs; - TF_ASSIGN_OR_RETURN(outs, OutputEdgesByIndex(n, index)); - for (const Edge* o : outs) user_to_source[o] = src_node_info; - } else { - (*source_to_path)[src_node_info].emplace(dst_node_info); - } - } - } + TF_RETURN_IF_ERROR(PropagateThroughCallOp(*n, function_name, call_depth, + lib_runtime, &user_to_source, + source_to_path)); continue; } - for (const Edge* o : n->out_edges()) { - if (o->IsControlEdge()) continue; - TF_ASSIGN_OR_RETURN(const Edge* e, WalkBackPassThroughEdge(o)); - if (!e || !user_to_source.contains(e)) continue; - user_to_source.emplace(std::make_pair(o, user_to_source[e])); + if (n->IsIdentity() || n->type_string() == kIdentityNOp) { + TF_RETURN_IF_ERROR(PropagateThroughIdentityOp(*n, &user_to_source)); } } @@ -260,8 +306,9 @@ Status AnalyzeResourceUsage( absl::flat_hash_map>* source_to_path) { return AnalyzeResourceUsage( - graph, lib_runtime, /*function_name=*/{}, /*call_depth=*/0, - /*resource_arg_indices=*/absl::flat_hash_set(), source_to_path); + graph, /*function_name=*/{}, /*call_depth=*/0, + /*resource_arg_indices=*/absl::flat_hash_set(), lib_runtime, + source_to_path); } } // namespace tensorflow From 18b767b991cb02bf995b9ec4550cb29426e693f3 Mon Sep 17 00:00:00 2001 From: Gaurav Jain Date: Tue, 9 Jul 2019 22:31:05 -0700 Subject: [PATCH 171/332] Unify sync and async eager execution code The code path for async & sync differed significantly making it harder to have changes that benefit all code paths. In addition, much of this code need careful reference counting (especially in the case of errors), and it is better to unify the code to avoid any gaps in the logic. As part of this change we always create Async or Unshaped handles and then call SetTensor or SetRemoteShape on them. Note ExecuteRecv still needs to be unified, but requires some changes to the mirroring code first. As part of this change: - Make reference counting in RemoteExecuteNode and ExecuteNode more similar. - Avoid calling resize on the retvals vector in execute.cc since callers like EagerService should be looking at num_retvals instead of the size of the vector. PiperOrigin-RevId: 257337271 --- .../core/common_runtime/eager/context.h | 4 +- .../common_runtime/eager/eager_executor.cc | 6 +- .../common_runtime/eager/eager_executor.h | 2 +- .../core/common_runtime/eager/execute.cc | 182 ++++++++---------- .../core/common_runtime/eager/execute.h | 3 +- .../core/common_runtime/eager/execute_node.h | 71 +++---- .../common_runtime/eager/tensor_handle.cc | 6 + .../eager/eager_service_impl.cc | 4 +- .../eager/remote_execute_node.h | 16 +- .../eager/remote_tensor_handle_data.cc | 19 +- 10 files changed, 149 insertions(+), 164 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h index a4fe6ad2a22..e67b5805270 100644 --- a/tensorflow/core/common_runtime/eager/context.h +++ b/tensorflow/core/common_runtime/eager/context.h @@ -181,8 +181,8 @@ class EagerContext : public core::RefCounted { GraphCollector* GetGraphCollector() { return &graph_collector_; } - void ExecutorAdd(std::unique_ptr node) { - executor_.Add(std::move(node)); + Status ExecutorAdd(std::unique_ptr node) { + return executor_.Add(std::move(node)); } Status AddFunctionDef(const FunctionDef& fdef); diff --git a/tensorflow/core/common_runtime/eager/eager_executor.cc b/tensorflow/core/common_runtime/eager/eager_executor.cc index 6e1ae00130c..765be6a41ca 100644 --- a/tensorflow/core/common_runtime/eager/eager_executor.cc +++ b/tensorflow/core/common_runtime/eager/eager_executor.cc @@ -32,12 +32,12 @@ void EagerExecutor::EnableAsync() { } } -void EagerExecutor::Add(std::unique_ptr node) { +Status EagerExecutor::Add(std::unique_ptr node) { tensorflow::mutex_lock l(node_queue_mutex_); DCHECK(thread_) << "EnableAsync should have been called before Add"; if (!status_.ok()) { // node will be automatically deleted - return; + return status_; } node_queue_.push(std::move(node)); @@ -47,6 +47,8 @@ void EagerExecutor::Add(std::unique_ptr node) { if (node_queue_.size() == 1) { nodes_pending_.notify_all(); } + + return Status::OK(); } tensorflow::Status EagerExecutor::WaitForAllPendingNodes() { diff --git a/tensorflow/core/common_runtime/eager/eager_executor.h b/tensorflow/core/common_runtime/eager/eager_executor.h index aea09ec1bb1..fb6a8e2b03a 100644 --- a/tensorflow/core/common_runtime/eager/eager_executor.h +++ b/tensorflow/core/common_runtime/eager/eager_executor.h @@ -75,7 +75,7 @@ class EagerExecutor { // Schedules `node` for execution. // Note that Add must be called in monotonically increasing order of node->id. - void Add(std::unique_ptr node); + Status Add(std::unique_ptr node); // Blocks till all currently pending ops are done. Status WaitForAllPendingNodes(); diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index 430e11ecc15..45c766576b1 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -466,8 +466,7 @@ Status ShouldCompileWithXLA(const EagerOperation* op, const EagerContext* ctx, // runtime. In this case, we don't select a device because running // a function with explicitly requested device has different behavior than // running without an explicitly requested device. -Status EagerLocalExecute(EagerOperation* op, - gtl::InlinedVector* retvals, +Status EagerLocalExecute(EagerOperation* op, TensorHandle** retvals, int* num_retvals) { profiler::TraceMe activity( [&] { return absl::StrCat("EagerLocalExecute: ", op->Name()); }, @@ -663,32 +662,30 @@ Status EagerLocalExecute(EagerOperation* op, maybe_stats->set_scheduled_nanos(now_nanos); // TODO(apassos) track referenced tensors } - retvals->resize(*num_retvals); - if (ctx->Async()) { - // Note that for async mode, execution order will make sure that all - // input handles are ready before executing them. - // TODO(agarwal): Consider executing "cheap" kernels inline for - // performance. - for (int i = 0; i < *num_retvals; ++i) { - TF_RETURN_IF_ERROR(TensorHandle::CreateAsyncLocalHandle( - /* d= */ kernel->OutputDevice(i), - /* op_device= */ kernel->device(), - /* resource_device= */ kernel->OutputResourceDevice(i), - output_dtypes[i], ctx, &(*retvals)[i])); - } - std::unique_ptr node(new ExecuteNode( - ctx, op->Inputs(), std::move(kernel), maybe_stats.release(), - maybe_step_stats, graph_collector, output_dtypes, *retvals)); - ctx->ExecutorAdd(std::move(node)); - return Status::OK(); - } else { - // Execute checks if retvals[i] is nullptr or not to figure if it needs to - // allocate it. - return EagerKernelExecute(ctx, op->Inputs(), kernel.get(), - maybe_stats.get(), maybe_step_stats, - graph_collector, retvals->data(), *num_retvals); + for (int i = 0; i < *num_retvals; ++i) { + TF_RETURN_IF_ERROR(TensorHandle::CreateAsyncLocalHandle( + /* d= */ kernel->OutputDevice(i), + /* op_device= */ kernel->device(), + /* resource_device= */ kernel->OutputResourceDevice(i), + output_dtypes[i], ctx, &retvals[i])); } + + std::unique_ptr node(new ExecuteNode( + ctx, op->Inputs(), std::move(kernel), maybe_stats.release(), + maybe_step_stats, graph_collector, output_dtypes, retvals, *num_retvals)); + // Note that for async mode, execution order will make sure that all + // input handles are ready before executing them. + // TODO(b/137118203): Consider executing "cheap" kernels inline for + // performance. + Status s = ctx->Async() ? ctx->ExecutorAdd(std::move(node)) : node->Run(); + if (!s.ok()) { + for (int i = 0; i < *num_retvals; ++i) { + retvals[i]->Unref(); + } + } + + return s; } #if !defined(IS_MOBILE_PLATFORM) @@ -793,8 +790,6 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, TF_RETURN_IF_ERROR(ctx->GetClient(op->GetDeviceName(), &eager_client)); std::unique_ptr request(new eager::EnqueueRequest); - eager::EnqueueResponse response; - request->set_context_id(context_id); eager::Operation* remote_op = request->add_queue()->mutable_operation(); @@ -839,10 +834,12 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, DataTypeVector output_dtypes; TF_RETURN_IF_ERROR(GetOutputDTypes(op, &output_dtypes)); - if (*num_retvals != output_dtypes.size()) { + const int output_dtypes_size = static_cast(output_dtypes.size()); + if (output_dtypes_size != *num_retvals) { return errors::InvalidArgument( "num_retvals does not match expected output dtypes"); } + *num_retvals = output_dtypes_size; tensorflow::Device* op_device = op->Device(); @@ -851,42 +848,34 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, << " (is async?: " << is_async << ")."; const tensorflow::uint64 id = remote_op->id(); - if (is_async) { - for (int i = 0; i < *num_retvals; i++) { - // TODO(nareshmodi): Change the callback to instead add the decref to a - // list of pending decrefs that we can send as a batch with the next - // execute. + for (int i = 0; i < *num_retvals; ++i) { + // TODO(nareshmodi): Change the callback to instead add the decref to a + // list of pending decrefs that we can send as a batch with the next + // execute. - // The device_ and resource_device_ of this TensorHandle might be - // incorrect. It is pretty hard to make it correct because for - // multi-device functions, we don't know the output device until the - // function is instantiated. Luckily, we don't need to know the correct - // remote device here. We just need to know that it is remote. If we need - // to copy this tensor to this process, the remote end will know the - // correct device of this handle. - TF_RETURN_IF_ERROR(TensorHandle::CreateUnshapedRemoteHandle( - id, i, eager_client, context_id, output_dtypes[i], op_device, - output_dtypes[i] == DT_RESOURCE ? op_device : nullptr, ctx, - &retvals[i])); - } + // The device_ and resource_device_ of this TensorHandle might be + // incorrect. It is pretty hard to make it correct because for + // multi-device functions, we don't know the output device until the + // function is instantiated. Luckily, we don't need to know the correct + // remote device here. We just need to know that it is remote. If we need + // to copy this tensor to this process, the remote end will know the + // correct device of this handle. + TF_RETURN_IF_ERROR(TensorHandle::CreateUnshapedRemoteHandle( + id, i, eager_client, context_id, output_dtypes[i], op_device, + output_dtypes[i] == DT_RESOURCE ? op_device : nullptr, ctx, + &retvals[i])); + } - // TODO(gjn): If the retval TensorHandle is simply going to be used as a - // mirror then there should be no need to call SetRemoteShape - std::unique_ptr node(new eager::RemoteExecuteNode( - std::move(request), eager_client, op->Inputs(), retvals, *num_retvals)); - ctx->ExecutorAdd(std::move(node)); - } else { - TF_RETURN_IF_ERROR(EnqueueAndWait(eager_client, request, &response)); - - for (int i = 0; i < *num_retvals; i++) { - TF_RETURN_IF_ERROR(TensorHandle::CreateRemoteHandle( - id, i, response.queue_response(0).shape(i), eager_client, context_id, - output_dtypes[i], op_device, - output_dtypes[i] == DT_RESOURCE ? op_device : nullptr, ctx, - &retvals[i])); + std::unique_ptr node(new eager::RemoteExecuteNode( + std::move(request), eager_client, op->Inputs(), retvals, *num_retvals)); + Status s = is_async ? ctx->ExecutorAdd(std::move(node)) : node->Run(); + if (!s.ok()) { + for (int i = 0; i < *num_retvals; ++i) { + retvals[i]->Unref(); } } - return Status::OK(); + + return s; } #endif // IS_MOBILE_PLATFORM @@ -1016,10 +1005,9 @@ Status EagerExecute(EagerOperation* op, if (op_is_local) { if (out_op) { - return EagerLocalExecute(out_op.get(), retvals, num_retvals); - } else { - return EagerLocalExecute(op, retvals, num_retvals); + op = out_op.get(); } + return EagerLocalExecute(op, retvals->data(), num_retvals); } if (op->EagerContext()->LogDevicePlacement() || VLOG_IS_ON(1)) { @@ -1036,16 +1024,17 @@ Status EagerExecute(EagerOperation* op, "Eager's remote execution is not available on mobile devices."); #else // !IS_MOBILE_PLATFORM if (out_op) { - return EagerRemoteExecute(out_op.get(), retvals->data(), num_retvals); - } else { - return EagerRemoteExecute(op, retvals->data(), num_retvals); + op = out_op.get(); } + return EagerRemoteExecute(op, retvals->data(), num_retvals); #endif // !IS_MOBILE_PLATFORM } +// TODO(gjn): Consider moving into ExecuteNode class Status EagerKernelExecute(EagerContext* ctx, const gtl::InlinedVector& op_inputs, - KernelAndDevice* kernel, NodeExecStats* maybe_stats, + const core::RefCountPtr& kernel, + NodeExecStats* maybe_stats, StepStats* maybe_step_stats, GraphCollector* graph_collector, TensorHandle** retvals, int num_retvals) { @@ -1153,19 +1142,10 @@ Status EagerKernelExecute(EagerContext* ctx, } DCHECK_EQ(num_retvals, outputs.size()); for (int i = 0; i < num_retvals; ++i) { - if (retvals[i] == nullptr) { - TF_RETURN_IF_ERROR(TensorHandle::CreateLocalHandle( - outputs[i], /* d= */ kernel->OutputDevice(i), - /* op_device= */ kernel->device(), ctx, &retvals[i])); - } else { - // In the async case, the retval is not a nullptr, and its device is - // already set since all TensorHandles always have their device set - // (potentially to nullptr) during construction. - DCHECK_EQ(kernel->device(), retvals[i]->op_device()); - DCHECK_EQ(kernel->OutputDevice(i), retvals[i]->device()); + DCHECK_EQ(kernel->device(), retvals[i]->op_device()); + DCHECK_EQ(kernel->OutputDevice(i), retvals[i]->device()); - TF_RETURN_IF_ERROR(retvals[i]->SetTensor(outputs[i])); - } + TF_RETURN_IF_ERROR(retvals[i]->SetTensor(outputs[i])); } return Status::OK(); } @@ -1175,23 +1155,19 @@ namespace { Status LocalEagerCopyToDevice(TensorHandle* h, EagerContext* ctx, Device* dstd, TensorHandle** result) { TF_RETURN_IF_ERROR(ctx->GetStatus()); - if (ctx->Async()) { - TF_RETURN_IF_ERROR(TensorHandle::CreateAsyncLocalHandle( - dstd, dstd, nullptr, h->dtype, ctx, result)); - // Note that `h` may not be currently ready. However execution order will - // make sure that `h` is ready before the copy is actually done. - std::unique_ptr node( - new CopyToDeviceNode(h, *result, dstd, ctx)); - // Note that calling Add makes `node` accessible by the EagerExecutor - // thread. So further accesses need to be thread-safe. - ctx->ExecutorAdd(std::move(node)); - return Status::OK(); - } else { - tensorflow::Tensor tensor; - TF_RETURN_IF_ERROR(h->CopyToDevice(ctx, dstd, &tensor)); - return TensorHandle::CreateLocalHandle(tensor, dstd, ctx, result); - return Status::OK(); + Device* resource_device = (h->dtype == DT_RESOURCE) ? dstd : nullptr; + TF_RETURN_IF_ERROR(TensorHandle::CreateAsyncLocalHandle( + dstd, dstd, resource_device, h->dtype, ctx, result)); + + // Note that `h` may not be currently ready. However execution order will + // make sure that `h` is ready before the copy is actually done. + std::unique_ptr node(new CopyToDeviceNode(h, *result, dstd, ctx)); + Status s = ctx->Async() ? ctx->ExecutorAdd(std::move(node)) : node->Run(); + if (!s.ok()) { + (*result)->Unref(); } + + return s; } #if !defined(IS_MOBILE_PLATFORM) @@ -1259,8 +1235,6 @@ Status ExecuteSend(EagerContext* ctx, Device* device, TensorHandle* h, TF_RETURN_IF_ERROR(ctx->GetClient(device, &eager_client)); std::unique_ptr request(new eager::EnqueueRequest); - eager::EnqueueResponse response; - request->set_context_id(context_id); auto* remote_op = request->add_queue()->mutable_operation(); @@ -1269,13 +1243,13 @@ Status ExecuteSend(EagerContext* ctx, Device* device, TensorHandle* h, PrepareRemoteOp(remote_op, &op); - if (ctx->Async()) { std::unique_ptr node(new eager::RemoteExecuteNode( std::move(request), eager_client, op.Inputs(), nullptr, 0)); - ctx->ExecutorAdd(std::move(node)); - } else { - TF_RETURN_IF_ERROR(EnqueueAndWait(eager_client, request, &response)); - } + if (ctx->Async()) { + TF_RETURN_IF_ERROR(ctx->ExecutorAdd(std::move(node))); + } else { + TF_RETURN_IF_ERROR(node->Run()); + } } return Status::OK(); @@ -1343,7 +1317,7 @@ Status ExecuteRecv(EagerContext* ctx, Device* device, DataType dtype, std::unique_ptr node(new eager::RemoteExecuteNode( std::move(request), eager_client, op.Inputs(), result, 1)); - ctx->ExecutorAdd(std::move(node)); + TF_RETURN_IF_ERROR(ctx->ExecutorAdd(std::move(node))); } else { TF_RETURN_IF_ERROR(EnqueueAndWait(eager_client, request, &response)); diff --git a/tensorflow/core/common_runtime/eager/execute.h b/tensorflow/core/common_runtime/eager/execute.h index d6fba09ed79..2cabf01ca95 100644 --- a/tensorflow/core/common_runtime/eager/execute.h +++ b/tensorflow/core/common_runtime/eager/execute.h @@ -45,7 +45,8 @@ Status EagerExecute( // `kernel->device()`, with the inputs op_inputs, in the context 'ctx'. Status EagerKernelExecute(EagerContext* ctx, const gtl::InlinedVector& op_inputs, - KernelAndDevice* kernel, NodeExecStats* maybe_stats, + const core::RefCountPtr& kernel, + NodeExecStats* maybe_stats, StepStats* maybe_step_stats, GraphCollector* graph_collector, TensorHandle** retvals, int num_retvals); diff --git a/tensorflow/core/common_runtime/eager/execute_node.h b/tensorflow/core/common_runtime/eager/execute_node.h index 3bbd291488a..81dced54987 100644 --- a/tensorflow/core/common_runtime/eager/execute_node.h +++ b/tensorflow/core/common_runtime/eager/execute_node.h @@ -32,70 +32,75 @@ namespace tensorflow { class ExecuteNode : public EagerNode { public: ExecuteNode(EagerContext* ctx, - const tensorflow::gtl::InlinedVector& inputs, + const gtl::InlinedVector& inputs, core::RefCountPtr kernel, NodeExecStats* maybe_stats, StepStats* maybe_step_stats, GraphCollector* graph_collector, - const DataTypeVector& output_dtypes, - const tensorflow::gtl::InlinedVector& retvals) + const DataTypeVector& output_dtypes, TensorHandle** retvals, + int num_retvals) : EagerNode(), ctx_(ctx), inputs_(inputs), kernel_(std::move(kernel)), maybe_stats_(maybe_stats), maybe_step_stats_(maybe_step_stats), - graph_collector_(graph_collector), - retvals_(retvals) { - for (auto handle : inputs_) { - handle->Ref(); + graph_collector_(graph_collector) { + // Copy the output handles, since the container for them might get + // destroyed. + for (int i = 0; i < num_retvals; i++) { + retvals_.push_back(retvals[i]); + retvals_[i]->Ref(); } - for (auto handle : retvals_) { - handle->Ref(); - } - } - ~ExecuteNode() override { + // This is required to ensure that the tensor handles stay alive across the + // execution. for (auto handle : inputs_) { - handle->Unref(); - } - for (auto handle : retvals_) { - handle->Unref(); + handle->Ref(); } } Status Run() override { const Status status = EagerKernelExecute( - ctx_, inputs_, kernel_.get(), maybe_stats_.get(), maybe_step_stats_, + ctx_, inputs_, kernel_, maybe_stats_.get(), maybe_step_stats_, graph_collector_, retvals_.begin(), retvals_.size()); - if (status.ok()) { - // If status is ok, EagerKernelExecute would have called SetTensor on - // all the output handles. + if (!status.ok()) { + Abort(status); return status; - } else { - Status s = - Status(status.code(), - strings::StrCat("Got error, \"", status.error_message(), - "\" while executing kernel ", - kernel_->kernel()->def().DebugString())); - Abort(s); - return s; } + + // If status is ok, EagerKernelExecute would have called SetTensor on + // all the output handles. + + for (auto handle : retvals_) { + handle->Unref(); + } + + for (auto handle : inputs_) { + handle->Unref(); + } + + return status; } void Abort(Status status) override { for (auto handle : retvals_) { handle->Poison(status); + handle->Unref(); + } + + for (auto handle : inputs_) { + handle->Unref(); } } private: - tensorflow::EagerContext* ctx_; - tensorflow::gtl::InlinedVector inputs_; - core::RefCountPtr kernel_; + EagerContext* ctx_; + gtl::InlinedVector inputs_; + core::RefCountPtr kernel_; std::unique_ptr maybe_stats_; StepStats* maybe_step_stats_; - tensorflow::GraphCollector* graph_collector_; - tensorflow::gtl::InlinedVector retvals_; + GraphCollector* graph_collector_; + gtl::InlinedVector retvals_; }; } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc index a6a2feef60a..e9e63633cd7 100644 --- a/tensorflow/core/common_runtime/eager/tensor_handle.cc +++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc @@ -365,6 +365,12 @@ Status TensorHandle::SetTensor(const tensorflow::Tensor& tensor) { DCHECK(!is_ready_notification_.HasBeenNotified()) << "SetTensor is only called on non-ready handles."; + VLOG(3) << "SetTensor on TensorHandle: " << this; + + if (tensor.dtype() == DT_RESOURCE) { + auto& resource_handle = tensor.flat()(0); + handle_dtypes_and_shapes_ = resource_handle.dtypes_and_shapes(); + } tensor_handle_data_ = absl::make_unique(tensor); is_poisoned_ = Status::OK(); is_ready_notification_.Notify(); diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc index a63386f70ba..ae2fd939bdb 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl.cc @@ -229,8 +229,10 @@ Status EagerServiceImpl::ExecuteOp(const Operation& operation, TF_RETURN_IF_ERROR(GetNumRetvals(server_context->Context(), operation.name(), operation.attrs(), &num_retvals)); - tensorflow::gtl::InlinedVector retvals; + tensorflow::gtl::InlinedVector retvals( + num_retvals); TF_RETURN_IF_ERROR(EagerExecute(op.get(), &retvals, &num_retvals)); + retvals.resize(num_retvals); server_context->Context()->RemoteMgr()->AddOperationOutputs(retvals, operation.id()); diff --git a/tensorflow/core/distributed_runtime/eager/remote_execute_node.h b/tensorflow/core/distributed_runtime/eager/remote_execute_node.h index c32894acbbc..0e844eeaf42 100644 --- a/tensorflow/core/distributed_runtime/eager/remote_execute_node.h +++ b/tensorflow/core/distributed_runtime/eager/remote_execute_node.h @@ -26,13 +26,13 @@ namespace eager { // RemoteExecuteNode is an implementation of EagerNode which enqueues // an operation via RPC in a remote EagerService. -class RemoteExecuteNode : public tensorflow::EagerNode { +class RemoteExecuteNode : public EagerNode { public: RemoteExecuteNode(std::unique_ptr request, EagerClient* eager_client, const gtl::InlinedVector& inputs, TensorHandle** retvals, int num_retvals) - : tensorflow::EagerNode(), + : EagerNode(), request_(std::move(request)), eager_client_(eager_client), inputs_(inputs) { @@ -45,7 +45,7 @@ class RemoteExecuteNode : public tensorflow::EagerNode { // This is required to ensure that the tensor handles stay alive across the // execution. - for (auto* handle : inputs_) { + for (auto handle : inputs_) { handle->Ref(); } } @@ -75,7 +75,7 @@ class RemoteExecuteNode : public tensorflow::EagerNode { retvals_[i]->Unref(); } - for (auto* handle : inputs_) { + for (auto handle : inputs_) { handle->Unref(); } @@ -83,12 +83,12 @@ class RemoteExecuteNode : public tensorflow::EagerNode { } void Abort(Status status) override { - for (int i = 0; i < retvals_.size(); i++) { - retvals_[i]->Poison(status); - retvals_[i]->Unref(); + for (auto handle : retvals_) { + handle->Poison(status); + handle->Unref(); } - for (auto* handle : inputs_) { + for (auto handle : inputs_) { handle->Unref(); } } diff --git a/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc b/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc index 2e245680a44..0b40da16232 100644 --- a/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc +++ b/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc @@ -43,18 +43,13 @@ void DestoryRemoteTensorHandle(EagerContext* ctx, handle_to_decref->set_op_id(op_id); handle_to_decref->set_output_num(output_num); - if (ctx->Async()) { - ctx->ExecutorAdd(absl::make_unique( - std::move(request), eager_client)); - } else { - eager::EnqueueRequest* actual_request = request.release(); - eager::EnqueueResponse* response = new eager::EnqueueResponse; - eager_client->EnqueueAsync( - actual_request, response, - [actual_request, response](const tensorflow::Status& s) { - delete actual_request; - delete response; - }); + std::unique_ptr node( + absl::make_unique(std::move(request), + eager_client)); + Status s = ctx->Async() ? ctx->ExecutorAdd(std::move(node)) : node->Run(); + if (!s.ok()) { + LOG(ERROR) << "Unable to destroy remote tensor handles: " + << s.error_message(); } } From a587e2406a76449ced38a283d9fe32e7973ca4c7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 02:02:35 -0700 Subject: [PATCH 172/332] compat: Update forward compatibility horizon to 2019-07-10 PiperOrigin-RevId: 257361049 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index db35b0790ae..b38483bbffb 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -27,7 +27,7 @@ import datetime from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 7, 9) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 7, 10) @tf_export("compat.forward_compatible") From 41336d143bf23dac8050a39f511cc48268dfd09d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 02:02:36 -0700 Subject: [PATCH 173/332] Update GraphDef version to 92. PiperOrigin-RevId: 257361064 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index f6bb1cb66fd..f55a1fcfea4 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 91 // Updated: 2019/7/9 +#define TF_GRAPH_DEF_VERSION 92 // Updated: 2019/7/10 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From 0fd8d48295436d14a885e6637674d2c58a3d4e15 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 03:03:53 -0700 Subject: [PATCH 174/332] Fixed the gradients for tf.gather with batch_dims. PiperOrigin-RevId: 257368772 --- .../python/kernel_tests/gather_op_test.py | 29 ++-- tensorflow/python/ops/array_grad.py | 125 +++++++++++++----- tensorflow/python/ops/array_ops.py | 29 +--- 3 files changed, 116 insertions(+), 67 deletions(-) diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py index f23b7d33664..6bd8f505f36 100644 --- a/tensorflow/python/kernel_tests/gather_op_test.py +++ b/tensorflow/python/kernel_tests/gather_op_test.py @@ -21,7 +21,7 @@ from __future__ import print_function from absl.testing import parameterized import numpy as np -from tensorflow.python.compat import compat +from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -339,15 +339,27 @@ class GatherTest(test.TestCase, parameterized.TestCase): ]) @test_util.run_in_graph_and_eager_modes def testBatchDims(self, params, indices, batch_dims, expected=None, - axis=None): + axis=None, expected_gradient_shape=None): result = array_ops.gather(params, indices, axis=axis, batch_dims=batch_dims) self.assertAllEqual(expected, result) - with compat.forward_compatibility_horizon(2019, 6, 11): + # Test the gradients shape. + if context.executing_eagerly(): + with backprop.GradientTape() as tape: + zeros = array_ops.zeros_like(params, dtype=dtypes.float32) + tape.watch(zeros) + values = zeros * 2 + zeros + result = array_ops.gather( + values, indices, axis=axis, batch_dims=batch_dims) + gradients = tape.gradient(result, zeros) + else: + zeros = array_ops.zeros_like(params, dtype=dtypes.float32) + values = zeros * 2 + zeros result = array_ops.gather( - params, indices, axis=axis, batch_dims=batch_dims) + values, indices, axis=axis, batch_dims=batch_dims) + gradients = gradients_impl.gradients(result, [zeros])[0] - self.assertAllEqual(expected, result) + self.assertAllEqual(array_ops.shape(params), array_ops.shape(gradients)) @parameterized.parameters([ dict( @@ -443,13 +455,6 @@ class GatherTest(test.TestCase, parameterized.TestCase): self.assertAllEqual(output_shape, result.shape.as_list()) self.assertAllEqual(expected, result) - with compat.forward_compatibility_horizon(2019, 6, 11): - result = array_ops.gather( - params, indices, axis=axis, batch_dims=batch_dims) - - self.assertAllEqual(output_shape, result.shape.as_list()) - self.assertAllEqual(expected, result) - def _batchNumpyGather(self, params, indices, axis, batch_dims): """Performs a batch gather by making recursive calls to np.take(). diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py index c51783387ab..efab2b13e4a 100644 --- a/tensorflow/python/ops/array_grad.py +++ b/tensorflow/python/ops/array_grad.py @@ -31,6 +31,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import control_flow_util from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import sparse_ops @@ -477,6 +478,61 @@ def _GatherGrad(op, grad): return [ops.IndexedSlices(values, indices, params_shape), None] +def _GetBatchIndices(params_shape, indices, batch_dims): + """Addds the batch offsets to the given indices and returns the results.""" + batch_indices = indices + indices_ndims = indices.shape.ndims + indices_dtype = indices.dtype.base_dtype + casted_params_shape = math_ops.cast(params_shape, indices_dtype) + accum_dim_value = array_ops.ones((), dtype=indices_dtype) + for dim in range(batch_dims, 0, -1): + dim_value = casted_params_shape[dim - 1] + accum_dim_value *= casted_params_shape[dim] + start = array_ops.zeros((), dtype=indices_dtype) + step = array_ops.ones((), dtype=indices_dtype) + dim_indices = math_ops.range(start, dim_value, step) + dim_indices *= accum_dim_value + dim_shape = array_ops.stack( + [1] * (dim - 1) + [dim_value] + [1] * (indices_ndims - dim), axis=0) + batch_indices += array_ops.reshape(dim_indices, dim_shape) + + return batch_indices + + +def _BatchGatherGrad( + params_shape, values, indices, batch_dims, gather_dim_size): + """Returns the gradient of GatherV2 with batch dimensions.""" + + # Axis is the first non-batch dimension. + indices_size = array_ops.expand_dims(array_ops.size(indices), 0) + if batch_dims: + values_shape = array_ops.shape(values) + # Add the batch offsets to indices and flatten the batch dimensions. + outer_shape = values_shape[:batch_dims] + inner_shape = values_shape[batch_dims:][1:] + batch_size = gen_math_ops.prod(outer_shape, [0], False) + flat_values_shape = array_ops.concat([[-1], inner_shape], 0) + gather_dim_size *= batch_size + + indices = _GetBatchIndices(params_shape, indices, batch_dims) + + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Converting sparse IndexedSlices to a dense Tensor.*") + values = array_ops.reshape(values, flat_values_shape) + + indices = array_ops.reshape(indices, indices_size) + params_grad = math_ops.unsorted_segment_sum(values, indices, gather_dim_size) + + if batch_dims: + # Put back the batch dimensions. + params_grad = array_ops.reshape( + params_grad, array_ops.concat([outer_shape, flat_values_shape], 0)) + + return params_grad + + @ops.RegisterGradient("GatherV2") def _GatherV2Grad(op, grad): """Gradient for GatherV2 op.""" @@ -495,6 +551,10 @@ def _GatherV2Grad(op, grad): indices_size = array_ops.expand_dims(array_ops.size(indices), 0) axis = op.inputs[2] axis_static = tensor_util.constant_value(axis) + batch_dims = int(op.get_attr("batch_dims")) + + if batch_dims < 0: + batch_dims += indices.shape.ndims # For axis 0 gathers, build an appropriately shaped IndexedSlices. if axis_static == 0: @@ -509,44 +569,45 @@ def _GatherV2Grad(op, grad): message="Converting sparse IndexedSlices to a dense Tensor.*") values = array_ops.reshape(grad, values_shape) indices = array_ops.reshape(indices, indices_size) - return [ops.IndexedSlices(values, indices, params_shape), None, None] + params_grad = ops.IndexedSlices(values, indices, params_shape) + else: + # Handle axis by transposing the axis dimension to be the first non-batch + # dimension, compute the gradiend and transpose the result back. + outer_shape = params_shape[:axis] + inner_shape = params_shape[axis:][1:] + values_shape = array_ops.concat([outer_shape, [-1], inner_shape], 0) - outer_shape = params_shape[:axis] - outer_dims = array_ops.size(outer_shape) - inner_shape = params_shape[axis:][1:] - inner_dims = array_ops.size(inner_shape) + values_dims = array_ops.size(values_shape) + axis_dims = array_ops.size(outer_shape) - outer_axes_indices = math_ops.range(outer_dims) - inner_axes_indices = math_ops.range(outer_dims + 1, - outer_dims + 1 + inner_dims) + outer_batches_indices = math_ops.range(batch_dims) + batch_axis_indices = math_ops.range(batch_dims, axis_dims) + inner_axes_indices = math_ops.range(axis_dims + 1, values_dims) - values_shape = array_ops.concat([outer_shape, indices_size, inner_shape], 0) - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - message="Converting sparse IndexedSlices to a dense Tensor.*") - values = array_ops.reshape(grad, values_shape) - indices = array_ops.reshape(indices, indices_size) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Converting sparse IndexedSlices to a dense Tensor.*") + values = array_ops.reshape(grad, values_shape) - # We need to sum up every slice `values[..., i, ....]` corresponding to - # `params[..., indices[i], ...]`. Since `unsorted_segment_sum` does not - # support an axis parameter, we transpose the gather dimension to the front, - # then use `unsorted_segment_sum` to build a - # [gather_axis, outer_axes, inner_axes] tensor with all the gradients - # affecting each index in `gather_axis` summed up. - transpose_dims = array_ops.concat( - [[outer_dims], outer_axes_indices, inner_axes_indices], 0) - values_transpose = array_ops.transpose(values, transpose_dims) - num_segments = params_shape[axis] + # Move values[axis] up to values[batch_dims] + transpose_dims = array_ops.concat( + [outer_batches_indices, [axis_dims], batch_axis_indices, + inner_axes_indices], + 0) + values_transpose = array_ops.transpose(values, transpose_dims) - params_grad = math_ops.unsorted_segment_sum(values_transpose, indices, - num_segments) + params_grad = _BatchGatherGrad(params_shape, values_transpose, indices, + batch_dims, params_shape[axis]) + + # Inverts the above transpose by moving dimension batch_dims back to its + # original position. + invert_transpose_dims = array_ops.concat( + [outer_batches_indices, batch_axis_indices + 1, [batch_dims], + inner_axes_indices], + 0) + params_grad = array_ops.transpose(params_grad, invert_transpose_dims) - # Inverts the above transpose by moving dimension 0 back to its original - # position. - invert_transpose_dims = array_ops.concat( - [outer_axes_indices + 1, [0], inner_axes_indices], 0) - params_grad = array_ops.transpose(params_grad, invert_transpose_dims) return [params_grad, None, None] diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 041ae2d8526..e7114f21204 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -3807,36 +3807,19 @@ def gather(params, A `Tensor`. Has the same type as `params`. """ del validate_indices - if compat.forward_compatible(2019, 8, 10): - if axis is None: - axis = batch_dims - if axis != 0: - return gen_array_ops.gather_v2( - params, indices, axis, batch_dims=batch_dims, name=name) - try: - # TODO(apassos) find a less bad way of detecting resource variables - # without introducing a circular dependency. - return params.sparse_read(indices, name=name) - except AttributeError: - return gen_array_ops.gather_v2( - params, indices, axis, name=name) - if batch_dims != 0: - with ops.name_scope(name, "Gather", [params, indices, axis]): - return _batch_gather(params, indices, batch_dims, axis) if axis is None: axis = batch_dims if axis != 0: - # Note that we do a sparse_read here to avoid snapshotting the entire - # resource variable and doing a gather, which can be inefficient and lead to - # subtle race conditions. TODO(apassos) implement axis != 0 on sparse_read - return gen_array_ops.gather_v2(params, indices, axis, name=name) + return gen_array_ops.gather_v2( + params, indices, axis, batch_dims=batch_dims, name=name) try: - # TODO(apassos) find a less bad way of detecting resource variables without - # introducing a circular dependency. + # TODO(apassos) find a less bad way of detecting resource variables + # without introducing a circular dependency. return params.sparse_read(indices, name=name) except AttributeError: - return gen_array_ops.gather_v2(params, indices, axis, name=name) + return gen_array_ops.gather_v2( + params, indices, axis, name=name) @tf_export("gather", v1=[]) From 15a56964c839ab9723550111a8d4a90477345e51 Mon Sep 17 00:00:00 2001 From: Rishabh Kabra Date: Wed, 10 Jul 2019 03:06:54 -0700 Subject: [PATCH 175/332] Correct example usage for `tf.strings.bytes_split`. Currently the examples call `tf.strings.to_bytes` which is deprecated. PiperOrigin-RevId: 257369177 --- tensorflow/python/ops/ragged/ragged_string_ops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/ops/ragged/ragged_string_ops.py b/tensorflow/python/ops/ragged/ragged_string_ops.py index e37b345ff4d..4b225da2edd 100644 --- a/tensorflow/python/ops/ragged/ragged_string_ops.py +++ b/tensorflow/python/ops/ragged/ragged_string_ops.py @@ -36,9 +36,9 @@ def string_bytes_split(input, name=None): # pylint: disable=redefined-builtin Examples: ```python - >>> tf.strings.to_bytes('hello') + >>> tf.strings.bytes_split('hello') ['h', 'e', 'l', 'l', 'o'] - >>> tf.strings.to_bytes(['hello', '123']) + >>> tf.strings.bytes_split(['hello', '123']) ``` @@ -53,7 +53,7 @@ def string_bytes_split(input, name=None): # pylint: disable=redefined-builtin name: A name for the operation (optional). Returns: - A `RaggedTensor` of rank `N+1`: the bytes that make up the soruce strings. + A `RaggedTensor` of rank `N+1`: the bytes that make up the source strings. """ with ops.name_scope(name, "StringsByteSplit", [input]): input = ragged_tensor.convert_to_tensor_or_ragged_tensor(input, From 4257acce5cee86b78a9d94f65a5c6378ae0266dc Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:32:00 +0530 Subject: [PATCH 176/332] Deprecated tf.Session removed in fully_connected_reader.py --- .../examples/how_tos/reading_data/fully_connected_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py b/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py index 5c52a2c8461..d701444b1ab 100644 --- a/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py +++ b/tensorflow/examples/how_tos/reading_data/fully_connected_reader.py @@ -153,7 +153,7 @@ def run_training(): tf.local_variables_initializer()) # Create a session for running operations in the Graph. - with tf.Session() as sess: + with tf.compat.v1.Session() as sess: # Initialize the variables (the trained variables and the # epoch counter). sess.run(init_op) From 82262e0244967ee0dae1b50305ee65aee036b1bb Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:32:13 +0530 Subject: [PATCH 177/332] Deprecated tf.Session removed in label_image.py --- tensorflow/examples/label_image/label_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/examples/label_image/label_image.py b/tensorflow/examples/label_image/label_image.py index fe5e0fc684a..f675ec35ec8 100644 --- a/tensorflow/examples/label_image/label_image.py +++ b/tensorflow/examples/label_image/label_image.py @@ -58,7 +58,7 @@ def read_tensor_from_image_file(file_name, dims_expander = tf.expand_dims(float_caster, 0) resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width]) normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std]) - sess = tf.Session() + sess = tf.compat.v1.Session() result = sess.run(normalized) return result @@ -128,7 +128,7 @@ if __name__ == "__main__": input_operation = graph.get_operation_by_name(input_name) output_operation = graph.get_operation_by_name(output_name) - with tf.Session(graph=graph) as sess: + with tf.compat.v1.Session(graph=graph) as sess: results = sess.run(output_operation.outputs[0], { input_operation.outputs[0]: t }) From 5ef5f683cafbf3d961cf900ea848621dee393625 Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:32:30 +0530 Subject: [PATCH 178/332] Deprecated tf.Session removed in input_data.py --- tensorflow/examples/speech_commands/input_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/examples/speech_commands/input_data.py b/tensorflow/examples/speech_commands/input_data.py index 983c0a3bafa..6c2ce3f13eb 100644 --- a/tensorflow/examples/speech_commands/input_data.py +++ b/tensorflow/examples/speech_commands/input_data.py @@ -122,7 +122,7 @@ def load_wav_file(filename): Returns: Numpy array holding the sample data as floats between -1.0 and 1.0. """ - with tf.Session(graph=tf.Graph()) as sess: + with tf.compat.v1.Session(graph=tf.Graph()) as sess: wav_filename_placeholder = tf.placeholder(tf.string, []) wav_loader = io_ops.read_file(wav_filename_placeholder) wav_decoder = contrib_audio.decode_wav(wav_loader, desired_channels=1) @@ -139,7 +139,7 @@ def save_wav_file(filename, wav_data, sample_rate): wav_data: 2D array of float PCM-encoded audio data. sample_rate: Samples per second to encode in the file. """ - with tf.Session(graph=tf.Graph()) as sess: + with tf.compat.v1.Session(graph=tf.Graph()) as sess: wav_filename_placeholder = tf.placeholder(tf.string, []) sample_rate_placeholder = tf.placeholder(tf.int32, []) wav_data_placeholder = tf.placeholder(tf.float32, [None, 1]) @@ -349,7 +349,7 @@ class AudioProcessor(object): background_dir = os.path.join(self.data_dir, BACKGROUND_NOISE_DIR_NAME) if not os.path.exists(background_dir): return self.background_data - with tf.Session(graph=tf.Graph()) as sess: + with tf.compat.v1.Session(graph=tf.Graph()) as sess: wav_filename_placeholder = tf.placeholder(tf.string, []) wav_loader = io_ops.read_file(wav_filename_placeholder) wav_decoder = contrib_audio.decode_wav(wav_loader, desired_channels=1) @@ -654,7 +654,7 @@ class AudioProcessor(object): words_list = self.words_list data = np.zeros((sample_count, desired_samples)) labels = [] - with tf.Session(graph=tf.Graph()) as sess: + with tf.compat.v1.Session(graph=tf.Graph()) as sess: wav_filename_placeholder = tf.placeholder(tf.string, []) wav_loader = io_ops.read_file(wav_filename_placeholder) wav_decoder = contrib_audio.decode_wav( From 7c2661fcfa449a75e11274c1a8b938696fcd9ec5 Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:32:40 +0530 Subject: [PATCH 179/332] Deprecated tf.Session removed in label_wav.py --- tensorflow/examples/speech_commands/label_wav.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/speech_commands/label_wav.py b/tensorflow/examples/speech_commands/label_wav.py index eb8323454c2..5af16691e82 100644 --- a/tensorflow/examples/speech_commands/label_wav.py +++ b/tensorflow/examples/speech_commands/label_wav.py @@ -59,7 +59,7 @@ def load_labels(filename): def run_graph(wav_data, labels, input_layer_name, output_layer_name, num_top_predictions): """Runs the audio data through the graph and prints predictions.""" - with tf.Session() as sess: + with tf.compat.v1.Session() as sess: # Feed the audio data as input to the graph. # predictions will contain a two-dimensional array, where one # dimension represents the input image count, and the other has From 1d13491083e75c24d4e894ca388f3696fee2658a Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:32:56 +0530 Subject: [PATCH 180/332] Deprecated tf.Session removed in label_wav_dir.py --- tensorflow/examples/speech_commands/label_wav_dir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/speech_commands/label_wav_dir.py b/tensorflow/examples/speech_commands/label_wav_dir.py index 2e1890c3e86..eb6fb757c10 100644 --- a/tensorflow/examples/speech_commands/label_wav_dir.py +++ b/tensorflow/examples/speech_commands/label_wav_dir.py @@ -60,7 +60,7 @@ def load_labels(filename): def run_graph(wav_dir, labels, input_layer_name, output_layer_name, num_top_predictions): """Runs the audio data through the graph and prints predictions.""" - with tf.Session() as sess: + with tf.compat.v1.Session() as sess: # Feed the audio data as input to the graph. # predictions will contain a two-dimensional array, where one # dimension represents the input image count, and the other has From ceaed427bd04bf4b0c1ad8cb4d9b9b897b17c7ab Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:33:09 +0530 Subject: [PATCH 181/332] Deprecated tf.Session removed in label_wav_test.py --- tensorflow/examples/speech_commands/label_wav_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/speech_commands/label_wav_test.py b/tensorflow/examples/speech_commands/label_wav_test.py index 77a88f98e16..3c833d66735 100644 --- a/tensorflow/examples/speech_commands/label_wav_test.py +++ b/tensorflow/examples/speech_commands/label_wav_test.py @@ -48,7 +48,7 @@ class LabelWavTest(test.TestCase): input_name = "test_input" output_name = "test_output" graph_filename = os.path.join(tmp_dir, "test_graph.pb") - with tf.Session() as sess: + with tf.compat.v1.Session() as sess: tf.placeholder(tf.string, name=input_name) tf.zeros([1, 3], name=output_name) with open(graph_filename, "wb") as f: From 1484ff209b74607dd3c02846d48848972577c88a Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:36:13 +0530 Subject: [PATCH 182/332] Deprecated tf.Session removed in mnist/fully_connected_feed.py --- tensorflow/examples/tutorials/mnist/fully_connected_feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/tutorials/mnist/fully_connected_feed.py b/tensorflow/examples/tutorials/mnist/fully_connected_feed.py index 35ca1b2f7f3..e61cbab6ef4 100644 --- a/tensorflow/examples/tutorials/mnist/fully_connected_feed.py +++ b/tensorflow/examples/tutorials/mnist/fully_connected_feed.py @@ -149,7 +149,7 @@ def run_training(): saver = tf.train.Saver() # Create a session for running Ops on the Graph. - sess = tf.Session() + sess = tf.compat.v1.Session() # Instantiate a SummaryWriter to output summaries and the Graph. summary_writer = tf.summary.FileWriter(FLAGS.log_dir, sess.graph) From 3cfa197e1c6f80949de9fb19aef224210368799c Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:36:26 +0530 Subject: [PATCH 183/332] Deprecated tf.Session removed in mnist/mnist_softmax_xla.py --- tensorflow/examples/tutorials/mnist/mnist_softmax_xla.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/tutorials/mnist/mnist_softmax_xla.py b/tensorflow/examples/tutorials/mnist/mnist_softmax_xla.py index 2945660dad5..28fdc2059f8 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_softmax_xla.py +++ b/tensorflow/examples/tutorials/mnist/mnist_softmax_xla.py @@ -64,7 +64,7 @@ def main(_): config.graph_options.optimizer_options.global_jit_level = jit_level run_metadata = tf.RunMetadata() - sess = tf.Session(config=config) + sess = tf.compat.v1.Session(config=config) tf.global_variables_initializer().run(session=sess) # Train train_loops = 1000 From 31d9fff142e664d7464718b8fab2ce2a168223f4 Mon Sep 17 00:00:00 2001 From: Siju Samuel Date: Wed, 10 Jul 2019 16:36:37 +0530 Subject: [PATCH 184/332] Deprecated tf.Session removed in word2vec/word2vec_basic.py --- tensorflow/examples/tutorials/word2vec/word2vec_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py index ebfaacb8a2c..380cd2be515 100644 --- a/tensorflow/examples/tutorials/word2vec/word2vec_basic.py +++ b/tensorflow/examples/tutorials/word2vec/word2vec_basic.py @@ -226,7 +226,7 @@ def word2vec_basic(log_dir): # Step 5: Begin training. num_steps = 100001 - with tf.Session(graph=graph) as session: + with tf.compat.v1.Session(graph=graph) as session: # Open a writer to write summaries. writer = tf.summary.FileWriter(log_dir, session.graph) From 114b8e76197285077819df046f9ed08311bf05c6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 05:16:51 -0700 Subject: [PATCH 185/332] [XLA] Refactor memory scheduler to return the peak memory. PiperOrigin-RevId: 257383333 --- .../xla/service/hlo_memory_scheduler.cc | 98 ++++++++++++------- .../xla/service/hlo_memory_scheduler.h | 36 ++++--- .../xla/service/hlo_memory_scheduler_test.cc | 26 ++++- 3 files changed, 110 insertions(+), 50 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc index d049d3f8d77..3a4b9d4802d 100644 --- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc +++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/heap_simulator.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_schedule.h" #include "tensorflow/compiler/xla/service/tuple_points_to_analysis.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -403,14 +404,17 @@ StatusOr ScheduleComputationHelper( const BufferValue::SizeFunction& size_function, const MemorySchedulerAlgorithm& algorithm, const absl::flat_hash_map& - memory_by_computation) { + memory_by_computation, + int64* peak_memory) { VLOG(2) << "Computation: " << computation->name(); + if (algorithm) { return algorithm(computation, points_to_analysis, alias_analysis, - size_function, memory_by_computation); + size_function, memory_by_computation, peak_memory); } return DefaultMemoryScheduler(computation, points_to_analysis, alias_analysis, - size_function, memory_by_computation); + size_function, memory_by_computation, + peak_memory); } } // namespace @@ -421,7 +425,8 @@ StatusOr DFSMemoryScheduler( const HloAliasAnalysis& alias_analysis, const BufferValue::SizeFunction& size_function, const absl::flat_hash_map& - memory_by_computation) { + memory_by_computation, + int64* peak_memory) { // These variables are a hack to prevent overflows. int64 cumulative_total_size = 0; int64 total_hlos = computation->parent()->instruction_count(); @@ -485,6 +490,12 @@ StatusOr DFSMemoryScheduler( return a->name() < b->name(); })); CHECK_EQ(sequence.size(), computation->instruction_count()); + if (peak_memory) { + TF_ASSIGN_OR_RETURN( + *peak_memory, HeapSimulator::MinimumMemoryForComputation( + *computation, sequence, alias_analysis, size_function, + &memory_by_computation)); + } return sequence; } // namespace xla @@ -494,9 +505,18 @@ StatusOr ListMemoryScheduler( const HloAliasAnalysis& alias_analysis, const BufferValue::SizeFunction& size_function, const absl::flat_hash_map& - memory_by_computation) { - return ListScheduler::Run(computation, points_to_analysis, size_function, - memory_by_computation); + memory_by_computation, + int64* peak_memory) { + TF_ASSIGN_OR_RETURN(HloInstructionSequence sequence, + ListScheduler::Run(computation, points_to_analysis, + size_function, memory_by_computation)); + if (peak_memory) { + TF_ASSIGN_OR_RETURN( + *peak_memory, HeapSimulator::MinimumMemoryForComputation( + *computation, sequence, alias_analysis, size_function, + &memory_by_computation)); + } + return sequence; } StatusOr PostOrderMemoryScheduler( @@ -505,8 +525,16 @@ StatusOr PostOrderMemoryScheduler( const HloAliasAnalysis& alias_analysis, const BufferValue::SizeFunction& size_function, const absl::flat_hash_map& - memory_by_computation) { - return HloInstructionSequence(computation->MakeInstructionPostOrder()); + memory_by_computation, + int64* peak_memory) { + HloInstructionSequence sequence(computation->MakeInstructionPostOrder()); + if (peak_memory) { + TF_ASSIGN_OR_RETURN( + *peak_memory, HeapSimulator::MinimumMemoryForComputation( + *computation, sequence, alias_analysis, size_function, + &memory_by_computation)); + } + return sequence; } StatusOr DefaultMemoryScheduler( @@ -515,7 +543,8 @@ StatusOr DefaultMemoryScheduler( const HloAliasAnalysis& alias_analysis, const BufferValue::SizeFunction& size_function, const absl::flat_hash_map& - memory_by_computation) { + memory_by_computation, + int64* peak_memory) { // We try a few schedulers and choose whichever returns a lower min-memory, // not accounting for fragmentation. // - List is a scheduler that uses greedy heuristics. @@ -524,38 +553,33 @@ StatusOr DefaultMemoryScheduler( // - Postorder does not use any heuristics. // List wins for most of our benchmarks; postorder-based schedulers win for // some RNNs. + int64 list_memory; TF_ASSIGN_OR_RETURN( HloInstructionSequence list_sequence, ListMemoryScheduler(computation, points_to_analysis, alias_analysis, - size_function, memory_by_computation)); - TF_ASSIGN_OR_RETURN(const int64 list_memory, - HeapSimulator::MinimumMemoryForComputation( - *computation, list_sequence, alias_analysis, - size_function, &memory_by_computation)); + size_function, memory_by_computation, &list_memory)); VLOG(2) << "Min-memory list sequence: " << HumanReadableNumBytes(list_memory); + int64 dfs_memory; TF_ASSIGN_OR_RETURN( HloInstructionSequence dfs_sequence, DFSMemoryScheduler(computation, points_to_analysis, alias_analysis, - size_function, memory_by_computation)); - TF_ASSIGN_OR_RETURN(const int64 dfs_memory, - HeapSimulator::MinimumMemoryForComputation( - *computation, dfs_sequence, alias_analysis, - size_function, &memory_by_computation)); + size_function, memory_by_computation, &dfs_memory)); VLOG(2) << "Min-memory dfs sequence: " << HumanReadableNumBytes(dfs_memory); + int64 post_order_memory; TF_ASSIGN_OR_RETURN( HloInstructionSequence post_order_sequence, PostOrderMemoryScheduler(computation, points_to_analysis, alias_analysis, - size_function, memory_by_computation)); - TF_ASSIGN_OR_RETURN(const int64 post_order_memory, - HeapSimulator::MinimumMemoryForComputation( - *computation, post_order_sequence, alias_analysis, - size_function, &memory_by_computation)); + size_function, memory_by_computation, + &post_order_memory)); VLOG(2) << "Min-memory post order sequence: " << HumanReadableNumBytes(post_order_memory); auto min_memory = std::min({dfs_memory, post_order_memory, list_memory}); + if (peak_memory) { + *peak_memory = min_memory; + } if (min_memory == list_memory) { VLOG(2) << "Chose min-memory list sequence: " @@ -574,7 +598,7 @@ StatusOr DefaultMemoryScheduler( StatusOr ScheduleModule( HloModule* module, const BufferValue::SizeFunction& size_function, - const MemorySchedulerAlgorithm& algorithm) { + const MemorySchedulerAlgorithm& algorithm, int64* peak_memory) { HloSchedule schedule(module); TF_ASSIGN_OR_RETURN(std::unique_ptr points_to_analysis, TuplePointsToAnalysis::Run(module)); @@ -584,15 +608,17 @@ StatusOr ScheduleModule( absl::flat_hash_map memory_by_computation; for (auto* computation : module->MakeComputationPostOrder()) { if (!computation->IsFusionComputation()) { - TF_ASSIGN_OR_RETURN(HloInstructionSequence computation_sequence, - ScheduleComputationHelper( - computation, *points_to_analysis, *alias_analysis, - size_function, algorithm, memory_by_computation)); - memory_by_computation[computation] = - HeapSimulator::MinimumMemoryForComputation( - *computation, computation_sequence, *alias_analysis, - size_function, &schedule) - .ValueOrDie(); + // peak_memory may be nullptr. + int64 peak_memory_internal; + TF_ASSIGN_OR_RETURN( + HloInstructionSequence computation_sequence, + ScheduleComputationHelper( + computation, *points_to_analysis, *alias_analysis, size_function, + algorithm, memory_by_computation, &peak_memory_internal)); + memory_by_computation[computation] = peak_memory_internal; + if (peak_memory) { + *peak_memory = peak_memory_internal; + } schedule.set_sequence(computation, std::move(computation_sequence)); } } @@ -614,7 +640,7 @@ StatusOr ScheduleComputation( absl::flat_hash_map empty_map; return ScheduleComputationHelper(computation, *points_to_analysis, *alias_analysis, size_function, nullptr, - empty_map); + empty_map, nullptr); } HloMemoryScheduler::HloMemoryScheduler( diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h index 7bc76dc5f7c..23f9721d539 100644 --- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h +++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h @@ -35,13 +35,16 @@ namespace xla { // A memory scheduler computes an execution sequence for the HLO instructions in // 'computation' that minimizes peak memory, given a points-to analysis result // that describes buffer aliasing, together with a target-specific size function -// that maps a tensor's logical size to its padded size. +// that maps a tensor's logical size to its padded size. peak_memory (may be +// nullptr) is set to the peak memory of the resulting schedule according to the +// HeapSimulator. // // TODO(yunxing): Cleanup usage of TuplePointsToAnalysis. typedef std::function( HloComputation*, const TuplePointsToAnalysis&, const HloAliasAnalysis&, const LogicalBuffer::SizeFunction&, - const absl::flat_hash_map&)> + const absl::flat_hash_map&, + /*peak_memory*/ int64*)> MemorySchedulerAlgorithm; // List scheduler @@ -51,7 +54,8 @@ StatusOr ListMemoryScheduler( const HloAliasAnalysis& alias_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& - memory_by_computation); + memory_by_computation, + int64* peak_memory); // DFS-order scheduler StatusOr DFSMemoryScheduler( @@ -60,7 +64,8 @@ StatusOr DFSMemoryScheduler( const HloAliasAnalysis& alias_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& - memory_by_computation); + memory_by_computation, + int64* peak_memory); // Naive Post Order scheduler StatusOr PostOrderMemoryScheduler( @@ -69,25 +74,30 @@ StatusOr PostOrderMemoryScheduler( const HloAliasAnalysis& alias_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& - memory_by_computation); + memory_by_computation, + int64* peak_memory); -// The default scheduling algorithm. Runs both the list scheduler -// and the DFS scheduler, and chooses whichever returns a lower min-memory, -// not accounting for fragmentation. +// The default scheduling algorithm. Runs the list scheduler, the DFS scheduler, +// and the post-order scheduler and chooses whichever returns a lower min- +// memory, not accounting for fragmentation. peak_memory (may be nullptr) is set +// to the peak memory of the resulting schedule according to the HeapSimulator. StatusOr DefaultMemoryScheduler( HloComputation* computation, const TuplePointsToAnalysis& points_to_analysis, const HloAliasAnalysis& alias_analysis, const LogicalBuffer::SizeFunction& size_function, const absl::flat_hash_map& - memory_by_computation); + memory_by_computation, + int64* peak_memory); -// Returns an HloSchedule which seeks to minimize the memory required for -// the computation. size_function is the function returning the number of bytes -// required for a LogicalBuffer. +// Returns an HloSchedule which seeks to minimize the memory required for the +// computation. size_function is the function returning the number of bytes +// required for a LogicalBuffer. peak_memory (if not nullptr) is set to the peak +// memory of the resulting schedule according to the HeapSimulator. StatusOr ScheduleModule( HloModule* module, const LogicalBuffer::SizeFunction& size_function, - const MemorySchedulerAlgorithm& algorithm = {}); + const MemorySchedulerAlgorithm& algorithm = {}, + int64* peak_memory = nullptr); // Computes the schedule for a single computation. // Currently only used by the GPU backend. diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc index 3ee0c114373..2b1e059c7e5 100644 --- a/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc +++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler_test.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_dce.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/hlo_ordering.h" #include "tensorflow/compiler/xla/service/hlo_parser.h" @@ -38,6 +39,25 @@ namespace { class HloSchedulingTest : public HloTestBase {}; +int64 PeakMemoryUseOfEntryComputation( + HloModule* module, LogicalBuffer::SizeFunction size_function) { + CHECK(module->has_entry_computation()); + CHECK(module->has_schedule()); + + std::unique_ptr alias_analysis = + HloAliasAnalysis::Run(module).ConsumeValueOrDie(); + + const HloSchedule& schedule = module->schedule(); + + HloComputation* computation = module->entry_computation(); + const HloInstructionSequence& sequence = schedule.sequence(computation); + return HeapSimulator::Run(absl::make_unique(), + *computation, sequence, *alias_analysis, + size_function) + .ValueOrDie() + .heap_size; +} + TEST_F(HloSchedulingTest, LastUseScheduledFirst) { // Tests scheduling of the following HLO code: // @@ -122,9 +142,11 @@ ENTRY root { auto size_fn = [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape(), /*pointer_size=*/8); }; + int64 peak_memory; TF_ASSERT_OK_AND_ASSIGN( HloSchedule schedule, - ScheduleModule(module.get(), size_fn, ListMemoryScheduler)); + ScheduleModule(module.get(), size_fn, ListMemoryScheduler, &peak_memory)); + TF_ASSERT_OK(module->set_schedule(schedule)); // Verify that all instructions are in the sequence. const std::vector& sequence = schedule.sequence(module->entry_computation()).instructions(); @@ -145,6 +167,8 @@ ENTRY root { SequentialHloOrdering ordering(schedule); EXPECT_TRUE(ordering.ExecutesBefore(instructions_by_name.at("d"), instructions_by_name.at("e"))); + EXPECT_EQ(PeakMemoryUseOfEntryComputation(module.get(), size_fn), + peak_memory); } TEST_F(HloSchedulingTest, HostSendDoneSchedule) { From a216c03fafa208392ff587cbdec2689448dfe0f8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 05:30:21 -0700 Subject: [PATCH 186/332] Automated rollback of commit 0fd8d48295436d14a885e6637674d2c58a3d4e15 PiperOrigin-RevId: 257384893 --- .../python/kernel_tests/gather_op_test.py | 29 ++-- tensorflow/python/ops/array_grad.py | 125 +++++------------- tensorflow/python/ops/array_ops.py | 29 +++- 3 files changed, 67 insertions(+), 116 deletions(-) diff --git a/tensorflow/python/kernel_tests/gather_op_test.py b/tensorflow/python/kernel_tests/gather_op_test.py index 6bd8f505f36..f23b7d33664 100644 --- a/tensorflow/python/kernel_tests/gather_op_test.py +++ b/tensorflow/python/kernel_tests/gather_op_test.py @@ -21,7 +21,7 @@ from __future__ import print_function from absl.testing import parameterized import numpy as np -from tensorflow.python.eager import backprop +from tensorflow.python.compat import compat from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -339,27 +339,15 @@ class GatherTest(test.TestCase, parameterized.TestCase): ]) @test_util.run_in_graph_and_eager_modes def testBatchDims(self, params, indices, batch_dims, expected=None, - axis=None, expected_gradient_shape=None): + axis=None): result = array_ops.gather(params, indices, axis=axis, batch_dims=batch_dims) self.assertAllEqual(expected, result) - # Test the gradients shape. - if context.executing_eagerly(): - with backprop.GradientTape() as tape: - zeros = array_ops.zeros_like(params, dtype=dtypes.float32) - tape.watch(zeros) - values = zeros * 2 + zeros - result = array_ops.gather( - values, indices, axis=axis, batch_dims=batch_dims) - gradients = tape.gradient(result, zeros) - else: - zeros = array_ops.zeros_like(params, dtype=dtypes.float32) - values = zeros * 2 + zeros + with compat.forward_compatibility_horizon(2019, 6, 11): result = array_ops.gather( - values, indices, axis=axis, batch_dims=batch_dims) - gradients = gradients_impl.gradients(result, [zeros])[0] + params, indices, axis=axis, batch_dims=batch_dims) - self.assertAllEqual(array_ops.shape(params), array_ops.shape(gradients)) + self.assertAllEqual(expected, result) @parameterized.parameters([ dict( @@ -455,6 +443,13 @@ class GatherTest(test.TestCase, parameterized.TestCase): self.assertAllEqual(output_shape, result.shape.as_list()) self.assertAllEqual(expected, result) + with compat.forward_compatibility_horizon(2019, 6, 11): + result = array_ops.gather( + params, indices, axis=axis, batch_dims=batch_dims) + + self.assertAllEqual(output_shape, result.shape.as_list()) + self.assertAllEqual(expected, result) + def _batchNumpyGather(self, params, indices, axis, batch_dims): """Performs a batch gather by making recursive calls to np.take(). diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py index efab2b13e4a..c51783387ab 100644 --- a/tensorflow/python/ops/array_grad.py +++ b/tensorflow/python/ops/array_grad.py @@ -31,7 +31,6 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import control_flow_util from tensorflow.python.ops import gen_array_ops -from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import sparse_ops @@ -478,61 +477,6 @@ def _GatherGrad(op, grad): return [ops.IndexedSlices(values, indices, params_shape), None] -def _GetBatchIndices(params_shape, indices, batch_dims): - """Addds the batch offsets to the given indices and returns the results.""" - batch_indices = indices - indices_ndims = indices.shape.ndims - indices_dtype = indices.dtype.base_dtype - casted_params_shape = math_ops.cast(params_shape, indices_dtype) - accum_dim_value = array_ops.ones((), dtype=indices_dtype) - for dim in range(batch_dims, 0, -1): - dim_value = casted_params_shape[dim - 1] - accum_dim_value *= casted_params_shape[dim] - start = array_ops.zeros((), dtype=indices_dtype) - step = array_ops.ones((), dtype=indices_dtype) - dim_indices = math_ops.range(start, dim_value, step) - dim_indices *= accum_dim_value - dim_shape = array_ops.stack( - [1] * (dim - 1) + [dim_value] + [1] * (indices_ndims - dim), axis=0) - batch_indices += array_ops.reshape(dim_indices, dim_shape) - - return batch_indices - - -def _BatchGatherGrad( - params_shape, values, indices, batch_dims, gather_dim_size): - """Returns the gradient of GatherV2 with batch dimensions.""" - - # Axis is the first non-batch dimension. - indices_size = array_ops.expand_dims(array_ops.size(indices), 0) - if batch_dims: - values_shape = array_ops.shape(values) - # Add the batch offsets to indices and flatten the batch dimensions. - outer_shape = values_shape[:batch_dims] - inner_shape = values_shape[batch_dims:][1:] - batch_size = gen_math_ops.prod(outer_shape, [0], False) - flat_values_shape = array_ops.concat([[-1], inner_shape], 0) - gather_dim_size *= batch_size - - indices = _GetBatchIndices(params_shape, indices, batch_dims) - - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - message="Converting sparse IndexedSlices to a dense Tensor.*") - values = array_ops.reshape(values, flat_values_shape) - - indices = array_ops.reshape(indices, indices_size) - params_grad = math_ops.unsorted_segment_sum(values, indices, gather_dim_size) - - if batch_dims: - # Put back the batch dimensions. - params_grad = array_ops.reshape( - params_grad, array_ops.concat([outer_shape, flat_values_shape], 0)) - - return params_grad - - @ops.RegisterGradient("GatherV2") def _GatherV2Grad(op, grad): """Gradient for GatherV2 op.""" @@ -551,10 +495,6 @@ def _GatherV2Grad(op, grad): indices_size = array_ops.expand_dims(array_ops.size(indices), 0) axis = op.inputs[2] axis_static = tensor_util.constant_value(axis) - batch_dims = int(op.get_attr("batch_dims")) - - if batch_dims < 0: - batch_dims += indices.shape.ndims # For axis 0 gathers, build an appropriately shaped IndexedSlices. if axis_static == 0: @@ -569,45 +509,44 @@ def _GatherV2Grad(op, grad): message="Converting sparse IndexedSlices to a dense Tensor.*") values = array_ops.reshape(grad, values_shape) indices = array_ops.reshape(indices, indices_size) - params_grad = ops.IndexedSlices(values, indices, params_shape) - else: - # Handle axis by transposing the axis dimension to be the first non-batch - # dimension, compute the gradiend and transpose the result back. - outer_shape = params_shape[:axis] - inner_shape = params_shape[axis:][1:] - values_shape = array_ops.concat([outer_shape, [-1], inner_shape], 0) + return [ops.IndexedSlices(values, indices, params_shape), None, None] - values_dims = array_ops.size(values_shape) - axis_dims = array_ops.size(outer_shape) + outer_shape = params_shape[:axis] + outer_dims = array_ops.size(outer_shape) + inner_shape = params_shape[axis:][1:] + inner_dims = array_ops.size(inner_shape) - outer_batches_indices = math_ops.range(batch_dims) - batch_axis_indices = math_ops.range(batch_dims, axis_dims) - inner_axes_indices = math_ops.range(axis_dims + 1, values_dims) + outer_axes_indices = math_ops.range(outer_dims) + inner_axes_indices = math_ops.range(outer_dims + 1, + outer_dims + 1 + inner_dims) - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - message="Converting sparse IndexedSlices to a dense Tensor.*") - values = array_ops.reshape(grad, values_shape) + values_shape = array_ops.concat([outer_shape, indices_size, inner_shape], 0) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message="Converting sparse IndexedSlices to a dense Tensor.*") + values = array_ops.reshape(grad, values_shape) + indices = array_ops.reshape(indices, indices_size) - # Move values[axis] up to values[batch_dims] - transpose_dims = array_ops.concat( - [outer_batches_indices, [axis_dims], batch_axis_indices, - inner_axes_indices], - 0) - values_transpose = array_ops.transpose(values, transpose_dims) + # We need to sum up every slice `values[..., i, ....]` corresponding to + # `params[..., indices[i], ...]`. Since `unsorted_segment_sum` does not + # support an axis parameter, we transpose the gather dimension to the front, + # then use `unsorted_segment_sum` to build a + # [gather_axis, outer_axes, inner_axes] tensor with all the gradients + # affecting each index in `gather_axis` summed up. + transpose_dims = array_ops.concat( + [[outer_dims], outer_axes_indices, inner_axes_indices], 0) + values_transpose = array_ops.transpose(values, transpose_dims) + num_segments = params_shape[axis] - params_grad = _BatchGatherGrad(params_shape, values_transpose, indices, - batch_dims, params_shape[axis]) - - # Inverts the above transpose by moving dimension batch_dims back to its - # original position. - invert_transpose_dims = array_ops.concat( - [outer_batches_indices, batch_axis_indices + 1, [batch_dims], - inner_axes_indices], - 0) - params_grad = array_ops.transpose(params_grad, invert_transpose_dims) + params_grad = math_ops.unsorted_segment_sum(values_transpose, indices, + num_segments) + # Inverts the above transpose by moving dimension 0 back to its original + # position. + invert_transpose_dims = array_ops.concat( + [outer_axes_indices + 1, [0], inner_axes_indices], 0) + params_grad = array_ops.transpose(params_grad, invert_transpose_dims) return [params_grad, None, None] diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index e7114f21204..041ae2d8526 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -3807,19 +3807,36 @@ def gather(params, A `Tensor`. Has the same type as `params`. """ del validate_indices + if compat.forward_compatible(2019, 8, 10): + if axis is None: + axis = batch_dims + if axis != 0: + return gen_array_ops.gather_v2( + params, indices, axis, batch_dims=batch_dims, name=name) + try: + # TODO(apassos) find a less bad way of detecting resource variables + # without introducing a circular dependency. + return params.sparse_read(indices, name=name) + except AttributeError: + return gen_array_ops.gather_v2( + params, indices, axis, name=name) + if batch_dims != 0: + with ops.name_scope(name, "Gather", [params, indices, axis]): + return _batch_gather(params, indices, batch_dims, axis) if axis is None: axis = batch_dims if axis != 0: - return gen_array_ops.gather_v2( - params, indices, axis, batch_dims=batch_dims, name=name) + # Note that we do a sparse_read here to avoid snapshotting the entire + # resource variable and doing a gather, which can be inefficient and lead to + # subtle race conditions. TODO(apassos) implement axis != 0 on sparse_read + return gen_array_ops.gather_v2(params, indices, axis, name=name) try: - # TODO(apassos) find a less bad way of detecting resource variables - # without introducing a circular dependency. + # TODO(apassos) find a less bad way of detecting resource variables without + # introducing a circular dependency. return params.sparse_read(indices, name=name) except AttributeError: - return gen_array_ops.gather_v2( - params, indices, axis, name=name) + return gen_array_ops.gather_v2(params, indices, axis, name=name) @tf_export("gather", v1=[]) From 3402d7118460857cf484f57338d14d9113597d15 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Wed, 10 Jul 2019 12:36:56 +0000 Subject: [PATCH 187/332] Fix numpy warning with numpy 1.17.0+ This fix tries to address the issue raised in 30427 where `import tensorflow` caused the following warning with numpy 1.17.0+: ``` /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'. _np_qint8 = np.dtype([("qint8", np.int8, 1)]) ``` The issue was cause by changes in numpy: https://github.com/numpy/numpy/commit/ad1e0600e45b9fa71096d0a0f10c1474e003f373 This fix fixes the warning. This fix fixes 30427. Signed-off-by: Yong Tang --- tensorflow/python/framework/dtypes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py index aad008ff05b..16403b266ca 100644 --- a/tensorflow/python/framework/dtypes.py +++ b/tensorflow/python/framework/dtypes.py @@ -513,16 +513,16 @@ _STRING_TO_TF["double_ref"] = float64_ref # quantized types. # TODO(mrry,keveman): Investigate Numpy type registration to replace this # hard-coding of names. -_np_qint8 = np.dtype([("qint8", np.int8, 1)]) -_np_quint8 = np.dtype([("quint8", np.uint8, 1)]) -_np_qint16 = np.dtype([("qint16", np.int16, 1)]) -_np_quint16 = np.dtype([("quint16", np.uint16, 1)]) -_np_qint32 = np.dtype([("qint32", np.int32, 1)]) +_np_qint8 = np.dtype([("qint8", np.int8)]) +_np_quint8 = np.dtype([("quint8", np.uint8)]) +_np_qint16 = np.dtype([("qint16", np.int16)]) +_np_quint16 = np.dtype([("quint16", np.uint16)]) +_np_qint32 = np.dtype([("qint32", np.int32)]) # _np_bfloat16 is defined by a module import. # Custom struct dtype for directly-fed ResourceHandles of supported type(s). -np_resource = np.dtype([("resource", np.ubyte, 1)]) +np_resource = np.dtype([("resource", np.ubyte)]) # Standard mappings between types_pb2.DataType values and numpy.dtypes. _NP_TO_TF = { From dd3ec9260259a5dba183f9271f517c06d7c69c82 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Wed, 10 Jul 2019 06:47:55 -0700 Subject: [PATCH 188/332] Optimize away the calculation of a range tensor for the pattern `for i in tf.range`. Along with the performance improvement, this is more compatible with XLA because it avoids generating dynamically-shaped tensors. Fixes #30182. PiperOrigin-RevId: 257394214 --- .../autograph/operators/control_flow.py | 66 +++++++++++++++++-- .../autograph/operators/control_flow_test.py | 34 ++++++++++ tensorflow/python/autograph/utils/misc.py | 12 ++++ .../python/autograph/utils/misc_test.py | 17 +++++ tensorflow/python/autograph/utils/tensors.py | 5 ++ .../python/autograph/utils/tensors_test.py | 8 +++ 6 files changed, 138 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py index faf98b43956..9e179f55c17 100644 --- a/tensorflow/python/autograph/operators/control_flow.py +++ b/tensorflow/python/autograph/operators/control_flow.py @@ -62,16 +62,19 @@ from __future__ import print_function from tensorflow.python.autograph.operators import py_builtins from tensorflow.python.autograph.operators import special_values from tensorflow.python.autograph.utils import ag_logging +from tensorflow.python.autograph.utils import misc from tensorflow.python.autograph.utils import tensors from tensorflow.python.data.experimental.ops import scan_ops from tensorflow.python.data.experimental.ops import take_while_ops from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.ops import iterator_ops from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import func_graph from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import tensor_array_ops LIMIT_PYTHON_ITERATIONS = True @@ -137,8 +140,12 @@ def for_stmt(iter_, extra_test, body, get_state, set_state, init_vars): Tuple containing the final state. """ if tensor_util.is_tensor(iter_): - return _known_len_tf_for_stmt(iter_, extra_test, body, get_state, set_state, - init_vars) + if tensors.is_range_tensor(iter_): + return _tf_range_for_stmt(iter_, extra_test, body, get_state, set_state, + init_vars) + else: + return _known_len_tf_for_stmt(iter_, extra_test, body, get_state, + set_state, init_vars) if isinstance(iter_, dataset_ops.DatasetV2): return _tf_dataset_for_stmt(iter_, extra_test, body, get_state, set_state, @@ -207,8 +214,59 @@ def _known_len_tf_for_stmt(iter_, extra_test, body, get_state, set_state, init_vars=(0,) + init_vars, opts=dict(maximum_iterations=n)) - # Dropping the iteration index because it's not syntactically visible. - # TODO(mdan): Don't. + # Note: the iteration index is not returned by the while loop, however + # if a symbol with the same name exists outside the loop, it will be captured + # by the loop variables and ultimately updated correctly. + if isinstance(results, (tuple, list)): + assert len(results) >= 1 # Has at least the iterate. + if len(results) > 1: + results = results[1:] + else: + results = () + + return results + + +def _tf_range_for_stmt(iter_, extra_test, body, get_state, set_state, + init_vars): + """Overload of for_stmt that iterates over a TF range (and elides it).""" + _disallow_undefs_into_loop(*init_vars) + + start, limit, delta = iter_.op.inputs + + def while_body(iterate, *loop_vars): + new_vars = body(iterate, *loop_vars) + + loop_vars = (iterate + delta,) + if new_vars: + loop_vars += new_vars + + return loop_vars + + def while_cond(iterate, *loop_vars): + main_test = math_ops.logical_or( + math_ops.logical_and(delta >= 0, iterate < limit), + math_ops.logical_and(delta < 0, iterate > limit)) + if extra_test is not None: + return control_flow_ops.cond( + main_test, lambda: extra_test(*loop_vars), lambda: False) + return main_test + + # This specific dtype is required by while_loop. + maximum_iterations = math_ops.cast( + misc.get_range_len(start, limit, delta), dtypes.int32) + + results = _tf_while_stmt( + while_cond, + while_body, + get_state, + set_state, + init_vars=(start,) + init_vars, + opts=dict(maximum_iterations=maximum_iterations)) + + # Note: the iteration index is not returned by the while loop, however + # if a symbol with the same name exists outside the loop, it will be captured + # by the loop variables and ultimately updated correctly. if isinstance(results, (tuple, list)): assert len(results) >= 1 # Has at least the iterate. if len(results) > 1: diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py index 0e62bbddfda..cf25075dfcd 100644 --- a/tensorflow/python/autograph/operators/control_flow_test.py +++ b/tensorflow/python/autograph/operators/control_flow_test.py @@ -33,6 +33,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -50,6 +51,39 @@ class ForLoopTest(test.TestCase): init_vars=(0,)) self.assertEqual(self.evaluate(s), (1234,)) + def test_range_tensor(self): + with ops.Graph().as_default(): + s = control_flow.for_stmt( + math_ops.range(5), + extra_test=lambda s: True, + body=lambda i, s: (s * 10 + i,), + get_state=lambda: (), + set_state=lambda _: None, + init_vars=(0,)) + self.assertEqual(self.evaluate(s), (1234,)) + + def test_range_tensor_explicit_limit_delta(self): + with ops.Graph().as_default(): + s = control_flow.for_stmt( + math_ops.range(-17, -3, 5), + extra_test=lambda s: True, + body=lambda i, s: (s * 100 + i,), + get_state=lambda: (), + set_state=lambda _: None, + init_vars=(0,)) + self.assertEqual(self.evaluate(s), (-171207,)) + + def test_range_tensor_negative_delta(self): + with ops.Graph().as_default(): + s = control_flow.for_stmt( + math_ops.range(17, 3, -5), + extra_test=lambda s: True, + body=lambda i, s: (s * 100 + i,), + get_state=lambda: (), + set_state=lambda _: None, + init_vars=(0,)) + self.assertEqual(self.evaluate(s), (171207,)) + def test_tensor_with_extra_test_only_python_state(self): class MutableObject(object): field_1 = constant_op.constant(0, dtype=dtypes.int32) diff --git a/tensorflow/python/autograph/utils/misc.py b/tensorflow/python/autograph/utils/misc.py index 046e6cf97dc..01c198e6278 100644 --- a/tensorflow/python/autograph/utils/misc.py +++ b/tensorflow/python/autograph/utils/misc.py @@ -20,6 +20,8 @@ from __future__ import print_function from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_math_ops +from tensorflow.python.ops import math_ops def alias_tensors(*args): @@ -55,3 +57,13 @@ def capitalize_initial(s): if s: return s[0].upper() + s[1:] return s + + +def get_range_len(start, limit, delta): + dist = ops.convert_to_tensor(limit - start) + unadjusted_len = dist // delta + adjustment = math_ops.cast( + gen_math_ops.not_equal(dist % delta, + array_ops.zeros_like(unadjusted_len)), dist.dtype) + final_len = unadjusted_len + adjustment + return gen_math_ops.maximum(final_len, array_ops.zeros_like(final_len)) diff --git a/tensorflow/python/autograph/utils/misc_test.py b/tensorflow/python/autograph/utils/misc_test.py index 24b5753a91a..67c1b827228 100644 --- a/tensorflow/python/autograph/utils/misc_test.py +++ b/tensorflow/python/autograph/utils/misc_test.py @@ -19,6 +19,8 @@ from __future__ import division from __future__ import print_function from tensorflow.python.autograph.utils import misc +from tensorflow.python.eager import def_function +from tensorflow.python.framework import constant_op from tensorflow.python.framework import test_util from tensorflow.python.framework.constant_op import constant from tensorflow.python.ops.variables import Variable @@ -61,6 +63,21 @@ class MiscTest(test.TestCase): with self.cached_session() as sess: self.assertEqual(1, self.evaluate(new_a)) + def test_get_range_len(self): + get_range_as_graph = def_function.function(misc.get_range_len) + test_range = [(i, constant_op.constant(i)) for i in range(-3, 3)] + results = [] + for i, ti in test_range: + for j, tj in test_range: + for k, tk in test_range: + if k == 0: + continue + results.append(((i, j, k), get_range_as_graph(ti, tj, tk))) + + for (i, j, k), result_tensor in results: + self.assertEqual( + len(list(range(i, j, k))), self.evaluate(result_tensor)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/autograph/utils/tensors.py b/tensorflow/python/autograph/utils/tensors.py index 352d67d2bd9..6ae2b947332 100644 --- a/tensorflow/python/autograph/utils/tensors.py +++ b/tensorflow/python/autograph/utils/tensors.py @@ -46,3 +46,8 @@ def is_tensor_list(t): # construct. return (tensor_util.is_tensor(t) and t.dtype == dtypes.variant and not t.shape.ndims) + + +def is_range_tensor(t): + """Returns True if a tensor is the result of a tf.range op. Best effort.""" + return tensor_util.is_tensor(t) and hasattr(t, 'op') and t.op.type == 'Range' diff --git a/tensorflow/python/autograph/utils/tensors_test.py b/tensorflow/python/autograph/utils/tensors_test.py index 1e7cfec9e1b..e561d7bbeff 100644 --- a/tensorflow/python/autograph/utils/tensors_test.py +++ b/tensorflow/python/autograph/utils/tensors_test.py @@ -22,6 +22,7 @@ from tensorflow.python.autograph.utils import tensors from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.ops import list_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.platform import test @@ -52,6 +53,13 @@ class TensorsTest(test.TestCase): self.assertFalse(tensors.is_tensor_list(self._simple_list_of_tensors())) self.assertFalse(tensors.is_tensor_list(None)) + def is_range_tensor(self): + self.assertTrue(tensors.is_range_tensor(math_ops.range(1))) + self.assertTrue(tensors.is_range_tensor(math_ops.range(1, 2))) + self.assertTrue(tensors.is_range_tensor(math_ops.range(1, 2, 3))) + self.assertFalse(tensors.is_range_tensor(None)) + self.assertFalse(tensors.is_range_tensor(constant_op.constant(range(1)))) + if __name__ == '__main__': test.main() From 9edc8fa0c6dd23474ab275d104acebe7d153014e Mon Sep 17 00:00:00 2001 From: Doe Hyun Yoon Date: Wed, 10 Jul 2019 06:52:39 -0700 Subject: [PATCH 189/332] Handle non-loop merge input in VirtualScheduler to correctly. The way VirtualScheduler currently checks node ready is to count how many input nodes are ready; it's possible that a certain pattern (e.g., Switch - Merge) can break this assumption; both outputs of Switch is triggered, and then Merge becomes ready twice. For those nodes that take Merge node as input, we check input node names to avoid this problem -- ignore if the same Merge input becomes ready more than once. PiperOrigin-RevId: 257394841 --- .../core/grappler/costs/virtual_scheduler.cc | 21 +++ .../core/grappler/costs/virtual_scheduler.h | 1 + .../grappler/costs/virtual_scheduler_test.cc | 164 +++++++++++++----- 3 files changed, 142 insertions(+), 44 deletions(-) diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc index 1a38df27dc5..b21e75c7d97 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler.cc +++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc @@ -788,6 +788,15 @@ void VirtualScheduler::AddOutputNodesToReadyQueue( for (auto* output_node : port_num_output_pair.second) { auto& output_state = node_map_[output_node]; + // Skip if this input node has already been seen. + if (IsMerge(*node) && (output_state.input_nodes_seen.find(node) != + output_state.input_nodes_seen.end())) { + LOG(WARNING) << "Output node [ " << output_node->name() + << " ] has alread seen this input node [ " << node->name() + << " -- possibly due to Swith-Merge in previous " + << "nodes. Skip to increment num_inputs_ready."; + continue; + } output_state.num_inputs_ready++; // Execute a node as soon as all its inputs are ready. Merge nodes are // special since they run as soon as one of their inputs becomes @@ -801,6 +810,18 @@ void VirtualScheduler::AddOutputNodesToReadyQueue( } } } + + // Update input_nodes_seen. We update it only for the outputs of Merge node, + // as in other cases, using only counter based checking works. + if (IsMerge(*node)) { + for (const auto& port_num_output_pair : node_state.outputs) { + if (slot >= 0 && port_num_output_pair.first != slot) continue; + for (auto* output_node : port_num_output_pair.second) { + auto& output_state = node_map_[output_node]; + output_state.input_nodes_seen.insert(node); + } + } + } } bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) { diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h index b50f61d155b..55f67e3ef33 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler.h +++ b/tensorflow/core/grappler/costs/virtual_scheduler.h @@ -41,6 +41,7 @@ struct NodeState { // List of output nodes (a list of nodes that takes this output port as input) // keyed by port_num. Note that port_num -1 is used for control dependency. std::unordered_map> outputs; + std::unordered_set input_nodes_seen; // Info from GraphProperties. std::vector input_properties; diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc index 21ca9ca4fc6..588bfce5e90 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc +++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc @@ -625,10 +625,10 @@ class TestVirtualScheduler : public VirtualScheduler { public: TestVirtualScheduler(const bool use_static_shapes, const bool use_aggressive_shape_inference, - Cluster* cluster) + ReadyNodeManager* ready_node_manager, Cluster* cluster) : VirtualScheduler( use_static_shapes, use_aggressive_shape_inference, cluster, - &ready_node_manager_, + ready_node_manager, absl::make_unique(cluster->GetDevices())) { enable_mem_usage_tracking(); } @@ -638,9 +638,6 @@ class TestVirtualScheduler : public VirtualScheduler { FRIEND_TEST(VirtualSchedulerTest, ComplexDependency); FRIEND_TEST(VirtualSchedulerTest, Variable); FRIEND_TEST(VirtualSchedulerTest, InterDeviceTransfer); - - protected: - FirstReadyManager ready_node_manager_; }; class VirtualSchedulerTest : public ::testing::Test { @@ -659,7 +656,8 @@ class VirtualSchedulerTest : public ::testing::Test { cluster_ = absl::make_unique(devices); scheduler_ = absl::make_unique( /*use_static_shapes=*/true, - /*use_aggressive_shape_inference=*/true, cluster_.get()); + /*use_aggressive_shape_inference=*/true, &first_ready_manager_, + cluster_.get()); } DeviceProperties GetDummyCPUDevice() { @@ -689,12 +687,10 @@ class VirtualSchedulerTest : public ::testing::Test { auto c0 = ops::Conv2D(s.WithOpName("c0"), x, f, strides, "SAME"); auto c1 = ops::Conv2D(s.WithOpName("c1"), y, f, strides, "SAME"); auto c2 = ops::Conv2D(s.WithOpName("c2"), z, f, strides, "SAME"); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_conv2d_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"c0", "c1"}; dependency_["c0"] = {"x", "f"}; @@ -710,12 +706,11 @@ class VirtualSchedulerTest : public ::testing::Test { {kernel_, kernel_, depth_in_, depth_out_}, DT_FLOAT); std::vector strides = {1, 1, 1, 1}; auto y = ops::Conv2D(s.WithOpName("y"), x, f, strides, "SAME"); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_conv2d_var_graph"; - grappler_item_->graph = def; + grappler_item_->fetch = {"y"}; dependency_["y"] = {"x", "f"}; @@ -740,12 +735,9 @@ class VirtualSchedulerTest : public ::testing::Test { auto abcd = ops::MatMul(s.WithOpName("abcd"), abc, d); auto abcde = ops::MatMul(s.WithOpName("abcde"), abcd, e); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_matmul_sequence_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"abcde"}; dependency_["ab"] = {"a", "b"}; @@ -763,12 +755,10 @@ class VirtualSchedulerTest : public ::testing::Test { auto w = ops::RandomUniform(s.WithOpName("w"), {10, 10, 10, 10}, DT_FLOAT); OutputList input_tensors = {x, y, z, w}; auto out = ops::AddN(s.WithOpName("out"), input_tensors); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_addn_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"out"}; dependency_["out"] = {"x", "y", "z", "w"}; @@ -780,12 +770,10 @@ class VirtualSchedulerTest : public ::testing::Test { auto unnecessary = ops::Placeholder(s.WithOpName("unnecessary"), DT_FLOAT); auto x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); - grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_extra_placeholders"; - grappler_item_->graph = def; grappler_item_->fetch = {"x"}; // Grappler Item Builder puts all placeholder nodes into the feed @@ -804,17 +792,62 @@ class VirtualSchedulerTest : public ::testing::Test { } auto out = ops::NoOp(s.WithControlDependencies(input_tensors).WithOpName("out")); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + grappler_item_->id = "test_control_dependency_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"out"}; dependency_["out"] = input_noop_names; } + void CreateGrapplerItemWithAddFromOneTensor() { + Scope s = Scope::NewRootScope().WithDevice(kCPU0); + auto x = tensorflow::ops::RandomUniform( + s.WithOpName("x"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); + + auto y = tensorflow::ops::Add(s.WithOpName("y"), x, x); + Output fetch = ops::Identity(s.WithOpName("fetch"), y); + + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + + grappler_item_->id = "test_add_from_one_tensor"; + grappler_item_->fetch = {"fetch"}; + + dependency_["fetch"] = {"y"}; + dependency_["y"] = {"x"}; + } + + void CreateGrapplerItemWithSwitchMergeInput() { + // sw = Switch(x, pred) + // a = Add(S:1, b) + // m = Merge(sw:0, a) + // y = Add(m, z) + + Scope s = Scope::NewRootScope().WithDevice(kCPU0); + auto x = ops::RandomUniform( + s.WithOpName("x"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); + auto pred = ops::Const(s.WithOpName("pred"), false, {}); + auto sw = ops::Switch(s.WithOpName("switch"), x, pred); + auto b = ops::RandomUniform( + s.WithOpName("b"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); + auto a = ops::Add(s.WithOpName("a"), sw.output_true, b); + auto m = ops::Merge(s.WithOpName("m"), {sw.output_false, a.z}); + auto z = ops::RandomUniform( + s.WithOpName("z"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); + auto y = ops::Add(s.WithOpName("y"), m.output, z); + + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + + grappler_item_->id = "test_add_merge_switch"; + grappler_item_->fetch = {"y"}; + + dependency_["y"] = {"m", "z"}; + } + // FusedBN [an op with multiple outputs] with multiple consumers (including // control dependency). void CreateGrapplerItemWithBatchNorm() { @@ -846,12 +879,10 @@ class VirtualSchedulerTest : public ::testing::Test { }; auto z4 = ops::NoOp(s.WithControlDependencies(batch_var).WithOpName("z4")); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); - grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_complex_dependency_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"z1", "z2", "z3", "z4"}; dependency_["bn"] = {"x", "scale", "offset", "mean", "var"}; @@ -975,7 +1006,8 @@ versions { } )EOF"; - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -1032,7 +1064,7 @@ versions { } )EOF"; - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -1428,7 +1460,7 @@ versions { } )EOF"; - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -2095,7 +2127,7 @@ versions { producer: 27 })EOF"; - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -2136,12 +2168,9 @@ versions { .WithControlDependencies(y) .WithDevice(kCPU1)); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_conv2d_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"y1", "y2", "batch_mean1", "batch_var1", "control_dep"}; @@ -2280,6 +2309,8 @@ versions { // cluster_ and scheduler_ are initialized in the c'tor. std::unique_ptr cluster_; std::unique_ptr scheduler_; + FirstReadyManager first_ready_manager_; + CompositeNodeManager composite_node_manager_; // grappler_item_ will be initialized differently for each test case. std::unique_ptr grappler_item_; @@ -2933,6 +2964,51 @@ TEST_F(VirtualSchedulerTest, GraphWihtOnlyRecv) { EXPECT_GT(ops_executed.count("Recv"), 0); } +TEST_F(VirtualSchedulerTest, AddMergeSwitch) { + // Override scheduler_ with CompositeNodeNamager. + scheduler_ = absl::make_unique( + /*use_static_shapes=*/true, + /*use_aggressive_shape_inference=*/true, &composite_node_manager_, + cluster_.get()); + CreateGrapplerItemWithSwitchMergeInput(); + InitScheduler(); + + // pred --+ z --+ + // | | + // V V + // x -> Switch --------> Merge ---> Add --> y + // | ^ + // | | + // +-----> Add -----+ + // ^ + // | + // b --------------+ + + // Run the scheduler. The current VirtualScheduler, w/o annotation, triggers + // both outputs of Switch; then Merge (as long as one input is ready, it's z + // is ready, if we just use num_inputs_ready counter, the final Add becomes + // ready. possible to skipt scheduling z. (Need to use CompositeNodeManager + // to test this case). + auto ops_executed = RunScheduler(""); + + EXPECT_GT(ops_executed.count("z"), 0); +} + +TEST_F(VirtualSchedulerTest, AddFromOneTensor) { + CreateGrapplerItemWithAddFromOneTensor(); + InitScheduler(); + + // x -+----> Add --> y + // | ^ + // | | + // +-------+ + + // Run the scheduler. + auto ops_executed = RunScheduler(""); + EXPECT_GT(ops_executed.count("y"), 0); + EXPECT_GT(ops_executed.count("x"), 0); +} + } // namespace } // end namespace grappler } // end namespace tensorflow From 03adff954b97325c031571171521b10652377949 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Wed, 10 Jul 2019 07:07:00 -0700 Subject: [PATCH 190/332] [XLA:Python] Delete PyLocalBuffer.FromPythonValues. We haven't ended up using this from JAX, so let's delete it until we have a use case. PiperOrigin-RevId: 257397061 --- .../compiler/xla/python/local_client.cc | 63 ------------------- tensorflow/compiler/xla/python/local_client.h | 6 -- tensorflow/compiler/xla/python/xla.cc | 1 - tensorflow/compiler/xla/python/xla_client.py | 4 -- 4 files changed, 74 deletions(-) diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index 2057788c12d..c34c01425b9 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -381,69 +381,6 @@ StatusOr> PyLocalBuffer::FromPython( return buffer; } -/*static */ StatusOr>> -PyLocalBuffer::FromPythonValues( - const std::vector>& arguments, - std::shared_ptr client) { - tensorflow::profiler::TraceMe traceme("PyLocalBuffer::FromPythonValues"); - int num_arguments = static_cast(arguments.size()); - std::vector> outputs(num_arguments); - if (num_arguments == 0) { - return outputs; - } - - struct H2DTransfer { - PythonBufferTree tree; - StatusOr> buffer; - PythonRefManager::ManagedPyObjects py_buffer_refs; - }; - - std::vector transfers(num_arguments); - for (int i = 0; i < num_arguments; ++i) { - TF_ASSIGN_OR_RETURN(transfers[i].tree, - GetPythonBufferTree(arguments[i].first)); - transfers[i].py_buffer_refs = client->py_ref_manager().ManageReferences( - absl::MakeSpan(transfers[i].tree.arrays)); - } - client->py_ref_manager().CollectGarbage(); - // We are done manipulating Python objects; release the GIL. - py::gil_scoped_release gil_release; - - auto transfer_h2d = [&](int i) -> StatusOr> { - int device_ordinal = arguments[i].second; - return TransferHostToDeviceAsync(transfers[i].tree, device_ordinal, client, - client->device(device_ordinal)); - }; - - // We perform the transfers on a thread pool in case XLA needs to do any - // host-side preprocessing of the input data. - if (num_arguments == 1) { - transfers[0].buffer = transfer_h2d(0); - } else { - absl::BlockingCounter counter(num_arguments); - for (int i = 0; i < num_arguments; ++i) { - client->h2d_transfer_pool()->Schedule([&, i]() { - transfers[i].buffer = transfer_h2d(i); - counter.DecrementCount(); - }); - } - counter.Wait(); - } - - // Release our references once the transfers have completed. - for (int i = 0; i < num_arguments; ++i) { - int device_ordinal = arguments[i].second; - const Device& device = client->device(device_ordinal); - device.ThenRelease(device.host_to_device_stream(), - std::move(transfers[i].py_buffer_refs)); - } - - for (int i = 0; i < num_arguments; ++i) { - TF_ASSIGN_OR_RETURN(outputs[i], std::move(transfers[i].buffer)); - } - return outputs; -} - /* static */ StatusOr> PyLocalBuffer::MakeTuple( const std::vector buffers, std::shared_ptr client, int device_ordinal) { diff --git a/tensorflow/compiler/xla/python/local_client.h b/tensorflow/compiler/xla/python/local_client.h index 4064e629c6f..38315d82e2c 100644 --- a/tensorflow/compiler/xla/python/local_client.h +++ b/tensorflow/compiler/xla/python/local_client.h @@ -241,12 +241,6 @@ class PyLocalBuffer { const pybind11::object& argument, std::shared_ptr client, int device_ordinal); - // Converts multiple (python object, device ordinal) pairs into - // PyLocalBuffers in parallel. - static StatusOr>> FromPythonValues( - const std::vector>& argument, - std::shared_ptr client); - static StatusOr> MakeTuple( const std::vector buffers, std::shared_ptr client, int device_ordinal); diff --git a/tensorflow/compiler/xla/python/xla.cc b/tensorflow/compiler/xla/python/xla.cc index 34ee5d7ea69..da4c847efb8 100644 --- a/tensorflow/compiler/xla/python/xla.cc +++ b/tensorflow/compiler/xla/python/xla.cc @@ -299,7 +299,6 @@ PYBIND11_MODULE(xla_extension, m) { py::class_(m, "PyLocalBuffer") .def_static("from_python", &PyLocalBuffer::FromPython) - .def_static("from_python_values", &PyLocalBuffer::FromPythonValues) .def_static("make_tuple", &PyLocalBuffer::MakeTuple) .def("copy_to_device", &PyLocalBuffer::CopyToDevice) .def("delete", &PyLocalBuffer::Delete) diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py index 0af9c0db62d..7e5692fef30 100644 --- a/tensorflow/compiler/xla/python/xla_client.py +++ b/tensorflow/compiler/xla/python/xla_client.py @@ -98,10 +98,6 @@ class LocalBackend(Backend): def buffer_from_pyval(self, pyval, device=0): return _xla.PyLocalBuffer.from_python(pyval, self.client, device) - def buffers_from_pyvals(self, pyvals_and_devices): - return _xla.PyLocalBuffer.from_python_values(pyvals_and_devices, - self.client) - def make_tuple(self, c_buffers, device_ordinal): return _xla.PyLocalBuffer.make_tuple(c_buffers, self.client, device_ordinal) From 5e01b339fa2698e4769f98a48fb2ab2fb4257e04 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Wed, 10 Jul 2019 07:21:48 -0700 Subject: [PATCH 191/332] [tf.data] Use memcpy for simple types in CopyElementToSlice(). This simplifies the codepath for `Dataset.batch()` and related transformations when batching tensors of simple values. PiperOrigin-RevId: 257399411 --- tensorflow/core/util/batch_util.cc | 55 +++++++++++++++++++----------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/tensorflow/core/util/batch_util.cc b/tensorflow/core/util/batch_util.cc index 8b694d9cc32..e1c32cd0069 100644 --- a/tensorflow/core/util/batch_util.cc +++ b/tensorflow/core/util/batch_util.cc @@ -42,42 +42,56 @@ Status ValidateInput(const Tensor& parent, const Tensor& element, int64 index) { } template -Status HandleElementToSlice(Tensor element, Tensor* parent, int64 index, +Status HandleElementToSlice(T* src, T* dest, int64 num_values, bool /* can_move */) { - parent->flat_outer_dims().chip(index, 0) = element.flat(); + static_assert(is_simple_type::value, "Memcpy requires a simple type."); + memcpy(dest, src, num_values * sizeof(T)); return Status::OK(); } template <> -Status HandleElementToSlice(Tensor element, Tensor* parent, int64 index, +Status HandleElementToSlice(string* src, string* dest, int64 num_values, bool can_move) { - auto parent_as_matrix = parent->flat_outer_dims(); - auto element_flat = element.flat(); if (can_move) { - for (int64 i = 0; i < element.NumElements(); ++i) { - parent_as_matrix(index, i) = std::move(element_flat(i)); + for (int64 i = 0; i < num_values; ++i) { + *dest++ = std::move(*src++); } } else { - parent_as_matrix.chip(index, 0) = element_flat; + std::copy_n(src, num_values, dest); } return Status::OK(); } template <> -Status HandleElementToSlice(Tensor element, Tensor* parent, - int64 index, bool can_move) { - auto parent_as_matrix = parent->flat_outer_dims(); - auto element_flat = element.flat(); +Status HandleElementToSlice(Variant* src, Variant* dest, + int64 num_values, bool can_move) { if (can_move) { - for (int64 i = 0; i < element.NumElements(); ++i) { - parent_as_matrix(index, i) = std::move(element_flat(i)); + for (int64 i = 0; i < num_values; ++i) { + *dest++ = std::move(*src++); } } else { - parent_as_matrix.chip(index, 0) = element_flat; + std::copy_n(src, num_values, dest); } return Status::OK(); } +template <> +Status HandleElementToSlice(ResourceHandle* src, + ResourceHandle* dest, + int64 num_values, + bool /* can_move */) { + std::copy_n(src, num_values, dest); + return Status::OK(); +} + +template <> +Status HandleElementToSlice(Eigen::half* src, Eigen::half* dest, + int64 num_values, + bool /* can_move */) { + std::copy_n(src, num_values, dest); + return Status::OK(); +} + // TODO(b/78245576): Consider removing this overload. template void HandleSliceToElement(const Tensor& parent, Tensor* element, int64 index) { @@ -123,12 +137,13 @@ void HandleSliceToElement(Tensor* parent, Tensor* element, int64 index, // Copies element into the index^th slice of parent (in the 0th dimension). Status CopyElementToSlice(Tensor element, Tensor* parent, int64 index) { TF_RETURN_IF_ERROR(ValidateInput(*parent, element, index)); - + const int64 num_values = element.NumElements(); bool can_move = element.RefCountIsOne(); -#define HANDLE_TYPE(T) \ - case DataTypeToEnum::value: { \ - return HandleElementToSlice(std::move(element), parent, index, \ - can_move); \ +#define HANDLE_TYPE(T) \ + case DataTypeToEnum::value: { \ + T* src = element.base(); \ + T* dest = parent->base() + (num_values * index); \ + return HandleElementToSlice(src, dest, num_values, can_move); \ } switch (element.dtype()) { From 825ca4b76624b81cb2917b4909be6486935dd2e6 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Wed, 10 Jul 2019 08:24:00 -0700 Subject: [PATCH 192/332] add deprecation notice. PiperOrigin-RevId: 257408291 --- tensorflow/tools/docs/generate2.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow/tools/docs/generate2.py b/tensorflow/tools/docs/generate2.py index a6d01ae9306..0e8cba27e14 100644 --- a/tensorflow/tools/docs/generate2.py +++ b/tensorflow/tools/docs/generate2.py @@ -150,6 +150,16 @@ _raw_ops_doc = textwrap.dedent("""\n if LooseVersion(tf.__version__) < LooseVersion('2'): tf.raw_ops.__doc__ = _raw_ops_doc + tf.contrib.__doc__ = """ + Contrib module containing volatile or experimental code. + + Warning: The `tf.contrib` module will not be included in TensorFlow 2.0. Many + of its submodules have been integrated into TensorFlow core, or spun-off into + other projects like [`tensorflow_io`](https://github.com/tensorflow/io), or + [`tensorflow_addons`](https://github.com/tensorflow/addons). For instructions + on how to upgrade see the + [Migration guide](https://www.tensorflow.org/beta/guide/migration_guide). + """ else: tf.raw_ops.__doc__ += _raw_ops_doc From d568ec7c7b005b82104e139b640ff7e520e41a84 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 10 Jul 2019 16:53:19 +0100 Subject: [PATCH 193/332] update gpu docker LD_LIBRARY_PATH as per README contributing guidelines --- .../tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile | 2 +- tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile | 2 +- tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile | 2 +- .../dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile | 2 +- .../dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile | 2 +- .../dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile | 2 +- .../dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile | 2 +- .../dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile | 2 +- .../tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile index 9f10a7f03a3..02d8f89919e 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu-jupyter.Dockerfile @@ -72,7 +72,7 @@ RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ # Configure the build for our CUDA configuration. ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 ENV TF_NEED_TENSORRT 1 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 diff --git a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile index 1d258e0404a..6d00ef3c115 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/devel-gpu.Dockerfile @@ -72,7 +72,7 @@ RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ # Configure the build for our CUDA configuration. ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 ENV TF_NEED_TENSORRT 1 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 diff --git a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile index 981d42c5759..fde7c9e8c39 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/gpu-jupyter.Dockerfile @@ -58,7 +58,7 @@ RUN [ ${ARCH} = ppc64le ] || (apt-get update && \ && rm -rf /var/lib/apt/lists/*) # For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile index 7dcaa72ab90..a05c718f6fb 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le-jupyter.Dockerfile @@ -72,7 +72,7 @@ RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ # Configure the build for our CUDA configuration. ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 ENV TF_NEED_TENSORRT 1 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile index 9a5a5c5cb9a..44d91ad067f 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/devel-gpu-ppc64le.Dockerfile @@ -72,7 +72,7 @@ RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ # Configure the build for our CUDA configuration. ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 ENV TF_NEED_TENSORRT 1 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile index f9a0aa8ab6c..b2f1ce152c2 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le-jupyter.Dockerfile @@ -58,7 +58,7 @@ RUN [ ${ARCH} = ppc64le ] || (apt-get update && \ && rm -rf /var/lib/apt/lists/*) # For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} diff --git a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile index 62f17046468..3422eadb60c 100644 --- a/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile +++ b/tensorflow/tools/dockerfiles/dockerfiles/ppc64le/gpu-ppc64le.Dockerfile @@ -58,7 +58,7 @@ RUN [ ${ARCH} = ppc64le ] || (apt-get update && \ && rm -rf /var/lib/apt/lists/*) # For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ARG USE_PYTHON_3_NOT_2 ARG _PY_SUFFIX=${USE_PYTHON_3_NOT_2:+3} diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile index ad68e0c8a5f..fc0976b023f 100644 --- a/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/devel-nvidia.partial.Dockerfile @@ -49,7 +49,7 @@ RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ # Configure the build for our CUDA configuration. ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH ENV TF_NEED_CUDA 1 ENV TF_NEED_TENSORRT 1 ENV TF_CUDA_COMPUTE_CAPABILITIES=3.5,5.2,6.0,6.1,7.0 diff --git a/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile index aeee9f7d689..b09c6456e9c 100644 --- a/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile +++ b/tensorflow/tools/dockerfiles/partials/ubuntu/nvidia.partial.Dockerfile @@ -35,4 +35,4 @@ RUN [ ${ARCH} = ppc64le ] || (apt-get update && \ && rm -rf /var/lib/apt/lists/*) # For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:$LD_LIBRARY_PATH From 112433f1c968ff721caf70e7846f5719a9e6053a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 08:52:01 -0700 Subject: [PATCH 194/332] Automated rollback of commit 21e648d52d74bc35105444dcb17584678819bcfe PiperOrigin-RevId: 257412841 --- tensorflow/python/kernel_tests/lookup_ops_test.py | 11 ----------- tensorflow/python/ops/lookup_ops.py | 9 +++++---- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/tensorflow/python/kernel_tests/lookup_ops_test.py b/tensorflow/python/kernel_tests/lookup_ops_test.py index 87eabf9e015..0db7d27e7e7 100644 --- a/tensorflow/python/kernel_tests/lookup_ops_test.py +++ b/tensorflow/python/kernel_tests/lookup_ops_test.py @@ -1998,17 +1998,6 @@ class IndexTableFromTensor(test.TestCase): ids = table.lookup(constant_op.constant(("salad", "surgery", "tarkus"))) self.assertAllEqual((1, 2, 3), self.evaluate(ids)) - def test_index_table_from_tensor_with_tensor_init_in_function(self): - @function.defun() - def lookup_fn(): - vocabulary_list = constant_op.constant(["brain", "salad", "surgery"]) - table = lookup_ops.index_table_from_tensor( - vocabulary_list=vocabulary_list, num_oov_buckets=1) - return table.lookup(constant_op.constant(("salad", "surgery", "tarkus"))) - - ids = lookup_fn() - self.assertAllEqual((1, 2, 3), self.evaluate(ids)) - def test_int32_index_table_from_tensor_with_tensor_init(self): with self.cached_session(): table = lookup_ops.index_table_from_tensor( diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index af7d4b0e4a1..3b726a611fa 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -171,7 +171,7 @@ class InitializableLookupTableBase(LookupInterface): self._initializer = self._track_trackable(initializer, "_initializer") with ops.init_scope(): self._resource_handle = self._create_resource() - self._init_op = self._initialize() + self._init_op = self._initialize() def _initialize(self): return self._initializer.initialize(self) @@ -420,9 +420,10 @@ class KeyValueTensorInitializer(TableInitializerBase): value_dtype: The `values` data type. Used when `values` is a python array. name: A name for the operation (optional). """ - self._keys = ops.convert_to_tensor(keys, dtype=key_dtype, name="keys") - self._values = ops.convert_to_tensor( - values, dtype=value_dtype, name="values") + with ops.init_scope(): + self._keys = ops.convert_to_tensor(keys, dtype=key_dtype, name="keys") + self._values = ops.convert_to_tensor( + values, dtype=value_dtype, name="values") self._name = name if name is not None else "key_value_init" if context.executing_eagerly(): # Ensure a unique name when eager execution is enabled to avoid spurious From a74fcc484e1909e1f1fed6ff6088611d1c7b7627 Mon Sep 17 00:00:00 2001 From: vbvg2008 Date: Wed, 10 Jul 2019 09:05:23 -0700 Subject: [PATCH 195/332] fix data type inconsistency for cifar10 --- tensorflow/python/keras/datasets/cifar10.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/python/keras/datasets/cifar10.py b/tensorflow/python/keras/datasets/cifar10.py index 36e1b83c10a..0a74ae8382d 100644 --- a/tensorflow/python/keras/datasets/cifar10.py +++ b/tensorflow/python/keras/datasets/cifar10.py @@ -58,5 +58,8 @@ def load_data(): if K.image_data_format() == 'channels_last': x_train = x_train.transpose(0, 2, 3, 1) x_test = x_test.transpose(0, 2, 3, 1) + + x_test = x_test.astype(x_train.dtype) + y_test = y_test.astype(y_train.dtype) return (x_train, y_train), (x_test, y_test) From 7c7c449f175db71cbcdf544bb4104f84f8cb91d9 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Wed, 10 Jul 2019 09:06:50 -0700 Subject: [PATCH 196/332] Fix 2.0 nightly package import error caused by lazy-loading change. PiperOrigin-RevId: 257415659 --- tensorflow/api_template.__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py index 8d5e43b672b..c6d39cd9f48 100644 --- a/tensorflow/api_template.__init__.py +++ b/tensorflow/api_template.__init__.py @@ -46,7 +46,7 @@ from tensorflow.python.tools import module_util as _module_util # Make sure directory containing top level submodules is in # the __path__ so that "from tensorflow.foo import bar" works. # We're using bitwise, but there's nothing special about that. -_API_MODULE = sys.modules[__name__].bitwise # pylint: disable=undefined-variable +_API_MODULE = _sys.modules[__name__].bitwise _tf_api_dir = _os.path.dirname(_os.path.dirname(_API_MODULE.__file__)) _current_module = _sys.modules[__name__] From d7fbbc00235c8d0c34de7b34a156fb9c576fb209 Mon Sep 17 00:00:00 2001 From: Bixia Zheng Date: Wed, 10 Jul 2019 09:14:31 -0700 Subject: [PATCH 197/332] [XLA] Enable truncated normal for double. Fix a problem in testTruncatedNormalIsInRange that causes the test not actually run. Add testTruncatedNormalIsNotConstant for double. PiperOrigin-RevId: 257417015 --- tensorflow/compiler/tests/random_ops_test.py | 6 ++++-- tensorflow/compiler/tf2xla/kernels/random_ops.cc | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/tests/random_ops_test.py b/tensorflow/compiler/tests/random_ops_test.py index 4ac6a82145d..6e21c2a2e46 100644 --- a/tensorflow/compiler/tests/random_ops_test.py +++ b/tensorflow/compiler/tests/random_ops_test.py @@ -116,12 +116,14 @@ class RandomOpsTest(xla_test.XLATestCase): def rng(dtype): return random_ops.truncated_normal(shape=[2], dtype=dtype) - self._testRngIsNotConstant(rng, dtypes.float32) + # TODO(b/34339814): make this test work with 16 bit float types. + for dtype in self._random_types() & {np.float32, np.float64}: + self._testRngIsNotConstant(rng, dtype) def testTruncatedNormalIsInRange(self): count = 10000000 # TODO(b/34339814): make this test work with 16 bit float types. - for dtype in self._random_types() & {dtypes.float32, dtypes.float64}: + for dtype in self._random_types() & {np.float32, np.float64}: with self.session() as sess: with self.test_scope(): x = random_ops.truncated_normal(shape=[count], dtype=dtype) diff --git a/tensorflow/compiler/tf2xla/kernels/random_ops.cc b/tensorflow/compiler/tf2xla/kernels/random_ops.cc index 0b54c88fae9..ed2ba9d1c47 100644 --- a/tensorflow/compiler/tf2xla/kernels/random_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/random_ops.cc @@ -293,7 +293,7 @@ class TruncatedNormalOp : public XlaOpKernel { REGISTER_XLA_OP(Name("TruncatedNormal") .CompileTimeConstantInput("shape") - .TypeConstraint("dtype", DT_FLOAT), + .TypeConstraint("dtype", {DT_FLOAT, DT_DOUBLE}), TruncatedNormalOp); } // namespace From 3ccb04a9a88681fc539813e62ded6b0a80dbf60a Mon Sep 17 00:00:00 2001 From: mdfaijul Date: Wed, 10 Jul 2019 09:24:57 -0700 Subject: [PATCH 198/332] Fixed quantized matmul unit test. --- tensorflow/core/kernels/mkl_qmatmul_op_test.cc | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/mkl_qmatmul_op_test.cc b/tensorflow/core/kernels/mkl_qmatmul_op_test.cc index e4db9653cdd..f018e12313a 100644 --- a/tensorflow/core/kernels/mkl_qmatmul_op_test.cc +++ b/tensorflow/core/kernels/mkl_qmatmul_op_test.cc @@ -51,7 +51,7 @@ class ConvMklToTF : public OpsTestBase { .Input(FakeInput(dtype)) // Input .Input(FakeInput(DT_UINT8)) // MKL second tensor .Attr("T", dtype) - .Attr("_kernel", "MklOp") + .Attr("_kernel", "MklLayoutDependentOp") .Finalize(node_def())); TF_EXPECT_OK(InitOp()); AddInputFromArray(first.shape(), first.flat()); @@ -444,10 +444,22 @@ TEST_F(QuantizedMatMulTest, Small_withBiasAndReluAndReq) { // After Bias addition // 74+10=84, 80-20=60, 86+30=116, 92-40=52, // 173+10=183, 188-20=168, 203+30=233, 218-40=178 - // After Relu and Requantize + // After Relu // 84, 60, 116, 52, 183, 168, 233, 178 + // After Requantize + // requantscale = scale_int32 / scale_eightbit / static_cast(1 << 23) + // requantscale = 2^31/255/2^23 ~= 1.00392 + // 84 * 1.00392 ~= 84.329 ~= 84 + // 60 * 1.00392 ~= 60.235 ~= 60 + // 116 * 1.00392 ~= 116.454 ~= 116 + // 52 * 1.00392 ~= 52.203 ~= 52 + // 183 * 1.00392 ~= 183.717 ~= 184 + // 168 * 1.00392 ~= 168.658 ~= 169 + // 233 * 1.00392 ~= 233.913 ~= 234 + // 178 * 1.00392 ~= 178.698 ~= 179 + Tensor expected(allocator(), DT_QUINT8, TensorShape({2, 4})); - test::FillValues(&expected, {84, 60, 116, 52, 183, 168, 233, 178}); + test::FillValues(&expected, {84, 60, 116, 52, 184, 169, 234, 179}); const Tensor& output = *GetOutput(0); const Tensor& mkl_shape_tensor = *GetOutput(3); From 3b66ba213222912357d0ea024eb09bf2267a91a4 Mon Sep 17 00:00:00 2001 From: Tae-Hwan Jung Date: Thu, 11 Jul 2019 01:27:39 +0900 Subject: [PATCH 199/332] edit #30526 'unsupported operand` when minus type --- tensorflow/python/framework/tensor_shape.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/framework/tensor_shape.py b/tensorflow/python/framework/tensor_shape.py index 54819b0357b..14fbddabd00 100644 --- a/tensorflow/python/framework/tensor_shape.py +++ b/tensorflow/python/framework/tensor_shape.py @@ -576,10 +576,7 @@ class Dimension(object): Returns: A Dimension whose value is `self` modulo `other`. """ - try: - other = as_dimension(other) - except (TypeError, ValueError): - return NotImplemented + other = as_dimension(other) if self._value is None or other.value is None: return Dimension(None) else: @@ -594,10 +591,7 @@ class Dimension(object): Returns: A Dimension whose value is `other` modulo `self`. """ - try: - other = as_dimension(other) - except (TypeError, ValueError): - return NotImplemented + other = as_dimension(other) return other % self def __lt__(self, other): From 0f5d0cfc2a738c5605cb23a1975ac4a1ceb11e24 Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Wed, 10 Jul 2019 09:25:53 -0700 Subject: [PATCH 200/332] [tfdbg] Fix gRPC message length limit issue in source remote Fixes https://github.com/tensorflow/tensorboard/issues/1103 PiperOrigin-RevId: 257419107 --- .../python/debug/lib/grpc_debug_server.py | 5 +- tensorflow/python/debug/lib/source_remote.py | 22 ++---- .../python/debug/lib/source_remote_test.py | 75 +++++++------------ 3 files changed, 36 insertions(+), 66 deletions(-) diff --git a/tensorflow/python/debug/lib/grpc_debug_server.py b/tensorflow/python/debug/lib/grpc_debug_server.py index 1b559f1f275..d7eece43310 100644 --- a/tensorflow/python/debug/lib/grpc_debug_server.py +++ b/tensorflow/python/debug/lib/grpc_debug_server.py @@ -346,7 +346,10 @@ class EventListenerBaseServicer(debug_service_pb2_grpc.EventListenerServicer): if self._server_started: raise ValueError("Server has already started running") - self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + no_max_message_sizes = [("grpc.max_receive_message_length", -1), + ("grpc.max_send_message_length", -1)] + self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10), + options=no_max_message_sizes) debug_service_pb2_grpc.add_EventListenerServicer_to_server(self, self.server) self.server.add_insecure_port("[::]:%d" % self._server_port) diff --git a/tensorflow/python/debug/lib/source_remote.py b/tensorflow/python/debug/lib/source_remote.py index 07de9f9cbd2..e0a3695df43 100644 --- a/tensorflow/python/debug/lib/source_remote.py +++ b/tensorflow/python/debug/lib/source_remote.py @@ -28,7 +28,6 @@ from tensorflow.python.debug.lib import common from tensorflow.python.debug.lib import debug_service_pb2_grpc from tensorflow.python.debug.lib import source_utils from tensorflow.python.platform import gfile -from tensorflow.python.platform import tf_logging from tensorflow.python.profiler import tfprof_logger @@ -96,11 +95,6 @@ def _source_file_paths_outside_tensorflow_py_library(code_defs, id_to_string): return non_tf_files -def grpc_message_length_bytes(): - """Maximum gRPC message length in bytes.""" - return 4 * 1024 * 1024 - - def _send_call_tracebacks(destinations, origin_stack, is_eager_execution=False, @@ -169,20 +163,14 @@ def _send_call_tracebacks(destinations, debugged_source_files.append(source_files) for destination in destinations: - channel = grpc.insecure_channel(destination) + no_max_message_sizes = [("grpc.max_receive_message_length", -1), + ("grpc.max_send_message_length", -1)] + channel = grpc.insecure_channel(destination, options=no_max_message_sizes) stub = debug_service_pb2_grpc.EventListenerStub(channel) stub.SendTracebacks(call_traceback) if send_source: - for path, source_files in zip( - source_file_paths, debugged_source_files): - if source_files.ByteSize() < grpc_message_length_bytes(): - stub.SendSourceFiles(source_files) - else: - tf_logging.warn( - "The content of the source file at %s is not sent to " - "gRPC debug server %s, because the message size exceeds " - "gRPC message length limit (%d bytes)." % ( - path, destination, grpc_message_length_bytes())) + for source_files in debugged_source_files: + stub.SendSourceFiles(source_files) def send_graph_tracebacks(destinations, diff --git a/tensorflow/python/debug/lib/source_remote_test.py b/tensorflow/python/debug/lib/source_remote_test.py index dce400c9ab0..14c8e744934 100644 --- a/tensorflow/python/debug/lib/source_remote_test.py +++ b/tensorflow/python/debug/lib/source_remote_test.py @@ -21,6 +21,8 @@ from __future__ import print_function import os import traceback +import grpc + from tensorflow.core.debug import debug_service_pb2 from tensorflow.python.client import session from tensorflow.python.debug.lib import grpc_debug_test_server @@ -129,9 +131,17 @@ class SendTracebacksTest(test_util.TensorFlowTestCase): send_traceback = traceback.extract_stack() send_lineno = line_number_above() - source_remote.send_graph_tracebacks( - [self._server_address, self._server_address_2], - "dummy_run_key", send_traceback, sess.graph) + + with test.mock.patch.object( + grpc, "insecure_channel", + wraps=grpc.insecure_channel) as mock_grpc_channel: + source_remote.send_graph_tracebacks( + [self._server_address, self._server_address_2], + "dummy_run_key", send_traceback, sess.graph) + mock_grpc_channel.assert_called_with( + test.mock.ANY, + options=[("grpc.max_receive_message_length", -1), + ("grpc.max_send_message_length", -1)]) servers = [self._server, self._server_2] for server in servers: @@ -157,51 +167,6 @@ class SendTracebacksTest(test_util.TensorFlowTestCase): self.assertEqual(["dummy_run_key"], server.query_call_keys()) self.assertEqual([sess.graph.version], server.query_graph_versions()) - def testSourceFileSizeExceedsGrpcMessageLengthLimit(self): - """In case source file size exceeds the grpc message length limit. - - it ought not to have been sent to the server. - """ - this_func_name = "testSourceFileSizeExceedsGrpcMessageLengthLimit" - - # Patch the method to simulate a very small message length limit. - with test.mock.patch.object( - source_remote, "grpc_message_length_bytes", return_value=2): - with session.Session() as sess: - a = variables.Variable(21.0, name="two/a") - a_lineno = line_number_above() - b = variables.Variable(2.0, name="two/b") - b_lineno = line_number_above() - x = math_ops.add(a, b, name="two/x") - x_lineno = line_number_above() - - send_traceback = traceback.extract_stack() - send_lineno = line_number_above() - source_remote.send_graph_tracebacks( - [self._server_address, self._server_address_2], - "dummy_run_key", send_traceback, sess.graph) - - servers = [self._server, self._server_2] - for server in servers: - # Even though the source file content is not sent, the traceback - # should have been sent. - tb = server.query_op_traceback("two/a") - self.assertIn((self._curr_file_path, a_lineno, this_func_name), tb) - tb = server.query_op_traceback("two/b") - self.assertIn((self._curr_file_path, b_lineno, this_func_name), tb) - tb = server.query_op_traceback("two/x") - self.assertIn((self._curr_file_path, x_lineno, this_func_name), tb) - - self.assertIn( - (self._curr_file_path, send_lineno, this_func_name), - server.query_origin_stack()[-1]) - - tf_trace_file_path = ( - self._findFirstTraceInsideTensorFlowPyLibrary(x.op)) - # Verify that the source content is not sent to the server. - with self.assertRaises(ValueError): - self._server.query_source_file_line(tf_trace_file_path, 0) - def testSendEagerTracebacksToSingleDebugServer(self): this_func_name = "testSendEagerTracebacksToSingleDebugServer" send_traceback = traceback.extract_stack() @@ -213,6 +178,20 @@ class SendTracebacksTest(test_util.TensorFlowTestCase): self.assertIn((self._curr_file_path, send_lineno, this_func_name), self._server.query_origin_stack()[-1]) + def testGRPCServerMessageSizeLimit(self): + """Assert gRPC debug server is started with unlimited message size.""" + with test.mock.patch.object( + grpc, "server", wraps=grpc.server) as mock_grpc_server: + (_, _, _, server_thread, + server) = grpc_debug_test_server.start_server_on_separate_thread( + poll_server=True) + mock_grpc_server.assert_called_with( + test.mock.ANY, + options=[("grpc.max_receive_message_length", -1), + ("grpc.max_send_message_length", -1)]) + server.stop_server().wait() + server_thread.join() + if __name__ == "__main__": googletest.main() From c8058612d04f90a5a7fe609a859247bbf871ec40 Mon Sep 17 00:00:00 2001 From: Ayush Dubey Date: Wed, 10 Jul 2019 09:27:10 -0700 Subject: [PATCH 201/332] Do not infer cross device ops via choose_the_best for collective strategy. PiperOrigin-RevId: 257419318 --- .../python/distribute/collective_all_reduce_strategy.py | 8 ++++---- tensorflow/python/distribute/mirrored_strategy.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/distribute/collective_all_reduce_strategy.py b/tensorflow/python/distribute/collective_all_reduce_strategy.py index e858b6a57fc..320dea1e840 100644 --- a/tensorflow/python/distribute/collective_all_reduce_strategy.py +++ b/tensorflow/python/distribute/collective_all_reduce_strategy.py @@ -157,12 +157,12 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended): self._host_input_device = numpy_dataset.SingleDevice(self._worker_device) self._collective_keys = cross_device_utils.CollectiveKeys() - super(CollectiveAllReduceExtended, self)._initialize_local(local_devices) # TODO(yuefengz): remove num_gpus_per_worker from CollectiveAllReduce. self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce( num_workers=self._num_workers, num_gpus_per_worker=num_gpus, collective_keys=self._collective_keys) + super(CollectiveAllReduceExtended, self)._initialize_local(local_devices) self._cluster_spec = None self._task_type = None @@ -257,13 +257,13 @@ class CollectiveAllReduceExtended(mirrored_strategy.MirroredExtended): local_devices = (self._worker_device,) self._collective_keys = cross_device_utils.CollectiveKeys() - super(CollectiveAllReduceExtended, self)._initialize_local(local_devices) - self._input_workers = input_lib.InputWorkers( - self._device_map, [(self._worker_device, self.worker_devices)]) self._cross_device_ops = cross_device_ops_lib.CollectiveAllReduce( num_workers=self._num_workers, num_gpus_per_worker=num_gpus, collective_keys=self._collective_keys) + super(CollectiveAllReduceExtended, self)._initialize_local(local_devices) + self._input_workers = input_lib.InputWorkers( + self._device_map, [(self._worker_device, self.worker_devices)]) # Add a default device so that ops without specified devices will not end up # on other workers. diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py index b3552171b30..dcdc70ce94c 100644 --- a/tensorflow/python/distribute/mirrored_strategy.py +++ b/tensorflow/python/distribute/mirrored_strategy.py @@ -400,8 +400,8 @@ class MirroredExtended(distribute_lib.StrategyExtendedV1): self._local_mode = True self._device_map = values.ReplicaDeviceMap(devices) self._input_workers = input_lib.InputWorkers(self._device_map) - self._inferred_cross_device_ops = cross_device_ops_lib.choose_the_best( - devices) + self._inferred_cross_device_ops = None if self._cross_device_ops else ( + cross_device_ops_lib.choose_the_best(devices)) self._host_input_device = numpy_dataset.SingleDevice("/cpu:0") def _initialize_multi_worker(self, devices): From 5c4fb1f142a2ee2126f6d4efe6ce1be174498ae8 Mon Sep 17 00:00:00 2001 From: Haoliang Zhang Date: Wed, 10 Jul 2019 09:27:44 -0700 Subject: [PATCH 202/332] Add `Tanh` support in TFLite MLIR converter. PiperOrigin-RevId: 257419427 --- tensorflow/compiler/mlir/lite/ir/tfl_ops.td | 5 ++-- .../compiler/mlir/lite/tests/legalize-tf.mlir | 8 ++++++ .../mlir/lite/transforms/legalize_patterns.td | 1 + .../mlir/tensorflow/ir/tf_generated_ops.td | 25 +++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/ir/tfl_ops.td b/tensorflow/compiler/mlir/lite/ir/tfl_ops.td index 0ced385ea2e..e2366d8cd80 100644 --- a/tensorflow/compiler/mlir/lite/ir/tfl_ops.td +++ b/tensorflow/compiler/mlir/lite/ir/tfl_ops.td @@ -1692,9 +1692,10 @@ def TFL_TanhOp: TFL_Op<"tanh", [ Computes element-wise Hyperbolic tangent of input }]; - let arguments = (ins AnyTensor:$x); + // TODO(haoliang): missing Uint8. + let arguments = (ins TensorOf<[F32, I16, I8]>:$x); - let results = (outs AnyTensor:$y); + let results = (outs TensorOf<[F32, I16, I8]>:$y); } def TFL_TileOp: TFL_Op<"tile", [NoSideEffect, diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir index 25ceede688e..f771e0b7504 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir @@ -849,3 +849,11 @@ func @mirror_pad_reflect(tensor<2x1x3xf32>, tensor<3x2xi32>) -> tensor // CHECK: %0 = "tfl.mirror_pad"(%arg0, %arg1) {mode = "REFLECT"} : (tensor<2x1x3xf32>, tensor<3x2xi32>) -> tensor // CHECK: return %0 : tensor } + +func @Tanh(%arg0: tensor<1xf32>) -> tensor<1xf32> { + %2 = "tf.Tanh"(%arg0) : (tensor<1xf32>) -> tensor<1xf32> + return %2: tensor<1xf32> + +// CHECK-LABEL: Tanh +// CHECK: %0 = "tfl.tanh"(%arg0) : (tensor<1xf32>) -> tensor<1xf32> +} diff --git a/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td b/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td index bcc00d12b01..0fac8a09f86 100644 --- a/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td +++ b/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td @@ -131,6 +131,7 @@ def : Pat<(TF_SinOp F32Tensor:$arg), (TFL_SinOp $arg)>; def : Pat<(TF_SliceOp $input, $begin, $size), (TFL_SliceOp $input, $begin, $size)>; def : Pat<(TF_SoftmaxOp $arg), (TFL_SoftmaxOp $arg, ConstF32Attr<"1.0">)>; def : Pat<(TF_SqueezeOp $arg, $squeeze_dims), (TFL_SqueezeOp $arg, $squeeze_dims)>; +def : Pat<(TF_TanhOp $arg), (TFL_TanhOp $arg)>; def : Pat<(TF_TransposeOp $arg, $perm), (TFL_TransposeOp $arg, $perm)>; def : Pat<(TF_ZerosLikeOp $arg), (TFL_ZerosLikeOp $arg)>; diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index 876b3f47e6b..05c87af9e11 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -2638,6 +2638,31 @@ retained with length 1. TF_DerivedOperandTypeAttr Tidx = TF_DerivedOperandTypeAttr<1>; } +def TF_TanhOp : TF_Op<"Tanh", [NoSideEffect, SameOperandsAndResultType]> { + let summary = "Computes hyperbolic tangent of `x` element-wise."; + + let description = [{ +Given an input tensor, this function computes hyperbolic tangent of every + element in the tensor. Input range is `[-inf, inf]` and + output range is `[-1,1]`. + + ```python + x = tf.constant([-float("inf"), -5, -0.5, 1, 1.2, 2, 3, float("inf")]) + tf.math.tanh(x) ==> [-1. -0.99990916 -0.46211717 0.7615942 0.8336547 0.9640276 0.9950547 1.] + ``` + }]; + + let arguments = (ins + TF_FpOrComplexTensor:$x + ); + + let results = (outs + TF_FpOrComplexTensor:$y + ); + + TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<0>; +} + def TF_TensorListFromTensorOp : TF_Op<"TensorListFromTensor", [NoSideEffect]> { let summary = [{ Creates a TensorList which, when stacked, has the value of `tensor`. From 00e016ae5d3640a2d19cac9f39539a2549c18fe7 Mon Sep 17 00:00:00 2001 From: Pooya Davoodi Date: Wed, 10 Jul 2019 09:37:58 -0700 Subject: [PATCH 203/332] Fix pylint issues --- .../python/compiler/tensorrt/test/const_broadcast_test.py | 1 + .../python/compiler/tensorrt/test/quantization_mnist_test.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py b/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py index 295c7aaca57..ccbaf9e52fa 100644 --- a/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py +++ b/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py @@ -29,6 +29,7 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase): """Test for Constant broadcasting in TF-TRT.""" def GraphFn(self, x): + """Return the expected graph to convert.""" dtype = x.dtype filt1 = constant_op.constant( 0.3, shape=(3, 3, 2, 1), dtype=dtype, name='filt1') diff --git a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py index 46a49cc3c5f..56994617b90 100644 --- a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py +++ b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py @@ -55,6 +55,7 @@ OUTPUT_NODE_NAME = 'output' class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase): + """Testing usage of quantization ranges inserted in graph.""" def _BuildGraph(self, x): @@ -239,7 +240,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase): if mode == ModeKeys.EVAL: return EstimatorSpec( mode, loss=loss, eval_metric_ops={'accuracy': accuracy}) - elif mode == ModeKeys.TRAIN: + if mode == ModeKeys.TRAIN: optimizer = AdamOptimizer(learning_rate=1e-2) train_op = optimizer.minimize(loss, global_step=get_global_step()) return EstimatorSpec(mode, loss=loss, train_op=train_op) From d20fe0b89893ebcc91b0e0b7bac02b4609b4b401 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 09:39:48 -0700 Subject: [PATCH 204/332] Add LogSoftMax support to NN API delegate PiperOrigin-RevId: 257421456 --- tensorflow/lite/delegates/nnapi/nnapi_delegate.cc | 14 ++++++++++++++ tensorflow/lite/kernels/BUILD | 1 + tensorflow/lite/nnapi/NeuralNetworksTypes.h | 1 + 3 files changed, 16 insertions(+) diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc index f23a9873486..28efbd19a2b 100644 --- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc +++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc @@ -1953,6 +1953,20 @@ class NNAPIDelegateKernel { }; } } break; + case kTfLiteBuiltinLogSoftmax: { + const auto input_type = context->tensors[node->inputs->data[0]].type; + if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI12 && + input_type == kTfLiteFloat32) { + return [](const NNAPIOpMappingArgs& mapping_args) + -> ANeuralNetworksOperationType { + // Scaling and axis are hardcoded to respectively 1 and -1 + // in TFLite. + mapping_args.builder->AddScalarFloat32Operand(1); + mapping_args.builder->AddScalarInt32Operand(-1); + return ANEURALNETWORKS_LOG_SOFTMAX; + }; + } + } break; default: // All other operators are not mapped. return nullptr; diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD index f74b2484de0..0e40f3fa0cd 100644 --- a/tensorflow/lite/kernels/BUILD +++ b/tensorflow/lite/kernels/BUILD @@ -1149,6 +1149,7 @@ cc_test( name = "log_softmax_test", size = "small", srcs = ["log_softmax_test.cc"], + tags = ["tflite_nnapi"], deps = [ ":builtin_ops", ":test_main", diff --git a/tensorflow/lite/nnapi/NeuralNetworksTypes.h b/tensorflow/lite/nnapi/NeuralNetworksTypes.h index 70f6c002df1..c5519b93c6d 100644 --- a/tensorflow/lite/nnapi/NeuralNetworksTypes.h +++ b/tensorflow/lite/nnapi/NeuralNetworksTypes.h @@ -105,6 +105,7 @@ enum { ANEURALNETWORKS_LOGICAL_AND = 61, ANEURALNETWORKS_LOGICAL_NOT = 62, ANEURALNETWORKS_LOGICAL_OR = 63, + ANEURALNETWORKS_LOG_SOFTMAX = 64, ANEURALNETWORKS_MAXIMUM = 65, ANEURALNETWORKS_MINIMUM = 66, ANEURALNETWORKS_NEG = 67, From 1d6ce978a54083c374a8de9e2e1408af40e46968 Mon Sep 17 00:00:00 2001 From: Doe Hyun Yoon Date: Wed, 10 Jul 2019 09:44:37 -0700 Subject: [PATCH 205/332] Automated rollback of commit 9edc8fa0c6dd23474ab275d104acebe7d153014e PiperOrigin-RevId: 257422311 --- .../core/grappler/costs/virtual_scheduler.cc | 21 --- .../core/grappler/costs/virtual_scheduler.h | 1 - .../grappler/costs/virtual_scheduler_test.cc | 164 +++++------------- 3 files changed, 44 insertions(+), 142 deletions(-) diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc index b21e75c7d97..1a38df27dc5 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler.cc +++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc @@ -788,15 +788,6 @@ void VirtualScheduler::AddOutputNodesToReadyQueue( for (auto* output_node : port_num_output_pair.second) { auto& output_state = node_map_[output_node]; - // Skip if this input node has already been seen. - if (IsMerge(*node) && (output_state.input_nodes_seen.find(node) != - output_state.input_nodes_seen.end())) { - LOG(WARNING) << "Output node [ " << output_node->name() - << " ] has alread seen this input node [ " << node->name() - << " -- possibly due to Swith-Merge in previous " - << "nodes. Skip to increment num_inputs_ready."; - continue; - } output_state.num_inputs_ready++; // Execute a node as soon as all its inputs are ready. Merge nodes are // special since they run as soon as one of their inputs becomes @@ -810,18 +801,6 @@ void VirtualScheduler::AddOutputNodesToReadyQueue( } } } - - // Update input_nodes_seen. We update it only for the outputs of Merge node, - // as in other cases, using only counter based checking works. - if (IsMerge(*node)) { - for (const auto& port_num_output_pair : node_state.outputs) { - if (slot >= 0 && port_num_output_pair.first != slot) continue; - for (auto* output_node : port_num_output_pair.second) { - auto& output_state = node_map_[output_node]; - output_state.input_nodes_seen.insert(node); - } - } - } } bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) { diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.h b/tensorflow/core/grappler/costs/virtual_scheduler.h index 55f67e3ef33..b50f61d155b 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler.h +++ b/tensorflow/core/grappler/costs/virtual_scheduler.h @@ -41,7 +41,6 @@ struct NodeState { // List of output nodes (a list of nodes that takes this output port as input) // keyed by port_num. Note that port_num -1 is used for control dependency. std::unordered_map> outputs; - std::unordered_set input_nodes_seen; // Info from GraphProperties. std::vector input_properties; diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc index 588bfce5e90..21ca9ca4fc6 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc +++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc @@ -625,10 +625,10 @@ class TestVirtualScheduler : public VirtualScheduler { public: TestVirtualScheduler(const bool use_static_shapes, const bool use_aggressive_shape_inference, - ReadyNodeManager* ready_node_manager, Cluster* cluster) + Cluster* cluster) : VirtualScheduler( use_static_shapes, use_aggressive_shape_inference, cluster, - ready_node_manager, + &ready_node_manager_, absl::make_unique(cluster->GetDevices())) { enable_mem_usage_tracking(); } @@ -638,6 +638,9 @@ class TestVirtualScheduler : public VirtualScheduler { FRIEND_TEST(VirtualSchedulerTest, ComplexDependency); FRIEND_TEST(VirtualSchedulerTest, Variable); FRIEND_TEST(VirtualSchedulerTest, InterDeviceTransfer); + + protected: + FirstReadyManager ready_node_manager_; }; class VirtualSchedulerTest : public ::testing::Test { @@ -656,8 +659,7 @@ class VirtualSchedulerTest : public ::testing::Test { cluster_ = absl::make_unique(devices); scheduler_ = absl::make_unique( /*use_static_shapes=*/true, - /*use_aggressive_shape_inference=*/true, &first_ready_manager_, - cluster_.get()); + /*use_aggressive_shape_inference=*/true, cluster_.get()); } DeviceProperties GetDummyCPUDevice() { @@ -687,10 +689,12 @@ class VirtualSchedulerTest : public ::testing::Test { auto c0 = ops::Conv2D(s.WithOpName("c0"), x, f, strides, "SAME"); auto c1 = ops::Conv2D(s.WithOpName("c1"), y, f, strides, "SAME"); auto c2 = ops::Conv2D(s.WithOpName("c2"), z, f, strides, "SAME"); + GraphDef def; + TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_conv2d_graph"; + grappler_item_->graph = def; grappler_item_->fetch = {"c0", "c1"}; dependency_["c0"] = {"x", "f"}; @@ -706,11 +710,12 @@ class VirtualSchedulerTest : public ::testing::Test { {kernel_, kernel_, depth_in_, depth_out_}, DT_FLOAT); std::vector strides = {1, 1, 1, 1}; auto y = ops::Conv2D(s.WithOpName("y"), x, f, strides, "SAME"); + GraphDef def; + TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_conv2d_var_graph"; - + grappler_item_->graph = def; grappler_item_->fetch = {"y"}; dependency_["y"] = {"x", "f"}; @@ -735,9 +740,12 @@ class VirtualSchedulerTest : public ::testing::Test { auto abcd = ops::MatMul(s.WithOpName("abcd"), abc, d); auto abcde = ops::MatMul(s.WithOpName("abcde"), abcd, e); - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + GraphDef def; + TF_CHECK_OK(s.ToGraphDef(&def)); + + grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_matmul_sequence_graph"; + grappler_item_->graph = def; grappler_item_->fetch = {"abcde"}; dependency_["ab"] = {"a", "b"}; @@ -755,10 +763,12 @@ class VirtualSchedulerTest : public ::testing::Test { auto w = ops::RandomUniform(s.WithOpName("w"), {10, 10, 10, 10}, DT_FLOAT); OutputList input_tensors = {x, y, z, w}; auto out = ops::AddN(s.WithOpName("out"), input_tensors); + GraphDef def; + TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_addn_graph"; + grappler_item_->graph = def; grappler_item_->fetch = {"out"}; dependency_["out"] = {"x", "y", "z", "w"}; @@ -770,10 +780,12 @@ class VirtualSchedulerTest : public ::testing::Test { auto unnecessary = ops::Placeholder(s.WithOpName("unnecessary"), DT_FLOAT); auto x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT); - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + GraphDef def; + TF_CHECK_OK(s.ToGraphDef(&def)); + grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_extra_placeholders"; + grappler_item_->graph = def; grappler_item_->fetch = {"x"}; // Grappler Item Builder puts all placeholder nodes into the feed @@ -792,62 +804,17 @@ class VirtualSchedulerTest : public ::testing::Test { } auto out = ops::NoOp(s.WithControlDependencies(input_tensors).WithOpName("out")); + GraphDef def; + TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); - + grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_control_dependency_graph"; + grappler_item_->graph = def; grappler_item_->fetch = {"out"}; dependency_["out"] = input_noop_names; } - void CreateGrapplerItemWithAddFromOneTensor() { - Scope s = Scope::NewRootScope().WithDevice(kCPU0); - auto x = tensorflow::ops::RandomUniform( - s.WithOpName("x"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); - - auto y = tensorflow::ops::Add(s.WithOpName("y"), x, x); - Output fetch = ops::Identity(s.WithOpName("fetch"), y); - - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); - - grappler_item_->id = "test_add_from_one_tensor"; - grappler_item_->fetch = {"fetch"}; - - dependency_["fetch"] = {"y"}; - dependency_["y"] = {"x"}; - } - - void CreateGrapplerItemWithSwitchMergeInput() { - // sw = Switch(x, pred) - // a = Add(S:1, b) - // m = Merge(sw:0, a) - // y = Add(m, z) - - Scope s = Scope::NewRootScope().WithDevice(kCPU0); - auto x = ops::RandomUniform( - s.WithOpName("x"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); - auto pred = ops::Const(s.WithOpName("pred"), false, {}); - auto sw = ops::Switch(s.WithOpName("switch"), x, pred); - auto b = ops::RandomUniform( - s.WithOpName("b"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); - auto a = ops::Add(s.WithOpName("a"), sw.output_true, b); - auto m = ops::Merge(s.WithOpName("m"), {sw.output_false, a.z}); - auto z = ops::RandomUniform( - s.WithOpName("z"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); - auto y = ops::Add(s.WithOpName("y"), m.output, z); - - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); - - grappler_item_->id = "test_add_merge_switch"; - grappler_item_->fetch = {"y"}; - - dependency_["y"] = {"m", "z"}; - } - // FusedBN [an op with multiple outputs] with multiple consumers (including // control dependency). void CreateGrapplerItemWithBatchNorm() { @@ -879,10 +846,12 @@ class VirtualSchedulerTest : public ::testing::Test { }; auto z4 = ops::NoOp(s.WithControlDependencies(batch_var).WithOpName("z4")); - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + GraphDef def; + TF_CHECK_OK(s.ToGraphDef(&def)); + grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_complex_dependency_graph"; + grappler_item_->graph = def; grappler_item_->fetch = {"z1", "z2", "z3", "z4"}; dependency_["bn"] = {"x", "scale", "offset", "mean", "var"}; @@ -1006,8 +975,7 @@ versions { } )EOF"; - grappler_item_ = absl::make_unique(); - + grappler_item_.reset(new GrapplerItem); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -1064,7 +1032,7 @@ versions { } )EOF"; - grappler_item_ = absl::make_unique(); + grappler_item_.reset(new GrapplerItem); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -1460,7 +1428,7 @@ versions { } )EOF"; - grappler_item_ = absl::make_unique(); + grappler_item_.reset(new GrapplerItem); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -2127,7 +2095,7 @@ versions { producer: 27 })EOF"; - grappler_item_ = absl::make_unique(); + grappler_item_.reset(new GrapplerItem); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -2168,9 +2136,12 @@ versions { .WithControlDependencies(y) .WithDevice(kCPU1)); - grappler_item_ = absl::make_unique(); - TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + GraphDef def; + TF_CHECK_OK(s.ToGraphDef(&def)); + + grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_conv2d_graph"; + grappler_item_->graph = def; grappler_item_->fetch = {"y1", "y2", "batch_mean1", "batch_var1", "control_dep"}; @@ -2309,8 +2280,6 @@ versions { // cluster_ and scheduler_ are initialized in the c'tor. std::unique_ptr cluster_; std::unique_ptr scheduler_; - FirstReadyManager first_ready_manager_; - CompositeNodeManager composite_node_manager_; // grappler_item_ will be initialized differently for each test case. std::unique_ptr grappler_item_; @@ -2964,51 +2933,6 @@ TEST_F(VirtualSchedulerTest, GraphWihtOnlyRecv) { EXPECT_GT(ops_executed.count("Recv"), 0); } -TEST_F(VirtualSchedulerTest, AddMergeSwitch) { - // Override scheduler_ with CompositeNodeNamager. - scheduler_ = absl::make_unique( - /*use_static_shapes=*/true, - /*use_aggressive_shape_inference=*/true, &composite_node_manager_, - cluster_.get()); - CreateGrapplerItemWithSwitchMergeInput(); - InitScheduler(); - - // pred --+ z --+ - // | | - // V V - // x -> Switch --------> Merge ---> Add --> y - // | ^ - // | | - // +-----> Add -----+ - // ^ - // | - // b --------------+ - - // Run the scheduler. The current VirtualScheduler, w/o annotation, triggers - // both outputs of Switch; then Merge (as long as one input is ready, it's z - // is ready, if we just use num_inputs_ready counter, the final Add becomes - // ready. possible to skipt scheduling z. (Need to use CompositeNodeManager - // to test this case). - auto ops_executed = RunScheduler(""); - - EXPECT_GT(ops_executed.count("z"), 0); -} - -TEST_F(VirtualSchedulerTest, AddFromOneTensor) { - CreateGrapplerItemWithAddFromOneTensor(); - InitScheduler(); - - // x -+----> Add --> y - // | ^ - // | | - // +-------+ - - // Run the scheduler. - auto ops_executed = RunScheduler(""); - EXPECT_GT(ops_executed.count("y"), 0); - EXPECT_GT(ops_executed.count("x"), 0); -} - } // namespace } // end namespace grappler } // end namespace tensorflow From 5982ad33eb2d492c5102dc728e43f139bccef52a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 10:07:49 -0700 Subject: [PATCH 206/332] NFC: Rename Module to ModuleOp. Module is a legacy name that only exists as a typedef of ModuleOp. PiperOrigin-RevId: 257427248 --- .../compiler/mlir/lite/emit_error_reporter.h | 4 ++-- .../compiler/mlir/lite/flatbuffer_translate.cc | 16 ++++++++-------- .../compiler/mlir/lite/flatbuffer_translate.h | 2 +- .../compiler/mlir/lite/tf_tfl_translate.cc | 4 ++-- .../compiler/mlir/lite/tf_to_tfl_flatbuffer.cc | 6 +++--- .../compiler/mlir/lite/tf_to_tfl_flatbuffer.h | 4 ++-- tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc | 4 ++-- .../transforms/functional_control_flow_to_cfg.cc | 4 ++-- .../transforms/tf_graph_optimization_pass.cc | 2 +- .../mlir/tensorflow/translate/export_graphdef.cc | 8 ++++---- .../mlir/tensorflow/translate/export_graphdef.h | 4 ++-- .../mlir/tensorflow/translate/import_graphdef.cc | 6 +++--- .../translate/tf_mlir_translate_registration.cc | 2 +- .../translate/translate_tf_dialect_op.cc | 5 +++-- 14 files changed, 36 insertions(+), 35 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/emit_error_reporter.h b/tensorflow/compiler/mlir/lite/emit_error_reporter.h index e32fa8d1b4e..40e89c5dec8 100644 --- a/tensorflow/compiler/mlir/lite/emit_error_reporter.h +++ b/tensorflow/compiler/mlir/lite/emit_error_reporter.h @@ -26,11 +26,11 @@ namespace tflite { // Error reporter that reports errors via the module's emitError. class EmitErrorReporter : public ErrorReporter { public: - explicit EmitErrorReporter(mlir::Module module) : module_(module) {} + explicit EmitErrorReporter(mlir::ModuleOp module) : module_(module) {} int Report(const char* format, va_list args) override; private: - mlir::Module module_; + mlir::ModuleOp module_; }; } // namespace tflite diff --git a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc index ba4c88f865a..34ee8814423 100644 --- a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc +++ b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc @@ -79,7 +79,7 @@ using mlir::Dialect; using mlir::ElementsAttr; using mlir::FuncOp; using mlir::MLIRContext; -using mlir::Module; +using mlir::ModuleOp; using mlir::NoneType; using mlir::openOutputFile; using mlir::Operation; @@ -243,7 +243,7 @@ static bool HasValidTFLiteType(Value* value, T& error_handler) { // TODO(hinsu): Now that translation is done by making a single pass over the // MLIR module, consider inlining these validation checks at the place where // these invariants are assumed instead of checking upfront. -static bool IsValidTFLiteMlirModule(Module module) { +static bool IsValidTFLiteMlirModule(ModuleOp module) { MLIRContext* context = module.getContext(); // Verify that module has a function named main. @@ -323,14 +323,14 @@ class Translator { // Translates the given MLIR module into TFLite FlatBuffer format and returns // the serialized output. Returns llvm::None on unsupported, invalid inputs or // internal error. - static Optional Translate(Module module, + static Optional Translate(ModuleOp module, bool emit_builtin_tflite_ops, bool emit_select_tf_ops, bool emit_custom_ops); private: enum class OpType : char { kTfliteBuiltin, kSelectTf, kCustomOp }; - explicit Translator(Module module, bool emit_builtin_tflite_ops, + explicit Translator(ModuleOp module, bool emit_builtin_tflite_ops, bool emit_select_tf_ops, bool emit_custom_ops) : module_(module), builder_(kInitialBufferSize) { // The first buffer must be empty according to the schema definition. @@ -403,7 +403,7 @@ class Translator { // Returns a unique name starting with a given prefix. std::string UniqueName(llvm::StringRef prefix); - Module module_; + ModuleOp module_; flatbuffers::FlatBufferBuilder builder_; BufferOffset empty_buffer_; @@ -926,7 +926,7 @@ Optional> Translator::BuildSubGraph(FuncOp fn) { /*name=*/builder_.CreateString(fn.getName().str())); } -Optional Translator::Translate(Module module, +Optional Translator::Translate(ModuleOp module, bool emit_builtin_tflite_ops, bool emit_select_tf_ops, bool emit_custom_ops) { @@ -991,7 +991,7 @@ Optional Translator::TranslateInternal() { // * Ops with variable tensors // bool tflite::MlirToFlatBufferTranslateFunction( - Module module, std::string* serialized_flatbuffer, + ModuleOp module, std::string* serialized_flatbuffer, bool emit_builtin_tflite_ops, bool emit_select_tf_ops, bool emit_custom_ops) { auto maybe_translated = Translator::Translate( @@ -1002,7 +1002,7 @@ bool tflite::MlirToFlatBufferTranslateFunction( } static mlir::LogicalResult MlirToFlatBufferFileTranslateFunction( - Module module, llvm::StringRef filename) { + ModuleOp module, llvm::StringRef filename) { std::string serialized_flatbuffer; if (tflite::MlirToFlatBufferTranslateFunction( module, &serialized_flatbuffer, emit_builtin_tflite_ops, diff --git a/tensorflow/compiler/mlir/lite/flatbuffer_translate.h b/tensorflow/compiler/mlir/lite/flatbuffer_translate.h index 7a0c60e27b1..820b2697e43 100644 --- a/tensorflow/compiler/mlir/lite/flatbuffer_translate.h +++ b/tensorflow/compiler/mlir/lite/flatbuffer_translate.h @@ -32,7 +32,7 @@ namespace tflite { // Translates the given MLIR `module` into a FlatBuffer and stores the // serialized flatbuffer into the string. -bool MlirToFlatBufferTranslateFunction(mlir::Module module, +bool MlirToFlatBufferTranslateFunction(mlir::ModuleOp module, std::string *serialized_flatbuffer, bool emit_builtin_tflite_ops, bool emit_select_tf_ops, diff --git a/tensorflow/compiler/mlir/lite/tf_tfl_translate.cc b/tensorflow/compiler/mlir/lite/tf_tfl_translate.cc index feeb91d3a04..5e8216dbc3b 100644 --- a/tensorflow/compiler/mlir/lite/tf_tfl_translate.cc +++ b/tensorflow/compiler/mlir/lite/tf_tfl_translate.cc @@ -33,7 +33,7 @@ limitations under the License. #include "tensorflow/stream_executor/lib/statusor.h" using mlir::MLIRContext; -using mlir::Module; +using mlir::ModuleOp; using stream_executor::port::StatusOr; using tensorflow::Status; @@ -47,7 +47,7 @@ static llvm::cl::opt print_function_result_mapping( enum TranslationStatus { kTrSuccess, kTrFailure }; static int PrintFunctionResultMapping(const std::string &result, - Module module) { + ModuleOp module) { // Build model from the resultant string to extract the return values from // their source of truth. auto model = diff --git a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc index 02e753e13d9..e13ca089a9d 100644 --- a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc +++ b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc @@ -34,7 +34,7 @@ limitations under the License. namespace tensorflow { using mlir::MLIRContext; -using mlir::Module; +using mlir::ModuleOp; using mlir::OwningModuleRef; using stream_executor::port::StatusOr; @@ -87,7 +87,7 @@ StatusOr LoadFromGraphdefOrMlirSource( context); } -bool ShouldRunQuantizePasses(mlir::Module m) { +bool ShouldRunQuantizePasses(mlir::ModuleOp m) { if (mlir::FuncOp main_fn = m.getNamedFunction("main")) { return main_fn.getAttrOfType("tf.quantize") != mlir::Attribute(); @@ -138,7 +138,7 @@ void AddTFToTFLConversionPasses(bool emit_builtin_tflite_ops, bool run_quantize, } Status ConvertTFControlFlowToTFLOrFlatbuffer( - mlir::Module module, bool export_to_mlir, bool emit_builtin_tflite_ops, + mlir::ModuleOp module, bool export_to_mlir, bool emit_builtin_tflite_ops, bool emit_select_tf_ops, bool emit_custom_ops, bool emit_quant_adaptor_ops, bool lower_tensor_list_ops, std::string *result) { mlir::StatusScopedDiagnosticHandler statusHandler(module.getContext(), diff --git a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.h b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.h index e4e9ce3ba00..68ab674872f 100644 --- a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.h +++ b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.h @@ -46,7 +46,7 @@ LoadFromGraphdefOrMlirSource( // attribute "tf.quantize" by the importer module. // TODO(fengliuai): switch to the cmd flag once the flags are moved to this // file with main method. -bool ShouldRunQuantizePasses(mlir::Module m); +bool ShouldRunQuantizePasses(mlir::ModuleOp m); // Add the MLIR passes that convert TF control flow dialect to TF Lite dialect // to a MLIR `pass_manager`. These passes first raise the control flow in the TF @@ -69,7 +69,7 @@ void AddTFToTFLConversionPasses(bool emit_builtin_tflite_ops, bool run_quantize, // main function, Quantization is applied. If `export_to_mlir` is true, the // result is exported in MLIR text format, otherwise exported in flat buffer. Status ConvertTFControlFlowToTFLOrFlatbuffer( - mlir::Module module, bool export_to_mlir, bool emit_builtin_tflite_ops, + mlir::ModuleOp module, bool export_to_mlir, bool emit_builtin_tflite_ops, bool emit_select_tf_ops, bool emit_custom_ops, bool emit_quant_adaptor_ops, bool lower_tensor_list_ops, std::string* result); } // namespace tensorflow diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc index c602e43ef16..cc32340c681 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc @@ -281,7 +281,7 @@ LogicalResult IfOp::verify() { auto elseAttr = getAttrOfType("else_branch"); if (!elseAttr) return emitOpError("requires else_branch attribute"); - auto module = getParentOfType(); + auto module = getParentOfType(); auto thenFn = module.getNamedFunction(thenAttr.getValue()); if (!thenFn) return emitOpError("then_branch refers to an undefined function : ") @@ -730,7 +730,7 @@ LogicalResult WhileOp::verify() { auto condAttr = getAttrOfType("cond"); if (!condAttr) return emitOpError("requires cond attribute"); - auto module = getParentOfType(); + auto module = getParentOfType(); auto condFn = module.getNamedFunction(condAttr.getValue()); auto condFuncType = condFn.getType(); diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc b/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc index 103e1442863..5be13da57b0 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc @@ -153,7 +153,7 @@ static LogicalResult LowerIfOp(IfOp op) { Value* cond_i1 = LowerCondition(loc, op.getCondition(), &builder); if (!cond_i1) return failure(); - auto module = op_inst->getParentOfType(); + auto module = op_inst->getParentOfType(); auto then_fn = module.getNamedFunction(op.getThen()); auto else_fn = module.getNamedFunction(op.getElse()); @@ -210,7 +210,7 @@ static LogicalResult LowerWhileOp(WhileOp op) { OpBuilder builder(op_inst); - auto module = op_inst->getParentOfType(); + auto module = op_inst->getParentOfType(); auto cond_fn = module.getNamedFunction(op.getCond()); auto body_fn = module.getNamedFunction(op.getBody()); diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.cc b/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.cc index c835ea64158..60f7ed35a0b 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/tf_graph_optimization_pass.cc @@ -89,7 +89,7 @@ std::vector GraphOptPass::FindPassIds() { } void GraphOptPass::runOnModule() { - mlir::Module module_in = getModule(); + mlir::ModuleOp module_in = getModule(); mlir::MLIRContext& ctx = getContext(); // Convert MLIR to Graph diff --git a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc index dc3108c3f77..f2eaa308deb 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc @@ -88,7 +88,7 @@ class Exporter { // one entry function, which is identified by name "main". This entry function // is converted to the base of the graph graph. The rest of the functions are // converted to the library functions in that graph. - static Status Convert(mlir::Module module, const ExporterConfigs& configs, + static Status Convert(mlir::ModuleOp module, const ExporterConfigs& configs, std::unique_ptr* graph, FunctionLibraryDefinition* flib_def); @@ -531,7 +531,7 @@ Status Exporter::ConvertLibFunction(const ExporterConfigs& configs, return Status::OK(); } -Status Exporter::Convert(mlir::Module module, const ExporterConfigs& configs, +Status Exporter::Convert(mlir::ModuleOp module, const ExporterConfigs& configs, std::unique_ptr* graph, FunctionLibraryDefinition* flib_def) { mlir::Identifier entry_func_id = @@ -567,14 +567,14 @@ Status Exporter::Convert(mlir::Module module, const ExporterConfigs& configs, } } // namespace -Status ConvertMlirToGraph(mlir::Module module, const ExporterConfigs& confs, +Status ConvertMlirToGraph(mlir::ModuleOp module, const ExporterConfigs& confs, std::unique_ptr* graph, FunctionLibraryDefinition* flib_def) { return Exporter::Convert(module, confs, graph, flib_def); } StatusOr> ConvertMlirToGraphdef( - mlir::Module module, const ExporterConfigs& confs) { + mlir::ModuleOp module, const ExporterConfigs& confs) { FunctionLibraryDefinition flib_def(OpRegistry::Global(), FunctionDefLibrary()); auto graph = absl::make_unique(flib_def); diff --git a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h index bfdceac9de0..93061a95239 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h +++ b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.h @@ -32,13 +32,13 @@ using stream_executor::port::StatusOr; // Given an MLIR module, returns a GraphDef. StatusOr> ConvertMlirToGraphdef( - mlir::Module module, const ExporterConfigs& configs); + mlir::ModuleOp module, const ExporterConfigs& configs); // Converts an MLIR module to TensorFlow graph and FunctionLibraryDefinition. // The "main" function of the module is stored in the graph and the rest of // functions are stored in the library. stream_executor::port::Status ConvertMlirToGraph( - mlir::Module module, const ExporterConfigs& confs, + mlir::ModuleOp module, const ExporterConfigs& confs, std::unique_ptr* graph, FunctionLibraryDefinition* flib_def); } // namespace tensorflow diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc index 774d18c7911..92cbfc70f62 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc @@ -86,7 +86,7 @@ class Importer { explicit Importer( const FunctionLibraryDefinition& flib, const GraphDebugInfo& debug_info, - const NodeSpecs& specs, mlir::Module module, + const NodeSpecs& specs, mlir::ModuleOp module, std::unordered_map* tf_name_to_mlir_name) : module_(module), context_(module.getContext()), @@ -262,7 +262,7 @@ class Importer { using NodeValueMap = absl::flat_hash_map; std::unique_ptr builder_; - mlir::Module module_; + mlir::ModuleOp module_; mlir::MLIRContext* context_; std::unordered_map* tf_name_to_mlir_name_; const FunctionLibraryDefinition& graph_flib_; @@ -1292,7 +1292,7 @@ StatusOr Importer::Convert( const GraphDebugInfo& debug_info, const FunctionLibraryDefinition& flib_def, const NodeSpecs& specs) { mlir::OwningModuleRef module = - mlir::Module::create(mlir::UnknownLoc::get(context)); + mlir::ModuleOp::create(mlir::UnknownLoc::get(context)); std::unordered_map tf_name_to_mlir_name; Importer importer(flib_def, debug_info, specs, module.get(), &tf_name_to_mlir_name); diff --git a/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate_registration.cc b/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate_registration.cc index 59d42fd1397..7d7632d7e82 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate_registration.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate_registration.cc @@ -63,7 +63,7 @@ static TranslateToMLIRRegistration GraphdefToSplattedMlirTranslate( "graphdef-to-splatted-mlir", GraphdefToSplattedMlirTranslateFunction); static LogicalResult MlirToGraphdefTranslateFunction( - Module module, llvm::StringRef output_filename) { + ModuleOp module, llvm::StringRef output_filename) { if (!module) return failure(); std::error_code error; diff --git a/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc b/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc index aa53ed9481f..5d33138c626 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc @@ -22,7 +22,7 @@ limitations under the License. #include "tensorflow/compiler/mlir/tensorflow/translate/export_tf_dialect_op.h" namespace mlir { -static mlir::Operation* ExtractOnlyOp(mlir::Module module) { +static mlir::Operation* ExtractOnlyOp(mlir::ModuleOp module) { mlir::FuncOp fn = module.getNamedFunction("main"); if (!fn) return nullptr; @@ -38,7 +38,8 @@ static mlir::Operation* ExtractOnlyOp(mlir::Module module) { return &block.front(); } -static LogicalResult MlirToTfNodeDef(Module module, llvm::StringRef filename) { +static LogicalResult MlirToTfNodeDef(ModuleOp module, + llvm::StringRef filename) { auto* context = module.getContext(); auto file = openOutputFile(filename); From f2b53e96cb03f6497889e1a468bcc11a5f813dc1 Mon Sep 17 00:00:00 2001 From: Haoliang Zhang Date: Wed, 10 Jul 2019 10:22:24 -0700 Subject: [PATCH 207/332] Add missing description (auto-generated) in TF MLIR ops specs. PiperOrigin-RevId: 257430295 --- .../mlir/tensorflow/ir/tf_generated_ops.td | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index 05c87af9e11..6b7be4435b4 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -79,6 +79,12 @@ def TF_AddNOp : TF_Op<"AddN", [Commutative, NoSideEffect]> { let summary = "Add all input tensors element wise."; let description = [{ +Inputs must be of same size and shape. + + ```python + x = [9, 7, 10] + tf.math.add_n(x) ==> 26 + ``` }]; let arguments = (ins @@ -467,6 +473,15 @@ def TF_CosOp : TF_Op<"Cos", [NoSideEffect, SameOperandsAndResultType]> { let summary = "Computes cos of x element-wise."; let description = [{ +Given an input tensor, this function computes cosine of every + element in the tensor. Input range is `(-inf, inf)` and + output range is `[-1,1]`. If input lies outside the boundary, `nan` + is returned. + + ```python + x = tf.constant([-float("inf"), -9, -0.5, 1, 1.2, 200, 10000, float("inf")]) + tf.math.cos(x) ==> [nan -0.91113025 0.87758255 0.5403023 0.36235774 0.48718765 -0.95215535 nan] + ``` }]; let arguments = (ins @@ -1027,6 +1042,43 @@ Invert (flip) each bit of supported types; for example, type `uint8` value 01010 let description = [{ Flip each bit of supported types. For example, type `int8` (decimal 2) binary 00000010 becomes (decimal -3) binary 11111101. This operation is performed on each element of the tensor argument `x`. + +Example: +```python +import tensorflow as tf +from tensorflow.python.ops import bitwise_ops + +# flip 2 (00000010) to -3 (11111101) +tf.assert_equal(-3, bitwise_ops.invert(2)) + +dtype_list = [dtypes.int8, dtypes.int16, dtypes.int32, dtypes.int64, + dtypes.uint8, dtypes.uint16, dtypes.uint32, dtypes.uint64] + +inputs = [0, 5, 3, 14] +for dtype in dtype_list: + # Because of issues with negative numbers, let's test this indirectly. + # 1. invert(a) and a = 0 + # 2. invert(a) or a = invert(0) + input_tensor = tf.constant([0, 5, 3, 14], dtype=dtype) + not_a_and_a, not_a_or_a, not_0 = [bitwise_ops.bitwise_and( + input_tensor, bitwise_ops.invert(input_tensor)), + bitwise_ops.bitwise_or( + input_tensor, bitwise_ops.invert(input_tensor)), + bitwise_ops.invert( + tf.constant(0, dtype=dtype))] + + expected = tf.constant([0, 0, 0, 0], dtype=tf.float32) + tf.assert_equal(tf.cast(not_a_and_a, tf.float32), expected) + + expected = tf.cast([not_0] * 4, tf.float32) + tf.assert_equal(tf.cast(not_a_or_a, tf.float32), expected) + + # For unsigned dtypes let's also check the result directly. + if dtype.is_unsigned: + inverted = bitwise_ops.invert(input_tensor) + expected = tf.constant([dtype.max - x for x in inputs], dtype=tf.float32) + tf.assert_equal(tf.cast(inverted, tf.float32), tf.cast(expected, tf.float32)) +``` }]; let arguments = (ins @@ -1394,7 +1446,6 @@ pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2] TF_DerivedOperandTypeAttr Tpaddings = TF_DerivedOperandTypeAttr<1>; } - def TF_MulOp : TF_Op<"Mul", [Broadcastable, Commutative, NoSideEffect]>, WithBroadcastableBinOpBuilder { let summary = "Returns x * y element-wise."; @@ -2241,9 +2292,17 @@ Specifically, `y = 1 / (1 + exp(-x))`. } def TF_SinOp : TF_Op<"Sin", [NoSideEffect, SameOperandsAndResultType]> { - let summary = "Computes sin of x element-wise."; + let summary = "Computes sine of x element-wise."; let description = [{ +Given an input tensor, this function computes sine of every + element in the tensor. Input range is `(-inf, inf)` and + output range is `[-1,1]`. + + ```python + x = tf.constant([-float("inf"), -9, -0.5, 1, 1.2, 200, 10, float("inf")]) + tf.math.sin(x) ==> [nan -0.4121185 -0.47942555 0.84147096 0.9320391 -0.87329733 -0.54402107 nan] + ``` }]; let arguments = (ins From 84c176726febd6f0b1eaae5b165af8b6a983b2f8 Mon Sep 17 00:00:00 2001 From: Sachin Joglekar Date: Wed, 10 Jul 2019 10:28:11 -0700 Subject: [PATCH 208/332] Adds support for cast in MLIR converter. Also adds zip test for cast. PiperOrigin-RevId: 257431619 --- tensorflow/compiler/mlir/lite/ir/tfl_ops.td | 19 +++++++++++++ .../compiler/mlir/lite/tests/legalize-tf.mlir | 8 ++++++ .../mlir/lite/transforms/legalize_patterns.td | 2 ++ tensorflow/lite/build_def.bzl | 1 + .../lite/testing/generate_examples_lib.py | 27 +++++++++++++++++++ 5 files changed, 57 insertions(+) diff --git a/tensorflow/compiler/mlir/lite/ir/tfl_ops.td b/tensorflow/compiler/mlir/lite/ir/tfl_ops.td index e2366d8cd80..b9972cebdce 100644 --- a/tensorflow/compiler/mlir/lite/ir/tfl_ops.td +++ b/tensorflow/compiler/mlir/lite/ir/tfl_ops.td @@ -1965,6 +1965,25 @@ def TFL_StridedSliceOp: TFL_Op<"strided_slice", let hasOptions = 1; } +def TFL_CastOp : TFL_Op<"cast", [NoSideEffect, SameOperandsAndResultShape]> { + let summary = "Cast operator"; + + let description = [{ + Casts input from input type to output type. + }]; + + // TODO(b/135538711): Add complex types here. + let arguments = (ins + TensorOf<[F32, I1, I32, I64]>:$input + ); + + let results = (outs TensorOf<[F32, I1, I32, I64]>:$output); + + // TFLite's cast op does not utilize CastOptions, instead derives types + // from the TfLiteTensors. + let hasOptions = 0; +} + def TFL_MirrorPadOp: TFL_Op<"mirror_pad", [ NoSideEffect, TFL_OperandHasRank<1, 2>]> { diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir index f771e0b7504..6b29869dde5 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir @@ -857,3 +857,11 @@ func @Tanh(%arg0: tensor<1xf32>) -> tensor<1xf32> { // CHECK-LABEL: Tanh // CHECK: %0 = "tfl.tanh"(%arg0) : (tensor<1xf32>) -> tensor<1xf32> } + +func @cast(%arg0: tensor<1x2x2x5xi32>) -> tensor<1x2x2x5xf32> { + %0 = "tf.Cast"(%arg0) : (tensor<1x2x2x5xi32>) -> tensor<1x2x2x5xf32> + return %0 : tensor<1x2x2x5xf32> + + // CHECK-LABEL: cast + // CHECK: "tfl.cast"(%arg0) : (tensor<1x2x2x5xi32>) -> tensor<1x2x2x5xf32> +} diff --git a/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td b/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td index 0fac8a09f86..d4866a4740e 100644 --- a/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td +++ b/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td @@ -235,6 +235,8 @@ def : Pat<(TF_MaxOp $arg0, $arg1, BoolAttr:$arg2), (TFL_ReduceMaxOp $arg0, $arg1 def : Pat<(TF_ProdOp $arg0, $arg1, BoolAttr:$arg2), (TFL_ReduceProdOp $arg0, $arg1, $arg2)>; +def : Pat<(TF_CastOp $arg0, BoolAttr:$arg1), (TFL_CastOp $arg0)>; + def : Pat<(TF_BatchToSpaceNDOp $input, $block_shape, $crops), (TFL_BatchToSpaceNdOp $input, $block_shape, $crops)>; def : Pat<(TF_SpaceToBatchNDOp $input, $block_shape, $paddings), (TFL_SpaceToBatchNdOp $input, $block_shape, $paddings)>; diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl index 9bdf0547d45..3c9337121f6 100644 --- a/tensorflow/lite/build_def.bzl +++ b/tensorflow/lite/build_def.bzl @@ -235,6 +235,7 @@ def generated_test_models(): "arg_min_max", "avg_pool", "batch_to_space_nd", + "cast", "ceil", "concat", "constant", diff --git a/tensorflow/lite/testing/generate_examples_lib.py b/tensorflow/lite/testing/generate_examples_lib.py index 431a9aa541b..31c8f94a075 100644 --- a/tensorflow/lite/testing/generate_examples_lib.py +++ b/tensorflow/lite/testing/generate_examples_lib.py @@ -3856,6 +3856,33 @@ def make_zeros_like_tests(options): make_zip_of_tests(options, test_parameters, build_graph, build_inputs) +@register_make_test_function() +def make_cast_tests(options): + """Generate examples for cast.""" + test_parameters = [{ + "input_dtype": [tf.int32], + "output_dtype": [tf.float32], + "input_shape": [[], [1], [1, 2], [5, 6, 7, 8], [3, 4, 5, 6]], + }] + + def build_graph(parameters): + """Build the cast testing graph.""" + input_value = tf.placeholder( + dtype=parameters["input_dtype"], + name="input", + shape=parameters["input_shape"]) + out = tf.cast(input_value, parameters["output_dtype"]) + return [input_value], [out] + + def build_inputs(parameters, sess, inputs, outputs): + input_value = create_tensor_data(parameters["input_dtype"], + parameters["input_shape"]) + return [input_value], sess.run( + outputs, feed_dict=dict(zip(inputs, [input_value]))) + + make_zip_of_tests(options, test_parameters, build_graph, build_inputs) + + def _make_elementwise_tests(op): """Make a set of tests to do element-wise operations.""" From 4b8b69e81ea701c9c554e3176ca26daea4f24fe6 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Wed, 10 Jul 2019 11:07:52 -0700 Subject: [PATCH 209/332] Remove the use of buffer liveness. Less files and simper codebase! PiperOrigin-RevId: 257441122 --- tensorflow/compiler/xla/service/BUILD | 48 - .../compiler/xla/service/buffer_assignment.cc | 23 +- .../compiler/xla/service/buffer_assignment.h | 10 +- .../compiler/xla/service/buffer_liveness.cc | 179 ---- .../compiler/xla/service/buffer_liveness.h | 114 --- .../xla/service/buffer_liveness_test.cc | 934 ------------------ .../compiler/xla/service/copy_insertion.h | 2 +- tensorflow/compiler/xla/service/cpu/BUILD | 1 - .../compiler/xla/service/cpu/cpu_compiler.cc | 1 - tensorflow/compiler/xla/service/gpu/BUILD | 1 - .../xla/service/gpu/nvptx_compiler.cc | 2 - .../xla/service/hlo_rematerialization.h | 1 - .../xla/service/reduce_precision_insertion.h | 1 - 13 files changed, 15 insertions(+), 1302 deletions(-) delete mode 100644 tensorflow/compiler/xla/service/buffer_liveness.cc delete mode 100644 tensorflow/compiler/xla/service/buffer_liveness.h delete mode 100644 tensorflow/compiler/xla/service/buffer_liveness_test.cc diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 880ea14fed5..ba1f43a6451 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -1096,50 +1096,6 @@ tf_cc_test( ], ) -cc_library( - name = "buffer_liveness", - srcs = [ - "buffer_liveness.cc", - ], - hdrs = [ - "buffer_liveness.h", - ], - deps = [ - ":hlo", - ":hlo_ordering", - ":logical_buffer", - ":tuple_points_to_analysis", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla:status_macros", - "//tensorflow/compiler/xla:statusor", - "//tensorflow/compiler/xla:types", - "//tensorflow/compiler/xla:util", - "//tensorflow/core:lib", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - ], -) - -tf_cc_test( - name = "buffer_liveness_test", - srcs = ["buffer_liveness_test.cc"], - deps = [ - ":buffer_liveness", - ":hlo", - ":hlo_dataflow_analysis", - ":hlo_parser", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla:types", - "//tensorflow/compiler/xla:util", - "//tensorflow/compiler/xla:xla_data_proto", - "//tensorflow/compiler/xla/tests:hlo_test_base", - "//tensorflow/compiler/xla/tests:xla_internal_test_main", - "//tensorflow/core:test", - "@com_google_absl//absl/memory", - ], -) - cc_library( name = "buffer_assignment", srcs = [ @@ -1149,7 +1105,6 @@ cc_library( "buffer_assignment.h", ], deps = [ - ":buffer_liveness", ":buffer_value_containers", ":heap_simulator", ":hlo", @@ -2786,7 +2741,6 @@ cc_library( srcs = ["copy_insertion.cc"], hdrs = ["copy_insertion.h"], deps = [ - ":buffer_liveness", ":dump", ":hlo", ":hlo_alias_analysis", @@ -2908,7 +2862,6 @@ cc_library( srcs = ["hlo_rematerialization.cc"], hdrs = ["hlo_rematerialization.h"], deps = [ - ":buffer_liveness", ":buffer_value", ":call_graph", ":flatten_call_graph", @@ -3627,7 +3580,6 @@ cc_library( srcs = ["reduce_precision_insertion.cc"], hdrs = ["reduce_precision_insertion.h"], deps = [ - ":buffer_liveness", ":hlo", ":hlo_pass", ":hlo_pass_pipeline", diff --git a/tensorflow/compiler/xla/service/buffer_assignment.cc b/tensorflow/compiler/xla/service/buffer_assignment.cc index 1e353c11445..3ae7235d887 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment.cc @@ -656,7 +656,7 @@ Status BufferAssignment::ComputeSummaryStats() { for (const auto& computation : module_->computations()) { if (!computation->IsFusionComputation()) { const HloInstructionSequence* sequence = - liveness_->hlo_ordering().SequentialOrder(*computation); + hlo_ordering().SequentialOrder(*computation); if (sequence == nullptr) { schedule_complete = false; } else { @@ -833,7 +833,7 @@ bool BufferAssigner::MaybeAssignBuffer(BufferAllocation* allocation, const HloValue& assigned_buffer = *CHECK_NOTNULL(dynamic_cast(buffer_offset_size.first)); for (const HloValue* new_value : hlo_buffer.values()) { - if (assignment->liveness().hlo_ordering().MayInterfere( + if (assignment->hlo_ordering().MayInterfere( assigned_buffer, *new_value, assignment->dataflow_analysis())) { VLOG(4) << "Can't assign: assignee " << assigned_buffer << " may interfere with " << new_value; @@ -917,7 +917,7 @@ Status BufferAssigner::MergeInplaceOpBuffers(BufferAssignment* assignment) { for (const HloValue* instruction_value : instruction_buffer.values()) { for (const HloValue* operand_value : operand_buffer.values()) { - if (assignment->liveness().hlo_ordering().MayInterfere( + if (assignment->hlo_ordering().MayInterfere( *instruction_value, *operand_value, assignment->dataflow_analysis())) { interfere = true; @@ -1047,8 +1047,7 @@ Status BufferAssigner::AssignSingleHloBuffer( for (const HloValue* hlo_value : hlo_buffer->values()) { HloComputation* computation = hlo_value->instruction()->parent(); const bool has_sequential_order = - assignment->liveness().hlo_ordering().SequentialOrder(*computation) != - nullptr; + assignment->hlo_ordering().SequentialOrder(*computation) != nullptr; all_computations_have_sequential_order &= has_sequential_order; } @@ -1125,10 +1124,9 @@ Status BufferAssigner::AssignBuffersForComputations( } } - const BufferLiveness& liveness = assignment->liveness(); for (const HloComputation* computation : computations) { const bool has_sequential_order = - liveness.hlo_ordering().SequentialOrder(*computation) != nullptr; + assignment->hlo_ordering().SequentialOrder(*computation) != nullptr; if (has_sequential_order && buffers_to_assign_sequentially != nullptr) { // Every sequential computation must get an entry in the // buffers_to_assign_sequentially map, even if we end up with an empty @@ -1197,7 +1195,7 @@ Status BufferAssigner::AssignBuffersWithSequentialOrdering( // Run the sequence of instructions through the heap simulator. The // heuristic that seems to give the best results is lazy-best-fit, with all // runs of alloc / free calls sorted in decreasing size order. - const HloOrdering& hlo_ordering = assignment->liveness().hlo_ordering(); + const HloOrdering& hlo_ordering = assignment->hlo_ordering(); // Returns a heap algorithm that chooses the best result from several // algorithms. @@ -1392,9 +1390,6 @@ StatusOr> BufferAssigner::CreateAssignment( BufferValue::SizeFunction buffer_size, LogicalBuffer::AlignmentFunction color_alignment, HloDataflowAnalysis::CanShareBuffer can_share_buffer) { - TF_ASSIGN_OR_RETURN(std::unique_ptr liveness, - BufferLiveness::Run(module, std::move(hlo_ordering))); - TF_ASSIGN_OR_RETURN(std::unique_ptr alias_analysis, HloAliasAnalysis::Run(module, can_share_buffer)); @@ -1408,11 +1403,11 @@ StatusOr> BufferAssigner::CreateAssignment( // Can't use absl::make_unique because BufferAssignment constructor is // private. std::unique_ptr assignment(new BufferAssignment( - module, std::move(liveness), std::move(buffer_size), + module, std::move(hlo_ordering), std::move(buffer_size), std::move(color_alignment), std::move(alias_analysis))); - TF_RETURN_IF_ERROR(colorer_(&assignment->alias_analysis(), - assignment->liveness().hlo_ordering())); + TF_RETURN_IF_ERROR( + colorer_(&assignment->alias_analysis(), assignment->hlo_ordering())); VLOG(3) << "After coloring:"; XLA_VLOG_LINES(3, assignment->alias_analysis().dataflow_analysis().ToString()); diff --git a/tensorflow/compiler/xla/service/buffer_assignment.h b/tensorflow/compiler/xla/service/buffer_assignment.h index 602b6b1b4fe..f60ad22fa51 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment.h +++ b/tensorflow/compiler/xla/service/buffer_assignment.h @@ -25,7 +25,6 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/types/span.h" -#include "tensorflow/compiler/xla/service/buffer_liveness.h" #include "tensorflow/compiler/xla/service/heap_simulator.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_alias_analysis.h" @@ -447,7 +446,7 @@ class BufferAssignment { HloAliasAnalysis& alias_analysis() const { return *alias_analysis_; } // Returns the BufferLiveness object used to construct this assignment. - const BufferLiveness& liveness() const { return *liveness_; } + const HloOrdering& hlo_ordering() const { return *hlo_ordering_; } string ToString() const; BufferAssignmentProto ToProto() const; @@ -478,12 +477,12 @@ class BufferAssignment { friend class BufferAssigner; BufferAssignment(const HloModule* module, - std::unique_ptr liveness, + std::unique_ptr hlo_ordering, BufferValue::SizeFunction buffer_size, LogicalBuffer::AlignmentFunction color_alignment, std::unique_ptr alias_analysis) : module_(module), - liveness_(std::move(liveness)), + hlo_ordering_(std::move(hlo_ordering)), buffer_size_(std::move(buffer_size)), color_alignment_(std::move(color_alignment)), alias_analysis_(std::move(alias_analysis)) {} @@ -535,7 +534,8 @@ class BufferAssignment { allocation_index_for_value_; const HloModule* module_; - const std::unique_ptr liveness_; + + const std::unique_ptr hlo_ordering_; // Function which returns the buffer size for a given logical buffer (shape). BufferValue::SizeFunction buffer_size_; diff --git a/tensorflow/compiler/xla/service/buffer_liveness.cc b/tensorflow/compiler/xla/service/buffer_liveness.cc deleted file mode 100644 index 3adf129a22d..00000000000 --- a/tensorflow/compiler/xla/service/buffer_liveness.cc +++ /dev/null @@ -1,179 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Defines the data returned by the XLA buffer assignment packages. - -#include "tensorflow/compiler/xla/service/buffer_liveness.h" - -#include -#include - -#include "absl/strings/str_format.h" -#include "absl/strings/str_join.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" -#include "tensorflow/compiler/xla/service/logical_buffer.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/status_macros.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/compiler/xla/util.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/platform/logging.h" - -namespace xla { - -/* static */ -StatusOr> BufferLiveness::Run( - const HloModule* module, std::unique_ptr hlo_ordering) { - std::unique_ptr liveness( - new BufferLiveness(module, std::move(hlo_ordering))); - TF_RETURN_IF_ERROR(liveness->Analyze()); - return std::move(liveness); -} - -Status BufferLiveness::Analyze() { - TF_ASSIGN_OR_RETURN(points_to_analysis_, TuplePointsToAnalysis::Run(module_)); - for (auto* computation : module_->computations()) { - if (computation->IsFusionComputation()) { - continue; - } - // Gather all instructions whose buffers might alias other instructions into - // the set aliased_buffers_. This includes those contained as a tuple - // element in other instruction's output. - for (const auto& instruction : computation->instructions()) { - for (const LogicalBuffer* aliased_buffer : - points_to_analysis_->GetPointsToSet(instruction) - .CreateFlattenedSet()) { - if (aliased_buffer->instruction() != instruction) { - aliased_buffers_.insert(aliased_buffer); - } - } - } - - if (computation == module_->entry_computation()) { - const HloInstruction* root = computation->root_instruction(); - maybe_live_out_buffers_ = - points_to_analysis_->GetPointsToSet(root).CreateFlattenedSet(); - } - } - - XLA_VLOG_LINES(3, ToString()); - return Status::OK(); -} - -string BufferLiveness::ToString() const { - std::vector pieces; - pieces.push_back( - absl::StrFormat("BufferLiveness(module=%s):", module_->name())); - pieces.push_back("HloOrdering:"); - pieces.push_back(hlo_ordering_->ToString()); - pieces.push_back("Aliased buffers:"); - for (const LogicalBuffer* buffer : aliased_buffers_) { - pieces.push_back(absl::StrFormat(" %s", buffer->ToString())); - } - pieces.push_back("Live out buffers:"); - for (const LogicalBuffer* buffer : maybe_live_out_buffers_) { - pieces.push_back(absl::StrFormat(" %s", buffer->ToString())); - } - return absl::StrJoin(pieces, "\n"); -} - -bool BufferLiveness::live_range_strictly_before(const LogicalBuffer& a, - const LogicalBuffer& b) const { - TF_DCHECK_OK(points_to_analysis_->VerifyBuffer(a)); - TF_DCHECK_OK(points_to_analysis_->VerifyBuffer(b)); - - if (!hlo_ordering_->ExecutesBefore(a.instruction(), b.instruction())) { - return false; - } - - for (const BufferAlias& alias : points_to_analysis_->GetBufferAliases(a)) { - // Every user of 'a' must be a predecessor of 'b' or 'b' itself. - for (auto user : alias.instruction()->users()) { - if (points_to_analysis().DoesNotUseOperandBuffer(alias.instruction(), - alias.index(), user)) { - continue; - } - if (user != b.instruction() && - !hlo_ordering_->ExecutesBefore(user, b.instruction())) { - return false; - } - } - - // If the root instruction aliases the buffer 'a', the live range of 'a' is - // until the end of the computation and can never be strictly before another - // buffer nested in the same computation. This is needed to prevent the root - // instruction's buffers from being reused by later instructions even when - // the root is not the last instruction in the schedule. - if (alias.instruction()->parent()->root_instruction() == - alias.instruction() && - hlo_ordering_->call_graph().InstructionIsNestedIn( - b.instruction(), alias.instruction()->parent())) { - return false; - } - } - - // If 'b' is a user of 'a' then the buffers interfere unless 'a.instruction' - // and 'b.instruction' emit the same shape/layout, and 'b.instruction' meets - // the qualifications specified in CanShareOperandBufferWithUser. - for (const BufferAlias& alias : points_to_analysis_->GetBufferAliases(a)) { - if (b.instruction()->IsUserOf(alias.instruction()) && - !points_to_analysis().CanShareOperandBufferWithUser( - alias.instruction(), alias.index(), b.instruction(), b.index())) { - return false; - } - } - return true; -} - -namespace { -bool IsEntryParameter(const HloInstruction* instruction) { - const HloComputation* computation = instruction->parent(); - return instruction->opcode() == HloOpcode::kParameter && - computation == computation->parent()->entry_computation(); -} -} // namespace - -bool BufferLiveness::MayInterfere(const LogicalBuffer& a, - const LogicalBuffer& b) const { - // Parameters live at the entry of the computation, thus always interfere with - // all other instructions inside the computation executing before them in the - // ordering. - const HloInstruction* a_instruction = a.instruction(); - const HloInstruction* b_instruction = b.instruction(); - if (a_instruction->opcode() == HloOpcode::kParameter && - hlo_ordering_->call_graph().InstructionIsNestedIn( - b_instruction, a_instruction->parent()) && - hlo_ordering_->ExecutesBefore(b_instruction, a_instruction)) { - return true; - } - if (b_instruction->opcode() == HloOpcode::kParameter && - hlo_ordering_->call_graph().InstructionIsNestedIn( - a_instruction, b_instruction->parent()) && - hlo_ordering_->ExecutesBefore(a_instruction, b_instruction)) { - return true; - } - // Buffers without disjoint liveness may interfere. - return !live_range_strictly_before(a, b) && !live_range_strictly_before(b, a); -} - -bool BufferLiveness::MaybeLiveOut(const LogicalBuffer& buffer) const { - // Verify that a buffer is actually defined at the given instruction/index - // (eg, its not an alias of another buffer such as occurs with a bitcast). - TF_CHECK_OK(points_to_analysis_->VerifyBuffer(buffer)); - return maybe_live_out_buffers_.count(&buffer); -} - -} // namespace xla diff --git a/tensorflow/compiler/xla/service/buffer_liveness.h b/tensorflow/compiler/xla/service/buffer_liveness.h deleted file mode 100644 index f939a426ead..00000000000 --- a/tensorflow/compiler/xla/service/buffer_liveness.h +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_BUFFER_LIVENESS_H_ -#define TENSORFLOW_COMPILER_XLA_SERVICE_BUFFER_LIVENESS_H_ - -#include -#include -#include - -#include "absl/container/flat_hash_set.h" -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/hlo_module.h" -#include "tensorflow/compiler/xla/service/hlo_ordering.h" -#include "tensorflow/compiler/xla/service/tuple_points_to_analysis.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/lib/core/status.h" - -namespace xla { - -// Class which computes liveness of the output buffers of HLOs and their -// interference. -class BufferLiveness { - public: - using Colorer = std::function; - - // Constructs a buffer liveness object for the given module assuming the given - // HLO instruction ordering. - static StatusOr> Run( - const HloModule* module, std::unique_ptr hlo_ordering); - - // Returns true if the live range of the buffer containing the output of 'a' - // may overlap with the live range of the buffer of 'b'. If instruction 'a' - // interferes with instruction 'b' then they cannot share the same buffer. - bool MayInterfere(const LogicalBuffer& a, const LogicalBuffer& b) const; - - // Returns true if the buffer for the given instruction may be live out of the - // module. That is, the instruction's buffer may be included in the output of - // the entry computation. - bool MaybeLiveOut(const LogicalBuffer& buffer) const; - - // Returns the complete set of buffers that may be live out of the module. - const PointsToSet::BufferSet& maybe_live_out_buffers() const { - return maybe_live_out_buffers_; - } - - // Returns the underlying points-to analysis used for this liveness analysis. - const TuplePointsToAnalysis& points_to_analysis() const { - return *points_to_analysis_; - } - - // Returns the underlying hlo ordering used for this liveness analysis. - const HloOrdering& hlo_ordering() const { return *hlo_ordering_; } - - const HloModule& module() const { return *module_; } - - string ToString() const; - - static Colorer DefaultColorer() { - return [](const BufferLiveness& buffer_liveness) { - for (LogicalBuffer::Id id = 0; - id < buffer_liveness.points_to_analysis().num_logical_buffers(); - id++) { - auto& buffer = buffer_liveness.points_to_analysis().logical_buffer(id); - buffer.set_color(LogicalBuffer::Color(0)); - } - return Status::OK(); - }; - } - - private: - explicit BufferLiveness(const HloModule* module, - std::unique_ptr hlo_ordering) - : module_(module), hlo_ordering_(std::move(hlo_ordering)) {} - - // Perform buffer liveness analysis. This method must be called prior to - // MayInterfere or MaybeLiveOut. - Status Analyze(); - - // Returns true if the live range of the buffer of 'a' is strictly before the - // live range of the buffer of 'b' (they do not overlap). - bool live_range_strictly_before(const LogicalBuffer& a, - const LogicalBuffer& b) const; - - const HloModule* module_; - std::unique_ptr hlo_ordering_; - - // Set of LogicalBuffers which are aliased in the output of other - // instructions. For example, a LogicalBuffer which is inserted into a tuple - // is considered to be aliased and will be in this set. - absl::flat_hash_set aliased_buffers_; - - // LogicalBuffers that may be live out of the entry computation. - PointsToSet::BufferSet maybe_live_out_buffers_; - - std::unique_ptr points_to_analysis_; -}; - -} // namespace xla - -#endif // TENSORFLOW_COMPILER_XLA_SERVICE_BUFFER_LIVENESS_H_ diff --git a/tensorflow/compiler/xla/service/buffer_liveness_test.cc b/tensorflow/compiler/xla/service/buffer_liveness_test.cc deleted file mode 100644 index d4b2268fd01..00000000000 --- a/tensorflow/compiler/xla/service/buffer_liveness_test.cc +++ /dev/null @@ -1,934 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/xla/service/buffer_liveness.h" - -#include -#include - -#include "absl/memory/memory.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" -#include "tensorflow/compiler/xla/service/hlo_dataflow_analysis.h" -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/hlo_opcode.h" -#include "tensorflow/compiler/xla/service/hlo_parser.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/tests/hlo_test_base.h" -#include "tensorflow/compiler/xla/types.h" -#include "tensorflow/compiler/xla/xla_data.pb.h" -#include "tensorflow/core/lib/core/status_test_util.h" - -namespace xla { -namespace { - -class BufferLivenessTest : public HloTestBase { - protected: - // Returns the LogicalBuffer defined at the given instruction and - // index. CHECKs if no buffer is defined at that point. - const LogicalBuffer& GetBuffer(const BufferLiveness& liveness, - const HloInstruction* instruction, - const ShapeIndex& index) { - const auto& pointed_to = liveness.points_to_analysis() - .GetPointsToSet(instruction) - .element(index); - CHECK_EQ(1, pointed_to.size()); - CHECK_EQ(instruction, pointed_to[0]->instruction()); - CHECK(index == pointed_to[0]->index()); - return *pointed_to[0]; - } - - // Returns true if the top-level buffers for instructions 'a' and 'b' may - // interfere. Precondition: 'a' and 'b' are array-shaped. - bool InstructionsMayInterfere(const BufferLiveness& liveness, - HloInstruction* a, HloInstruction* b) { - EXPECT_FALSE(a->shape().IsTuple()); - EXPECT_FALSE(b->shape().IsTuple()); - return liveness.MayInterfere( - GetBuffer(liveness, /*instruction=*/a, /*index=*/{}), - GetBuffer(liveness, /*instruction=*/b, /*index=*/{})); - } - - // Returns true if the tuple elements at 'index' for instructions 'a' and 'b' - // may interfere. Precondition: 'a' and 'b' are tuple-shaped, with equal - // tuple element sub-shapes. - bool TupleElementsMayInterfere(const BufferLiveness& liveness, - HloInstruction* a, HloInstruction* b, - const ShapeIndex& index) { - // Check that top-level shapes are tuple and tuple element shapes are equal. - EXPECT_TRUE(a->shape().IsTuple()); - EXPECT_TRUE(b->shape().IsTuple()); - EXPECT_TRUE( - ShapeUtil::Compatible(ShapeUtil::GetSubshape(a->shape(), index), - ShapeUtil::GetSubshape(b->shape(), index))); - // Lookup PointsTo set for instructions 'a' and 'b'. - auto& points_to_analysis = liveness.points_to_analysis(); - const auto& points_to_a = - points_to_analysis.GetPointsToSet(a).element(index); - const auto& points_to_b = - points_to_analysis.GetPointsToSet(b).element(index); - // Make sure PointsTo sets for 'a' and 'b' are unambiguous. - EXPECT_EQ(1, points_to_a.size()); - EXPECT_EQ(points_to_a.size(), points_to_b.size()); - // Check interference. - return liveness.MayInterfere(*points_to_a[0], *points_to_b[0]); - } - - // Returns true if the top-level buffers for the given instruction maybe - // liveout of the entry computation. - // Precondition: instruction is array-shaped. - bool InstructionMaybeLiveOut(const BufferLiveness& liveness, - HloInstruction* instruction) { - return liveness.MaybeLiveOut( - GetBuffer(liveness, instruction, /*index=*/{})); - } - - std::unique_ptr BuildDummyComputation() { - auto builder = HloComputation::Builder(TestName() + "_dummy"); - builder.AddInstruction(HloInstruction::CreateParameter(0, vec_, "param")); - return builder.Build(); - } - - const Shape vec_ = ShapeUtil::MakeShape(xla::F32, {42}); -}; - -TEST_F(BufferLivenessTest, ElementwiseChain) { - // A simple chain of elementwise operations. No buffers should interfere. - // - // param --> negate -> exp -> log - // - auto builder = HloComputation::Builder(TestName()); - auto param = - builder.AddInstruction(HloInstruction::CreateParameter(0, vec_, "param")); - auto negate = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kNegate, param)); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kExp, negate)); - auto log = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kLog, exp)); - - auto module = CreateNewVerifiedModule(); - module->AddEntryComputation(builder.Build()); - - auto liveness = - BufferLiveness::Run( - module.get(), absl::make_unique(module.get())) - .ConsumeValueOrDie(); - - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, exp)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, log)); - - // No buffers should interfere. - EXPECT_FALSE(InstructionsMayInterfere(*liveness, negate, exp)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, negate, log)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, exp, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, exp, log)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, log, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, log, exp)); - - // Buffers should interfere with itself. - EXPECT_TRUE(InstructionsMayInterfere(*liveness, exp, exp)); - - // Only log is live out. - EXPECT_FALSE(InstructionMaybeLiveOut(*liveness, param)); - EXPECT_FALSE(InstructionMaybeLiveOut(*liveness, negate)); - EXPECT_FALSE(InstructionMaybeLiveOut(*liveness, exp)); - EXPECT_TRUE(InstructionMaybeLiveOut(*liveness, log)); -} - -TEST_F(BufferLivenessTest, MultipleEntryParameters_Sequential) { - // Two entry params, which interfere with each other. - // - // param0 --> negate ---------------\ - // param1 --> exp --> add - auto builder = HloComputation::Builder(TestName()); - auto param0 = builder.AddInstruction( - HloInstruction::CreateParameter(0, vec_, "param0")); - auto param1 = builder.AddInstruction( - HloInstruction::CreateParameter(1, vec_, "param1")); - auto negate = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kNegate, param0)); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kExp, param1)); - auto add = builder.AddInstruction( - HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp)); - - auto module = CreateNewVerifiedModule(); - HloComputation* entry = module->AddEntryComputation(builder.Build()); - - HloSchedule schedule(module.get()); - schedule.set_sequence(entry, {param0, negate, param1, exp, add}); - auto liveness = - BufferLiveness::Run(module.get(), - absl::make_unique(schedule)) - .ConsumeValueOrDie(); - - // Entry parameters interfere as if they are defined simultaneously at - // the very beginning. - EXPECT_TRUE(InstructionsMayInterfere(*liveness, param0, param1)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param0, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param0, exp)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param0, add)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, param1, param0)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, param1, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param1, exp)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param1, add)); - - // Negate and exp still interfere. - EXPECT_TRUE(InstructionsMayInterfere(*liveness, negate, exp)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, exp, negate)); - - // But {negate, add} and {exp, add} don't interfere. - EXPECT_FALSE(InstructionsMayInterfere(*liveness, negate, add)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, add, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, exp, add)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, add, exp)); -} - -TEST_F(BufferLivenessTest, EmbeddedComputationParameters) { - absl::string_view hlo_string = R"( -HloModule EmbeddedComputationParameters, is_scheduled=true - -%EmbeddedComputationParameters_embedded (embedded_param0: f32[42], embedded_param1: f32[42]) -> (f32[42], f32[42]) { - %embedded_param0 = f32[42]{0} parameter(0) - %log = f32[42]{0} log(f32[42]{0} %embedded_param0) - %add = f32[42]{0} add(f32[42]{0} %log, f32[42]{0} %log) - %embedded_param1 = f32[42]{0} parameter(1) - ROOT %tuple = (f32[42]{0}, f32[42]{0}) tuple(f32[42]{0} %add, f32[42]{0} %embedded_param1) -} - -ENTRY %EmbeddedComputationParameters (param0: f32[42], param1: f32[42]) -> (f32[42], f32[42]) { - %param0 = f32[42]{0} parameter(0) - %param1 = f32[42]{0} parameter(1) - ROOT %call = (f32[42]{0}, f32[42]{0}) call(f32[42]{0} %param0, f32[42]{0} %param1), to_apply=%EmbeddedComputationParameters_embedded -} -)"; - HloModuleConfig hlo_config; - TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr module, - ParseAndReturnUnverifiedModule(hlo_string, hlo_config)); - auto liveness = - BufferLiveness::Run( - module.get(), - absl::make_unique(module->schedule())) - .ConsumeValueOrDie(); - - auto embedded_log = FindInstruction(module.get(), "log"); - auto embedded_param0 = FindInstruction(module.get(), "embedded_param0"); - auto embedded_param1 = FindInstruction(module.get(), "embedded_param1"); - auto param0 = FindInstruction(module.get(), "param0"); - auto param1 = FindInstruction(module.get(), "param1"); - - // Parameters should interfere with other instructions inside the computation. - EXPECT_TRUE( - InstructionsMayInterfere(*liveness, embedded_log, embedded_param1)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, embedded_log, param0)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, embedded_log, param1)); - EXPECT_TRUE( - InstructionsMayInterfere(*liveness, embedded_param0, embedded_param1)); -} - -TEST_F(BufferLivenessTest, InterferenceWithOuterRoot) { - absl::string_view hlo_string = R"( -HloModule InterferenceWithOuterRoot, is_scheduled=true - -Emmbedded (embedded_param: f32[42]) -> f32[42] { - embedded_param = f32[42]{0} parameter(0) - multiply = f32[42]{0} multiply(embedded_param, embedded_param) - ROOT log = f32[42]{0} log(multiply) -} - -ENTRY InterferenceWithOuterRoot { - param = f32[4096,4096]{1,0} parameter(0) - ROOT add = f32[4096,4096]{1,0} add(param, param) - call = f32[42]{0} call(param), to_apply=Emmbedded -} - -)"; - HloModuleConfig hlo_config; - TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr module, - ParseAndReturnUnverifiedModule(hlo_string, hlo_config)); - auto liveness = - BufferLiveness::Run( - module.get(), - absl::make_unique(module->schedule())) - .ConsumeValueOrDie(); - - auto multiply = FindInstruction(module.get(), "multiply"); - auto add = FindInstruction(module.get(), "add"); - - EXPECT_TRUE(InstructionsMayInterfere(*liveness, multiply, add)); -} - -TEST_F(BufferLivenessTest, NonElementwiseOperand) { - // A chain of operations with two elementwise and one non-elementwise. The - // elementwise op should not interfere with its operand, while the - // non-elementwise op should interfere. Entry params always interfere. - // - // param --> exp -> negate -> reverse - // - auto builder = HloComputation::Builder(TestName()); - auto param = - builder.AddInstruction(HloInstruction::CreateParameter(0, vec_, "param")); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kExp, param)); - auto negate = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kNegate, exp)); - auto reverse = - builder.AddInstruction(HloInstruction::CreateReverse(vec_, negate, {0})); - - auto module = CreateNewVerifiedModule(); - module->AddEntryComputation(builder.Build()); - - auto liveness = - BufferLiveness::Run( - module.get(), absl::make_unique(module.get())) - .ConsumeValueOrDie(); - - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, exp)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, reverse)); - - // Negate is elementwise, so doesn't interfere with its operand. - // Reverse is non-elementwise, so does interfere with its operand. - EXPECT_FALSE(InstructionsMayInterfere(*liveness, exp, negate)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, negate, reverse)); -} - -TEST_F(BufferLivenessTest, OverlappedBuffers) { - // Verify simultaneously live buffers interfere (exp and negate). - // - // param --> negate -> add - // \---> exp -----/ - // - auto builder = HloComputation::Builder(TestName()); - auto param = - builder.AddInstruction(HloInstruction::CreateParameter(0, vec_, "param")); - auto negate = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kNegate, param)); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kExp, param)); - auto add = builder.AddInstruction( - HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp)); - - auto module = CreateNewVerifiedModule(); - module->AddEntryComputation(builder.Build()); - - auto liveness = - BufferLiveness::Run( - module.get(), absl::make_unique(module.get())) - .ConsumeValueOrDie(); - - EXPECT_TRUE(InstructionsMayInterfere(*liveness, param, negate)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, param, exp)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, add)); - - // Negate and exp interfere with each other, but not with add. - EXPECT_TRUE(InstructionsMayInterfere(*liveness, negate, exp)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, exp, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, negate, add)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, add, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, exp, add)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, add, exp)); -} - -TEST_F(BufferLivenessTest, OverlappedBuffersSequentialOrder) { - // Identical to the test OverlappedBuffer but using a sequential ordering of - // HLO instructions. - // - // param --> negate -> add - // \---> exp -----/ - // - // Sequential order: - // param, negate, exp, add - // - // Liveness is identical to the DependencyHloOrdering. - auto builder = HloComputation::Builder(TestName()); - auto param = - builder.AddInstruction(HloInstruction::CreateParameter(0, vec_, "param")); - auto negate = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kNegate, param)); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kExp, param)); - auto add = builder.AddInstruction( - HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, negate, exp)); - - auto module = CreateNewVerifiedModule(); - auto computation = module->AddEntryComputation(builder.Build()); - - HloSchedule schedule(module.get()); - schedule.set_sequence(computation, {param, negate, exp, add}); - auto liveness = - BufferLiveness::Run(module.get(), - absl::make_unique(schedule)) - .ConsumeValueOrDie(); - - EXPECT_TRUE(InstructionsMayInterfere(*liveness, param, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, exp)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, add)); - - // Negate and exp interfere with each other, but not with add. - EXPECT_TRUE(InstructionsMayInterfere(*liveness, negate, exp)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, exp, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, negate, add)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, add, negate)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, exp, add)); - EXPECT_FALSE(InstructionsMayInterfere(*liveness, add, exp)); -} - -TEST_F(BufferLivenessTest, RootInstructionIsNotLastInSequentialOrder) { - // Tests that when the root instruction is not the last instruction in the - // schedule, the live range of its buffers interfere with the buffers of the - // later instructions. - // - // Two sets of independent instructions are executed in the computation. - // param --> add (root) - // recv --> recv-done --> send --> send-done - // - // Sequential order: - // param, add (root), recv, recv-done, send, send-done - auto builder = HloComputation::Builder(TestName()); - auto param = - builder.AddInstruction(HloInstruction::CreateParameter(0, vec_, "param")); - auto add = builder.AddInstruction( - HloInstruction::CreateBinary(vec_, HloOpcode::kAdd, param, param)); - auto token = builder.AddInstruction(HloInstruction::CreateToken()); - auto recv = builder.AddInstruction( - HloInstruction::CreateRecv(vec_, token, /*channel_id=*/0)); - auto recv_done = builder.AddInstruction(HloInstruction::CreateRecvDone(recv)); - auto send = builder.AddInstruction( - HloInstruction::CreateSend(recv_done, token, /*channel_id=*/1)); - auto send_done = builder.AddInstruction(HloInstruction::CreateSendDone(send)); - - auto module = CreateNewVerifiedModule(); - auto computation = module->AddEntryComputation(builder.Build(add)); - - HloSchedule schedule(module.get()); - schedule.set_sequence(computation, - {param, add, token, recv, recv_done, send, send_done}); - TF_ASSERT_OK(schedule.Verify()); - auto liveness = - BufferLiveness::Run(module.get(), - absl::make_unique(schedule)) - .ConsumeValueOrDie(); - - EXPECT_FALSE(InstructionsMayInterfere(*liveness, param, add)); - // Check the root instruction (add) buffer interferes with the recv buffer. - EXPECT_TRUE( - liveness->MayInterfere(GetBuffer(*liveness, add, /*index=*/{}), - GetBuffer(*liveness, recv, /*index=*/{0}))); -} - -TEST_F(BufferLivenessTest, TupleLiveOut) { - // Verify MaybeLiveOut with nested tuples. Result of computation looks like: - // - // Tuple({Tuple({Negate(Param)}, Exp(Negate(Param)))}) - // - // All values should be live out except Param. - auto builder = HloComputation::Builder(TestName()); - auto param = - builder.AddInstruction(HloInstruction::CreateParameter(0, vec_, "param")); - auto negate = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kNegate, param)); - auto inner_tuple = - builder.AddInstruction(HloInstruction::CreateTuple({negate})); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kExp, negate)); - auto outer_tuple = - builder.AddInstruction(HloInstruction::CreateTuple({inner_tuple, exp})); - - auto module = CreateNewVerifiedModule(); - module->AddEntryComputation(builder.Build()); - - auto liveness = - BufferLiveness::Run( - module.get(), absl::make_unique(module.get())) - .ConsumeValueOrDie(); - - // All buffers should be live out except the param - EXPECT_FALSE(InstructionMaybeLiveOut(*liveness, param)); - EXPECT_TRUE(InstructionMaybeLiveOut(*liveness, negate)); - EXPECT_TRUE(InstructionMaybeLiveOut(*liveness, inner_tuple)); - EXPECT_TRUE(InstructionMaybeLiveOut(*liveness, exp)); - EXPECT_TRUE(InstructionMaybeLiveOut(*liveness, outer_tuple)); -} - -// bitcast liveout. - -TEST_F(BufferLivenessTest, EmbeddedComputation) { - // Test MaybeLiveOut and MayInterfere for embedded computation. - auto module = CreateNewVerifiedModule(); - - auto embedded_builder = HloComputation::Builder(TestName() + "_embedded"); - auto embedded_param = embedded_builder.AddInstruction( - HloInstruction::CreateParameter(0, vec_, "embedded_param")); - auto embedded_log = embedded_builder.AddInstruction( - HloInstruction::CreateUnary(vec_, HloOpcode::kLog, embedded_param)); - - auto embedded_computation = - module->AddEmbeddedComputation(embedded_builder.Build()); - - auto builder = HloComputation::Builder(TestName()); - auto param = - builder.AddInstruction(HloInstruction::CreateParameter(0, vec_, "param")); - auto call = builder.AddInstruction( - HloInstruction::CreateCall(vec_, {param}, embedded_computation)); - - module->AddEntryComputation(builder.Build()); - - auto liveness = - BufferLiveness::Run( - module.get(), absl::make_unique(module.get())) - .ConsumeValueOrDie(); - - // Buffers in different computations should always interfere. - EXPECT_TRUE(InstructionsMayInterfere(*liveness, embedded_log, call)); - EXPECT_TRUE(InstructionsMayInterfere(*liveness, embedded_param, param)); - EXPECT_FALSE( - InstructionsMayInterfere(*liveness, embedded_param, embedded_log)); - - // The only buffers for which MaybeLiveOut == true are those live out - // of the entry computation. Buffers live out of embedded computations should - // return false for this method. - EXPECT_FALSE(InstructionMaybeLiveOut(*liveness, embedded_log)); - EXPECT_TRUE(InstructionMaybeLiveOut(*liveness, call)); -} - -TEST_F(BufferLivenessTest, TupleConstantLiveOut) { - // Verify non top-level elements of a nested tuple constant are properly - // marked as liveout. Computation: - // - // GetTupleElement(0, TupleConstant({{0, 1}, {3}}) - // - // Only the array buffers containing 0 and 1 are liveout of the - // computation. The buffer containing {0, 1} is copied by GetTupleElement, and - // the buffers containing {3} and 3 are dead. - auto builder = HloComputation::Builder(TestName()); - Literal elements0[] = {LiteralUtil::CreateR0(0), - LiteralUtil::CreateR0(1)}; - auto inner_tuple0 = LiteralUtil::MakeTuple({&elements0[0], &elements0[1]}); - Literal element1 = LiteralUtil::CreateR0(3); - auto inner_tuple1 = LiteralUtil::MakeTuple({&element1}); - auto tuple_constant = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::MakeTuple({&inner_tuple0, &inner_tuple1}))); - builder.AddInstruction(HloInstruction::CreateGetTupleElement( - inner_tuple0.shape(), tuple_constant, 0)); - - auto module = CreateNewVerifiedModule(); - module->AddEntryComputation(builder.Build()); - - auto liveness = - BufferLiveness::Run( - module.get(), absl::make_unique(module.get())) - .ConsumeValueOrDie(); - - // Only the element buffers of the tuple constant which are pointed to by - // the GetTupleElement instruction should be liveout. - EXPECT_FALSE(liveness->MaybeLiveOut( - GetBuffer(*liveness, tuple_constant, /*index=*/{}))); - EXPECT_TRUE(liveness->MaybeLiveOut( - GetBuffer(*liveness, tuple_constant, /*index=*/{0}))); - EXPECT_TRUE(liveness->MaybeLiveOut( - GetBuffer(*liveness, tuple_constant, /*index=*/{0, 0}))); - EXPECT_TRUE(liveness->MaybeLiveOut( - GetBuffer(*liveness, tuple_constant, /*index=*/{0, 1}))); - EXPECT_FALSE(liveness->MaybeLiveOut( - GetBuffer(*liveness, tuple_constant, /*index=*/{1}))); - EXPECT_FALSE(liveness->MaybeLiveOut( - GetBuffer(*liveness, tuple_constant, /*index=*/{1, 0}))); - EXPECT_FALSE(liveness->MaybeLiveOut( - GetBuffer(*liveness, tuple_constant, /*index=*/{1, 0}))); -} - -TEST_F(BufferLivenessTest, IndependentTupleElements) { - auto builder = HloComputation::Builder(TestName()); - // Create param0 Tuple. - auto tuple_param0 = builder.AddInstruction(HloInstruction::CreateParameter( - 0, - ShapeUtil::MakeTupleShape( - {ShapeUtil::MakeShape(F32, {8}), ShapeUtil::MakeShape(S32, {4})}), - "param0")); - // Create independent computations for each tuple elememt. - - // Tuple element0 computation: - // Add(GetTupleElement(tuple_param0, 0), const0) - auto tuple_element0_shape = - ShapeUtil::GetSubshape(tuple_param0->shape(), {0}); - auto tuple_element0 = - builder.AddInstruction(HloInstruction::CreateGetTupleElement( - tuple_element0_shape, tuple_param0, 0)); - auto const0 = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR1({1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f}))); - auto add0 = builder.AddInstruction(HloInstruction::CreateBinary( - tuple_element0_shape, HloOpcode::kAdd, tuple_element0, const0)); - - // Tuple element1 computation: - // Add(GetTupleElement(tuple_param0, 1), const1) - auto tuple_element1_shape = - ShapeUtil::GetSubshape(tuple_param0->shape(), {1}); - auto tuple_element1 = - builder.AddInstruction(HloInstruction::CreateGetTupleElement( - tuple_element1_shape, tuple_param0, 1)); - auto const1 = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR1({2.f, 2.f, 2.f, 2.f, 2.f, 2.f, 2.f, 2.f}))); - auto add1 = builder.AddInstruction(HloInstruction::CreateBinary( - tuple_element1_shape, HloOpcode::kAdd, tuple_element1, const1)); - - // Create output tuple. - auto tuple_root = - builder.AddInstruction(HloInstruction::CreateTuple({add0, add1})); - - auto module = CreateNewUnverifiedModule(); - module->AddEntryComputation(BuildDummyComputation()); - module->AddEmbeddedComputation(builder.Build()); - - auto liveness = - BufferLiveness::Run( - module.get(), absl::make_unique(module.get())) - .ConsumeValueOrDie(); - - // We compare tuple element pairs that are input/output to the computation: - // 1) (input_tuple_element, output_tuple_element) = ('tuple_element0', 'add0') - // 2) (input_tuple_element, output_tuple_element) = ('tuple_element1', 'add1') - - // Tuple output element 'add0' does not depend on input 'tuple_element1'. - // Tuple output element 'add1' does not depend on input 'tuple_element0'. - - // Both element pair does not interfere, because there is no other dependency - // on the pairs tuple input element, and so liveness can compute that all - // users of the input tuple element execute before the associated output - // tuple element. - EXPECT_FALSE( - TupleElementsMayInterfere(*liveness, tuple_param0, tuple_root, {0})); - EXPECT_FALSE( - TupleElementsMayInterfere(*liveness, tuple_param0, tuple_root, {1})); -} - -TEST_F(BufferLivenessTest, DependentTupleElements) { - auto builder = HloComputation::Builder(TestName()); - // Create param0 Tuple. - auto tuple_param0 = builder.AddInstruction(HloInstruction::CreateParameter( - 0, - ShapeUtil::MakeTupleShape( - {ShapeUtil::MakeShape(F32, {8}), ShapeUtil::MakeShape(F32, {8})}), - "param0")); - // Create dependent computations for each tuple elememt. - - // Tuple element0 computation: - // Add(GetTupleElement(tuple_param0, 0), const0) - auto tuple_element0_shape = - ShapeUtil::GetSubshape(tuple_param0->shape(), {0}); - auto tuple_element0 = - builder.AddInstruction(HloInstruction::CreateGetTupleElement( - tuple_element0_shape, tuple_param0, 0)); - auto const0 = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR1({1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f, 1.f}))); - auto add0 = builder.AddInstruction(HloInstruction::CreateBinary( - tuple_element0_shape, HloOpcode::kAdd, tuple_element0, const0)); - - // Tuple element1 computation: - // Add(GetTupleElement(tuple_param0, 0), GetTupleElement(tuple_param0, 1)) - auto tuple_element1_shape = - ShapeUtil::GetSubshape(tuple_param0->shape(), {1}); - auto tuple_element1 = - builder.AddInstruction(HloInstruction::CreateGetTupleElement( - tuple_element1_shape, tuple_param0, 1)); - auto add1 = builder.AddInstruction(HloInstruction::CreateBinary( - tuple_element1_shape, HloOpcode::kAdd, tuple_element0, tuple_element1)); - - // Create output tuple. - auto tuple_root = - builder.AddInstruction(HloInstruction::CreateTuple({add0, add1})); - - auto module = CreateNewVerifiedModule(); - module->AddEntryComputation(BuildDummyComputation()); - module->AddEmbeddedComputation(builder.Build()); - - auto liveness = - BufferLiveness::Run( - module.get(), absl::make_unique(module.get())) - .ConsumeValueOrDie(); - - // We compare tuple element pairs that are input/output to the computation: - // 1) (input_tuple_element, output_tuple_element) = ('tuple_element0', 'add0') - // 2) (input_tuple_element, output_tuple_element) = ('tuple_element1', 'add1') - - // The first tuple element pair output 'add0', has no dependency on second - // tuple element pairs input 'tuple_element1'. - - // The second tuple element pair output 'add1', has a dependency on first - // tuple element pairs input 'tuple_element0'. - - // The first tuple element pair does interfere, because liveness cannot - // compute that all references to 'tuple_element0' are executed before 'add0' - // (because of the depenency of 'add1' on 'tuple_element0'). - EXPECT_TRUE( - TupleElementsMayInterfere(*liveness, tuple_param0, tuple_root, {0})); - - // The second tuple element pair does not interfere, because there is no - // other dependency on 'tuple_element1', and so liveness can compute that - // all users execute before 'add1'. - EXPECT_FALSE( - TupleElementsMayInterfere(*liveness, tuple_param0, tuple_root, {1})); -} - -class FusedDynamicUpdateSliceLivenessTest : public BufferLivenessTest { - protected: - // Builds and runs a computation (see test case computation graphs below). - std::unique_ptr BuildModule( - const bool update_uses_tuple_element1, const bool fuse_gte0) { - auto builder = HloComputation::Builder(TestName()); - // Create param0 Tuple. - Shape data_shape = ShapeUtil::MakeShape(F32, {8}); - Shape update_shape = ShapeUtil::MakeShape(F32, {3}); - auto tuple_param0 = builder.AddInstruction(HloInstruction::CreateParameter( - 0, ShapeUtil::MakeTupleShape({data_shape, data_shape}), "param0")); - - auto gte0 = builder.AddInstruction( - HloInstruction::CreateGetTupleElement(data_shape, tuple_param0, 0)); - - auto gte1 = builder.AddInstruction( - HloInstruction::CreateGetTupleElement(data_shape, tuple_param0, 1)); - - auto update = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR1({2.f, 2.f, 2.f}))); - HloInstruction* slice = nullptr; - if (update_uses_tuple_element1) { - // Create a slice instruction as an additional user of 'gte1'. - slice = builder.AddInstruction( - HloInstruction::CreateSlice(update_shape, gte1, {0}, {3}, {1})); - update = builder.AddInstruction(HloInstruction::CreateBinary( - update_shape, HloOpcode::kAdd, update, slice)); - } - // Create a DynamicUpdateSlice instruction of tuple element 1 with 'update'. - auto starts = builder.AddInstruction( - HloInstruction::CreateConstant(LiteralUtil::CreateR0(2))); - auto dynamic_update_slice = - builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice( - data_shape, gte1, update, {starts})); - // Create output tuple. - builder.AddInstruction( - HloInstruction::CreateTuple({gte0, dynamic_update_slice})); - // Build module and get reference to entry computation. - auto module = CreateNewVerifiedModule(); - module->AddEntryComputation(builder.Build()); - auto* computation = module->entry_computation(); - // Create fusion instruction based on number of tuple element 1 users. - if (update_uses_tuple_element1) { - computation->CreateFusionInstruction( - {dynamic_update_slice, starts, update, CHECK_NOTNULL(slice), gte1}, - HloInstruction::FusionKind::kLoop); - } else { - computation->CreateFusionInstruction( - {dynamic_update_slice, starts, update, gte1}, - HloInstruction::FusionKind::kLoop); - } - // Create fusion instruction for tuple element 0 (if requested). - if (fuse_gte0) { - computation->CreateFusionInstruction({gte0}, - HloInstruction::FusionKind::kLoop); - } - return module; - } - - // Returns whether buffer interference is detected between tuple-shaped - // parameter and root instructions at tuple element 1. - bool Run(const bool update_uses_tuple_element1, - const bool fuse_gte0 = false) { - auto module = BuildModule(update_uses_tuple_element1, fuse_gte0); - // Run BufferLiveness on 'module'. - auto liveness = BufferLiveness::Run( - module.get(), - absl::make_unique(module.get())) - .ConsumeValueOrDie(); - // Return whether or not buffers interference is detected between - // 'tuple_param0' and 'tuple_root' at shape index '{1}'. - auto tuple_param0 = FindInstruction(module.get(), "param0"); - auto tuple_root = module->entry_computation()->root_instruction(); - return TupleElementsMayInterfere(*liveness, tuple_param0, tuple_root, {1}); - } - bool RunWithHloDataflowAnalysis(const bool update_uses_tuple_element1, - const bool fuse_gte0 = false) { - auto module = BuildModule(update_uses_tuple_element1, fuse_gte0); - // Run BufferLiveness on 'module'. - auto dataflow = HloDataflowAnalysis::Run(*module).ConsumeValueOrDie(); - auto hlo_ordering = absl::make_unique(module.get()); - // Return whether or not buffers interference is detected between - // 'tuple_param0' and 'tuple_root' at shape index '{1}'. - auto tuple_param0 = FindInstruction(module.get(), "param0"); - auto tuple_root = module->entry_computation()->root_instruction(); - return hlo_ordering->MayInterfere( - dataflow->GetUniqueValueAt(tuple_param0, {1}), - dataflow->GetUniqueValueAt(tuple_root, {1}), *dataflow); - } -}; - -// Tests that live ranges of buffers Param0[1] and Tuple[1] (which alias fusion) -// do not overlap with the following computation: -// -// Param0 -// / \ -// GTE(0) Fusion -----------> FusionParam -// | | | -// | | GTE(1) Const Const -// | | \ | / -// | | DynamicUpdateSlice // fused root -// \ / -// Tuple // computation root -// -TEST_F(FusedDynamicUpdateSliceLivenessTest, NoInterference) { - EXPECT_FALSE(Run(/*update_uses_tuple_element1=*/false)); - EXPECT_FALSE( - RunWithHloDataflowAnalysis(/*update_uses_tuple_element1=*/false)); -} - -// Tests that live ranges of buffers Param0[1] and Tuple[1] (which aliases -// 'fusion1') do not overlap in the presence of another fusion instruction -// (which is a user of 'param0' at a different tuple index). -// BufferLiveness should detect no uses of Param0 at index {1} in Fusion0 -// (because Fusion0 only uses Param0 at index {0}). -// -// Param0 -// / \ -// FusionParam <----- Fusion0 Fusion1 ------> FusionParam -// | | | | -// GTE(0) | | GTE(1) Const Const -// | | \ | / -// \ / DynamicUpdateSlice -// Tuple -// -TEST_F(FusedDynamicUpdateSliceLivenessTest, NoInterferenceWithUnrelatedFusion) { - EXPECT_FALSE(Run(/*update_uses_tuple_element1=*/false, /*fuse_gte0=*/true)); - EXPECT_FALSE(RunWithHloDataflowAnalysis(/*update_uses_tuple_element1=*/false, - /*fuse_gte0=*/true)); -} - -// Tests that live ranges of buffers Param0[1] and Tuple[1] (which alias fusion) -// do overlap because GTE(1) has two users: -// 1) DynamicUpdateSlice at operand 0. -// 2) Slice at operand 0. -// -// Param0 -// / \ Const -// / \ / -// GTE(0) Fusion -----------> FusionParam FusionParam -// | | | | -// | | GTE(1) / -// | | | \ / -// | | | Slice / -// | | | \ / -// | | | Add Const -// | | | | | -// | | DynamicUpdateSlice // fused root -// \ / -// Tuple // computation root -// -TEST_F(FusedDynamicUpdateSliceLivenessTest, WithInterference) { - EXPECT_TRUE(Run(/*update_uses_tuple_element1=*/true)); - EXPECT_TRUE(RunWithHloDataflowAnalysis(/*update_uses_tuple_element1=*/true)); -} - -class DynamicUpdateSliceLivenessTest : public BufferLivenessTest { - protected: - // Builds and runs a computation (see test case computation graphs below). - // Runs BufferLiveness on this computation. - // Returns whether buffer interference is detected between tuple-shaped - // parameter and root instructions at tuple element 1. - bool Run(const bool tuple_element1_has_two_uses) { - auto builder = HloComputation::Builder(TestName()); - // Create param0 Tuple. - Shape data_shape = ShapeUtil::MakeShape(F32, {8}); - Shape update_shape = ShapeUtil::MakeShape(F32, {3}); - auto tuple_param0 = builder.AddInstruction(HloInstruction::CreateParameter( - 0, ShapeUtil::MakeTupleShape({data_shape, data_shape}), "param0")); - - auto gte0 = builder.AddInstruction( - HloInstruction::CreateGetTupleElement(data_shape, tuple_param0, 0)); - - auto gte1 = builder.AddInstruction( - HloInstruction::CreateGetTupleElement(data_shape, tuple_param0, 1)); - - auto update = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR1({2.f, 2.f, 2.f}))); - - if (tuple_element1_has_two_uses) { - // Add 'gte0' and 'gte1' to create another user of 'gte1'. - gte0 = builder.AddInstruction(HloInstruction::CreateBinary( - data_shape, HloOpcode::kAdd, gte0, gte1)); - } - // Create a DynamicUpdateSlice instruction of tuple element 1 with 'update'. - auto starts = builder.AddInstruction( - HloInstruction::CreateConstant(LiteralUtil::CreateR0(2))); - auto dynamic_update_slice = - builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice( - data_shape, gte1, update, {starts})); - // Create output tuple. - auto tuple_root = builder.AddInstruction( - HloInstruction::CreateTuple({gte0, dynamic_update_slice})); - // Build module and get reference to entry computation. - auto module = CreateNewVerifiedModule(); - module->AddEntryComputation(BuildDummyComputation()); - module->AddEmbeddedComputation(builder.Build()); - // Run BufferLiveness on 'module'. - auto liveness = BufferLiveness::Run( - module.get(), - absl::make_unique(module.get())) - .ConsumeValueOrDie(); - // Return whether or not buffers interference is detected between - // 'tuple_param0' and 'tuple_root' at shape index '{1}'. - return TupleElementsMayInterfere(*liveness, tuple_param0, tuple_root, {1}); - } -}; - -// Tests that live ranges of buffers Param0[1] and Tuple[1] do not overlap in -// the following computation (because DynamicUpdateSlice (at operand 0) is the -// unique user): -// -// Parameter0 -// | | -// GTE(0) GTE(1) Const Const -// | \ | / -// | DynamicUpdateSlice -// \ / -// Tuple -// -TEST_F(DynamicUpdateSliceLivenessTest, NoInterference) { - EXPECT_FALSE(Run(/*tuple_element1_has_two_uses=*/false)); -} - -// Tests that live ranges of buffers Param0[1] and Tuple[1] do overlap because -// GTE(1) has two users: -// 1) DynamicUpdateSlice at operand 0. -// 2) Add at operand 1. -// -// Parameter0 -// | | -// GTE(0) GTE(1) -// | / | -// | / | -// Add | Const Const -// | | | | -// | DynamicUpdateSlice -// \ / -// Tuple -// -TEST_F(DynamicUpdateSliceLivenessTest, WithInterference) { - EXPECT_TRUE(Run(/*tuple_element1_has_two_uses=*/true)); -} - -} // namespace - -} // namespace xla diff --git a/tensorflow/compiler/xla/service/copy_insertion.h b/tensorflow/compiler/xla/service/copy_insertion.h index 72a245bc04c..703942662b1 100644 --- a/tensorflow/compiler/xla/service/copy_insertion.h +++ b/tensorflow/compiler/xla/service/copy_insertion.h @@ -16,7 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_COPY_INSERTION_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_COPY_INSERTION_H_ -#include "tensorflow/compiler/xla/service/buffer_liveness.h" +#include "tensorflow/compiler/xla/service/hlo_alias_analysis.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_module.h" diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index 41a2d7f34bc..37baf0e36df 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -111,7 +111,6 @@ cc_library( "//tensorflow/compiler/xla/service:batch_dot_simplification", "//tensorflow/compiler/xla/service:batchnorm_expander", "//tensorflow/compiler/xla/service:buffer_assignment", - "//tensorflow/compiler/xla/service:buffer_liveness", "//tensorflow/compiler/xla/service:call_inliner", "//tensorflow/compiler/xla/service:cholesky_expander", "//tensorflow/compiler/xla/service:conditional_simplifier", diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 4b09fa93f79..e7134325907 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -50,7 +50,6 @@ limitations under the License. #include "tensorflow/compiler/xla/service/batch_dot_simplification.h" #include "tensorflow/compiler/xla/service/batchnorm_expander.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" -#include "tensorflow/compiler/xla/service/buffer_liveness.h" #include "tensorflow/compiler/xla/service/call_inliner.h" #include "tensorflow/compiler/xla/service/cholesky_expander.h" #include "tensorflow/compiler/xla/service/conditional_simplifier.h" diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 476d4fdeb2e..6ba53814326 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -998,7 +998,6 @@ cc_library( "//tensorflow/compiler/xla/service:algebraic_simplifier", "//tensorflow/compiler/xla/service:batchnorm_expander", "//tensorflow/compiler/xla/service:buffer_assignment", - "//tensorflow/compiler/xla/service:buffer_liveness", "//tensorflow/compiler/xla/service:call_inliner", "//tensorflow/compiler/xla/service:conditional_simplifier", "//tensorflow/compiler/xla/service:convolution_group_converter", diff --git a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc index bc25527238a..d50a0d4baa0 100644 --- a/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/nvptx_compiler.cc @@ -34,7 +34,6 @@ limitations under the License. #include "tensorflow/compiler/xla/service/algebraic_simplifier.h" #include "tensorflow/compiler/xla/service/batchnorm_expander.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" -#include "tensorflow/compiler/xla/service/buffer_liveness.h" #include "tensorflow/compiler/xla/service/call_inliner.h" #include "tensorflow/compiler/xla/service/conditional_simplifier.h" #include "tensorflow/compiler/xla/service/convolution_group_converter.h" @@ -493,7 +492,6 @@ void WarnIfBadDriverJITVersion() { }); } - } // namespace NVPTXCompiler::NVPTXCompiler() diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.h b/tensorflow/compiler/xla/service/hlo_rematerialization.h index 8172f0d3a15..350cf0f8e8f 100644 --- a/tensorflow/compiler/xla/service/hlo_rematerialization.h +++ b/tensorflow/compiler/xla/service/hlo_rematerialization.h @@ -17,7 +17,6 @@ #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" -#include "tensorflow/compiler/xla/service/buffer_liveness.h" #include "tensorflow/compiler/xla/service/call_graph.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" diff --git a/tensorflow/compiler/xla/service/reduce_precision_insertion.h b/tensorflow/compiler/xla/service/reduce_precision_insertion.h index 76c6a87f176..05990afb625 100644 --- a/tensorflow/compiler/xla/service/reduce_precision_insertion.h +++ b/tensorflow/compiler/xla/service/reduce_precision_insertion.h @@ -16,7 +16,6 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_SERVICE_REDUCE_PRECISION_INSERTION_H_ #define TENSORFLOW_COMPILER_XLA_SERVICE_REDUCE_PRECISION_INSERTION_H_ -#include "tensorflow/compiler/xla/service/buffer_liveness.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_module.h" From f79c91e6df318bf6017d308dce254c3fdc6b4431 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 11:12:27 -0700 Subject: [PATCH 210/332] NFC: Remove Function::getModule. There is already a more general 'getParentOfType' method, and 'getModule' is likely to be misused as functions get placed within different regions than ModuleOp. PiperOrigin-RevId: 257442243 --- .../compiler/mlir/tensorflow/translate/export_graphdef.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc index f2eaa308deb..b60bbe2c4b9 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc @@ -448,7 +448,8 @@ StatusOr> Exporter::Convert(const ExporterConfigs& confs, // definition library // TODO(prakalps): If two functions have cyclic dependence, this will // introduce an infinite loop. - auto func = function.getModule().getNamedFunction(op_name.ValueOrDie()); + auto func = function.getParentOfType().getNamedFunction( + op_name.ValueOrDie()); if (func != nullptr) { TF_RETURN_IF_ERROR(ConvertLibFunction(confs, tf_dialect, func, flib)); TF_RETURN_IF_ERROR(graph->AddFunctionLibrary(*flib)); @@ -511,7 +512,9 @@ Status Exporter::ConvertLibFunction(const ExporterConfigs& configs, // and populates the GradientDef. auto grad_string = mlir::TF::TensorFlowDialect::GetGradientAttrName(); if (auto attr = function.getAttrOfType(grad_string)) { - auto grad_func = function.getModule().getNamedFunction(attr.getValue()); + auto grad_func = + function.getParentOfType().getNamedFunction( + attr.getValue()); TF_RETURN_IF_ERROR( ConvertLibFunction(configs, tf_dialect, grad_func, flib)); GradientDef grad; From 3b1c5e378b5a81092d5cfb4d4950d85fa8f528b3 Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Wed, 10 Jul 2019 11:19:44 -0700 Subject: [PATCH 211/332] Adding a TPU registration for DeleteIteratorOp PiperOrigin-RevId: 257444075 --- tensorflow/compiler/jit/xla_device_ops.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/compiler/jit/xla_device_ops.h b/tensorflow/compiler/jit/xla_device_ops.h index fd46034ec99..2c8203b1c5d 100644 --- a/tensorflow/compiler/jit/xla_device_ops.h +++ b/tensorflow/compiler/jit/xla_device_ops.h @@ -183,6 +183,9 @@ class XlaAssignVariableOp : public OpKernel { REGISTER_KERNEL_BUILDER( \ Name("AnonymousIteratorV2").Device(DEVICE).HostMemory("deleter"), \ data::AnonymousIteratorHandleOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("DeleteIterator").Device(DEVICE).HostMemory("deleter"), \ + data::DeleteIteratorOp); \ REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE), \ data::IteratorGetNextOp); \ REGISTER_KERNEL_BUILDER(Name("IteratorGetNextAsOptional").Device(DEVICE), \ From 1dffb00347a80d41c91eb43d0416d2d4ee0d9bc1 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Wed, 10 Jul 2019 11:32:24 -0700 Subject: [PATCH 212/332] Fix the device placement test - check the variable's device, instead of the read tensor's device. PiperOrigin-RevId: 257447029 --- tensorflow/python/eager/def_function_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/eager/def_function_test.py b/tensorflow/python/eager/def_function_test.py index 46f45395272..4a7d6fe4e9e 100644 --- a/tensorflow/python/eager/def_function_test.py +++ b/tensorflow/python/eager/def_function_test.py @@ -580,7 +580,7 @@ class DefFunctionTest(test.TestCase): # TODO(b/137148281): reenable @test_util.run_gpu_only - def DISABLED_testDeviceAnnotationRespected(self): + def testDeviceAnnotationRespected(self): a = [] @def_function.function() @@ -590,13 +590,13 @@ class DefFunctionTest(test.TestCase): (2, 2), maxval=1000000, dtype=dtypes.int64) if not a: - with ops.device("CPU:0"): + with ops.device('CPU:0'): a.append(resource_variable_ops.ResourceVariable(initial_value)) return a[0].read_value() created_variable_read = create_variable() - self.assertRegexpMatches(created_variable_read.device, "CPU") + self.assertRegexpMatches(a[0].device, 'CPU') def testDecorate(self): func = def_function.function(lambda: 1) From 5aff636fd85da389ea0052aa2cf5cc566fb8d482 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Wed, 10 Jul 2019 11:43:23 -0700 Subject: [PATCH 213/332] Update MLIR rev. PiperOrigin-RevId: 257449463 --- third_party/mlir/mlir_configure.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/third_party/mlir/mlir_configure.bzl b/third_party/mlir/mlir_configure.bzl index f0855be3c0a..2a6cd531962 100644 --- a/third_party/mlir/mlir_configure.bzl +++ b/third_party/mlir/mlir_configure.bzl @@ -1,7 +1,7 @@ """Repository rule to setup the external MLIR repository.""" -_MLIR_REV = "510870483fa9f897db59c59a983c6774fb522197" -_MLIR_SHA256 = "eda40ec8cfb9de60f6ab7165aa722e8b82d7c83bb6d256ac176e37d3bed5d412" +_MLIR_REV = "5f2159dab14169f8878d76d42a9367866c1b8d8d" +_MLIR_SHA256 = "c1d429d53dda2e38fd24ac895b6395965c53d9b6e3a29e20fa86e73005a3a86e" def _mlir_autoconf_impl(repository_ctx): """Implementation of the mlir_configure repository rule.""" From 25b71066b50071a0cc26cdd78da976aa76732238 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 11:52:48 -0700 Subject: [PATCH 214/332] PR #28079: Moved strided slide ops from reference to its own. Imported from GitHub PR #28079 This is to remove the gemmlowp dependency. Copybara import of the project: - a972909b44dc492bb534efe56e4b81f8bb0558c1 Moved strided slide ops from reference to its own. by Amit Srivastava - cee8e4bc29e274ff12949cc045f13d8c1f4740d1 Merge a972909b44dc492bb534efe56e4b81f8bb0558c1 into 8db90... by Amit <30853054+amitsrivastava78@users.noreply.github.com> COPYBARA_INTEGRATE_REVIEW=https://github.com/tensorflow/tensorflow/pull/28079 from amitsrivastava78:move_strided_slice a972909b44dc492bb534efe56e4b81f8bb0558c1 PiperOrigin-RevId: 257451365 --- tensorflow/lite/kernels/internal/BUILD | 2 + .../internal/reference/reference_ops.h | 54 +------------ .../internal/reference/strided_slice.h | 80 +++++++++++++++++++ 3 files changed, 83 insertions(+), 53 deletions(-) create mode 100644 tensorflow/lite/kernels/internal/reference/strided_slice.h diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD index 9b4e65c6eb7..f9b62c724ee 100644 --- a/tensorflow/lite/kernels/internal/BUILD +++ b/tensorflow/lite/kernels/internal/BUILD @@ -367,6 +367,7 @@ cc_library( "reference/pooling.h", "reference/reference_ops.h", "reference/softmax.h", + "reference/strided_slice.h", ], deps = [ ":common", @@ -406,6 +407,7 @@ cc_library( "reference/pooling.h", "reference/reference_ops.h", "reference/softmax.h", + "reference/strided_slice.h", ], deps = [ ":common", diff --git a/tensorflow/lite/kernels/internal/reference/reference_ops.h b/tensorflow/lite/kernels/internal/reference/reference_ops.h index 8889b00a7f2..ce34f525c37 100644 --- a/tensorflow/lite/kernels/internal/reference/reference_ops.h +++ b/tensorflow/lite/kernels/internal/reference/reference_ops.h @@ -36,6 +36,7 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/reference/fully_connected.h" #include "tensorflow/lite/kernels/internal/reference/pooling.h" #include "tensorflow/lite/kernels/internal/reference/softmax.h" +#include "tensorflow/lite/kernels/internal/reference/strided_slice.h" #include "tensorflow/lite/kernels/internal/round.h" #include "tensorflow/lite/kernels/internal/strided_slice_logic.h" #include "tensorflow/lite/kernels/internal/tensor.h" @@ -3071,59 +3072,6 @@ inline void PadImageStyle(const tflite::PadParams& op_params, output_data); } -template -inline void StridedSlice(const tflite::StridedSliceParams& op_params, - const RuntimeShape& unextended_input_shape, - const T* input_data, - const RuntimeShape& unextended_output_shape, - T* output_data) { - // Note that the output_shape is not used herein. - tflite::StridedSliceParams params_copy = op_params; - - TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4); - TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4); - const RuntimeShape input_shape = - RuntimeShape::ExtendedShape(4, unextended_input_shape); - const RuntimeShape output_shape = - RuntimeShape::ExtendedShape(4, unextended_output_shape); - - // Reverse and pad to 4 dimensions because that is what the runtime code - // requires (ie. all shapes must be 4D and are given backwards). - strided_slice::StridedSlicePadIndices(¶ms_copy, 4); - - const int start_b = strided_slice::StartForAxis(params_copy, input_shape, 0); - const int stop_b = - strided_slice::StopForAxis(params_copy, input_shape, 0, start_b); - const int start_h = strided_slice::StartForAxis(params_copy, input_shape, 1); - const int stop_h = - strided_slice::StopForAxis(params_copy, input_shape, 1, start_h); - const int start_w = strided_slice::StartForAxis(params_copy, input_shape, 2); - const int stop_w = - strided_slice::StopForAxis(params_copy, input_shape, 2, start_w); - const int start_d = strided_slice::StartForAxis(params_copy, input_shape, 3); - const int stop_d = - strided_slice::StopForAxis(params_copy, input_shape, 3, start_d); - - T* out_ptr = output_data; - for (int in_b = start_b; - !strided_slice::LoopCondition(in_b, stop_b, params_copy.strides[0]); - in_b += params_copy.strides[0]) { - for (int in_h = start_h; - !strided_slice::LoopCondition(in_h, stop_h, params_copy.strides[1]); - in_h += params_copy.strides[1]) { - for (int in_w = start_w; - !strided_slice::LoopCondition(in_w, stop_w, params_copy.strides[2]); - in_w += params_copy.strides[2]) { - for (int in_d = start_d; !strided_slice::LoopCondition( - in_d, stop_d, params_copy.strides[3]); - in_d += params_copy.strides[3]) { - *out_ptr++ = input_data[Offset(input_shape, in_b, in_h, in_w, in_d)]; - } - } - } - } -} - template inline void Slice(const tflite::SliceParams& op_params, const RuntimeShape& input_shape, diff --git a/tensorflow/lite/kernels/internal/reference/strided_slice.h b/tensorflow/lite/kernels/internal/reference/strided_slice.h new file mode 100644 index 00000000000..921c49ea77b --- /dev/null +++ b/tensorflow/lite/kernels/internal/reference/strided_slice.h @@ -0,0 +1,80 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_STRIDED_SLICE_H_ +#define TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_STRIDED_SLICE_H_ + +#include "tensorflow/lite/kernels/internal/common.h" +#include "tensorflow/lite/kernels/internal/strided_slice_logic.h" +#include "tensorflow/lite/kernels/internal/types.h" + +namespace tflite { + +namespace reference_ops { +template +inline void StridedSlice(const tflite::StridedSliceParams& op_params, + const RuntimeShape& unextended_input_shape, + const T* input_data, + const RuntimeShape& unextended_output_shape, + T* output_data) { + // Note that the output_shape is not used herein. + tflite::StridedSliceParams params_copy = op_params; + + TFLITE_DCHECK_LE(unextended_input_shape.DimensionsCount(), 4); + TFLITE_DCHECK_LE(unextended_output_shape.DimensionsCount(), 4); + const RuntimeShape input_shape = + RuntimeShape::ExtendedShape(4, unextended_input_shape); + const RuntimeShape output_shape = + RuntimeShape::ExtendedShape(4, unextended_output_shape); + + // Reverse and pad to 4 dimensions because that is what the runtime code + // requires (ie. all shapes must be 4D and are given backwards). + strided_slice::StridedSlicePadIndices(¶ms_copy, 4); + + const int start_b = strided_slice::StartForAxis(params_copy, input_shape, 0); + const int stop_b = + strided_slice::StopForAxis(params_copy, input_shape, 0, start_b); + const int start_h = strided_slice::StartForAxis(params_copy, input_shape, 1); + const int stop_h = + strided_slice::StopForAxis(params_copy, input_shape, 1, start_h); + const int start_w = strided_slice::StartForAxis(params_copy, input_shape, 2); + const int stop_w = + strided_slice::StopForAxis(params_copy, input_shape, 2, start_w); + const int start_d = strided_slice::StartForAxis(params_copy, input_shape, 3); + const int stop_d = + strided_slice::StopForAxis(params_copy, input_shape, 3, start_d); + + T* out_ptr = output_data; + for (int in_b = start_b; + !strided_slice::LoopCondition(in_b, stop_b, params_copy.strides[0]); + in_b += params_copy.strides[0]) { + for (int in_h = start_h; + !strided_slice::LoopCondition(in_h, stop_h, params_copy.strides[1]); + in_h += params_copy.strides[1]) { + for (int in_w = start_w; + !strided_slice::LoopCondition(in_w, stop_w, params_copy.strides[2]); + in_w += params_copy.strides[2]) { + for (int in_d = start_d; !strided_slice::LoopCondition( + in_d, stop_d, params_copy.strides[3]); + in_d += params_copy.strides[3]) { + *out_ptr++ = input_data[Offset(input_shape, in_b, in_h, in_w, in_d)]; + } + } + } + } +} +} // namespace reference_ops +} // namespace tflite + +#endif // TENSORFLOW_LITE_KERNELS_INTERNAL_REFERENCE_STRIDED_SLICE_H_ From b6d846490be7a8f818c8433c3dea627e375f9e18 Mon Sep 17 00:00:00 2001 From: Nupur Garg Date: Wed, 10 Jul 2019 12:05:52 -0700 Subject: [PATCH 215/332] Add support for non-lowered If ops to freezing in 2.0. PiperOrigin-RevId: 257454161 --- tensorflow/core/framework/node_def_util.cc | 9 +- .../python/framework/convert_to_constants.py | 190 ++++++++++++++---- .../framework/convert_to_constants_test.py | 123 ++++++++++-- 3 files changed, 264 insertions(+), 58 deletions(-) diff --git a/tensorflow/core/framework/node_def_util.cc b/tensorflow/core/framework/node_def_util.cc index 681ce8bba5c..656202df631 100644 --- a/tensorflow/core/framework/node_def_util.cc +++ b/tensorflow/core/framework/node_def_util.cc @@ -814,9 +814,14 @@ Status AddPrefixAndSuffixToNode(StringPiece prefix, StringPiece suffix, } // Update colocation constraints. - auto class_attr = node_def->mutable_attr()->find("_class"); + constexpr char kClassAttr[] = "_class"; + auto class_attr = node_def->mutable_attr()->find(kClassAttr); if (class_attr != node_def->mutable_attr()->end()) { - class_attr->second.set_s(strings::StrCat(prefix, class_attr->second.s())); + AttrValue new_value; + new_value.mutable_list()->add_s( + strings::StrCat(prefix, class_attr->second.s())); + node_def->mutable_attr()->erase(kClassAttr); + node_def->mutable_attr()->insert({kClassAttr, new_value}); } return Status::OK(); diff --git a/tensorflow/python/framework/convert_to_constants.py b/tensorflow/python/framework/convert_to_constants.py index 8443946e5ac..94576fdcfc1 100644 --- a/tensorflow/python/framework/convert_to_constants.py +++ b/tensorflow/python/framework/convert_to_constants.py @@ -24,14 +24,35 @@ from tensorflow.core.framework import variable_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.python.eager import wrap_function +from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_util from tensorflow.python.grappler import tf_optimizer from tensorflow.python.ops import array_ops -from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training.saver import export_meta_graph -def _run_inline_graph_optimization(func): +def _disable_lower_using_switch_merge(graph_def): + """Set '_lower_using_switch_merge' attributes to False in If and While ops. + + Args: + graph_def: GraphDef proto. + + Returns: + GraphDef + """ + output_graph_def = graph_pb2.GraphDef() + output_graph_def.library.CopyFrom(graph_def.library) + output_graph_def.versions.CopyFrom(graph_def.versions) + + for input_node in graph_def.node: + output_node = output_graph_def.node.add() + output_node.CopyFrom(input_node) + if output_node.op in ("If", "While"): + output_node.attr["_lower_using_switch_merge"].b = False + return output_graph_def + + +def _run_inline_graph_optimization(func, lower_control_flow): """Apply function inline optimization to the graph. Returns the GraphDef after Grappler's function inlining optimization is @@ -39,12 +60,16 @@ def _run_inline_graph_optimization(func): Args: func: ConcreteFunction. + lower_control_flow: Boolean indicating whether or not to lower control flow + ops such as If and While. (default True) Returns: GraphDef """ - meta_graph = export_meta_graph( - graph_def=func.graph.as_graph_def(), graph=func.graph) + graph_def = func.graph.as_graph_def() + if not lower_control_flow: + graph_def = _disable_lower_using_switch_merge(graph_def) + meta_graph = export_meta_graph(graph_def=graph_def, graph=func.graph) # Clear the initializer_name for the variables collections, since they are not # needed after saved to saved_model. @@ -73,25 +98,6 @@ def _run_inline_graph_optimization(func): return tf_optimizer.OptimizeGraph(config, meta_graph) -def _get_tensors_from_graph(graph, tensors): - """Gets the Tensors in `graph` with the name of the tensors in `tensors`. - - Args: - graph: TensorFlow Graph. - tensors: List of Tensors. - - Returns: - List of Tensors. - """ - new_tensors = [] - for orig_tensor in tensors: - new_tensor = graph.get_tensor_by_name(orig_tensor.name) - if new_tensor.shape.rank is None: - new_tensor.set_shape(orig_tensor.shape) - new_tensors.append(new_tensor) - return new_tensors - - def _get_tensor_name(name): """Returns the name of the input tensor. @@ -140,6 +146,39 @@ def _get_tensor_data(func): return tensor_data +def _get_control_flow_function_types(graph_def, tensor_data): + """Gets the types for the parameters to the function. + + Creates a map from function name to a list of types that correspond with the + function arguments. The type is primarily determined from the corresponding + "If" op. If the argument is a resource variable, then the type is determined + from the type of the data contained within the Tensor. + + Args: + graph_def: GraphDef proto. + tensor_data: {str name : Tensor}. + + Returns: + {str function name : [int representing DataType]} + """ + # TODO(b/133793620): Support the "While" op. + func_types = {} + for node in graph_def.node: + if node.op == "If": + arg_types = [dtype for dtype in node.attr["Tin"].list.type] + + for idx in range(len(arg_types)): + if arg_types[idx] == dtypes.resource: + # Skip first index which represents the condition. + input_name = node.input[idx + 1] + numpy_type = tensor_data[input_name]["data"].dtype + arg_types[idx] = dtypes.as_dtype(numpy_type).as_datatype_enum + + func_types[node.attr["then_branch"].func.name] = arg_types + func_types[node.attr["else_branch"].func.name] = arg_types + return func_types + + def _populate_const_op(output_node, node_name, dtype, data, data_shape): """Creates a Const op. @@ -158,6 +197,21 @@ def _populate_const_op(output_node, node_name, dtype, data, data_shape): output_node.attr["value"].tensor.CopyFrom(tensor) +def _populate_identity_op(output_node, input_node): + """Creates an Identity op from a ReadVariable op. + + Args: + output_node: TensorFlow NodeDef. + input_node: TensorFlow NodeDef. + """ + output_node.op = "Identity" + output_node.name = input_node.name + output_node.input.append(input_node.input[0]) + output_node.attr["T"].CopyFrom(input_node.attr["dtype"]) + if "_class" in input_node.attr: + output_node.attr["_class"].CopyFrom(input_node.attr["_class"]) + + def _construct_concrete_function(func, output_graph_def, converted_input_indices): """Constructs a concrete function from the `output_graph_def`. @@ -193,7 +247,7 @@ def _construct_concrete_function(func, output_graph_def, return new_func -def convert_variables_to_constants_v2(func): +def convert_variables_to_constants_v2(func, lower_control_flow=True): """Replaces all the variables in a graph with constants of the same values. TensorFlow 2.0 function for converting all Variable ops into Const ops holding @@ -207,13 +261,15 @@ def convert_variables_to_constants_v2(func): Args: func: ConcreteFunction. + lower_control_flow: Boolean indicating whether or not to lower control flow + ops such as If and While. (default True) Returns: ConcreteFunction containing a simplified version of the original. """ # TODO(nupurgarg): Replace ResourceGather with Gather. - # TODO(nupurgarg): Change attr for Variables in control flow and functions. - graph_def = _run_inline_graph_optimization(func) + # Inline the graph in order to remove functions when possible. + graph_def = _run_inline_graph_optimization(func, lower_control_flow) # Get mapping from node name to node. name_to_node = {_get_tensor_name(node.name): node for node in graph_def.node} @@ -221,6 +277,10 @@ def convert_variables_to_constants_v2(func): # Get mapping from node name to variable value. tensor_data = _get_tensor_data(func) + # Get mapping from function name to argument types. + get_new_func_name = lambda func_name: func_name + "_frozen" + function_types = _get_control_flow_function_types(graph_def, tensor_data) + # Get variable data. reference_variables = {} resource_identities = {} @@ -235,6 +295,15 @@ def convert_variables_to_constants_v2(func): converted_input_indices.add(tensor_data[node_name]["index"]) for node in graph_def.node: + if node.op == "If": + # Get dtype and data for resource Placeholders. + then_func = node.attr["then_branch"].func.name + arg_types = function_types[then_func] + for idx, input_tensor in enumerate(node.input[1:]): + input_name = _get_tensor_name(input_tensor) + if input_name in tensor_data: + dtype = attr_value_pb2.AttrValue(type=arg_types[idx]) + _save_placeholder(_get_tensor_name(input_tensor), dtype) if node.op == "VariableV2": # Get data for VariableV2 ops (reference variables) that cannot be lifted. with func.graph.as_default(): @@ -278,21 +347,74 @@ def convert_variables_to_constants_v2(func): dtype = placeholders[input_node.name]["dtype"] _populate_const_op(output_node, input_node.name, dtype, data, data.shape) how_many_converted += 1 - # Change the dtype for Identity ops that are inputs to ReadVariableOps. + # Update the dtype for Identity ops that are inputs to ReadVariableOps. elif input_node.name in resource_identities: output_node.CopyFrom(input_node) output_node.attr["T"].CopyFrom(resource_identities[input_node.name]) # Convert ReadVariableOps to Identity ops. elif input_node.op == "ReadVariableOp": - output_node.op = "Identity" - output_node.name = input_node.name - output_node.input.extend([input_node.input[0]]) - output_node.attr["T"].CopyFrom(input_node.attr["dtype"]) - if "_class" in input_node.attr: - output_node.attr["_class"].CopyFrom(input_node.attr["_class"]) + _populate_identity_op(output_node, input_node) + # Update the function names and function's arguments types for the If ops. + elif input_node.op == "If": + output_node.CopyFrom(input_node) + then_func = input_node.attr["then_branch"].func.name + output_node.attr["then_branch"].func.name = get_new_func_name(then_func) + output_node.attr["else_branch"].func.name = get_new_func_name( + input_node.attr["else_branch"].func.name) + output_node.attr["Tin"].list.CopyFrom( + attr_value_pb2.AttrValue.ListValue(type=function_types[then_func])) else: output_node.CopyFrom(input_node) - logging.info("Converted %d variables to const ops.", how_many_converted) + # Add functions to reconstructed graph. + if graph_def.library: + library = output_graph_def.library + + for input_library_func in graph_def.library.function: + orig_func_name = input_library_func.signature.name + new_func_name = get_new_func_name(orig_func_name) + + # Do not copy any functions that aren't being used in the graph. Any + # functions that are not used by control flow should have been inlined. + if orig_func_name not in function_types: + continue + + output_library_func = library.function.add() + for key, value in input_library_func.ret.items(): + output_library_func.ret[key] = value + for key, value in input_library_func.control_ret.items(): + output_library_func.control_ret[key] = value + + # Update the input types in the function signature. + output_library_func.signature.CopyFrom(input_library_func.signature) + output_library_func.signature.name = new_func_name + for dtype, arg in zip(function_types[orig_func_name], + output_library_func.signature.input_arg): + arg.type = dtype + + # Update the NodeDefs. + func_variables = { + node.name: node.input[0] + for node in input_library_func.node_def + if node.op == "ReadVariableOp" + } + + for input_node in input_library_func.node_def: + output_node = output_library_func.node_def.add() + # Convert ReadVariableOps to Identity ops. + if input_node.op == "ReadVariableOp": + _populate_identity_op(output_node, input_node) + else: + output_node.CopyFrom(input_node) + # Convert :value to :output for ops that use the ReadVariableOp. + for idx, full_name in enumerate(input_node.input): + input_name = _get_tensor_name(full_name) + if input_name in func_variables: + full_name_parts = full_name.split(":") + full_name_parts[1] = "output" + input_name = ":".join(full_name_parts) + output_node.input[idx] = input_name + + output_graph_def.versions.CopyFrom(graph_def.versions) return _construct_concrete_function(func, output_graph_def, converted_input_indices) diff --git a/tensorflow/python/framework/convert_to_constants_test.py b/tensorflow/python/framework/convert_to_constants_test.py index 2ec340063fd..e06508804b8 100644 --- a/tensorflow/python/framework/convert_to_constants_test.py +++ b/tensorflow/python/framework/convert_to_constants_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import os +import numpy as np from tensorflow.python import keras from tensorflow.python.client import session as session_lib @@ -27,8 +28,13 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import convert_to_constants from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import rnn +from tensorflow.python.ops import rnn_cell_impl from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.saved_model import simple_save @@ -38,7 +44,6 @@ from tensorflow.python.training.tracking import tracking from tensorflow.python.util import nest -# TODO(nupurgarg): Simplify the test cases to use the ConcreteFunction. class VariablesToConstantsTest(test.TestCase): def _hasStatefulPartitionedCallOp(self, graph_def): @@ -56,29 +61,36 @@ class VariablesToConstantsTest(test.TestCase): input_data): # Check that the converted ConcreteFunction produces the same result as the # original Function. - expected_value = nest.flatten(func(input_data)) - actual_value = nest.flatten(converted_concrete_func(input_data)) - self.assertEqual(expected_value[0].numpy(), actual_value) + expected_value = nest.flatten(func(**input_data)) + actual_value = nest.flatten(converted_concrete_func(**input_data)) + + for expected, actual in zip(expected_value, actual_value): + np.testing.assert_almost_equal(expected.numpy(), actual.numpy()) # Ensure the shape is retained. - self.assertEqual(converted_concrete_func.inputs[0].shape, input_data.shape) + for tensor in converted_concrete_func.inputs: + actual_shape = input_data[tensor.name.split(":")[0]].shape + self.assertEqual(tensor.shape, actual_shape) # Save the converted ConcreteFunction as a signature. save_dir = os.path.join(self.get_temp_dir(), "frozen_saved_model") - save(obj, save_dir, {"mykey": converted_concrete_func}) + root = tracking.AutoTrackable() + root.f = converted_concrete_func + save(root, save_dir, {"mykey": converted_concrete_func}) # Load it back and make sure it works. loaded_obj = load(save_dir) - actual_value = nest.flatten(loaded_obj.signatures["mykey"](input_data)) - self.assertEqual(expected_value[0].numpy(), actual_value) + actual_value = nest.flatten(loaded_obj.signatures["mykey"](**input_data)) + for expected, actual in zip(expected_value, actual_value): + np.testing.assert_almost_equal(expected.numpy(), actual.numpy()) @test_util.run_v2_only def testConstSavedModel(self): """Test a basic model with functions to make sure functions are inlined.""" - input_data = constant_op.constant(1., shape=[1]) + input_data = {"x": constant_op.constant(1., shape=[1])} root = tracking.AutoTrackable() root.f = def_function.function(lambda x: 2. * x) - to_save = root.f.get_concrete_function(input_data) + to_save = root.f.get_concrete_function(input_data["x"]) save_dir = os.path.join(self.get_temp_dir(), "saved_model") save(root, save_dir, to_save) @@ -100,12 +112,12 @@ class VariablesToConstantsTest(test.TestCase): @test_util.run_v2_only def testVariableModel(self): """Test a basic model with Variables.""" - input_data = constant_op.constant(1., shape=[1]) + input_data = {"x": constant_op.constant(1., shape=[1])} root = tracking.AutoTrackable() root.v1 = variables.Variable(3.) root.v2 = variables.Variable(2.) root.f = def_function.function(lambda x: root.v1 * root.v2 * x) - input_func = root.f.get_concrete_function(input_data) + input_func = root.f.get_concrete_function(input_data["x"]) variable_graph_def = input_func.graph.as_graph_def() self.assertEqual(2, self._getNumVariables(variable_graph_def)) @@ -121,12 +133,12 @@ class VariablesToConstantsTest(test.TestCase): @test_util.run_v2_only def testScalarModel(self): """Test a basic model with Variables.""" - input_data = constant_op.constant(1., shape=[]) + input_data = {"x": constant_op.constant(1., shape=[])} root = tracking.AutoTrackable() root.v1 = variables.Variable(3.) root.v2 = variables.Variable(2.) root.f = def_function.function(lambda x: root.v1 * root.v2 * x) - input_func = root.f.get_concrete_function(input_data) + input_func = root.f.get_concrete_function(input_data["x"]) variable_graph_def = input_func.graph.as_graph_def() self.assertEqual(2, self._getNumVariables(variable_graph_def)) @@ -142,12 +154,12 @@ class VariablesToConstantsTest(test.TestCase): @test_util.run_v2_only def testVariableSavedModel(self): """Test a basic model with Variables with saving/loading the SavedModel.""" - input_data = constant_op.constant(1., shape=[1]) + input_data = {"x": constant_op.constant(1., shape=[1])} root = tracking.AutoTrackable() root.v1 = variables.Variable(3.) root.v2 = variables.Variable(2.) root.f = def_function.function(lambda x: root.v1 * root.v2 * x) - to_save = root.f.get_concrete_function(input_data) + to_save = root.f.get_concrete_function(input_data["x"]) save_dir = os.path.join(self.get_temp_dir(), "saved_model") save(root, save_dir, to_save) @@ -187,9 +199,9 @@ class VariablesToConstantsTest(test.TestCase): self.z = variables.Variable(3.) return x - self.z - input_data = constant_op.constant(1., shape=[1]) + input_data = {"x": constant_op.constant(1., shape=[1])} root = BasicModel() - input_func = root.add.get_concrete_function(input_data) + input_func = root.add.get_concrete_function(input_data["x"]) variable_graph_def = input_func.graph.as_graph_def() self.assertEqual(1, self._getNumVariables(variable_graph_def)) @@ -236,7 +248,7 @@ class VariablesToConstantsTest(test.TestCase): actual_value = nest.flatten(output_func(input_data)) self.assertEqual(expected_value.numpy(), actual_value) - def _v1_single_metagraph_saved_model(self): + def _singleMetaGraphSavedModel(self): export_graph = ops.Graph() with export_graph.as_default(): start = array_ops.placeholder( @@ -262,8 +274,8 @@ class VariablesToConstantsTest(test.TestCase): return path @test_util.run_v2_only - def test_ref_variable_import(self): - saved = self._v1_single_metagraph_saved_model() + def testRefVariableImport(self): + saved = self._singleMetaGraphSavedModel() imported = load(saved) fn = imported.signatures["serving_default"] output_func = convert_to_constants.convert_variables_to_constants_v2(fn) @@ -271,9 +283,76 @@ class VariablesToConstantsTest(test.TestCase): self.assertEqual(0, self._getNumVariables(constant_graph_def)) self.assertFalse(self._hasStatefulPartitionedCallOp(constant_graph_def)) - input_data = constant_op.constant(1., shape=[1, 1]) + input_data = {"start": constant_op.constant(1., shape=[1, 1])} root = tracking.AutoTrackable() self._testConvertedFunction(root, fn, output_func, input_data) + @test_util.run_v2_only + def testControlFlow(self): + input_data = { + "x": constant_op.constant([1., 2.], shape=[1, 2]), + "b": constant_op.constant(True) + } + + weights = variables.Variable([[0.1, 0.2], [0.3, 0.4]], dtype=dtypes.float32) + + def true_fn(x): + return math_ops.matmul(x, weights) + + def false_fn(x): + return math_ops.add(x, weights) + + @def_function.function(input_signature=[ + tensor_spec.TensorSpec(shape=[1, 2], dtype=dtypes.float32), + tensor_spec.TensorSpec(shape=(), dtype=dtypes.bool) + ]) + def model(x, b): + return control_flow_ops.cond( + b, true_fn=lambda: true_fn(x), false_fn=lambda: false_fn(x)) + + root = tracking.AutoTrackable() + root.f = model + input_func = root.f.get_concrete_function() + input_func(**input_data) + + output_func = convert_to_constants.convert_variables_to_constants_v2( + input_func, lower_control_flow=False) + constant_graph_def = output_func.graph.as_graph_def() + self.assertEqual(0, self._getNumVariables(constant_graph_def)) + self.assertFalse(self._hasStatefulPartitionedCallOp(constant_graph_def)) + + self._testConvertedFunction(root, root.f, output_func, input_data) + + @test_util.run_v2_only + def testStaticRnn(self): + input_data = { + "x": + constant_op.constant( + np.array(np.random.random_sample((3, 10)), dtype=np.float32)) + } + + cell = rnn_cell_impl.LSTMCell(10) + + @def_function.function(input_signature=[ + tensor_spec.TensorSpec(shape=[3, 10], dtype=dtypes.float32) + ]) + def model(x): + seq = array_ops.split(x, 3, 0) + return rnn.static_rnn( + cell, seq, dtype=dtypes.float32, sequence_length=[1]) + + root = tracking.AutoTrackable() + root.f = model + input_func = root.f.get_concrete_function() + + output_func = convert_to_constants.convert_variables_to_constants_v2( + input_func, lower_control_flow=False) + constant_graph_def = output_func.graph.as_graph_def() + self.assertEqual(0, self._getNumVariables(constant_graph_def)) + self.assertFalse(self._hasStatefulPartitionedCallOp(constant_graph_def)) + + self._testConvertedFunction(root, root.f, output_func, input_data) + + if __name__ == "__main__": test.main() From a7ecc1b1a03ea02cd140d88146b741a4672dba4d Mon Sep 17 00:00:00 2001 From: Bixia Zheng Date: Wed, 10 Jul 2019 12:11:17 -0700 Subject: [PATCH 216/332] [XLA] Fix CreateFromProto to return an error status for illegal reshape instructions. Previously, CreateFromProto would crash when handling a reshape instruction with non-array shapes. Fix it to return an error status instead, so that the fuzzer can ignore the input module. PiperOrigin-RevId: 257455123 --- tensorflow/compiler/xla/service/hlo_instruction.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 7274099ad97..9a083a46ec0 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -614,8 +614,9 @@ StatusOr> HloInstruction::CreateFromProto( if (!proto.dimensions().empty()) { inferred_dimension = proto.dimensions()[0]; } - TF_RET_CHECK(ShapeUtil::ElementsIn(shape) == - ShapeUtil::ElementsIn(operands(0)->shape())) + TF_RET_CHECK(shape.IsArray() && operands(0)->shape().IsArray() && + ShapeUtil::ElementsIn(shape) == + ShapeUtil::ElementsIn(operands(0)->shape())) << "shape: " << ShapeUtil::HumanString(shape) << " operand: " << ShapeUtil::HumanString(operands(0)->shape()); instruction = CreateReshape(shape, operands(0), inferred_dimension); From 36e969ee0b105b39c3c2d45cab5961f74348a22a Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Wed, 10 Jul 2019 12:14:17 -0700 Subject: [PATCH 217/332] Remove TuplePointsToAnalysis::CanShareOperandBufferWithUser. The split-brain state of this complicated function and its counter part in DataflowAnalysis, after a few syncs, got eventually out of sync. This has become the source of many nasty bugs. This CL removes the one in tuple points to analysis let the one in data flow analysis become the source of truth. PiperOrigin-RevId: 257455657 --- .../xla/service/tuple_points_to_analysis.cc | 143 ------- .../xla/service/tuple_points_to_analysis.h | 10 +- .../service/tuple_points_to_analysis_test.cc | 378 ------------------ 3 files changed, 1 insertion(+), 530 deletions(-) diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc index 3868bbe22e4..9ff819437b3 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc @@ -708,147 +708,4 @@ bool TuplePointsToAnalysis::HasUniqueFusedUseOfOperandAt( fused_param_uses[0].first == fusion->fused_expression_root() && fused_param_uses[0].second == use_operand_index; } - -// User and operand can share buffers iff both instructions emit the same shape -// and layout, and 'user' meets one of the following qualifications: -// -// (1) Is element-wise. Or... -// (2) Is a loop fusion instruction where the only use of 'operand' at 'index' -// in the set 'user.fused_instructions' is a DynamicUpdateSlice fused root -// at operand 0. Or... -// (3) Is a kDot -> kAdd output fusion instruction where the only use of -// 'operand' at 'index' in the set 'user.fused_instructions' is a kAdd fused -// root at operand 0 or 1. Or... -// (4) The 'user' of 'operand' is DynamicUpdateSlice or While at operand index -// 0. -// (5) The 'user' of 'operand' is Sort, and it is the only user. -// (6) The 'user' of 'operand' is TriangularSolve, it is the second operand, -// and it is the only user. -// -// (2) and (3) can only be determined if points-to analysis is available. -bool TuplePointsToAnalysis::CanShareOperandBufferWithUser( - HloInstruction* operand, const ShapeIndex& operand_index, - HloInstruction* user, const ShapeIndex& user_index) const { - CHECK(user->IsUserOf(operand)) - << "user: " << user->ToString() << " operand: " << operand->ToString(); - const Shape& operand_subshape = - ShapeUtil::GetSubshape(operand->shape(), operand_index); - const Shape& user_subshape = - ShapeUtil::GetSubshape(user->shape(), user_index); - // Check that operand and user emit the same shape and layout. - if (!ShapeUtil::Equal(operand_subshape, user_subshape)) { - return false; - } - if (user->opcode() == HloOpcode::kFusion) { - if (user->IsLoopFusion() || user->IsInputFusion()) { - if (user->fused_expression_root()->opcode() == - HloOpcode::kDynamicUpdateSlice) { - // Loop fusion with kDynamicUpdateSlice fused root. - // - // Returns true iff there is exactly one use of 'operand' at shape index - // 'operand_index', and this singleton use is the fused root at operand - // index 0. - return HasUniqueFusedUseOfOperandAt(operand, operand_index, user, 0); - } else { - HloInstruction* fusion_param = - user->fused_parameter(user->operand_index(operand)); - return HloDataflowAnalysis::AreTransitiveUsesElementwiseOrTuple( - fusion_param); - } - } else if (user->IsOutputFusion() && - user->fused_expression_root()->opcode() == HloOpcode::kAdd) { - // Output fusion with kAdd fused root. - - // Check if one operand of kAdd fused root is kDot or kConvolution. - auto* add = user->fused_expression_root(); - auto add_operand_it = - absl::c_find_if(add->operands(), [&](HloInstruction* operand) { - return operand->opcode() == HloOpcode::kConvolution || - operand->opcode() == HloOpcode::kDot; - }); - if (add_operand_it == add->operands().end()) { - return false; - } - auto* matched_add_operand = *add_operand_it; - // Calculate operand index of 'add' operand which was not matched above. - const int64 other_add_operand_index = - matched_add_operand == add->operand(0) ? 1 : 0; - // Returns true iff there is exactly one use of 'operand' at shape index - // 'operand_index', and this singleton use is the fused root (at operand - // index 'other_add_operand_index'). - return HasUniqueFusedUseOfOperandAt(operand, operand_index, user, - other_add_operand_index); - } else if (user->IsCustomFusion()) { - std::vector operand_indices = user->OperandIndices(operand); - return operand_indices.size() == 1 && operand_indices[0] == 0 && - absl::c_any_of( - user->fused_instructions_computation()->instructions(), - [](const HloInstruction* hlo) { - return hlo->opcode() == HloOpcode::kScatter; - }); - } - } - if (user->opcode() == HloOpcode::kDynamicUpdateSlice || - user->opcode() == HloOpcode::kScatter || - user->opcode() == HloOpcode::kWhile) { - // We eliminated other users in BufferLiveness::live_range_strictly_before, - // so here we just need to check that the use is at operand index 0. - std::vector operand_indices = user->OperandIndices(operand); - return operand_indices.size() == 1 && operand_indices[0] == 0; - } - if (user->opcode() == HloOpcode::kSort) { - // Only valid if there are no other users. - if (operand->users().size() != 1) { - return false; - } - // If we only sort keys, the output of sort is not a tuple, so we can always - // share the buffer. - if (user->operand_count() == 1) { - return true; - } - CHECK(!user_index.empty()); - // Only share with the right tuple element buffer. - std::vector operand_indices = user->OperandIndices(operand); - return operand_indices.size() == 1 && user_index[0] == operand_indices[0]; - } - if (user->opcode() == HloOpcode::kTriangularSolve) { - // Only valid if there are no other users. - if (operand->users().size() != 1) { - return false; - } - std::vector operand_indices = user->OperandIndices(operand); - return operand_indices.size() == 1 && operand_indices[0] == 1; - } - if (user->opcode() == HloOpcode::kCall) { - // TODO(b/62548313): Remove when buffer assignment is module scoped and - // does not assign buffers to calls. - // Find called computation parameter associated with 'operand'. - const std::vector operand_indices = user->OperandIndices(operand); - if (operand_indices.size() > 1) { - return false; - } - CHECK_EQ(1, operand_indices.size()); - auto* param = user->to_apply()->parameter_instruction(operand_indices[0]); - // Get all uses of 'operand' at 'index' in called computation. - auto param_uses = GetAllUsesOfInstructionAtIndex(param, operand_index); - - // Return true iff: - // *) There exists exactly one use of 'operand' in called computation. - // *) The unique use is by the root instruction of called computation. - // (Note: we check the root of the called computation, because the - // root result buffer is required to alias with the Call result buffer). - // *) The root instruction of the called computation is element-wise on - // 'operand'. - auto* callee_root = user->to_apply()->root_instruction(); - return param_uses.size() == 1 && param_uses[0].first == callee_root && - callee_root->IsElementwiseOnOperand(param_uses[0].second); - } - // Loop fusions that contain transposing copies won't reach here as they have - // different layouts, which fails the check in the beginning of this function. - // - // Multi-output fusion will fail the check here as tuples are not considered - // an elementwise operation. - return user->IsElementwiseOnOperand(user->operand_index(operand)); -} - } // namespace xla diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h index ec0f7eeea2d..cb589326ba7 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_TUPLE_POINTS_TO_ANALYSIS_H_ #include + #include #include #include @@ -265,15 +266,6 @@ class TuplePointsToAnalysis : public DfsHloVisitorWithDefault { const ShapeIndex& index, const HloInstruction* user) const; - // Returns true if 'user' (at 'user_index') can share a buffer with its - // operand 'operand' (at 'operand_index'). Returns false otherwise. - // - // REQUIRES: 'operand' is an operand of 'user'. - bool CanShareOperandBufferWithUser(HloInstruction* operand, - const ShapeIndex& operand_index, - HloInstruction* user, - const ShapeIndex& user_index) const; - private: explicit TuplePointsToAnalysis( const HloModule* module, diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc index 4f8d1b92a98..d0515fb5825 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis_test.cc @@ -928,383 +928,5 @@ TEST_F(DoesNotUseOperandBufferTest, FusedDynamicUpdateSlice) { EXPECT_FALSE( points_to_analysis_->DoesNotUseOperandBuffer(tuple, {1}, fusion)); } - -class CanShareOperandBufferWithUserTest : public PointsToAnalysisTestBase {}; - -TEST_F(CanShareOperandBufferWithUserTest, ElementWiseSameShape) { - auto builder = HloComputation::Builder(TestName()); - - Shape shape = ShapeUtil::MakeShape(F32, {8}); - auto param = builder.AddInstruction( - HloInstruction::CreateParameter(0, shape, "param")); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(shape, HloOpcode::kExp, param)); - auto log = builder.AddInstruction( - HloInstruction::CreateUnary(shape, HloOpcode::kLog, exp)); - - BuildModuleAndRunAnalysis(builder.Build()); - - EXPECT_TRUE( - points_to_analysis_->CanShareOperandBufferWithUser(param, {}, exp, {})); - EXPECT_TRUE( - points_to_analysis_->CanShareOperandBufferWithUser(exp, {}, log, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, ElementWiseDifferentShape) { - auto builder = HloComputation::Builder(TestName()); - - Shape in_shape = ShapeUtil::MakeShape(F32, {8}); - Shape out_shape = ShapeUtil::MakeShape(PRED, {8}); - auto param0 = builder.AddInstruction( - HloInstruction::CreateParameter(0, in_shape, "param0")); - auto param1 = builder.AddInstruction( - HloInstruction::CreateParameter(1, in_shape, "param1")); - auto result = builder.AddInstruction(HloInstruction::CreateCompare( - out_shape, param0, param1, ComparisonDirection::kEq)); - - BuildModuleAndRunAnalysis(builder.Build()); - - EXPECT_FALSE(points_to_analysis_->CanShareOperandBufferWithUser(param0, {}, - result, {})); - EXPECT_FALSE(points_to_analysis_->CanShareOperandBufferWithUser(param1, {}, - result, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, CopyShares) { - auto builder = HloComputation::Builder(TestName()); - - Shape shape = ShapeUtil::MakeShape(F32, {8}); - auto param = builder.AddInstruction( - HloInstruction::CreateParameter(0, shape, "param")); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(shape, HloOpcode::kExp, param)); - auto copy = builder.AddInstruction( - HloInstruction::CreateUnary(shape, HloOpcode::kCopy, exp)); - - BuildModuleAndRunAnalysis(builder.Build()); - - EXPECT_TRUE( - points_to_analysis_->CanShareOperandBufferWithUser(param, {}, exp, {})); - EXPECT_TRUE( - points_to_analysis_->CanShareOperandBufferWithUser(exp, {}, copy, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, FusedDynamicUpdateSlice) { - auto builder = HloComputation::Builder(TestName()); - - Shape data_shape = ShapeUtil::MakeShape(F32, {8}); - auto tuple = builder.AddInstruction(HloInstruction::CreateParameter( - 0, ShapeUtil::MakeTupleShape({data_shape, data_shape}), "tuple")); - auto gte0 = builder.AddInstruction( - HloInstruction::CreateGetTupleElement(data_shape, tuple, 0)); - auto gte1 = builder.AddInstruction( - HloInstruction::CreateGetTupleElement(data_shape, tuple, 1)); - - // Create a DynamicUpdateSlice instruction of tuple element 1. - auto starts = builder.AddInstruction( - HloInstruction::CreateConstant(LiteralUtil::CreateR0(2))); - auto update = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR1({2.f, 2.f, 2.f}))); - auto dynamic_update_slice = - builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice( - data_shape, gte1, update, {starts})); - builder.AddInstruction( - HloInstruction::CreateTuple({gte0, dynamic_update_slice})); - - BuildModule(builder.Build()); - auto fusion = computation_->CreateFusionInstruction( - {dynamic_update_slice, starts, update, gte1}, - HloInstruction::FusionKind::kLoop); - RunAnalysis(); - - // The fusion instruction can share with tuple element 1. - EXPECT_FALSE(points_to_analysis_->CanShareOperandBufferWithUser(tuple, {0}, - fusion, {})); - EXPECT_TRUE(points_to_analysis_->CanShareOperandBufferWithUser(tuple, {1}, - fusion, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, DynamicUpdateSliceCanShare) { - auto builder = HloComputation::Builder(TestName()); - - Shape data_shape = ShapeUtil::MakeShape(F32, {8}); - Shape update_shape = ShapeUtil::MakeShape(F32, {4}); - Shape starts_shape = ShapeUtil::MakeShape(S32, {}); - auto data = builder.AddInstruction( - HloInstruction::CreateParameter(0, data_shape, "data")); - auto update = builder.AddInstruction( - HloInstruction::CreateParameter(1, update_shape, "update")); - auto starts = builder.AddInstruction( - HloInstruction::CreateParameter(2, starts_shape, "starts")); - auto dus = builder.AddInstruction(HloInstruction::CreateDynamicUpdateSlice( - data_shape, data, update, {starts})); - - BuildModuleAndRunAnalysis(builder.Build()); - - // The DynamicUpdateSlice instruction can share with the data operand, but not - // with update or starts. - EXPECT_TRUE( - points_to_analysis_->CanShareOperandBufferWithUser(data, {}, dus, {})); - EXPECT_FALSE( - points_to_analysis_->CanShareOperandBufferWithUser(update, {}, dus, {})); - EXPECT_FALSE( - points_to_analysis_->CanShareOperandBufferWithUser(starts, {}, dus, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, ScatterCanShare) { - const char* hlo_text = R"( - HloModule TensorFlowScatterV1 - - update_s32 (lhs: s32[], rhs: s32[]) -> s32[] { - lhs = s32[] parameter(0) - ROOT rhs = s32[] parameter(1) - } - - ENTRY main { - operand = s32[3,3] parameter(0) - indices = s32[2] parameter(1) - updates = s32[2,3] parameter(2) - ROOT scatter = s32[3,3] scatter(operand, indices, updates), - to_apply=update_s32, - update_window_dims={1}, - inserted_window_dims={0}, - scatter_dims_to_operand_dims={0}, - index_vector_dim=1 - } - )"; - TF_ASSERT_OK_AND_ASSIGN(module_, ParseAndReturnUnverifiedModule(hlo_text)); - computation_ = module_->entry_computation(); - RunAnalysis(); - - HloInstruction* operand_param = computation_->parameter_instruction(0); - HloInstruction* indices_param = computation_->parameter_instruction(1); - HloInstruction* updates_param = computation_->parameter_instruction(2); - HloInstruction* scatter = computation_->root_instruction(); - - EXPECT_TRUE(points_to_analysis_->CanShareOperandBufferWithUser( - operand_param, {}, scatter, {})); - EXPECT_FALSE(points_to_analysis_->CanShareOperandBufferWithUser( - indices_param, {}, scatter, {})); - EXPECT_FALSE(points_to_analysis_->CanShareOperandBufferWithUser( - updates_param, {}, scatter, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, SortCanShare) { - auto builder = HloComputation::Builder(TestName()); - module_ = CreateNewVerifiedModule(); - - Shape keys_shape = ShapeUtil::MakeShape(F32, {8}); - auto keys = builder.AddInstruction( - HloInstruction::CreateParameter(0, keys_shape, "keys")); - TF_ASSERT_OK_AND_ASSIGN( - auto* sort, MakeSortHlo(keys_shape, {keys}, 0, /*is_stable=*/false, - &builder, module_.get())); - - computation_ = module_->AddEntryComputation(builder.Build()); - RunAnalysis(); - - EXPECT_TRUE( - points_to_analysis_->CanShareOperandBufferWithUser(keys, {}, sort, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, SortCanShareWithTupleUser) { - auto builder = HloComputation::Builder(TestName()); - module_ = CreateNewVerifiedModule(); - - Shape keys_shape = ShapeUtil::MakeShape(F32, {8}); - Shape values_shape = ShapeUtil::MakeShape(F32, {8}); - auto keys = builder.AddInstruction( - HloInstruction::CreateParameter(0, keys_shape, "keys")); - auto values = builder.AddInstruction( - HloInstruction::CreateParameter(1, values_shape, "values")); - TF_ASSERT_OK_AND_ASSIGN( - auto* sort, - MakeSortHlo(ShapeUtil::MakeTupleShape({keys_shape, values_shape}), - {keys, values}, 0, /*is_stable=*/false, &builder, - module_.get())); - - computation_ = module_->AddEntryComputation(builder.Build()); - RunAnalysis(); - - // The buffer for the keys can be shared with the first tuple entry. - EXPECT_TRUE( - points_to_analysis_->CanShareOperandBufferWithUser(keys, {}, sort, {0})); - // The buffer for the values can be shared with the second tuple entry. - EXPECT_TRUE(points_to_analysis_->CanShareOperandBufferWithUser(values, {}, - sort, {1})); - // Verify that the buffers are not shared with the "wrong" tuple entry. - EXPECT_FALSE( - points_to_analysis_->CanShareOperandBufferWithUser(keys, {}, sort, {1})); - EXPECT_FALSE(points_to_analysis_->CanShareOperandBufferWithUser(values, {}, - sort, {0})); -} - -TEST_F(CanShareOperandBufferWithUserTest, FusedDotAdd) { - auto builder = HloComputation::Builder(TestName()); - Shape data_shape = ShapeUtil::MakeShape(F32, {2, 2}); - - auto a = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR2({{1.0, 0.0}, {0.0, 1.0}}))); - auto b = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR2({{2.0, 2.0}, {2.0, 2.0}}))); - - DotDimensionNumbers dot_dnums; - dot_dnums.add_lhs_contracting_dimensions(1); - dot_dnums.add_rhs_contracting_dimensions(0); - PrecisionConfig precision_config; - precision_config.mutable_operand_precision()->Resize( - /*new_size=*/2, PrecisionConfig::DEFAULT); - auto dot = builder.AddInstruction( - HloInstruction::CreateDot(data_shape, a, b, dot_dnums, precision_config)); - - auto one = builder.AddInstruction( - HloInstruction::CreateConstant(LiteralUtil::CreateR0(1.0))); - auto add_operand = builder.AddInstruction( - HloInstruction::CreateBroadcast(data_shape, one, {1})); - - auto add = builder.AddInstruction(HloInstruction::CreateBinary( - data_shape, HloOpcode::kAdd, dot, add_operand)); - - BuildModule(builder.Build()); - auto fusion = computation_->CreateFusionInstruction( - {add, dot}, HloInstruction::FusionKind::kOutput); - RunAnalysis(); - - // Output fused dot add should be able to share buffer with 'add_operand'. - EXPECT_TRUE(points_to_analysis_->CanShareOperandBufferWithUser( - add_operand, {}, fusion, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, OutputFusionCantAliasOperandBuffer) { - auto builder = HloComputation::Builder(TestName()); - Shape data_shape = ShapeUtil::MakeShape(F32, {2, 2}); - - auto one = builder.AddInstruction( - HloInstruction::CreateConstant(LiteralUtil::CreateR0(1.0))); - auto operand = builder.AddInstruction( - HloInstruction::CreateBroadcast(data_shape, one, {1})); - - auto reverse = builder.AddInstruction( - HloInstruction::CreateReverse(data_shape, operand, {0, 1})); - - auto two = builder.AddInstruction(HloInstruction::CreateConstant( - LiteralUtil::CreateR2({{2.0, 2.0}, {2.0, 2.0}}))); - - auto add = builder.AddInstruction( - HloInstruction::CreateBinary(data_shape, HloOpcode::kAdd, reverse, two)); - - BuildModule(builder.Build()); - auto fusion = computation_->CreateFusionInstruction( - {add, two, reverse}, HloInstruction::FusionKind::kOutput); - RunAnalysis(); - - // Output fused operand->reverse->add cannot alias operand buffer 'operand'. - EXPECT_FALSE(points_to_analysis_->CanShareOperandBufferWithUser(operand, {}, - fusion, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, WhileCanShare) { - Shape data_shape = ShapeUtil::MakeShape(F32, {8}); - - auto make_cond = [&data_shape]() { - auto builder = HloComputation::Builder(TestName() + ".Cond"); - auto data = builder.AddInstruction( - HloInstruction::CreateParameter(0, data_shape, "data")); - builder.AddInstruction(HloInstruction::CreateCompare( - ShapeUtil::MakeShape(PRED, {}), data, data, ComparisonDirection::kEq)); - return builder.Build(); - }; - - auto make_body = [&data_shape]() { - auto builder = HloComputation::Builder(TestName() + ".Body"); - auto data = builder.AddInstruction( - HloInstruction::CreateParameter(0, data_shape, "data")); - builder.AddInstruction( - HloInstruction::CreateBinary(data_shape, HloOpcode::kAdd, data, data)); - return builder.Build(); - }; - - module_ = CreateNewUnverifiedModule(); - HloComputation* cond_computation = - module_->AddEmbeddedComputation(make_cond()); - HloComputation* body_computation = - module_->AddEmbeddedComputation(make_body()); - - auto builder = HloComputation::Builder(TestName()); - auto data = builder.AddInstruction( - HloInstruction::CreateParameter(0, data_shape, "data")); - auto whil = builder.AddInstruction(HloInstruction::CreateWhile( - data_shape, cond_computation, body_computation, data)); - computation_ = module_->AddEntryComputation(builder.Build()); - - RunAnalysis(); - - // The While instruction can share with the data operand. - EXPECT_TRUE( - points_to_analysis_->CanShareOperandBufferWithUser(data, {}, whil, {})); -} - -// Tests that Call can alias operand buffer if the only use of the operand -// in the called computation is an elementwise instruction. -TEST_F(CanShareOperandBufferWithUserTest, CallToComputationWithFusionRoot) { - Shape shape = ShapeUtil::MakeShape(F32, {8}); - // Build sub-computation with fusion root. - auto sub_builder = HloComputation::Builder(TestName() + "_sub"); - auto sub_param = sub_builder.AddInstruction( - HloInstruction::CreateParameter(0, shape, "sub_param")); - auto one = sub_builder.AddInstruction( - HloInstruction::CreateConstant(LiteralUtil::CreateR0(1.0))); - auto ones = sub_builder.AddInstruction( - HloInstruction::CreateBroadcast(shape, one, {1})); - auto add = sub_builder.AddInstruction( - HloInstruction::CreateBinary(shape, HloOpcode::kAdd, sub_param, ones)); - - module_ = CreateNewUnverifiedModule(); - auto sub_computation = module_->AddEmbeddedComputation(sub_builder.Build()); - sub_computation->CreateFusionInstruction({add, ones}, - HloInstruction::FusionKind::kLoop); - - // Build entry-computation with kCall which calls 'sub_computation'. - auto builder = HloComputation::Builder(TestName()); - - auto param = builder.AddInstruction( - HloInstruction::CreateParameter(0, shape, "param")); - auto reverse = - builder.AddInstruction(HloInstruction::CreateReverse(shape, param, {0})); - auto call = builder.AddInstruction( - HloInstruction::CreateCall(shape, {reverse}, sub_computation)); - computation_ = module_->AddEntryComputation(builder.Build()); - - RunAnalysis(); - - EXPECT_TRUE(points_to_analysis_->CanShareOperandBufferWithUser(reverse, {}, - call, {})); -} - -TEST_F(CanShareOperandBufferWithUserTest, LoopFusionWithElementwiseOperand) { - Shape full_shape = ShapeUtil::MakeShape(F32, {16, 32}); - Shape broadcast_shape = ShapeUtil::MakeShape(F32, {16}); - - auto builder = HloComputation::Builder(TestName() + "_fusion"); - auto param0 = builder.AddInstruction( - HloInstruction::CreateParameter(0, full_shape, "full")); - auto param1 = builder.AddInstruction( - HloInstruction::CreateParameter(1, broadcast_shape, "small")); - auto broadcast = builder.AddInstruction( - HloInstruction::CreateBroadcast(full_shape, param1, {0})); - auto add = builder.AddInstruction(HloInstruction::CreateBinary( - full_shape, HloOpcode::kAdd, param0, broadcast)); - - BuildModule(builder.Build()); - auto fusion = computation_->CreateFusionInstruction( - {add, broadcast}, HloInstruction::FusionKind::kLoop); - RunAnalysis(); - - EXPECT_TRUE(points_to_analysis_->CanShareOperandBufferWithUser(param0, {}, - fusion, {})); - EXPECT_FALSE(points_to_analysis_->CanShareOperandBufferWithUser(param1, {}, - fusion, {})); -} - } // namespace } // namespace xla From 17521bcffd230abe7f7f5ceec6279b9a5504529a Mon Sep 17 00:00:00 2001 From: Yu-Cheng Ling Date: Wed, 10 Jul 2019 12:37:54 -0700 Subject: [PATCH 218/332] Fix invoking while multiple times when body is dynamic. PiperOrigin-RevId: 257459721 --- tensorflow/lite/kernels/while.cc | 55 +++++++++++++++++++-------- tensorflow/lite/kernels/while_test.cc | 9 +++-- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/tensorflow/lite/kernels/while.cc b/tensorflow/lite/kernels/while.cc index b3f00d3fe13..a6438558458 100644 --- a/tensorflow/lite/kernels/while.cc +++ b/tensorflow/lite/kernels/while.cc @@ -28,21 +28,36 @@ namespace { // Propagate tensor shapes and types from `src_tensor_indices` in `src_subgraph` // to `dst_tensor_indices` in `dst_subgraph`. +// +// When `resize_subgraph_inputs` is true, the function calls subgraphs's +// `ResizeInputTensor` function, and it may trigger the memory planner to +// reallocate memory. +// When `resize_subgraph_inputs` is false, it implies `context` belongs to +// `dst_subgraph`. The function calls `context->ResizeTensor`. This happens +// when resizing `While` op's outputs. template TfLiteStatus CopyTensorsShapeAndType(TfLiteContext* context, Subgraph* src_subgraph, const SrcVector& src_tensor_indices, Subgraph* dst_subgraph, - const DstVector& dst_tensor_indices) { + const DstVector& dst_tensor_indices, + bool resize_subgraph_inputs) { TF_LITE_ENSURE_EQ(context, src_tensor_indices.size(), dst_tensor_indices.size()); for (int i = 0; i < src_tensor_indices.size(); ++i) { const TfLiteTensor* src_tensor = src_subgraph->tensor(src_tensor_indices[i]); - std::vector dims(src_tensor->dims->data, - src_tensor->dims->data + src_tensor->dims->size); - dst_subgraph->ResizeInputTensor(dst_tensor_indices[i], dims); + TfLiteTensor* dst_tensor = dst_subgraph->tensor(dst_tensor_indices[i]); + if (resize_subgraph_inputs) { + std::vector dims(src_tensor->dims->data, + src_tensor->dims->data + src_tensor->dims->size); + dst_subgraph->ResizeInputTensor(dst_tensor_indices[i], dims); + } else { + TF_LITE_ENSURE_OK( + context, context->ResizeTensor(context, dst_tensor, + TfLiteIntArrayCopy(src_tensor->dims))); + } dst_tensor->type = src_tensor->type; } return kTfLiteOk; @@ -130,9 +145,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { // Prepare and check the condition subgraph. TF_LITE_ENSURE_OK( - context, CopyTensorsShapeAndType(context, this_subgraph, - TfLiteIntArrayView(node->inputs), - cond_subgraph, cond_subgraph->inputs())); + context, CopyTensorsShapeAndType( + context, this_subgraph, TfLiteIntArrayView(node->inputs), + cond_subgraph, cond_subgraph->inputs(), true)); TF_LITE_ENSURE_OK(context, cond_subgraph->AllocateTensors()); TfLiteTensor* cond_output = cond_subgraph->tensor(cond_subgraph->outputs()[0]); @@ -148,9 +163,9 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { // Prepare and check the body subgraph. TF_LITE_ENSURE_OK( - context, CopyTensorsShapeAndType(context, this_subgraph, - TfLiteIntArrayView(node->inputs), - body_subgraph, body_subgraph->inputs())); + context, CopyTensorsShapeAndType( + context, this_subgraph, TfLiteIntArrayView(node->inputs), + body_subgraph, body_subgraph->inputs(), true)); TF_LITE_ENSURE_OK(context, body_subgraph->AllocateTensors()); if (body_subgraph->HasDynamicTensors()) { op_data->body_has_dynamic_output_tensors = true; @@ -232,6 +247,16 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { // boundary. Currently we copy the input / output between the subgraphs. This // isn't optimized yet and a lot of redundant copies are made. // TODO(b/120234921): Optimize and avoid copying tensors between subgraphs. + + if (op_data->body_has_dynamic_output_tensors) { + // If body subgraph has dynamic outputs, the input of condition subgraph may + // be changed in the last invocation and may need resizing. + TF_LITE_ENSURE_OK( + context, CopyTensorsShapeAndType( + context, this_subgraph, TfLiteIntArrayView(node->inputs), + cond_subgraph, cond_subgraph->inputs(), true)); + TF_LITE_ENSURE_OK(context, cond_subgraph->AllocateTensors()); + } TF_LITE_ENSURE_OK( context, CopyTensorsData(context, this_subgraph, TfLiteIntArrayView(node->inputs), @@ -254,7 +279,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_OK(context, CopyTensorsShapeAndType( context, cond_subgraph, cond_subgraph->inputs(), - body_subgraph, body_subgraph->inputs())); + body_subgraph, body_subgraph->inputs(), true)); TF_LITE_ENSURE_OK(context, body_subgraph->AllocateTensors()); } @@ -273,7 +298,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_OK(context, CopyTensorsShapeAndType( context, body_subgraph, body_subgraph->outputs(), - cond_subgraph, cond_subgraph->inputs())); + cond_subgraph, cond_subgraph->inputs(), true)); TF_LITE_ENSURE_OK(context, cond_subgraph->AllocateTensors()); } @@ -287,9 +312,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { // TODO(b/120234921): Optimize and avoid copying tensors between subgraphs. if (op_data->body_has_dynamic_output_tensors) { TF_LITE_ENSURE_OK( - context, CopyTensorsShapeAndType(context, cond_subgraph, - cond_subgraph->inputs(), this_subgraph, - TfLiteIntArrayView(node->outputs))); + context, CopyTensorsShapeAndType( + context, cond_subgraph, cond_subgraph->inputs(), + this_subgraph, TfLiteIntArrayView(node->outputs), false)); } TF_LITE_ENSURE_OK( diff --git a/tensorflow/lite/kernels/while_test.cc b/tensorflow/lite/kernels/while_test.cc index a3a80ea6f50..1745f585ed0 100644 --- a/tensorflow/lite/kernels/while_test.cc +++ b/tensorflow/lite/kernels/while_test.cc @@ -59,8 +59,6 @@ TEST_F(WhileTest, TestTriangularNumberSequence) { } } -// This requires dynamic sized subgraphs and it's not supported right now. -// TODO(ycling): Support dynamic sized subgraphs. TEST_F(WhileTest, TestPadLoop) { interpreter_.reset(new Interpreter); interpreter_->AddSubgraphs(2); @@ -70,8 +68,6 @@ TEST_F(WhileTest, TestPadLoop) { interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {1}); interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {2}); - // This is not supported yet. The test ensures thatit doesn't crash and raises - // an error properly. ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); FillIntTensor(interpreter_->tensor(interpreter_->inputs()[0]), {1}); @@ -82,6 +78,11 @@ TEST_F(WhileTest, TestPadLoop) { CheckIntTensor(output1, {1}, {4}); TfLiteTensor* output2 = interpreter_->tensor(interpreter_->outputs()[1]); CheckIntTensor(output2, {11}, {0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0}); + + // The extra invocation serves as a regiression test: There was a bug that + // invoking a while loop with dynamic shaped body makes the interpreter + // state uninvokable. + ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk); } } // namespace From fc65d8b83a308027399338e052c426524943accf Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 10 Jul 2019 13:11:17 -0700 Subject: [PATCH 219/332] Use archive.debian.org for jessie-backports as they got removed from main. PiperOrigin-RevId: 257466395 --- tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu index 438ffa4821d..cf04ebb4620 100644 --- a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu +++ b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu @@ -5,7 +5,7 @@ LABEL maintainer="Jan Prach " # Copy and run the install scripts. COPY install/*.sh /install/ RUN /install/install_bootstrap_deb_packages.sh -RUN echo "deb http://archive.debian.org/debian jessie-backports main" | \ +RUN echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" | \ tee -a /etc/apt/sources.list # Workaround bug in Jessie backport repository deb packages # http://serverfault.com/questions/830636/cannot-install-openjdk-8-jre-headless-on-debian-jessie From 60e79a2689cb90ea83b56517faa90da5daa95eb5 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Wed, 10 Jul 2019 20:24:58 +0000 Subject: [PATCH 220/332] Adding ROCm support for the bias op --- tensorflow/core/kernels/bias_op.cc | 9 +++-- tensorflow/core/kernels/bias_op_gpu.cu.cc | 48 ++++++++++++++--------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/tensorflow/core/kernels/bias_op.cc b/tensorflow/core/kernels/bias_op.cc index 6a407f5551a..00b920c5f45 100644 --- a/tensorflow/core/kernels/bias_op.cc +++ b/tensorflow/core/kernels/bias_op.cc @@ -18,7 +18,6 @@ limitations under the License. #define EIGEN_USE_THREADS #include "tensorflow/core/kernels/bias_op.h" - #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/numeric_op.h" @@ -28,9 +27,11 @@ limitations under the License. #include "tensorflow/core/kernels/redux_functor.h" #include "tensorflow/core/util/tensor_format.h" -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #include "tensorflow/core/kernels/bias_op_gpu.h" #include "tensorflow/core/platform/stream_executor.h" +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM +#if GOOGLE_CUDA #include "tensorflow/stream_executor/cuda/cuda_stream.h" #endif // GOOGLE_CUDA @@ -319,7 +320,7 @@ REGISTER_KERNEL(double); #undef REGISTER_KERNEL #endif // TENSORFLOW_USE_SYCL -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM template class BiasOp : public BinaryOp { public: @@ -618,6 +619,6 @@ class BiasGradOp : public OpKernel { TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL); #undef REGISTER_GPU_KERNEL -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM } // namespace tensorflow diff --git a/tensorflow/core/kernels/bias_op_gpu.cu.cc b/tensorflow/core/kernels/bias_op_gpu.cu.cc index 764d087f45d..843aedc3e0f 100644 --- a/tensorflow/core/kernels/bias_op_gpu.cu.cc +++ b/tensorflow/core/kernels/bias_op_gpu.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -53,7 +53,7 @@ struct AccumulatorType { template __global__ void BiasNHWCKernel(int32 nthreads, const T* input, const T* bias, T* output, int32 bias_size) { - CUDA_1D_KERNEL_LOOP(index, nthreads) { + GPU_1D_KERNEL_LOOP(index, nthreads) { int32 bias_offset = index % bias_size; output[index] = ldg(input + index) + ldg(bias + bias_offset); } @@ -62,7 +62,7 @@ __global__ void BiasNHWCKernel(int32 nthreads, const T* input, const T* bias, template __global__ void BiasNCHWKernel(int32 nthreads, const T* input, const T* bias, T* output, int32 bias_size, int32 image_size) { - CUDA_1D_KERNEL_LOOP(index, nthreads) { + GPU_1D_KERNEL_LOOP(index, nthreads) { int32 index2 = index / image_size; int32 bias_offset = index2 % bias_size; output[index] = ldg(input + index) + ldg(bias + bias_offset); @@ -99,9 +99,9 @@ void BiasGPU::compute(const GPUDevice& d, const T* input, const T* bias, template __global__ void BiasGradNHWC_Naive(int32 nthreads, const T* output_backprop, T* bias_backprop, int32 bias_size) { - CUDA_1D_KERNEL_LOOP(index, nthreads) { + GPU_1D_KERNEL_LOOP(index, nthreads) { int32 bias_offset = index % bias_size; - CudaAtomicAdd(bias_backprop + bias_offset, ldg(output_backprop + index)); + GpuAtomicAdd(bias_backprop + bias_offset, ldg(output_backprop + index)); } } @@ -110,14 +110,13 @@ template __global__ void BiasGradNCHW_Naive(int32 nthreads, const T* output_backprop, T* bias_backprop, int32 bias_size, int32 image_size) { - CUDA_1D_KERNEL_LOOP(index, nthreads) { + GPU_1D_KERNEL_LOOP(index, nthreads) { int32 index2 = index / image_size; int32 bias_offset = index2 % bias_size; - CudaAtomicAdd(bias_backprop + bias_offset, ldg(output_backprop + index)); + GpuAtomicAdd(bias_backprop + bias_offset, ldg(output_backprop + index)); } } -extern __shared__ char s_buf[]; template __global__ void BiasGradNHWC_SharedAtomics(int32 nthreads, @@ -134,12 +133,12 @@ __global__ void BiasGradNHWC_SharedAtomics(int32 nthreads, for (int32 index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads; index += blockDim.x * gridDim.x) { int32 bias_offset = index % bias_size; - CudaAtomicAdd(s_data + bias_offset, AccT(ldg(output_backprop + index))); + GpuAtomicAdd(s_data + bias_offset, AccT(ldg(output_backprop + index))); } __syncthreads(); for (int32 index = threadIdx.x; index < bias_size; index += blockDim.x) { - CudaAtomicAdd(bias_backprop + index, T(s_data[index])); + GpuAtomicAdd(bias_backprop + index, T(s_data[index])); } } @@ -175,21 +174,34 @@ __global__ void BiasGradNCHW_SharedAtomics(const T* output_backprop, // Write the accumulated sum in this thread to the shared memory. Each thread // shifts their write location to avoid bank conflict. int bias_offset = threadIdx.x % 32; - CudaAtomicAdd(s_data + bias_offset, sum); + GpuAtomicAdd(s_data + bias_offset, sum); __syncthreads(); // Accumulate the results in the shared memory into the first element. // No syncthreads is needed since this is only in the same warp. int32 thread_index = threadIdx.x; +#if GOOGLE_CUDA if (thread_index < 32) { AccT data = s_data[thread_index]; for (int32 delta = warpSize / 2; delta > 0; delta /= 2) { - data += CudaShuffleXorSync(kCudaWarpAll, data, delta); + data += GpuShuffleXorSync(kCudaWarpAll, data, delta); } if (thread_index == 0) { - CudaAtomicAdd(bias_backprop + bias_index, T(data)); + GpuAtomicAdd(bias_backprop + bias_index, T(data)); } } +#elif TENSORFLOW_USE_ROCM + if (thread_index < 16) s_data[thread_index] += s_data[thread_index + 16]; + if (thread_index < 8) s_data[thread_index] += s_data[thread_index + 8]; + if (thread_index < 4) s_data[thread_index] += s_data[thread_index + 4]; + if (thread_index < 2) s_data[thread_index] += s_data[thread_index + 2]; + if (thread_index < 1) s_data[thread_index] += s_data[thread_index + 1]; + + // The first thread writes out the accumulated result to the global location. + if (thread_index == 0) { + GpuAtomicAdd(bias_backprop + bias_index, T(s_data[0])); + } +#endif } template @@ -252,8 +264,8 @@ void BiasGradGPU::DoRowReduction(OpKernelContext* context, T* output, const T* input, int rows, int cols) { typedef const Eigen::array::Tensor::Index, 1>& ReductionAxes; Constants constants; - cub::Sum op; - functor::ReduceImpl( + gpuprim::Sum op; + functor::ReduceImpl( context, output, input, 2, rows, cols, 1, 1, constants.kOne, op); } @@ -262,8 +274,8 @@ void BiasGradGPU::DoColReduction(OpKernelContext* context, T* output, const T* input, int rows, int cols) { typedef const Eigen::array::Tensor::Index, 1>& ReductionAxes; Constants constants; - cub::Sum op; - functor::ReduceImpl( + gpuprim::Sum op; + functor::ReduceImpl( context, output, input, 2, rows, cols, 1, 1, constants.kZero, op); } @@ -275,4 +287,4 @@ TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS); } // end namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM From 381c9ae0cbdb082cd570937b63a00d3d2c3a2d18 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Wed, 10 Jul 2019 13:30:25 -0700 Subject: [PATCH 221/332] [tf.data] Graph rewrite that adds `prefetch(AUTOTUNE)` after `MapAndBatch`, `ParallelMap`, and `ParallelInterleaveV2`. This will allow us to tune the buffer sizes for these datasets that have parallelism. PiperOrigin-RevId: 257470267 --- .../core/grappler/optimizers/data/BUILD | 21 ++++ .../optimizers/data/inject_prefetch.cc | 93 ++++++++++++++++++ .../optimizers/data/inject_prefetch.h | 51 ++++++++++ .../optimizers/data/meta_optimizer.cc | 5 +- .../kernel_tests/optimization/BUILD | 20 ++++ .../optimization/inject_prefetch_test.py | 96 +++++++++++++++++++ .../experimental/ops/optimization_options.py | 11 +++ ...a.experimental.-optimization-options.pbtxt | 4 + ...a.experimental.-optimization-options.pbtxt | 4 + 9 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 tensorflow/core/grappler/optimizers/data/inject_prefetch.cc create mode 100644 tensorflow/core/grappler/optimizers/data/inject_prefetch.h create mode 100644 tensorflow/python/data/experimental/kernel_tests/optimization/inject_prefetch_test.py diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD index 7a31127d8f9..8fffe36e84d 100644 --- a/tensorflow/core/grappler/optimizers/data/BUILD +++ b/tensorflow/core/grappler/optimizers/data/BUILD @@ -17,6 +17,7 @@ cc_library( ":filter_fusion", ":filter_with_random_uniform_fusion", ":hoist_random_uniform", + ":inject_prefetch", ":latency_all_edges", ":make_sloppy", ":map_and_batch_fusion", @@ -288,6 +289,26 @@ tf_cc_test( ] + tf_protos_all(), ) +cc_library( + name = "inject_prefetch", + srcs = ["inject_prefetch.cc"], + hdrs = ["inject_prefetch.h"], + deps = [ + ":graph_utils", + ":optimizer_base", + "//tensorflow/core/grappler:mutable_graph_view", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core/grappler:grappler_item", + "//tensorflow/core/grappler:op_types", + "//tensorflow/core/grappler:utils", + "//tensorflow/core/grappler/clusters:cluster", + "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry", + "//tensorflow/core:lib_internal", + ] + tf_protos_all(), + alwayslink = 1, +) + cc_library( name = "latency_all_edges", srcs = ["latency_all_edges.cc"], diff --git a/tensorflow/core/grappler/optimizers/data/inject_prefetch.cc b/tensorflow/core/grappler/optimizers/data/inject_prefetch.cc new file mode 100644 index 00000000000..4050d099d74 --- /dev/null +++ b/tensorflow/core/grappler/optimizers/data/inject_prefetch.cc @@ -0,0 +1,93 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/grappler/optimizers/data/inject_prefetch.h" + +#include "tensorflow/core/framework/model.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/grappler/clusters/cluster.h" +#include "tensorflow/core/grappler/grappler_item.h" +#include "tensorflow/core/grappler/mutable_graph_view.h" +#include "tensorflow/core/grappler/op_types.h" +#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h" +#include "tensorflow/core/grappler/optimizers/data/graph_utils.h" +#include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { +namespace grappler { +namespace { + +constexpr std::array kAsyncDatasetOps = { + "ParallelMapDataset", "ParallelInterleaveDatasetV2", "MapAndBatchDataset", + "ExperimentalMapAndBatchDataset"}; + +} // namespace + +Status InjectPrefetch::OptimizeAndCollectStats(Cluster* cluster, + const GrapplerItem& item, + GraphDef* output, + OptimizationStats* stats) { + *output = item.graph; + MutableGraphView graph(output); + + std::vector async_datasets; + for (const NodeDef& node : item.graph.node()) { + for (const auto& async_dataset_op : kAsyncDatasetOps) { + if (node.op() == async_dataset_op) { + async_datasets.push_back(&node); + break; + } + } + } + + if (async_datasets.empty()) return Status::OK(); + + // Add a const node with value kAutotune + NodeDef* autotune_value = + graph_utils::AddScalarConstNode(data::model::kAutotune, &graph); + + for (const NodeDef* async_dataset_node : async_datasets) { + NodeDef prefetch_node; + graph_utils::SetUniqueGraphNodeName( + strings::StrCat("autotune/prefetch_", async_dataset_node->name()), + graph.graph(), &prefetch_node); + prefetch_node.set_op("PrefetchDataset"); + // `input_dataset` input + *prefetch_node.mutable_input()->Add() = async_dataset_node->name(); + // `buffer_size` input + *prefetch_node.mutable_input()->Add() = autotune_value->name(); + + for (const auto& attr_name : {"output_types", "output_shapes"}) { + graph_utils::CopyAttribute(attr_name, *async_dataset_node, + &prefetch_node); + } + + auto* added_node = graph.AddNode(std::move(prefetch_node)); + TF_RETURN_IF_ERROR( + graph.UpdateFanouts(async_dataset_node->name(), added_node->name())); + } + return Status::OK(); +} + +void InjectPrefetch::Feedback(Cluster* cluster, const GrapplerItem& item, + const GraphDef& optimize_output, double result) { + // no-op +} + +REGISTER_GRAPH_OPTIMIZER_AS(InjectPrefetch, "inject_prefetch"); + +} // namespace grappler +} // namespace tensorflow diff --git a/tensorflow/core/grappler/optimizers/data/inject_prefetch.h b/tensorflow/core/grappler/optimizers/data/inject_prefetch.h new file mode 100644 index 00000000000..8f51dab4d9f --- /dev/null +++ b/tensorflow/core/grappler/optimizers/data/inject_prefetch.h @@ -0,0 +1,51 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_INJECT_PREFETCH_H_ +#define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_INJECT_PREFETCH_H_ + +#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h" + +namespace tensorflow { +namespace grappler { + +// This optimization adds `Prefetch(AUTOTUNE)` after all asynchronous tf.data +// transformations. This reduces the problem of tuning buffer sizes of these +// asynchronous transformations to tuning buffer sizes of the prefetch +// transformation. +class InjectPrefetch : public TFDataOptimizerBase { + public: + InjectPrefetch() = default; + ~InjectPrefetch() override = default; + + string name() const override { return "autotune_buffers"; }; + + Status Init( + const tensorflow::RewriterConfig_CustomGraphOptimizer* config) override { + return Status::OK(); + } + + Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item, + GraphDef* output, + OptimizationStats* stats) override; + + void Feedback(Cluster* cluster, const GrapplerItem& item, + const GraphDef& optimize_output, double result) override; +}; + +} // namespace grappler +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_INJECT_PREFETCH_H_ diff --git a/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc index 9e6b7f2bdef..b364296d9a9 100644 --- a/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc @@ -36,7 +36,7 @@ using ConfigMap = std::map; // tf.data optimizations, in the order we want to perform them. -constexpr std::array kTFDataOptimizations = { +constexpr std::array kTFDataOptimizations = { "noop_elimination", "shuffle_and_repeat_fusion", "map_fusion", @@ -50,7 +50,8 @@ constexpr std::array kTFDataOptimizations = { "latency_all_edges", "make_sloppy", "parallel_batch", - "slack"}; + "slack", + "inject_prefetch"}; // Standard grappler optimizations, in the order we want to perform them. constexpr std::array kGrapplerOptimizations = { diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD index d45e7e07122..26c213fb8a0 100644 --- a/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/BUILD @@ -27,6 +27,26 @@ py_test( ], ) +py_test( + name = "inject_prefetch_test", + size = "small", + srcs = ["inject_prefetch_test.py"], + python_version = "PY2", + srcs_version = "PY2AND3", + tags = [ + "no_oss", + "no_pip", + "no_windows", + ], + deps = [ + "//tensorflow/python:client_testlib", + "//tensorflow/python:errors", + "//tensorflow/python/data/experimental/ops:optimization", + "//tensorflow/python/data/kernel_tests:test_base", + "//tensorflow/python/data/ops:dataset_ops", + ], +) + py_test( name = "filter_fusion_test", size = "medium", diff --git a/tensorflow/python/data/experimental/kernel_tests/optimization/inject_prefetch_test.py b/tensorflow/python/data/experimental/kernel_tests/optimization/inject_prefetch_test.py new file mode 100644 index 00000000000..89f61f141b0 --- /dev/null +++ b/tensorflow/python/data/experimental/kernel_tests/optimization/inject_prefetch_test.py @@ -0,0 +1,96 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the `AutotuneBuffers` rewrite.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.data.experimental.ops import optimization +from tensorflow.python.data.kernel_tests import test_base +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +@test_util.run_all_in_graph_and_eager_modes +class InjectPrefetchTest(test_base.DatasetTestBase): + + def _enable_autotune_buffers(self, dataset): + options = dataset_ops.Options() + options.experimental_optimization.autotune_buffers = True + return dataset.with_options(options) + + def testParallelMap(self): + dataset = dataset_ops.Dataset.range(100) + dataset = dataset.apply( + optimization.assert_next(["ParallelMap", "Prefetch", "FiniteTake"])) + dataset = dataset.map( + lambda x: x + 1, num_parallel_calls=dataset_ops.AUTOTUNE) + dataset = dataset.take(50) + dataset = self._enable_autotune_buffers(dataset) + self.assertDatasetProduces(dataset, range(1, 51)) + + def testMapAndBatch(self): + dataset = dataset_ops.Dataset.range(100) + dataset = dataset.apply( + optimization.assert_next(["MapAndBatch", "Prefetch", "FiniteTake"])) + dataset = dataset.map( + lambda x: x + 1, num_parallel_calls=dataset_ops.AUTOTUNE) + dataset = dataset.batch(10) + dataset = dataset.take(5) + dataset = self._enable_autotune_buffers(dataset) + self.assertDatasetProduces( + dataset, [list(range(i + 1, i + 11)) for i in range(0, 50, 10)]) + + def testParallelInterleaveV2(self): + dataset = dataset_ops.Dataset.range(100) + dataset = dataset.apply( + optimization.assert_next( + ["ParallelInterleaveV2", "Prefetch", "FiniteTake"])) + dataset = dataset.interleave( + lambda x: dataset_ops.Dataset.from_tensors(x + 1), + num_parallel_calls=dataset_ops.AUTOTUNE) + dataset = dataset.take(50) + dataset = self._enable_autotune_buffers(dataset) + self.assertDatasetProduces(dataset, range(1, 51)) + + def testChainedParallelDatasets(self): + dataset = dataset_ops.Dataset.range(100) + dataset = dataset.apply( + optimization.assert_next([ + "ParallelMap", "Prefetch", "ParallelInterleaveV2", "Prefetch", + "MapAndBatch", "Prefetch", "FiniteTake" + ])) + dataset = dataset.map( + lambda x: x + 1, num_parallel_calls=dataset_ops.AUTOTUNE) + dataset = dataset.interleave( + lambda x: dataset_ops.Dataset.from_tensors(x + 1), + num_parallel_calls=dataset_ops.AUTOTUNE) + dataset = dataset.map( + lambda x: x + 1, num_parallel_calls=dataset_ops.AUTOTUNE) + dataset = dataset.batch(1) + dataset = dataset.take(50) + dataset = self._enable_autotune_buffers(dataset) + self.assertDatasetProduces(dataset, [[i] for i in range(3, 53)]) + + def testNoRegularMap(self): + dataset = dataset_ops.Dataset.range(100) + dataset = dataset.apply(optimization.assert_next(["Map", "FiniteTake"])) + dataset = dataset.map(lambda x: x + 1).take(50) + dataset = self._enable_autotune_buffers(dataset) + self.assertDatasetProduces(dataset, range(1, 51)) + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/data/experimental/ops/optimization_options.py b/tensorflow/python/data/experimental/ops/optimization_options.py index 5168effaf9f..094b11c76fb 100644 --- a/tensorflow/python/data/experimental/ops/optimization_options.py +++ b/tensorflow/python/data/experimental/ops/optimization_options.py @@ -90,6 +90,14 @@ class OptimizationOptions(options.OptionsBase): "When autotuning is enabled (through `autotune`), identifies the " "algorithm to use for the autotuning optimization.") + autotune_buffers = options.create_option( + name="autotune_buffers", + ty=bool, + docstring= + "When autotuning is enabled (through `autotune`), determines whether to " + "also autotune buffer sizes for datasets with parallelism. If None," + " defaults to False.") + autotune_cpu_budget = options.create_option( name="autotune_cpu_budget", ty=int, @@ -205,6 +213,9 @@ class OptimizationOptions(options.OptionsBase): if self.map_vectorization is not None: result.update(self.map_vectorization._static_optimizations()) # pylint: disable=protected-access + + if self.autotune is not False and self.autotune_buffers: # pylint: disable=g-bool-id-comparison + result.add("inject_prefetch") return sorted(list(result)) def _static_optimization_configs(self): diff --git a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt index 3d009186c7e..f7301ff180c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.data.experimental.-optimization-options.pbtxt @@ -15,6 +15,10 @@ tf_class { name: "autotune_algorithm" mtype: "" } + member { + name: "autotune_buffers" + mtype: "" + } member { name: "autotune_cpu_budget" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt index 3d009186c7e..f7301ff180c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.-optimization-options.pbtxt @@ -15,6 +15,10 @@ tf_class { name: "autotune_algorithm" mtype: "" } + member { + name: "autotune_buffers" + mtype: "" + } member { name: "autotune_cpu_budget" mtype: "" From 222ca11e0e28de021cf0b5514b2c4a0c334a5c76 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 10 Jul 2019 13:39:24 -0700 Subject: [PATCH 222/332] Add set -ex to ci_parameterized_build.sh This helps in debugging build jobs PiperOrigin-RevId: 257472023 --- tensorflow/tools/ci_build/ci_parameterized_build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/tools/ci_build/ci_parameterized_build.sh b/tensorflow/tools/ci_build/ci_parameterized_build.sh index 62c1e014d5e..49bb51465a4 100755 --- a/tensorflow/tools/ci_build/ci_parameterized_build.sh +++ b/tensorflow/tools/ci_build/ci_parameterized_build.sh @@ -98,6 +98,8 @@ # # This script can be used by Jenkins parameterized / matrix builds. +set -ex + # Helper function: Convert to lower case to_lower () { echo "$1" | tr '[:upper:]' '[:lower:]' From aa783e1cc6a5151a495a6eb6d7ad12c7fbc5132d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 13:39:50 -0700 Subject: [PATCH 223/332] [XLA] Fix peak_memory computation in ScheduleModule. PiperOrigin-RevId: 257472114 --- .../xla/service/hlo_memory_scheduler.cc | 18 +++++++++++------- .../xla/service/hlo_memory_scheduler.h | 6 +++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc index 3a4b9d4802d..da82b599a6a 100644 --- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc +++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_memory_scheduler.h" +#include #include #include #include @@ -608,22 +609,25 @@ StatusOr ScheduleModule( absl::flat_hash_map memory_by_computation; for (auto* computation : module->MakeComputationPostOrder()) { if (!computation->IsFusionComputation()) { - // peak_memory may be nullptr. - int64 peak_memory_internal; + int64 computation_peak_memory; TF_ASSIGN_OR_RETURN( HloInstructionSequence computation_sequence, ScheduleComputationHelper( computation, *points_to_analysis, *alias_analysis, size_function, - algorithm, memory_by_computation, &peak_memory_internal)); - memory_by_computation[computation] = peak_memory_internal; - if (peak_memory) { - *peak_memory = peak_memory_internal; - } + algorithm, memory_by_computation, &computation_peak_memory)); + memory_by_computation[computation] = computation_peak_memory; schedule.set_sequence(computation, std::move(computation_sequence)); } } VLOG(1) << "Module schedule:\n" << schedule; + if (peak_memory) { + *peak_memory = 0; + for (const auto& computation_and_peak : memory_by_computation) { + *peak_memory = std::max(*peak_memory, computation_and_peak.second); + } + } + TF_RETURN_IF_ERROR(schedule.Verify()); return std::move(schedule); diff --git a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h index 23f9721d539..fd416e9413e 100644 --- a/tensorflow/compiler/xla/service/hlo_memory_scheduler.h +++ b/tensorflow/compiler/xla/service/hlo_memory_scheduler.h @@ -91,9 +91,9 @@ StatusOr DefaultMemoryScheduler( int64* peak_memory); // Returns an HloSchedule which seeks to minimize the memory required for the -// computation. size_function is the function returning the number of bytes -// required for a LogicalBuffer. peak_memory (if not nullptr) is set to the peak -// memory of the resulting schedule according to the HeapSimulator. +// module. size_function is the function returning the number of bytes required +// for a LogicalBuffer. peak_memory (if not nullptr) is set to the largest peak +// memory (according to the HeapSimulator) of all computations in the module. StatusOr ScheduleModule( HloModule* module, const LogicalBuffer::SizeFunction& size_function, const MemorySchedulerAlgorithm& algorithm = {}, From ad249152a4b7653b14d62d75a4694239f62cebe5 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Wed, 10 Jul 2019 11:10:01 -0700 Subject: [PATCH 224/332] Address the comments --- tensorflow/core/kernels/data/BUILD | 19 -- .../core/kernels/data/dataset_test_base.cc | 67 ++---- .../core/kernels/data/dataset_test_base.h | 35 +-- .../kernels/data/text_line_dataset_op_test.cc | 215 +++++++++++------- 4 files changed, 171 insertions(+), 165 deletions(-) diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD index a6dacab6893..250d3221db3 100644 --- a/tensorflow/core/kernels/data/BUILD +++ b/tensorflow/core/kernels/data/BUILD @@ -1017,25 +1017,6 @@ tf_cc_test( ], ) -tf_cc_test( - name = "fixed_length_record_dataset_op_test", - size = "small", - srcs = ["fixed_length_record_dataset_op_test.cc"], - deps = [ - ":dataset_test_base", - ":dataset_utils", - ":iterator_ops", - ":reader_dataset_ops", - "//tensorflow/core:core_cpu_internal", - "//tensorflow/core:dataset_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:lib_internal", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - tf_kernel_library( name = "iterator_ops", srcs = ["iterator_ops.cc"], diff --git a/tensorflow/core/kernels/data/dataset_test_base.cc b/tensorflow/core/kernels/data/dataset_test_base.cc index 61424bb3458..0c419adecee 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.cc +++ b/tensorflow/core/kernels/data/dataset_test_base.cc @@ -22,63 +22,55 @@ limitations under the License. namespace tensorflow { namespace data { -constexpr char kNewLine[] = "\n"; - -string CompressionName(CompressionType compression_type) { +string ToString(CompressionType compression_type) { switch (compression_type) { - case ZLIB: + case CompressionType::ZLIB: return "ZLIB"; - case GZIP: + case CompressionType::GZIP: return "GZIP"; - case RAW: + case CompressionType::RAW: return "RAW"; - case UNCOMPRESSED: + case CompressionType::UNCOMPRESSED: return ""; - default: - LOG(WARNING) << "Undefined compression type: " << compression_type; - return "UNDEFINED"; } + return "UNDEFINED"; } io::ZlibCompressionOptions GetZlibCompressionOptions( CompressionType compression_type) { switch (compression_type) { - case ZLIB: + case CompressionType::ZLIB: return io::ZlibCompressionOptions::DEFAULT(); - case GZIP: + case CompressionType::GZIP: return io::ZlibCompressionOptions::GZIP(); - case RAW: + case CompressionType::RAW: return io::ZlibCompressionOptions::RAW(); default: - LOG(WARNING) << "Undefined zlib compression type: " << compression_type; - return io::ZlibCompressionOptions::DEFAULT(); + LOG(WARNING) << "ZlibCompressionOptions does not have an option for " + << ToString(compression_type); } + return io::ZlibCompressionOptions::DEFAULT(); } -Status CreateFile(const string& filename, const std::vector& text, - int input_buffer_size, int output_buffer_size, - const CompressionType& compression_type) { +Status WriteDataToFile(const string& filename, const char* data, + int input_buffer_size, int output_buffer_size, + const CompressionType& compression_type) { Env* env = Env::Default(); std::unique_ptr file_writer; TF_RETURN_IF_ERROR(env->NewWritableFile(filename, &file_writer)); - if (compression_type == UNCOMPRESSED) { - for (const string& line : text) { - TF_RETURN_IF_ERROR(file_writer->Append(line)); - TF_RETURN_IF_ERROR(file_writer->Append(kNewLine)); - } - } else if (compression_type == ZLIB || compression_type == GZIP) { + if (compression_type == CompressionType::UNCOMPRESSED) { + TF_RETURN_IF_ERROR(file_writer->Append(data)); + } else if (compression_type == CompressionType::ZLIB || + compression_type == CompressionType::GZIP) { auto zlib_compression_options = GetZlibCompressionOptions(compression_type); io::ZlibOutputBuffer out(file_writer.get(), input_buffer_size, output_buffer_size, zlib_compression_options); TF_RETURN_IF_ERROR(out.Init()); - for (const string& line : text) { - TF_RETURN_IF_ERROR(out.Append(line)); - TF_RETURN_IF_ERROR(out.Append(kNewLine)); - } + TF_RETURN_IF_ERROR(out.Append(data)); TF_RETURN_IF_ERROR(out.Close()); } else { return tensorflow::errors::InvalidArgument("Unsupported compression_type: ", - compression_type); + ToString(compression_type)); } TF_RETURN_IF_ERROR(file_writer->Flush()); @@ -87,23 +79,6 @@ Status CreateFile(const string& filename, const std::vector& text, return Status::OK(); } -Status CreateMultiTextFiles(const std::vector& filenames, - const std::vector& multi_texts, - int input_buffer_size, int output_buffer_size, - const CompressionType& compression_type) { - int lines_per_file = multi_texts.size() / filenames.size(); - auto first = multi_texts.begin(); - for (int i = 0; i < filenames.size(); ++i) { - auto last = - i == filenames.size() - 1 ? multi_texts.end() : first + lines_per_file; - std::vector text(first, last); - TF_RETURN_IF_ERROR(CreateFile(filenames[i], text, input_buffer_size, - output_buffer_size, compression_type)); - first += lines_per_file; - } - return Status::OK(); -} - template Status IsEqual(const Tensor& t1, const Tensor& t2) { if (t1.dtype() != t2.dtype()) { diff --git a/tensorflow/core/kernels/data/dataset_test_base.h b/tensorflow/core/kernels/data/dataset_test_base.h index e014c9d3390..b033a5590af 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.h +++ b/tensorflow/core/kernels/data/dataset_test_base.h @@ -45,36 +45,23 @@ limitations under the License. namespace tensorflow { namespace data { -enum CompressionType : int { ZLIB = 0, GZIP = 1, RAW = 2, UNCOMPRESSED = 3 }; +enum class CompressionType { ZLIB = 0, GZIP = 1, RAW = 2, UNCOMPRESSED = 3 }; -// Gets the compression name. -string CompressionName(CompressionType compression_type); +// Returns a string representation for the given compression type. +string ToString(CompressionType compression_type); // Gets the specified zlib compression options according to the compression -// type. Note that `CompressionType : UNCOMPRESSED` is not supported because -// `ZlibCompressionOptions` does not have the option for uncompressed data. +// type. Note that `CompressionType::UNCOMPRESSED` is not supported because +// `ZlibCompressionOptions` does not have an option. io::ZlibCompressionOptions GetZlibCompressionOptions( CompressionType compression_type); -// Saves the input text into the file with the specified compression. -Status CreateFile(const string& filename, const std::vector& text, - int input_buffer_size, int output_buffer_size, - const CompressionType& compression_type); - -// Saves the input texts into multiple text files with the specified -// compression. -Status CreateMultiTextFiles(const std::vector& filenames, - const std::vector& multi_texts, - int input_buffer_size, int output_buffer_size, - const CompressionType& compression_type); - -// Saves the input contents into multiple fixed length record files with the -// specified compression. -Status CreateMultiFixedLengthRecordFiles( - const std::vector& filenames, const string& header, - const string& footer, const std::vector& contents, - int input_buffer_size, int output_buffer_size, - const CompressionType& compression_type); +// Writes the input data into the file with the specified compression. +// `input_buffer_size` is the size of z_stream.next_in buffer; +// `output_buffer_size` is the size of z_stream.next_out buffer. +Status WriteDataToFile(const string& filename, const char* data, + int input_buffer_size, int output_buffer_size, + const CompressionType& compression_type); // Helpful functions to test Dataset op kernels. class DatasetOpsTestBase : public ::testing::Test { diff --git a/tensorflow/core/kernels/data/text_line_dataset_op_test.cc b/tensorflow/core/kernels/data/text_line_dataset_op_test.cc index 112b8536ba6..a3b5c7d3847 100644 --- a/tensorflow/core/kernels/data/text_line_dataset_op_test.cc +++ b/tensorflow/core/kernels/data/text_line_dataset_op_test.cc @@ -47,6 +47,7 @@ class TextLineDatasetOpTest : public DatasetOpsTestBase { struct TestCase { std::vector filenames; + std::vector texts; CompressionType compression_type; int64 buffer_size; std::vector expected_outputs; @@ -71,7 +72,10 @@ TestCase TestCase1() { return { /*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_1"), absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_2")}, - /*compression_type*/ ZLIB, + /*texts*/ + {absl::StrCat("hello world\n", "11223334455\n"), + absl::StrCat("abcd, EFgH\n", " \n", "$%^&*()\n")}, + /*compression_type*/ CompressionType::ZLIB, /*buffer_size*/ 10, /*expected_outputs*/ {DatasetOpsTestBase::CreateTensor(TensorShape({}), @@ -91,40 +95,55 @@ TestCase TestCase1() { // Test case 2: multiple text files with GZIP compression. TestCase TestCase2() { - return {/*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_GZIP_1"), - absl::StrCat(testing::TmpDir(), "/text_line_GZIP_2")}, - /*compression_type*/ GZIP, - /*buffer_size*/ 10, - /*expected_outputs*/ - {DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"hello world"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"11223334455"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"abcd, EFgH"})}, - /*expected_output_dtypes*/ {DT_STRING}, - /*expected_output_shapes*/ {PartialTensorShape({})}, - /*expected_cardinality*/ kUnknownCardinality, - /*breakpoints*/ {0, 2, 6}}; + return { + /*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_GZIP_1"), + absl::StrCat(testing::TmpDir(), "/text_line_GZIP_2")}, + /*texts*/ + {absl::StrCat("hello world\n", "11223334455\n"), + absl::StrCat("abcd, EFgH\n", " \n", "$%^&*()\n")}, + /*compression_type*/ CompressionType::GZIP, + /*buffer_size*/ 10, + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"hello world"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"11223334455"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"abcd, EFgH"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {" "}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), {"$%^&*()"})}, + /*expected_output_dtypes*/ {DT_STRING}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ kUnknownCardinality, + /*breakpoints*/ {0, 2, 6}}; } -// Test case 3: a single text file without compression. +// Test case 3: multiple text files without compression. TestCase TestCase3() { - return {/*filenames*/ { - absl::StrCat(testing::TmpDir(), "/text_line_UNCOMPRESSED")}, - /*compression_type*/ UNCOMPRESSED, - /*buffer_size*/ 10, - /*expected_outputs*/ - {DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"hello world"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"11223334455"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"abcd, EFgH"})}, - /*expected_output_dtypes*/ {DT_STRING}, - /*expected_output_shapes*/ {PartialTensorShape({})}, - /*expected_cardinality*/ kUnknownCardinality, - /*breakpoints*/ {0, 2, 6}}; + return { + /*filenames*/ { + absl::StrCat(testing::TmpDir(), "/text_line_UNCOMPRESSED_1"), + absl::StrCat(testing::TmpDir(), "/text_line_UNCOMPRESSED_2")}, + /*texts*/ + {absl::StrCat("hello world\n", "11223334455\n"), + absl::StrCat("abcd, EFgH\n", " \n", "$%^&*()\n")}, + /*compression_type*/ CompressionType::UNCOMPRESSED, + /*buffer_size*/ 10, + /*expected_outputs*/ + {DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"hello world"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"11223334455"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {"abcd, EFgH"}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), + {" "}), + DatasetOpsTestBase::CreateTensor(TensorShape({}), {"$%^&*()"})}, + /*expected_output_dtypes*/ {DT_STRING}, + /*expected_output_shapes*/ {PartialTensorShape({})}, + /*expected_cardinality*/ kUnknownCardinality, + /*breakpoints*/ {0, 2, 6}}; } class ParameterizedTextLineDatasetOpTest @@ -138,9 +157,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, GetNext) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -149,7 +172,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, GetNext) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -191,9 +214,13 @@ TEST_F(TextLineDatasetOpTest, DatasetNodeName) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -202,7 +229,7 @@ TEST_F(TextLineDatasetOpTest, DatasetNodeName) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -227,9 +254,13 @@ TEST_F(TextLineDatasetOpTest, DatasetTypeString) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -238,7 +269,7 @@ TEST_F(TextLineDatasetOpTest, DatasetTypeString) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -264,9 +295,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetOutputDtypes) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -275,7 +310,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetOutputDtypes) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -301,9 +336,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetOutputShapes) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -312,7 +351,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetOutputShapes) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -338,9 +377,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, Cardinality) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -349,7 +392,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, Cardinality) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -374,9 +417,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetSave) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -385,7 +432,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetSave) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -416,9 +463,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputDtypes) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -427,7 +478,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputDtypes) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -461,9 +512,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputShapes) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -472,7 +527,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputShapes) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -506,9 +561,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputPrefix) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -517,7 +576,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputPrefix) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), @@ -552,9 +611,13 @@ TEST_P(ParameterizedTextLineDatasetOpTest, Roundtrip) { TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); + EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), + test_case.buffer_size, test_case.buffer_size, + test_case.compression_type)); + } std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -563,7 +626,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, Roundtrip) { Tensor filenames = CreateTensor(TensorShape({num_files}), test_case.filenames); Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); + TensorShape({}), {ToString(test_case.compression_type)}); Tensor buffer_size = CreateTensor(TensorShape({}), {test_case.buffer_size}); gtl::InlinedVector inputs{TensorValue(&filenames), From 840f32a26f50b945d6601f3a50afde914e94bf6a Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Wed, 10 Jul 2019 14:00:24 -0700 Subject: [PATCH 225/332] Split out multiprocessing specific test to a different file so that we can disable the test on windows. PiperOrigin-RevId: 257476078 --- tensorflow/python/BUILD | 16 ++++- .../lib/io/tf_record_multiprocessing_test.py | 64 +++++++++++++++++++ tensorflow/python/lib/io/tf_record_test.py | 38 ----------- 3 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 tensorflow/python/lib/io/tf_record_multiprocessing_test.py diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 5f38820fd87..93f43ab338a 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -5299,7 +5299,21 @@ tf_py_test( ":lib", ":util", ], - tags = ["no_windows"], # b/137097422 +) + +tf_py_test( + name = "tf_record_multiprocessing_test", + size = "small", + srcs = ["lib/io/tf_record_multiprocessing_test.py"], + additional_deps = [ + ":client_testlib", + ":errors", + ":lib", + ":util", + ], + # The multiprocessing module behaves differently on windows, so we + # disable this test on windows. + tags = ["no_windows"], ) cuda_py_test( diff --git a/tensorflow/python/lib/io/tf_record_multiprocessing_test.py b/tensorflow/python/lib/io/tf_record_multiprocessing_test.py new file mode 100644 index 00000000000..eaecffe9ff4 --- /dev/null +++ b/tensorflow/python/lib/io/tf_record_multiprocessing_test.py @@ -0,0 +1,64 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Multiprocessing tests for TFRecordWriter and tf_record_iterator.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import multiprocessing +import os + +from tensorflow.python.lib.io import tf_record +from tensorflow.python.platform import test +from tensorflow.python.util import compat + +TFRecordCompressionType = tf_record.TFRecordCompressionType + + +def ChildProcess(writer, rs): + for r in rs: + writer.write(r) + writer.flush() + + +class TFRecordWriterCloseAndFlushTests(test.TestCase): + """TFRecordWriter close and flush tests.""" + + # pylint: disable=arguments-differ + def setUp(self, compression_type=TFRecordCompressionType.NONE): + super(TFRecordWriterCloseAndFlushTests, self).setUp() + self._fn = os.path.join(self.get_temp_dir(), "tf_record_writer_test.txt") + self._options = tf_record.TFRecordOptions(compression_type) + self._writer = tf_record.TFRecordWriter(self._fn, self._options) + self._num_records = 20 + + def _Record(self, r): + return compat.as_bytes("Record %d" % r) + + def testFlush(self): + """test Flush.""" + records = [self._Record(i) for i in range(self._num_records)] + + write_process = multiprocessing.Process( + target=ChildProcess, args=(self._writer, records)) + write_process.start() + write_process.join() + actual = list(tf_record.tf_record_iterator(self._fn, self._options)) + self.assertListEqual(actual, records) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/lib/io/tf_record_test.py b/tensorflow/python/lib/io/tf_record_test.py index c3000c09432..280c2b10918 100644 --- a/tensorflow/python/lib/io/tf_record_test.py +++ b/tensorflow/python/lib/io/tf_record_test.py @@ -19,7 +19,6 @@ from __future__ import division from __future__ import print_function import gzip -import multiprocessing import os import random import string @@ -34,9 +33,7 @@ from tensorflow.python.util import compat prefix_path = "third_party/tensorflow/core/lib" -# pylint: disable=invalid-name TFRecordCompressionType = tf_record.TFRecordCompressionType -# pylint: enable=invalid-name # Edgar Allan Poe's 'Eldorado' _TEXT = b"""Gaily bedight, @@ -174,7 +171,6 @@ class TFRecordWriterTest(TFCompressionTestCase): # Negative number => better compression. return os.path.getsize(fn_a) - os.path.getsize(fn_b) -# pylint: disable=invalid-name def testWriteReadZLibFiles(self): """test Write Read ZLib Files""" @@ -198,7 +194,6 @@ class TFRecordWriterTest(TFCompressionTestCase): for i, fn in enumerate(compressed_files) ] self._AssertFilesEqual(uncompressed_files, files, True) -# pylint: disable=invalid-name def testWriteReadGzipFiles(self): """test Write Read Gzip Files""" @@ -224,7 +219,6 @@ class TFRecordWriterTest(TFCompressionTestCase): for i, fn in enumerate(compressed_files) ] self._AssertFilesEqual(uncompressed_files, files, True) -# pylint: disable=invalid-name def testNoCompressionType(self): """test No Compression Type""" @@ -243,7 +237,6 @@ class TFRecordWriterTest(TFCompressionTestCase): with self.assertRaises(ValueError): tf_record.TFRecordOptions("BZ2") -# pylint: disable=invalid-name def testZlibCompressionType(self): """test Zlib Compression Type""" @@ -263,7 +256,6 @@ class TFRecordWriterTest(TFCompressionTestCase): "ZLIB", tf_record.TFRecordOptions.get_compression_type_string( tf_record.TFRecordOptions(tf_record.TFRecordOptions(zlib_t)))) -# pylint: disable=invalid-name def testCompressionOptions(self): """Create record with mix of random and repeated data to test compression on.""" @@ -304,7 +296,6 @@ class TFRecordWriterTest(TFCompressionTestCase): class TFRecordWriterZlibTest(TFCompressionTestCase): """TFRecordWriter Zlib test""" - # pylint: disable=invalid-name def testZLibFlushRecord(self): """test ZLib Flush Record""" original = [b"small record"] @@ -333,7 +324,6 @@ class TFRecordWriterZlibTest(TFCompressionTestCase): options = tf_record.TFRecordOptions(TFRecordCompressionType.ZLIB) actual = list(tf_record.tf_record_iterator(fn, options=options)) self.assertEqual(actual, original) -# pylint: disable=invalid-name def testZlibReadWrite(self): """Verify that files produced are zlib compatible.""" @@ -345,7 +335,6 @@ class TFRecordWriterZlibTest(TFCompressionTestCase): options = tf_record.TFRecordOptions(TFRecordCompressionType.ZLIB) actual = list(tf_record.tf_record_iterator(zfn, options=options)) self.assertEqual(actual, original) -# pylint: disable=invalid-name def testZlibReadWriteLarge(self): """Verify that writing large contents also works.""" @@ -358,7 +347,6 @@ class TFRecordWriterZlibTest(TFCompressionTestCase): options = tf_record.TFRecordOptions(TFRecordCompressionType.ZLIB) actual = list(tf_record.tf_record_iterator(zfn, options=options)) self.assertEqual(actual, original) -# pylint: disable=invalid-name def testGzipReadWrite(self): """Verify that files produced are gzip compatible.""" @@ -377,7 +365,6 @@ class TFRecordIteratorTest(TFCompressionTestCase): def setUp(self): super(TFRecordIteratorTest, self).setUp() self._num_records = 7 -# pylint: disable=invalid-name def testIterator(self): """test Iterator""" @@ -391,7 +378,6 @@ class TFRecordIteratorTest(TFCompressionTestCase): self.assertAllEqual(expected, record) with self.assertRaises(StopIteration): record = next(reader) -# pylint: disable=invalid-name def testWriteZlibRead(self): """Verify compression with TFRecordWriter is zlib library compatible.""" @@ -403,7 +389,6 @@ class TFRecordIteratorTest(TFCompressionTestCase): zfn = self._ZlibDecompressFile(fn, "write_zlib_read.tfrecord") actual = list(tf_record.tf_record_iterator(zfn)) self.assertEqual(actual, original) -# pylint: disable=invalid-name def testWriteZlibReadLarge(self): """Verify compression for large records is zlib library compatible.""" @@ -415,7 +400,6 @@ class TFRecordIteratorTest(TFCompressionTestCase): zfn = self._ZlibDecompressFile(fn, "write_zlib_read_large.tfrecord") actual = list(tf_record.tf_record_iterator(zfn)) self.assertEqual(actual, original) -# pylint: disable=invalid-name def testWriteGzipRead(self): original = [b"foo", b"bar"] @@ -426,7 +410,6 @@ class TFRecordIteratorTest(TFCompressionTestCase): gzfn = self._GzipDecompressFile(fn, "write_gzip_read.tfrecord") actual = list(tf_record.tf_record_iterator(gzfn)) self.assertEqual(actual, original) -# pylint: disable=invalid-name def testBadFile(self): """Verify that tf_record_iterator throws an exception on bad TFRecords.""" @@ -443,10 +426,6 @@ class TFRecordIteratorTest(TFCompressionTestCase): for _ in tf_record.tf_record_iterator(fn_truncated): pass -def ChildProcess(writer, rs): - for r in rs: - writer.write(r) - writer.flush() class TFRecordWriterCloseAndFlushTests(test.TestCase): """TFRecordWriter close and flush tests""" @@ -461,7 +440,6 @@ class TFRecordWriterCloseAndFlushTests(test.TestCase): def _Record(self, r): return compat.as_bytes("Record %d" % r) -# pylint: disable=invalid-name def testWriteAndLeaveOpen(self): records = list(map(self._Record, range(self._num_records))) @@ -469,7 +447,6 @@ class TFRecordWriterCloseAndFlushTests(test.TestCase): self._writer.write(record) # Verify no segfault if writer isn't explicitly closed. -# pylint: disable=invalid-name def testWriteAndRead(self): records = list(map(self._Record, range(self._num_records))) @@ -479,25 +456,11 @@ class TFRecordWriterCloseAndFlushTests(test.TestCase): actual = list(tf_record.tf_record_iterator(self._fn, self._options)) self.assertListEqual(actual, records) -# pylint: disable=invalid-name - - def testFlush(self): - """test Flush""" - records = list(map(self._Record, range(self._num_records))) - - write_process = multiprocessing.Process( - target=ChildProcess, args=(self._writer, records)) - write_process.start() - write_process.join() - actual = list(tf_record.tf_record_iterator(self._fn, self._options)) - self.assertListEqual(actual, records) -# pylint: disable=invalid-name def testDoubleClose(self): self._writer.write(self._Record(0)) self._writer.close() self._writer.close() -# pylint: disable=invalid-name def testFlushAfterCloseIsError(self): self._writer.write(self._Record(0)) @@ -505,7 +468,6 @@ class TFRecordWriterCloseAndFlushTests(test.TestCase): with self.assertRaises(errors_impl.FailedPreconditionError): self._writer.flush() -# pylint: disable=invalid-name def testWriteAfterCloseIsError(self): self._writer.write(self._Record(0)) From 3d9fab8a3112557a3c3dcc0da9d46a210fcfeb97 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 14:02:46 -0700 Subject: [PATCH 226/332] NFC: Remove useless CHECK-EMPTY lines. These don't test anything and are brittle to printer changes. PiperOrigin-RevId: 257476676 --- .../tensorflow/tests/graphdef2mlir/graph-custom-operation.pbtxt | 2 -- .../tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt | 1 - 2 files changed, 3 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-custom-operation.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-custom-operation.pbtxt index 1c371586b2d..82146716fff 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-custom-operation.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-custom-operation.pbtxt @@ -2176,14 +2176,12 @@ versions { # CHECK: %73:2 = "_tf.mul_2_Da30D05wlPU0"(%58#0, %72#0, %47#1) {_tpu_replicate = "cluster", device = "", name = "while/mul_2_Da30D05wlPU"} : (tensor<*xf32>, tensor<*xf32>, !_tf.control) -> (tensor<*xf32>, !_tf.control) # CHECK: return # CHECK-NEXT: } -# CHECK-EMPTY: # CHECK: func @less_than_5_If8q4vKg9jA0(%arg0: tensor<*xf32>) -> tensor<*xi1> # CHECK-NEXT: attributes {tf._noinline = true} { # CHECK-NEXT: %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_FLOAT", name = "Less/y", value = dense<5.000000e+00> : tensor} : () -> (tensor, !_tf.control) # CHECK-NEXT: %1:2 = "_tf.Less"(%arg0, %0#0) {T = "tfdtype$DT_FLOAT", device = "", name = "Less"} : (tensor<*xf32>, tensor) -> (tensor<*xi1>, !_tf.control) # CHECK-NEXT: return %1#0 : tensor<*xi1> # CHECK-NEXT: } -# CHECK-EMPTY: # CHECK: func @mul_2_Da30D05wlPU0(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> # CHECK-NEXT: attributes {tf._noinline = true} { # CHECK-NEXT: %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_FLOAT", name = "mul/y", value = dense<2.000000e+00> : tensor<1x1xf32>} : () -> (tensor<1x1xf32>, !_tf.control) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt index fe91ebe41e9..40392a6954a 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-function-defs.pbtxt @@ -536,4 +536,3 @@ versions { # CHECK-NEXT: %1:2 = "_tf.Identity"(%arg1) {T = "tfdtype$DT_INT32", device = "", name = "Identity_1"} : (tensor<*xi32>) -> (tensor<*xi32>, !_tf.control) # CHECK-NEXT: return %0#0, %1#0 : tensor<*xi32>, tensor<*xi32> # CHECK-NEXT: } -# CHECK-EMPTY: From d51ed905fca5d0ae188408438003a0041ba7b3b1 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Wed, 10 Jul 2019 14:54:10 -0700 Subject: [PATCH 227/332] [tf.data] Temporarily silence a noisy warning. PiperOrigin-RevId: 257486974 --- tensorflow/core/kernels/data/iterator_ops.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/data/iterator_ops.cc b/tensorflow/core/kernels/data/iterator_ops.cc index f28386ccd0b..a5a49b2c7eb 100644 --- a/tensorflow/core/kernels/data/iterator_ops.cc +++ b/tensorflow/core/kernels/data/iterator_ops.cc @@ -617,7 +617,14 @@ void DeleteIteratorOp::Compute(OpKernelContext* ctx) { // The iterator resource is guaranteed to exist because the variant tensor // wrapping the deleter is provided as an unused input to this op, which // guarantees that it has not run yet. - OP_REQUIRES_OK(ctx, ctx->resource_manager()->Delete(handle)); + Status s = ctx->resource_manager()->Delete(handle); + if (errors::IsNotFound(s)) { + // TODO(b/135948230): Investigate why is the above statement not true and + // then get rid of the special case. + ctx->SetStatus(Status::OK()); + return; + } + ctx->SetStatus(s); } namespace { From c66d06503cba43b999fd2e4fb7ff0a18751d5b3b Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Wed, 10 Jul 2019 15:03:51 -0700 Subject: [PATCH 228/332] [XLA] Delete dead function. PiperOrigin-RevId: 257489127 --- .../compiler/xla/service/gpu/instruction_fusion.cc | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc index f35bce404da..b3ad5719e83 100644 --- a/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/gpu/instruction_fusion.cc @@ -28,19 +28,6 @@ limitations under the License. namespace xla { namespace gpu { -namespace { - -bool IsIEEEFloatingPointScalarConstant(const HloInstruction* constant) { - if (constant->opcode() != HloOpcode::kConstant || - !ShapeUtil::IsScalar(constant->shape())) { - return false; - } - auto type = constant->shape().element_type(); - return type == F16 || type == F32 || type == F64; -} - -} // namespace - /*static*/ bool GpuInstructionFusion::IsExpensive( const HloInstruction& instruction) { // We say that floating-point division is cheap on the GPU. From 05f0a6c5e0db00b792713f0e477a00f25b682309 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 15:11:06 -0700 Subject: [PATCH 229/332] Expanded the restrictions for the softmax op to take 1D tensors as input (in addition to 2D) PiperOrigin-RevId: 257490731 --- tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc | 7 ++++--- tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc index cc32340c681..150eead08be 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc @@ -627,9 +627,10 @@ OpFoldResult ShapeOp::fold(ArrayRef operands) { //===----------------------------------------------------------------------===// static LogicalResult Verify(SoftmaxOp op) { - if (!IsOfRankOrUnranked(op.logits(), 2)) - return op.emitOpError("requires operand to be 2D tensor"); - + if (!IsOfRankOrUnranked(op.logits(), 1) && + !IsOfRankOrUnranked(op.logits(), 2)) { + return op.emitOpError("requires operand to be 1D/2D tensor"); + } return success(); } diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir index 52a14cd60c6..5e7d733bb57 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir @@ -588,7 +588,7 @@ func @testSoftmax(tensor<8x16xf32>) -> tensor<8x16xf32> { // Test invalid tf.Softmax func @testSoftmax(tensor<8x8x8xf32>) -> tensor<8x8x8xf32> { ^bb0(%arg0: tensor<8x8x8xf32>): - // expected-error @+1 {{requires operand to be 2D tensor}} + // expected-error @+1 {{requires operand to be 1D/2D tensor}} %0 = "tf.Softmax"(%arg0) {T = "tfdtype$DT_FLOAT"} : (tensor<8x8x8xf32>) -> tensor<8x8x8xf32> return %0 : tensor<8x8x8xf32> } From 22f1546586bc47d02c10c5a3ecf087562dddf760 Mon Sep 17 00:00:00 2001 From: Katherine Wu Date: Wed, 10 Jul 2019 15:17:20 -0700 Subject: [PATCH 230/332] Fix error with tracing function with extra argument. This appeared when training argument is passed in args instead of kwargs. PiperOrigin-RevId: 257492170 --- .../keras/saving/saved_model/constants.py | 4 +++ .../python/keras/saving/saved_model/save.py | 35 ++++++++++++------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/tensorflow/python/keras/saving/saved_model/constants.py b/tensorflow/python/keras/saving/saved_model/constants.py index 093de2884f5..3f1eca9c500 100644 --- a/tensorflow/python/keras/saving/saved_model/constants.py +++ b/tensorflow/python/keras/saving/saved_model/constants.py @@ -22,3 +22,7 @@ from __future__ import print_function # e.g. the list of layers can be accessed using `loaded.keras_api.layers`, in an # object loaded from `tf.saved_model.load()`. KERAS_ATTR = 'keras_api' + +# Keys for the serialization cache. +# Maps to the keras serialization dict {Layer --> SerializedAttributes object} +KERAS_CACHE_KEY = 'keras_serialized_attributes' diff --git a/tensorflow/python/keras/saving/saved_model/save.py b/tensorflow/python/keras/saving/saved_model/save.py index 89c16433764..1bf80e6ae13 100644 --- a/tensorflow/python/keras/saving/saved_model/save.py +++ b/tensorflow/python/keras/saving/saved_model/save.py @@ -28,6 +28,7 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.keras.engine import input_spec from tensorflow.python.keras.saving import saving_utils +from tensorflow.python.keras.saving.saved_model import constants from tensorflow.python.keras.saving.saved_model import load as keras_load from tensorflow.python.keras.saving.saved_model import serialized_attributes from tensorflow.python.keras.saving.saved_model import utils @@ -38,6 +39,7 @@ from tensorflow.python.training.tracking import base as trackable from tensorflow.python.training.tracking import data_structures from tensorflow.python.training.tracking import layer_utils as trackable_layer_utils from tensorflow.python.util import nest +from tensorflow.python.util import tf_inspect from tensorflow.python.util.lazy_loader import LazyLoader # To avoid circular dependencies between keras/engine and keras/saving, @@ -86,23 +88,18 @@ def save(model, filepath, overwrite, include_optimizer): model.optimizer = orig_optimizer -# Keys for the serialization cache. -# Maps to the keras serialization dict {Layer --> SerializedAttributes object} -_KERAS_CACHE_KEY = 'keras_serialized_attributes' - - def serialize_all_attributes(layer, serialization_cache): """Serialize all attributes in the layer.""" save_model_default_signature = False - if _KERAS_CACHE_KEY not in serialization_cache: - keras_cache = serialization_cache[_KERAS_CACHE_KEY] = {} + if constants.KERAS_CACHE_KEY not in serialization_cache: + keras_cache = serialization_cache[constants.KERAS_CACHE_KEY] = {} if isinstance(layer, training_lib.Model): # Only trace default signature if the root object is a Model. Since the # keras cache key is only created in this method, we know that the object # is root if the key does not yet exist in the cache. save_model_default_signature = True else: - keras_cache = serialization_cache[_KERAS_CACHE_KEY] + keras_cache = serialization_cache[constants.KERAS_CACHE_KEY] if layer in keras_cache: return keras_cache[layer] @@ -320,11 +317,12 @@ def _replace_child_layer_functions(layer, serialization_cache): # pylint: disable=protected-access original_fns = {} for child_layer in _list_all_layers(layer): - if child_layer not in serialization_cache[_KERAS_CACHE_KEY]: + if child_layer not in serialization_cache[constants.KERAS_CACHE_KEY]: layer_fns = (serialize_all_attributes(child_layer, serialization_cache) .functions) else: - layer_fns = serialization_cache[_KERAS_CACHE_KEY][child_layer].functions + layer_fns = ( + serialization_cache[constants.KERAS_CACHE_KEY][child_layer].functions) if not layer_fns: # This indicates either: # - circular dependency, which means the current layer's functions @@ -442,15 +440,28 @@ class LayerCallCollection(object): *args: Positional args passed to the original function. **kwargs: Keyword args passed to the original function. """ + args = list(args) kwargs = kwargs.copy() self.tracing = True for fn in self._functions.values(): # TODO(kathywu): Replace arguments with broader shapes defined in the # input signature. if self._expects_training_arg: - kwargs['training'] = False + arg_list = tf_inspect.getfullargspec(fn.python_function).args + if 'training' in arg_list: + training_arg_index = arg_list.index('training') + else: + training_arg_index = -1 + + def set_training_arg(training, index=training_arg_index): + if index >= 0 and len(args) > index: + args[index] = training + else: + kwargs['training'] = training + + set_training_arg(False) fn.original_get_concrete_function(*args, **kwargs) - kwargs['training'] = True + set_training_arg(True) fn.original_get_concrete_function(*args, **kwargs) else: fn.original_get_concrete_function(*args, **kwargs) From 3aa7f231399b3399ccb3d1e7a0e42c5e5357b56b Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Wed, 10 Jul 2019 15:17:24 -0700 Subject: [PATCH 231/332] [XLA] Fix typo PiperOrigin-RevId: 257492187 --- tensorflow/compiler/xla/service/multi_output_fusion.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/multi_output_fusion.cc b/tensorflow/compiler/xla/service/multi_output_fusion.cc index 12a4a5ad5e9..582e59349e8 100644 --- a/tensorflow/compiler/xla/service/multi_output_fusion.cc +++ b/tensorflow/compiler/xla/service/multi_output_fusion.cc @@ -225,7 +225,7 @@ bool MultiOutputFusion::LegalToFuse(HloInstruction* instr1, return false; } - // Fusing nodes with 0 user makes no sense and the rest of the implementation + // Fusing nodes with 0 users makes no sense and the rest of the implementation // doesn't support it either. if (instr1->user_count() == 0 || instr2->user_count() == 0) { return false; From 943a1757d523bfe46d54ee4f51e67d948ee06bb6 Mon Sep 17 00:00:00 2001 From: Yifei Feng Date: Wed, 10 Jul 2019 15:37:15 -0700 Subject: [PATCH 232/332] Fix import error for tf.summary.create_file_writer for 2.0. tensorboard/summary/_tf/summary/__init__.py's reexport_tf_summary depends on tensorflow.compat.v2 being already imported (in sys.modules). PiperOrigin-RevId: 257496023 --- tensorflow/api_template.__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/api_template.__init__.py b/tensorflow/api_template.__init__.py index c6d39cd9f48..3d8d92c63e7 100644 --- a/tensorflow/api_template.__init__.py +++ b/tensorflow/api_template.__init__.py @@ -56,6 +56,10 @@ elif _tf_api_dir not in __path__: __path__.append(_tf_api_dir) # Hook external TensorFlow modules. + +# Import compat before trying to import summary from tensorboard, so that +# reexport_tf_summary can get compat from sys.modules +_current_module.compat.v2.compat.v1 = _current_module.compat.v1 try: from tensorboard.summary._tf import summary _current_module.__path__ = ( @@ -150,6 +154,4 @@ if hasattr(_current_module, 'keras'): setattr(_current_module, "metrics", metrics) setattr(_current_module, "optimizers", optimizers) setattr(_current_module, "initializers", initializers) - -_current_module.compat.v2.compat.v1 = _current_module.compat.v1 # pylint: enable=undefined-variable From 1c94a5bed5402c4f764ec9b6b25d00af529c541b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 15:42:27 -0700 Subject: [PATCH 233/332] Set the indices in TopK op for an edge case where k=1 and input is all NaN. The returned indices aren't set in this case because input.maximum() returns -Inf and NaN != -Inf. Even if input.maximum() returns NaN, the indices still won't be set due to NaN != NaN. This behavior surprises users and is likely to cause index-out-of-range error in downstream operations. PiperOrigin-RevId: 257497013 --- tensorflow/core/kernels/topk_op.cc | 2 ++ tensorflow/python/kernel_tests/topk_op_test.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/tensorflow/core/kernels/topk_op.cc b/tensorflow/core/kernels/topk_op.cc index f51deb20196..02b99e44880 100644 --- a/tensorflow/core/kernels/topk_op.cc +++ b/tensorflow/core/kernels/topk_op.cc @@ -123,12 +123,14 @@ struct TopKFunctor { input.maximum(/*dims=*/reduce_on_cols).eval().reshape(rows_by_one); // Get the indices of the maximum values. for (int r = 0; r < num_rows; ++r) { + indices(r, 0) = 0; for (int c = 0; c < num_cols; ++c) { if (values(r, 0) == input(r, c)) { indices(r, 0) = c; break; } } + values(r, 0) = input(r, indices(r, 0)); } return Status::OK(); diff --git a/tensorflow/python/kernel_tests/topk_op_test.py b/tensorflow/python/kernel_tests/topk_op_test.py index 32ac9a41569..7872e62050a 100644 --- a/tensorflow/python/kernel_tests/topk_op_test.py +++ b/tensorflow/python/kernel_tests/topk_op_test.py @@ -108,6 +108,10 @@ class TopKTest(test.TestCase): values = -np.sort(-inputs)[:k] self._validateTopK(inputs, k, values, indices) + def testTop1AllNan(self): + inputs = [[np.NaN, np.NaN], [np.NaN, np.NaN]] + self._validateTopK(inputs, 1, [[np.NaN], [np.NaN]], [[0], [0]]) + def _testLargeSort(self, dtype): b = 10 n = 5000 From f7b566f4fd070d6c04a7dc3ef5b7c0a2a4c8a76b Mon Sep 17 00:00:00 2001 From: Penporn Koanantakool Date: Wed, 10 Jul 2019 15:43:55 -0700 Subject: [PATCH 234/332] Automated rollback of commit 6a8c897298c12299e85518291a4cb6d92e646888. Revert #30375. PiperOrigin-RevId: 257497283 --- tensorflow/core/kernels/scatter_functor.h | 36 +++++++--------------- tensorflow/core/kernels/scatter_op_test.cc | 33 ++------------------ 2 files changed, 13 insertions(+), 56 deletions(-) diff --git a/tensorflow/core/kernels/scatter_functor.h b/tensorflow/core/kernels/scatter_functor.h index 517f0e314b9..755f8f8dc55 100644 --- a/tensorflow/core/kernels/scatter_functor.h +++ b/tensorflow/core/kernels/scatter_functor.h @@ -26,7 +26,6 @@ limitations under the License. #include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/kernels/dense_update_functor.h" #include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" namespace tensorflow { @@ -206,30 +205,17 @@ struct ScatterFunctorBase { // indices and params sizes were validated in DoCompute(). const Index N = static_cast(indices.size()); const Index limit = static_cast(params.dimension(0)); - mutex mu_; - Index bad_index GUARDED_BY(mu_) = -1; - auto ParallelScatter = [&](Index start, Index end) LOCKS_EXCLUDED(mu_) { - for (Index i = start; i < end; i++) { - // Grab the index and check its validity. Do this carefully, - // to avoid checking the value and grabbing it again from - // memory a second time (a security risk since it may change in - // between). - const Index index = ::tensorflow::internal::SubtleMustCopy(indices(i)); - if (!FastBoundsCheck(index, limit)) { - mutex_lock lock(mu_); - bad_index = i; - return; - } - // Copy last Ndim-1 dimensions of updates[i] to params[index] - scatter_op::internal::Assign::Run(params.template chip<0>(index), - updates.template chip<0>(i)); - } - }; - const DeviceBase::CpuWorkerThreads& worker_threads = - *(c->device()->tensorflow_cpu_worker_threads()); - Shard(worker_threads.num_threads, worker_threads.workers, N, 35.0, - ParallelScatter); // Cost is arbitrary for now. - return bad_index; + for (Index i = 0; i < N; i++) { + // Grab the index and check its validity. Do this carefully, + // to avoid checking the value and grabbing it again from + // memory a second time (a security risk since it may change in between). + const Index index = ::tensorflow::internal::SubtleMustCopy(indices(i)); + if (!FastBoundsCheck(index, limit)) return i; + // Copy last Ndim-1 dimensions of updates[i] to params[index] + scatter_op::internal::Assign::Run(params.template chip<0>(index), + updates.template chip<0>(i)); + } + return -1; } }; diff --git a/tensorflow/core/kernels/scatter_op_test.cc b/tensorflow/core/kernels/scatter_op_test.cc index 27286521091..ae6548e9ef2 100644 --- a/tensorflow/core/kernels/scatter_op_test.cc +++ b/tensorflow/core/kernels/scatter_op_test.cc @@ -48,18 +48,6 @@ class ScatterUpdateOpTest : public OpsTestBase { } }; -class ScatterSubOpTest : public OpsTestBase { - protected: - void MakeOp(DataType variable_ref_type, DataType index_type) { - TF_ASSERT_OK(NodeDefBuilder("myop", "ScatterSub") - .Input(FakeInput(variable_ref_type)) - .Input(FakeInput(index_type)) - .Input(FakeInput(RemoveRefType(variable_ref_type))) - .Finalize(node_def())); - TF_ASSERT_OK(InitOp()); - } -}; - TEST_F(ScatterUpdateOpTest, Simple_StringType) { MakeOp(DT_STRING_REF, DT_INT32); AddInputFromArray(TensorShape({1}), {"Brain"}); @@ -187,19 +175,6 @@ TEST_F(ScatterUpdateOpTest, Error_IndexOutOfRange) { << s; } -TEST_F(ScatterSubOpTest, Error_IndexOutOfRange) { - MakeOp(DT_FLOAT_REF, DT_INT32); - // Feed and run - AddInputFromArray(TensorShape({14}), - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); - AddInputFromArray(TensorShape({3}), {0, 1, 99}); - AddInputFromArray(TensorShape({3}), {100, 101, 102}); - Status s = RunOpKernel(); - EXPECT_TRUE( - absl::StrContains(s.ToString(), "indices[2] = 99 is not in [0, 14)")) - << s; -} - TEST_F(ScatterUpdateOpTest, Error_WrongDimsIndices) { MakeOp(DT_FLOAT_REF, DT_INT32); @@ -263,8 +238,7 @@ class ScatterUpdateBM : public ScatterUpdateOpTest { }; template -static void BM_ScatterHelper(int iters, int embedding_size, const char* op, - bool big_num_updates = false) { +static void BM_ScatterHelper(int iters, int embedding_size, const char* op) { testing::StopTiming(); const int kRows = 10000000 / embedding_size; std::vector values; @@ -272,7 +246,7 @@ static void BM_ScatterHelper(int iters, int embedding_size, const char* op, for (int i = 0; i < kRows * embedding_size; i++) { values.push_back(i); } - const int kNumUpdates = big_num_updates ? 1000000 : 1000; + const int kNumUpdates = 1000; random::PhiloxRandom philox(301, 17); random::SimplePhilox rnd(&philox); std::vector indices; @@ -308,9 +282,7 @@ static void BM_ScatterUpdateInt64(int iters, int embedding_size) { static void BM_ScatterAddInt32(int iters, int embedding_size) { BM_ScatterHelper(iters, embedding_size, "ScatterAdd"); - BM_ScatterHelper(iters, embedding_size, "ScatterAdd", true); } - static void BM_ScatterAddInt64(int iters, int embedding_size) { BM_ScatterHelper(iters, embedding_size, "ScatterAdd"); } @@ -367,7 +339,6 @@ BENCHMARK(BM_ScatterUpdateInt64) ->Arg(100000); BENCHMARK(BM_ScatterAddInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); - BENCHMARK(BM_ScatterAddInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); BENCHMARK(BM_ScatterMulInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024); From 166662af04142a7554de915ed4f5928e5e40944b Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Wed, 10 Jul 2019 15:46:20 -0700 Subject: [PATCH 235/332] Forward gradients: fixes for higher-order forward-only gradients One fix for non-trainable target handling when using a tape. Arrays passed to the tape could previously be misaligned. Also fixes the order of op recording so that "outer" accumulators record ops first and so can always trace inner accumulators' jvp computation if necessary. PiperOrigin-RevId: 257497741 --- tensorflow/c/eager/tape.h | 33 ++++---- tensorflow/python/eager/forwardprop.py | 21 +++++ tensorflow/python/eager/forwardprop_test.py | 47 +++++++++++- tensorflow/python/eager/pywrap_tfe_src.cc | 85 +++++++++++++++------ 4 files changed, 147 insertions(+), 39 deletions(-) diff --git a/tensorflow/c/eager/tape.h b/tensorflow/c/eager/tape.h index 1e288667121..2353ddc9b20 100644 --- a/tensorflow/c/eager/tape.h +++ b/tensorflow/c/eager/tape.h @@ -890,9 +890,19 @@ ForwardAccumulator::ForwardpropFromTape( // Stop the tape from recording pop_backward_tape.release()(); + if (grad.size() != in_grads.size()) { + return tensorflow::errors::Internal("Wrong number of gradients returned."); + } + std::vector targets; + std::vector used_in_grads; + // We may end up with slightly fewer elements than we reserve, but grad.size() + // should be a reasonably tight upper bound. + targets.reserve(grad.size()); + used_in_grads.reserve(grad.size()); std::unordered_map sources_that_are_targets; - for (Gradient* grad_tensor : grad) { + for (int grad_index = 0; grad_index < grad.size(); ++grad_index) { + Gradient* grad_tensor = grad[grad_index]; if (grad_tensor != nullptr) { int64 tensor_id = vspace_.TensorId(grad_tensor); targets.push_back(tensor_id); @@ -900,23 +910,18 @@ ForwardAccumulator::ForwardpropFromTape( sources_that_are_targets.emplace( tensor_id, vspace_.TapeTensorFromGradient(grad_tensor)); } - } - } - if (targets.size() > in_grads.size()) { - return tensorflow::errors::Internal("Too many gradients returned."); - } - - for (int target_index = 0; target_index < targets.size(); ++target_index) { - Gradient* in_grad = in_grads[target_index]; - Gradient* grad_tensor = grad[target_index]; - if (grad_tensor != nullptr && in_grad != nullptr) { - // ComputeGradient steals a reference - vspace_.MarkAsResult(in_grad); + Gradient* in_grad = in_grads[grad_index]; + if (in_grad != nullptr) { + // ComputeGradient steals a reference + vspace_.MarkAsResult(in_grad); + } + used_in_grads.push_back(in_grad); } } return tape->ComputeGradient(vspace_, targets, sources, - sources_that_are_targets, in_grads, out_grads); + sources_that_are_targets, used_in_grads, + out_grads); } template diff --git a/tensorflow/python/eager/forwardprop.py b/tensorflow/python/eager/forwardprop.py index e2ebb594aea..ae29d4c63ce 100644 --- a/tensorflow/python/eager/forwardprop.py +++ b/tensorflow/python/eager/forwardprop.py @@ -40,6 +40,24 @@ class ForwardGradientAccumulator(object): y = tf.reduce_sum(tf.sin(x) * tf.tan(x), axis=1) jvp = acc.jvp(y) ``` + + Note that `ForwardGradientAccumulator`s are always applied in creation order, + so inner accumulators may not see JVP computation from outer + accumulators. Take higher-order gradients from outer accumulators: + + ``` + primal = tf.constant(1.1) + with ForwardGradientAccumulator() as outer_acc: + outer_acc.watch(primal, tf.constant(1.)) + with ForwardGradientAccumulator() as acc: + acc.watch(primal, tf.constant(1.)) + primal_out = primal ** tf.constant(3.5) + inner_jvp = acc.jvp(primal_out) + outer_jvp = outer_acc.jvp(inner_jvp) + ``` + + Reversing the collection in the last two lines to instead retrieve + `acc.jvp(outer_acc.jvp(primal_out))` will not work. """ def __init__(self): @@ -70,6 +88,9 @@ class ForwardGradientAccumulator(object): pywrap_tensorflow.TFE_Py_ForwardAccumulatorSetRemove(self._accumulator) self._recording = False + # TODO(allenl): Does this need to be public, or should the constructor instead + # take all watched Tensors? Write a realistic usage example (e.g. Hessian-free + # optimization) and decide. def watch(self, tensor, tangents): """Ensures that `tensor` is being traced by this tape. diff --git a/tensorflow/python/eager/forwardprop_test.py b/tensorflow/python/eager/forwardprop_test.py index d3896a6200f..fdba6f8122c 100644 --- a/tensorflow/python/eager/forwardprop_test.py +++ b/tensorflow/python/eager/forwardprop_test.py @@ -191,6 +191,50 @@ class ForwardpropTest(test.TestCase): _test_gradients(self, f, [constant_op.constant([1.])], order=3) + @test_util.assert_no_new_pyobjects_executing_eagerly + def testHigherOrderPureForward(self): + + def _forwardgrad(f): + def _compute_forwardgrad(primal): + tangent = constant_op.constant(1.) + with forwardprop.ForwardGradientAccumulator() as acc: + acc.watch(primal, tangent) + primal_out = f(primal) + return acc.jvp(primal_out) + return _compute_forwardgrad + + def _forward(x): + return x ** 3.5 + + f = _forward + primal = constant_op.constant(1.1) + for expected in [1.1 ** 3.5, + 3.5 * 1.1 ** 2.5, + 3.5 * 2.5 * 1.1 ** 1.5, + 3.5 * 2.5 * 1.5 * 1.1 ** 0.5, + 3.5 * 2.5 * 1.5 * 0.5 * 1.1 ** -0.5]: + self.assertAllClose(expected, f(primal)) + f = _forwardgrad(f) + + def testFunctionGradPureForward(self): + + @def_function.function + def f(x): + return x ** 3.5 + + primal = constant_op.constant(1.1) + with forwardprop.ForwardGradientAccumulator() as outer_acc: + outer_acc.watch(primal, constant_op.constant(1.)) + with forwardprop.ForwardGradientAccumulator() as acc: + acc.watch(primal, constant_op.constant(1.)) + primal_out = f(primal) + inner_jvp = acc.jvp(primal_out) + outer_jvp = outer_acc.jvp(inner_jvp) + self.assertAllClose(1.1 ** 3.5, primal_out) + self.assertAllClose(3.5 * 1.1 ** 2.5, inner_jvp) + self.assertAllClose(3.5 * 2.5 * 1.1 ** 1.5, outer_jvp) + self.assertIsNone(acc.jvp(outer_acc.jvp(primal_out))) + def testFunctionGrad(self): @def_function.function @@ -201,8 +245,7 @@ class ForwardpropTest(test.TestCase): self, f, [constant_op.constant([1., 2.])], - # TODO(allenl): figure out why functions aren't N times differentiable - order=1) + order=3) @test_util.assert_no_new_pyobjects_executing_eagerly def testHVPMemory(self): diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index 4b959d9f17e..643891ecf8b 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -1367,14 +1367,57 @@ tensorflow::gtl::CompactPointerSet* GetTapeSet() { return tape_set.get(); } -tensorflow::gtl::CompactPointerSet* -GetAccumulatorSet() { - thread_local std::unique_ptr< - tensorflow::gtl::CompactPointerSet> - accumulator_set{nullptr}; +// A linked hash set, where iteration is in insertion order. +// +// Nested accumulators rely on op recording happening in insertion order, so an +// unordered data structure like CompactPointerSet is not suitable. Outer +// accumulators need to observe operations first so they know to watch the inner +// accumulator's jvp computation. +// +// Not thread safe. +class AccumulatorSet { + public: + void insert(TFE_Py_ForwardAccumulator* element) { + if (map_.find(element) != map_.end()) { + return; + } + ListType::iterator it = ordered_.insert(ordered_.end(), element); + map_.insert(std::make_pair(element, it)); + } + + void erase(TFE_Py_ForwardAccumulator* element) { + MapType::iterator existing = map_.find(element); + if (existing == map_.end()) { + return; + } + ListType::iterator list_position = existing->second; + map_.erase(existing); + ordered_.erase(list_position); + } + + bool empty() const { return ordered_.empty(); } + + private: + typedef std::list ListType; + typedef tensorflow::gtl::FlatMap + MapType; + + public: + typedef ListType::const_iterator const_iterator; + const_iterator begin() const { return ordered_.begin(); } + + const_iterator end() const { return ordered_.end(); } + + private: + MapType map_; + ListType ordered_; +}; + +AccumulatorSet* GetAccumulatorSet() { + thread_local std::unique_ptr accumulator_set{nullptr}; if (accumulator_set == nullptr) { - accumulator_set.reset( - new tensorflow::gtl::CompactPointerSet); + accumulator_set.reset(new AccumulatorSet); } return accumulator_set.get(); } @@ -1385,12 +1428,10 @@ inline bool HasTape() { return !GetTapeSet()->empty() || HasAccumulator(); } // A safe copy of a set, used for tapes and accumulators. The copy is not // affected by other python threads changing the set of active tapes. -template +template class SafeSetCopy { public: - explicit SafeSetCopy( - const tensorflow::gtl::CompactPointerSet& to_copy) - : set_copy_(to_copy) { + explicit SafeSetCopy(const ContainerType& to_copy) : set_copy_(to_copy) { for (auto* member : set_copy_) { Py_INCREF(member); } @@ -1402,31 +1443,29 @@ class SafeSetCopy { } } - typename tensorflow::gtl::CompactPointerSet::const_iterator - begin() const { + typename ContainerType::const_iterator begin() const { return set_copy_.begin(); } - typename tensorflow::gtl::CompactPointerSet::const_iterator end() - const { - return set_copy_.end(); - } + typename ContainerType::const_iterator end() const { return set_copy_.end(); } bool empty() const { return set_copy_.empty(); } private: - tensorflow::gtl::CompactPointerSet set_copy_; + ContainerType set_copy_; }; -class SafeTapeSet : public SafeSetCopy { +class SafeTapeSet + : public SafeSetCopy> { public: - SafeTapeSet() : SafeSetCopy(*GetTapeSet()) {} + SafeTapeSet() + : SafeSetCopy>( + *GetTapeSet()) {} }; -class SafeAccumulatorSet : public SafeSetCopy { +class SafeAccumulatorSet : public SafeSetCopy { public: - SafeAccumulatorSet() - : SafeSetCopy(*GetAccumulatorSet()) {} + SafeAccumulatorSet() : SafeSetCopy(*GetAccumulatorSet()) {} }; bool* ThreadTapeIsStopped() { From cd2b389d534717651b4619ea6764d96e8918e78f Mon Sep 17 00:00:00 2001 From: Gaurav Jain Date: Wed, 10 Jul 2019 15:48:50 -0700 Subject: [PATCH 236/332] Add support for unshaped remote handle mirrors As part of unifying the sync and async code path in ExecuteRecv, we need to add support for unshaped remote handle mirrors that get converted to regular remote handle mirror during the SetRemoteShape call. This should be non-functional change. Additionally: - Ensure that Abort() is always called if a node is not successfully added. - Make ExecuteNode & RemoteExecuteNode take absl::Span instead of raw double pointer. PiperOrigin-RevId: 257498187 --- tensorflow/core/common_runtime/eager/BUILD | 1 + .../common_runtime/eager/eager_executor.cc | 37 +++--- .../common_runtime/eager/eager_executor.h | 5 +- .../core/common_runtime/eager/execute.cc | 112 ++++++++---------- .../core/common_runtime/eager/execute.h | 3 +- .../core/common_runtime/eager/execute_node.h | 13 +- .../common_runtime/eager/tensor_handle.cc | 86 +++++++++++--- .../core/common_runtime/eager/tensor_handle.h | 24 ++-- .../core/distributed_runtime/eager/BUILD | 1 + .../eager/remote_execute_node.h | 17 +-- 10 files changed, 186 insertions(+), 113 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD index 70c79b1dc8f..5d771703409 100644 --- a/tensorflow/core/common_runtime/eager/BUILD +++ b/tensorflow/core/common_runtime/eager/BUILD @@ -248,6 +248,7 @@ cc_library( ":kernel_and_device", ":tensor_handle", "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:span", "//tensorflow/core/profiler/lib:traceme", ] + select({ "//tensorflow:android": [ diff --git a/tensorflow/core/common_runtime/eager/eager_executor.cc b/tensorflow/core/common_runtime/eager/eager_executor.cc index 765be6a41ca..ae3369dfbc0 100644 --- a/tensorflow/core/common_runtime/eager/eager_executor.cc +++ b/tensorflow/core/common_runtime/eager/eager_executor.cc @@ -33,22 +33,31 @@ void EagerExecutor::EnableAsync() { } Status EagerExecutor::Add(std::unique_ptr node) { - tensorflow::mutex_lock l(node_queue_mutex_); - DCHECK(thread_) << "EnableAsync should have been called before Add"; - if (!status_.ok()) { - // node will be automatically deleted - return status_; + Status status; + + // If we are unable to add the node to the queue, we must call Abort. However, + // we want to do that outside of the scope of the lock since the Abort may + // try to call EagerExecutor::Add() + { + tensorflow::mutex_lock l(node_queue_mutex_); + DCHECK(thread_) << "EnableAsync should have been called before Add"; + status = status_; + if (status.ok()) { + node_queue_.push(std::move(node)); + + // If there were no previous nodes pending, wake the run thread to start + // processing requests again. + if (node_queue_.size() == 1) { + nodes_pending_.notify_all(); + } + + return Status::OK(); + } } - node_queue_.push(std::move(node)); - - // If there were no previous nodes pending, wake the run thread to start - // processing requests again. - if (node_queue_.size() == 1) { - nodes_pending_.notify_all(); - } - - return Status::OK(); + // Node needs to be aborted since it was not added to the queue + node->Abort(status); + return status; } tensorflow::Status EagerExecutor::WaitForAllPendingNodes() { diff --git a/tensorflow/core/common_runtime/eager/eager_executor.h b/tensorflow/core/common_runtime/eager/eager_executor.h index fb6a8e2b03a..9a5aee313b6 100644 --- a/tensorflow/core/common_runtime/eager/eager_executor.h +++ b/tensorflow/core/common_runtime/eager/eager_executor.h @@ -42,6 +42,10 @@ namespace tensorflow { class EagerNode { public: EagerNode() {} + // Nodes should not do any work in their destructor. This is because if the + // node is being destructed by the EagerExecutor, then the node queue lock may + // be held. Instead opt for calling clean-up code as part of Run() or Abort(), + // since one of those are guaranteed to be run. virtual ~EagerNode() {} // Runs the computation corresponding to this node and blocks till the @@ -74,7 +78,6 @@ class EagerExecutor { void EnableAsync(); // Schedules `node` for execution. - // Note that Add must be called in monotonically increasing order of node->id. Status Add(std::unique_ptr node); // Blocks till all currently pending ops are done. diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index 45c766576b1..d39c7af8727 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -632,13 +632,13 @@ Status EagerLocalExecute(EagerOperation* op, TensorHandle** retvals, ctx->AddKernelToCache(cache_key, kernel.get()); } const DataTypeVector& output_dtypes = kernel->output_dtypes(); - const int output_dtypes_size = static_cast(output_dtypes.size()); - if (output_dtypes_size > *num_retvals) { - return errors::InvalidArgument("Expecting ", output_dtypes.size(), + const size_t num_outputs = static_cast(output_dtypes.size()); + if (num_outputs > *num_retvals) { + return errors::InvalidArgument("Expecting ", num_outputs, " outputs, but *num_retvals is ", *num_retvals); } - *num_retvals = output_dtypes_size; + *num_retvals = num_outputs; TF_RETURN_IF_ERROR(ValidateInputTypeAndPlacement( ctx, op, kernel, ctx->ShouldStoreStepStats() ? ctx->RunMetadataProto() : nullptr)); @@ -663,7 +663,7 @@ Status EagerLocalExecute(EagerOperation* op, TensorHandle** retvals, // TODO(apassos) track referenced tensors } - for (int i = 0; i < *num_retvals; ++i) { + for (int i = 0; i < num_outputs; ++i) { TF_RETURN_IF_ERROR(TensorHandle::CreateAsyncLocalHandle( /* d= */ kernel->OutputDevice(i), /* op_device= */ kernel->device(), @@ -671,16 +671,19 @@ Status EagerLocalExecute(EagerOperation* op, TensorHandle** retvals, output_dtypes[i], ctx, &retvals[i])); } - std::unique_ptr node(new ExecuteNode( - ctx, op->Inputs(), std::move(kernel), maybe_stats.release(), - maybe_step_stats, graph_collector, output_dtypes, retvals, *num_retvals)); + std::unique_ptr node( + new ExecuteNode(ctx, op->Inputs(), std::move(kernel), + maybe_stats.release(), maybe_step_stats, graph_collector, + output_dtypes, {retvals, num_outputs})); // Note that for async mode, execution order will make sure that all // input handles are ready before executing them. // TODO(b/137118203): Consider executing "cheap" kernels inline for // performance. Status s = ctx->Async() ? ctx->ExecutorAdd(std::move(node)) : node->Run(); + // Since the operation failed, we need to Unref any outputs that were + // allocated. if (!s.ok()) { - for (int i = 0; i < *num_retvals; ++i) { + for (int i = 0; i < num_outputs; ++i) { retvals[i]->Unref(); } } @@ -747,21 +750,6 @@ Status EagerRemoteSendTensor(EagerContext* ctx, TensorHandle* h, return status; } -Status EnqueueAndWait(eager::EagerClient* eager_client, - const std::unique_ptr& request, - eager::EnqueueResponse* response) { - Notification n; - Status status; - eager_client->EnqueueAsync(request.get(), response, - [&n, &status](const Status& s) { - status = s; - n.Notify(); - }); - n.WaitForNotification(); - - return status; -} - void PrepareRemoteOp(eager::Operation* remote_op, EagerOperation* op) { EagerContext* ctx = op->EagerContext(); @@ -834,12 +822,12 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, DataTypeVector output_dtypes; TF_RETURN_IF_ERROR(GetOutputDTypes(op, &output_dtypes)); - const int output_dtypes_size = static_cast(output_dtypes.size()); - if (output_dtypes_size != *num_retvals) { + const size_t num_outputs = static_cast(output_dtypes.size()); + if (num_outputs != *num_retvals) { return errors::InvalidArgument( "num_retvals does not match expected output dtypes"); } - *num_retvals = output_dtypes_size; + *num_retvals = num_outputs; tensorflow::Device* op_device = op->Device(); @@ -848,7 +836,7 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, << " (is async?: " << is_async << ")."; const tensorflow::uint64 id = remote_op->id(); - for (int i = 0; i < *num_retvals; ++i) { + for (int i = 0; i < num_outputs; ++i) { // TODO(nareshmodi): Change the callback to instead add the decref to a // list of pending decrefs that we can send as a batch with the next // execute. @@ -861,16 +849,18 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, // to copy this tensor to this process, the remote end will know the // correct device of this handle. TF_RETURN_IF_ERROR(TensorHandle::CreateUnshapedRemoteHandle( - id, i, eager_client, context_id, output_dtypes[i], op_device, - output_dtypes[i] == DT_RESOURCE ? op_device : nullptr, ctx, + id, i, eager_client, context_id, output_dtypes[i], op_device, ctx, &retvals[i])); } - std::unique_ptr node(new eager::RemoteExecuteNode( - std::move(request), eager_client, op->Inputs(), retvals, *num_retvals)); + std::unique_ptr node( + new eager::RemoteExecuteNode(std::move(request), op_device, eager_client, + op->Inputs(), {retvals, num_outputs})); Status s = is_async ? ctx->ExecutorAdd(std::move(node)) : node->Run(); + // Since the operation failed, we need to Unref any outputs that were + // allocated. if (!s.ok()) { - for (int i = 0; i < *num_retvals; ++i) { + for (int i = 0; i < num_outputs; ++i) { retvals[i]->Unref(); } } @@ -1037,7 +1027,7 @@ Status EagerKernelExecute(EagerContext* ctx, NodeExecStats* maybe_stats, StepStats* maybe_step_stats, GraphCollector* graph_collector, - TensorHandle** retvals, int num_retvals) { + absl::Span retvals) { profiler::TraceMe activity("EagerKernelExecute", profiler::TraceMeLevel::kInfo); std::vector outputs(1); @@ -1140,8 +1130,8 @@ Status EagerKernelExecute(EagerContext* ctx, } } } - DCHECK_EQ(num_retvals, outputs.size()); - for (int i = 0; i < num_retvals; ++i) { + DCHECK_EQ(retvals.size(), outputs.size()); + for (int i = 0; i < retvals.size(); ++i) { DCHECK_EQ(kernel->device(), retvals[i]->op_device()); DCHECK_EQ(kernel->OutputDevice(i), retvals[i]->device()); @@ -1163,6 +1153,8 @@ Status LocalEagerCopyToDevice(TensorHandle* h, EagerContext* ctx, Device* dstd, // make sure that `h` is ready before the copy is actually done. std::unique_ptr node(new CopyToDeviceNode(h, *result, dstd, ctx)); Status s = ctx->Async() ? ctx->ExecutorAdd(std::move(node)) : node->Run(); + // Since the operation failed, we need to Unref any outputs that were + // allocated. if (!s.ok()) { (*result)->Unref(); } @@ -1243,13 +1235,13 @@ Status ExecuteSend(EagerContext* ctx, Device* device, TensorHandle* h, PrepareRemoteOp(remote_op, &op); - std::unique_ptr node(new eager::RemoteExecuteNode( - std::move(request), eager_client, op.Inputs(), nullptr, 0)); - if (ctx->Async()) { - TF_RETURN_IF_ERROR(ctx->ExecutorAdd(std::move(node))); - } else { - TF_RETURN_IF_ERROR(node->Run()); - } + std::unique_ptr node(new eager::RemoteExecuteNode( + std::move(request), nullptr, eager_client, op.Inputs(), {nullptr, 0})); + if (ctx->Async()) { + TF_RETURN_IF_ERROR(ctx->ExecutorAdd(std::move(node))); + } else { + TF_RETURN_IF_ERROR(node->Run()); + } } return Status::OK(); @@ -1310,30 +1302,24 @@ Status ExecuteRecv(EagerContext* ctx, Device* device, DataType dtype, PrepareRemoteOp(remote_op, &op); const uint64 id = remote_op->id(); - if (ctx->Async()) { + auto tensor_handle_data = absl::make_unique( + id, 0, eager_client, context_id, ctx); + if (mirror_dst != nullptr) { + TF_RETURN_IF_ERROR(mirror_dst->AddUnshapedRemoteMirror( + std::move(tensor_handle_data), device)); + mirror_dst->Ref(); + *result = mirror_dst; + } else { TF_RETURN_IF_ERROR(TensorHandle::CreateUnshapedRemoteHandle( - id, 0, eager_client, context_id, dtype, device, - dtype == DT_RESOURCE ? device : nullptr, ctx, result)); + std::move(tensor_handle_data), dtype, device, ctx, result)); + } - std::unique_ptr node(new eager::RemoteExecuteNode( - std::move(request), eager_client, op.Inputs(), result, 1)); + std::unique_ptr node(new eager::RemoteExecuteNode( + std::move(request), device, eager_client, op.Inputs(), {result, 1})); + if (ctx->Async()) { TF_RETURN_IF_ERROR(ctx->ExecutorAdd(std::move(node))); } else { - TF_RETURN_IF_ERROR(EnqueueAndWait(eager_client, request, &response)); - - auto tensor_handle_data = absl::make_unique( - id, 0, response.queue_response(0).shape(0), eager_client, context_id, - ctx); - if (mirror_dst != nullptr) { - TF_RETURN_IF_ERROR( - mirror_dst->AddRemoteMirror(std::move(tensor_handle_data), device)); - mirror_dst->Ref(); - *result = mirror_dst; - } else { - TF_RETURN_IF_ERROR(TensorHandle::CreateRemoteHandle( - std::move(tensor_handle_data), dtype, device, - dtype == DT_RESOURCE ? device : nullptr, ctx, result)); - } + TF_RETURN_IF_ERROR(node->Run()); } } diff --git a/tensorflow/core/common_runtime/eager/execute.h b/tensorflow/core/common_runtime/eager/execute.h index 2cabf01ca95..0e7e1641adf 100644 --- a/tensorflow/core/common_runtime/eager/execute.h +++ b/tensorflow/core/common_runtime/eager/execute.h @@ -15,6 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_H_ #define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_H_ +#include "absl/types/span.h" #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/common_runtime/eager/context.h" #include "tensorflow/core/common_runtime/eager/eager_operation.h" @@ -49,7 +50,7 @@ Status EagerKernelExecute(EagerContext* ctx, NodeExecStats* maybe_stats, StepStats* maybe_step_stats, GraphCollector* graph_collector, - TensorHandle** retvals, int num_retvals); + absl::Span retvals); // Low-level utility to copy a tensor handle from one device to another. If // successful, result TensorHandle will be populated. If the caller requests for diff --git a/tensorflow/core/common_runtime/eager/execute_node.h b/tensorflow/core/common_runtime/eager/execute_node.h index 81dced54987..0e6e5241a56 100644 --- a/tensorflow/core/common_runtime/eager/execute_node.h +++ b/tensorflow/core/common_runtime/eager/execute_node.h @@ -15,6 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_NODE_H_ #define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_NODE_H_ +#include "absl/types/span.h" #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/common_runtime/eager/context.h" #include "tensorflow/core/common_runtime/eager/eager_executor.h" @@ -36,8 +37,8 @@ class ExecuteNode : public EagerNode { core::RefCountPtr kernel, NodeExecStats* maybe_stats, StepStats* maybe_step_stats, GraphCollector* graph_collector, - const DataTypeVector& output_dtypes, TensorHandle** retvals, - int num_retvals) + const DataTypeVector& output_dtypes, + absl::Span retvals) : EagerNode(), ctx_(ctx), inputs_(inputs), @@ -47,9 +48,9 @@ class ExecuteNode : public EagerNode { graph_collector_(graph_collector) { // Copy the output handles, since the container for them might get // destroyed. - for (int i = 0; i < num_retvals; i++) { - retvals_.push_back(retvals[i]); - retvals_[i]->Ref(); + for (auto handle : retvals) { + handle->Ref(); + retvals_.push_back(handle); } // This is required to ensure that the tensor handles stay alive across the @@ -62,7 +63,7 @@ class ExecuteNode : public EagerNode { Status Run() override { const Status status = EagerKernelExecute( ctx_, inputs_, kernel_, maybe_stats_.get(), maybe_step_stats_, - graph_collector_, retvals_.begin(), retvals_.size()); + graph_collector_, absl::MakeSpan(retvals_)); if (!status.ok()) { Abort(status); return status; diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc index e9e63633cd7..8f68ee4bb99 100644 --- a/tensorflow/core/common_runtime/eager/tensor_handle.cc +++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc @@ -209,25 +209,29 @@ TensorHandle::TensorHandle(std::unique_ptr t, } Status TensorHandle::CreateUnshapedRemoteHandle( - int64 op_id, int32 output_num, eager::EagerClient* eager_client, - uint64 context_id, DataType dtype, Device* d, Device* resource_device, - EagerContext* ctx, TensorHandle** h) { - DCHECK(dtype == DT_RESOURCE ? resource_device != nullptr - : resource_device == nullptr); + std::unique_ptr t, DataType dtype, + Device* d, EagerContext* ctx, TensorHandle** h) { + *h = new TensorHandle(std::move(t), dtype, d, ctx); + return Status::OK(); +} + +Status TensorHandle::CreateUnshapedRemoteHandle( + int64 op_id, int32 output_num, eager::EagerClient* eager_client, + uint64 context_id, DataType dtype, Device* device, EagerContext* ctx, + TensorHandle** h) { *h = new TensorHandle(absl::make_unique( op_id, output_num, eager_client, context_id, ctx), - dtype, d, resource_device, ctx); + dtype, device, ctx); return Status::OK(); } TensorHandle::TensorHandle(std::unique_ptr t, - DataType dtype, Device* d, Device* resource_device, - EagerContext* ctx) + DataType dtype, Device* device, EagerContext* ctx) : dtype(dtype), - device_(d), - op_device_(d), - resource_device_(resource_device), + device_(device), + op_device_(device), + resource_device_(dtype == DT_RESOURCE ? device : nullptr), remote_op_id_(t->op_id()), remote_output_num_(t->output_num()), remote_eager_client_(t->eager_client()), @@ -303,7 +307,7 @@ Status TensorHandle::NumElements(int64* num_elements) { Status TensorHandle::RemoteAddress(Device* d, int64* op_id, int32* output_num) const { if (d != device_) { - mutex_lock l(remote_mirrors_mutex_); + tf_shared_lock l(remote_mirrors_mutex_); auto mirror = remote_mirrors_.find(d); if (mirror != remote_mirrors_.end()) { *op_id = mirror->second->op_id(); @@ -311,6 +315,13 @@ Status TensorHandle::RemoteAddress(Device* d, int64* op_id, return Status::OK(); } + auto unshaped_mirror = unshaped_remote_mirrors_.find(d); + if (unshaped_mirror != unshaped_remote_mirrors_.end()) { + *op_id = unshaped_mirror->second->op_id(); + *output_num = unshaped_mirror->second->output_num(); + return Status::OK(); + } + return errors::FailedPrecondition( "Could not find remote mirror for specified device"); } @@ -321,15 +332,36 @@ Status TensorHandle::RemoteAddress(Device* d, int64* op_id, } bool TensorHandle::HasRemoteMirror(Device* d) { - mutex_lock l(remote_mirrors_mutex_); + tf_shared_lock l(remote_mirrors_mutex_); auto mirror = remote_mirrors_.find(d); if (mirror != remote_mirrors_.end()) { return true; } + auto unshaped_mirror = unshaped_remote_mirrors_.find(d); + if (unshaped_mirror != unshaped_remote_mirrors_.end()) { + return true; + } + return false; } +Status TensorHandle::AddUnshapedRemoteMirror( + std::unique_ptr t, Device* d) { + mutex_lock l(remote_mirrors_mutex_); + if (remote_mirrors_.find(d) != remote_mirrors_.end()) { + return errors::Internal("Attempted to duplicate a remote mirror."); + } + + auto ret = unshaped_remote_mirrors_.insert(std::make_pair(d, std::move(t))); + if (!ret.second) { + return errors::Internal( + "Attempted to duplicate an unshaped remote mirror."); + } + + return Status::OK(); +} + Status TensorHandle::AddRemoteMirror(std::unique_ptr t, Device* d) { mutex_lock l(remote_mirrors_mutex_); @@ -341,7 +373,33 @@ Status TensorHandle::AddRemoteMirror(std::unique_ptr t, return Status::OK(); } -Status TensorHandle::SetRemoteShape(const TensorShape& shape) { +Status TensorHandle::SetRemoteShape(const TensorShape& shape, + tensorflow::Device* d) { + VLOG(3) << "SetRemoteShape on TensorHandle: " << this << " device: " << d; + + if (d != device_) { + mutex_lock l(remote_mirrors_mutex_); + if (remote_mirrors_.find(d) != remote_mirrors_.end()) { + return errors::Internal( + "Attempted to set remote shape for existing mirror."); + } + + auto elem = unshaped_remote_mirrors_.find(d); + if (elem == unshaped_remote_mirrors_.end()) { + return errors::Internal( + "Attempted to set remote shape for non-waiting mirror."); + } + + auto& data = elem->second; + data->ReleaseRemoteTensorHandle(); + remote_mirrors_[d] = absl::make_unique( + data->op_id(), data->output_num(), shape, data->eager_client(), + data->context_id(), data->ctx()); + unshaped_remote_mirrors_.erase(elem); + + return Status::OK(); + } + DCHECK(is_remote_) << "SeRemoteShape is only called on remote handles."; DCHECK(!is_ready_notification_.HasBeenNotified()) << "SetRemoteShape is only called on non-ready handles."; diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h index 4dc6b6f1f5f..58d10192dc0 100644 --- a/tensorflow/core/common_runtime/eager/tensor_handle.h +++ b/tensorflow/core/common_runtime/eager/tensor_handle.h @@ -82,8 +82,7 @@ class TensorHandle : public core::RefCounted { TensorHandle(std::unique_ptr t, DataType dtype, Device* d, Device* resource_device, EagerContext* ctx); TensorHandle(std::unique_ptr t, - DataType dtype, Device* d, Device* resource_device, - EagerContext* ctx); + DataType dtype, Device* device, EagerContext* ctx); #endif // IS_MOBILE_PLATFORM public: @@ -112,8 +111,11 @@ class TensorHandle : public core::RefCounted { static Status CreateUnshapedRemoteHandle(int64 op_id, int32 output_num, eager::EagerClient* eager_client, uint64 context_id, DataType dtype, - Device* d, Device* resource_device, - EagerContext* ctx, TensorHandle** h); + Device* device, EagerContext* ctx, + TensorHandle** h); + static Status CreateUnshapedRemoteHandle( + std::unique_ptr t, DataType dtype, + Device* device, EagerContext* ctx, TensorHandle** h); #endif // IS_MOBILE_PLATFORM // Symbolic tensor constructor. @@ -139,8 +141,9 @@ class TensorHandle : public core::RefCounted { #if !defined(IS_MOBILE_PLATFORM) bool HasRemoteMirror(Device* d); - // TODO(gjn): Add Unshaped remote mirrors once EagerRemoteSendTensor supports - // async execution and EagerRemoteExecute is mirror-aware. + + Status AddUnshapedRemoteMirror( + std::unique_ptr t, Device* d); Status AddRemoteMirror(std::unique_ptr t, Device* d); // Return the op_id and output num if the handle refers to a remote tensor. @@ -152,7 +155,7 @@ class TensorHandle : public core::RefCounted { // queried. // This method or Poison must be called exactly once for remote tensors that // were created without a known shape. - Status SetRemoteShape(const TensorShape& shape); + Status SetRemoteShape(const TensorShape& shape, tensorflow::Device* d); #endif // Sets the `tensor` for this async non-ready handle making it ready. @@ -224,6 +227,13 @@ class TensorHandle : public core::RefCounted { #if !defined(IS_MOBILE_PLATFORM) mutable mutex remote_mirrors_mutex_; + // TODO(gjn): Unshaped remote mirrors are long expected to be long-lived. + // Consider replacing the unshaped_remote_mirrors_ map with something more + // efficient. + std::map> + unshaped_remote_mirrors_ GUARDED_BY(remote_mirrors_mutex_); + // TODO(gjn): Is std::map the most optimal choice here? Perhaps this should be + // a fixed size map. std::map> remote_mirrors_ GUARDED_BY(remote_mirrors_mutex_); diff --git a/tensorflow/core/distributed_runtime/eager/BUILD b/tensorflow/core/distributed_runtime/eager/BUILD index aa7adb88b0e..bffe4dcca83 100644 --- a/tensorflow/core/distributed_runtime/eager/BUILD +++ b/tensorflow/core/distributed_runtime/eager/BUILD @@ -52,6 +52,7 @@ cc_library( "//tensorflow/core:protos_all_cc", "//tensorflow/core/common_runtime/eager:eager_executor", "//tensorflow/core/common_runtime/eager:tensor_handle", + "@com_google_absl//absl/types:span", ], ) diff --git a/tensorflow/core/distributed_runtime/eager/remote_execute_node.h b/tensorflow/core/distributed_runtime/eager/remote_execute_node.h index 0e844eeaf42..761efff0796 100644 --- a/tensorflow/core/distributed_runtime/eager/remote_execute_node.h +++ b/tensorflow/core/distributed_runtime/eager/remote_execute_node.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_EAGER_REMOTE_EXECUTE_NODE_H_ #define TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_EAGER_REMOTE_EXECUTE_NODE_H_ +#include "absl/types/span.h" #include "tensorflow/core/common_runtime/eager/eager_executor.h" #include "tensorflow/core/common_runtime/eager/tensor_handle.h" #include "tensorflow/core/distributed_runtime/eager/eager_client.h" @@ -28,19 +29,20 @@ namespace eager { // an operation via RPC in a remote EagerService. class RemoteExecuteNode : public EagerNode { public: - RemoteExecuteNode(std::unique_ptr request, + RemoteExecuteNode(std::unique_ptr request, Device* device, EagerClient* eager_client, const gtl::InlinedVector& inputs, - TensorHandle** retvals, int num_retvals) + absl::Span retvals) : EagerNode(), request_(std::move(request)), + device_(device), eager_client_(eager_client), inputs_(inputs) { // Copy the output handles, since the container for them might get // destroyed. - for (int i = 0; i < num_retvals; i++) { - retvals_.push_back(retvals[i]); - retvals_[i]->Ref(); + for (auto handle : retvals) { + handle->Ref(); + retvals_.push_back(handle); } // This is required to ensure that the tensor handles stay alive across the @@ -67,8 +69,8 @@ class RemoteExecuteNode : public EagerNode { } for (int i = 0; i < retvals_.size(); i++) { - Status s = - retvals_[i]->SetRemoteShape(response.queue_response(0).shape(i)); + Status s = retvals_[i]->SetRemoteShape( + response.queue_response(0).shape(i), device_); if (!s.ok()) { retvals_[i]->Poison(s); } @@ -95,6 +97,7 @@ class RemoteExecuteNode : public EagerNode { private: std::unique_ptr request_; + Device* device_; // Not owned EagerClient* eager_client_; // Not owned, and must outlive this node. gtl::InlinedVector inputs_; gtl::InlinedVector retvals_; From 7df87194e2335b33b8085ae06fb5f4f3fbef3ea9 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Wed, 10 Jul 2019 16:22:55 -0700 Subject: [PATCH 237/332] Revise WriteDataToFile API and the related parts --- .../core/kernels/data/dataset_test_base.cc | 33 +- .../core/kernels/data/dataset_test_base.h | 17 +- .../fixed_length_record_dataset_op_test.cc | 656 ------------------ .../kernels/data/text_line_dataset_op_test.cc | 123 +--- 4 files changed, 62 insertions(+), 767 deletions(-) delete mode 100644 tensorflow/core/kernels/data/fixed_length_record_dataset_op_test.cc diff --git a/tensorflow/core/kernels/data/dataset_test_base.cc b/tensorflow/core/kernels/data/dataset_test_base.cc index 0c419adecee..dfc8733e250 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.cc +++ b/tensorflow/core/kernels/data/dataset_test_base.cc @@ -33,7 +33,6 @@ string ToString(CompressionType compression_type) { case CompressionType::UNCOMPRESSED: return ""; } - return "UNDEFINED"; } io::ZlibCompressionOptions GetZlibCompressionOptions( @@ -45,32 +44,40 @@ io::ZlibCompressionOptions GetZlibCompressionOptions( return io::ZlibCompressionOptions::GZIP(); case CompressionType::RAW: return io::ZlibCompressionOptions::RAW(); - default: + case CompressionType::UNCOMPRESSED: LOG(WARNING) << "ZlibCompressionOptions does not have an option for " << ToString(compression_type); + return io::ZlibCompressionOptions::DEFAULT(); } - return io::ZlibCompressionOptions::DEFAULT(); +} + +Status WriteDataToFile(const string& filename, const char* data) { + CompressionParams params; + params.compression_type = CompressionType::UNCOMPRESSED; + return WriteDataToFile(filename, data, params); } Status WriteDataToFile(const string& filename, const char* data, - int input_buffer_size, int output_buffer_size, - const CompressionType& compression_type) { + const CompressionParams& params) { Env* env = Env::Default(); std::unique_ptr file_writer; TF_RETURN_IF_ERROR(env->NewWritableFile(filename, &file_writer)); - if (compression_type == CompressionType::UNCOMPRESSED) { + if (params.compression_type == CompressionType::UNCOMPRESSED) { TF_RETURN_IF_ERROR(file_writer->Append(data)); - } else if (compression_type == CompressionType::ZLIB || - compression_type == CompressionType::GZIP) { - auto zlib_compression_options = GetZlibCompressionOptions(compression_type); - io::ZlibOutputBuffer out(file_writer.get(), input_buffer_size, - output_buffer_size, zlib_compression_options); + } else if (params.compression_type == CompressionType::ZLIB || + params.compression_type == CompressionType::GZIP || + params.compression_type == CompressionType::RAW) { + auto zlib_compression_options = + GetZlibCompressionOptions(params.compression_type); + io::ZlibOutputBuffer out(file_writer.get(), params.input_buffer_size, + params.output_buffer_size, + zlib_compression_options); TF_RETURN_IF_ERROR(out.Init()); TF_RETURN_IF_ERROR(out.Append(data)); TF_RETURN_IF_ERROR(out.Close()); } else { - return tensorflow::errors::InvalidArgument("Unsupported compression_type: ", - ToString(compression_type)); + return tensorflow::errors::InvalidArgument( + "Unsupported compression_type: ", ToString(params.compression_type)); } TF_RETURN_IF_ERROR(file_writer->Flush()); diff --git a/tensorflow/core/kernels/data/dataset_test_base.h b/tensorflow/core/kernels/data/dataset_test_base.h index b033a5590af..8caff593834 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.h +++ b/tensorflow/core/kernels/data/dataset_test_base.h @@ -56,12 +56,21 @@ string ToString(CompressionType compression_type); io::ZlibCompressionOptions GetZlibCompressionOptions( CompressionType compression_type); +// Used to specify parameters when writing data into files with compression. +// `input_buffer_size` and `output_buffer_size` specify the input and output +// buffer size when ZLIB and GZIP compression is used. +struct CompressionParams { + CompressionType compression_type; + int32 input_buffer_size; + int32 output_buffer_size; +}; + +// Writes the input data into the file without compression. +Status WriteDataToFile(const string& filename, const char* data); + // Writes the input data into the file with the specified compression. -// `input_buffer_size` is the size of z_stream.next_in buffer; -// `output_buffer_size` is the size of z_stream.next_out buffer. Status WriteDataToFile(const string& filename, const char* data, - int input_buffer_size, int output_buffer_size, - const CompressionType& compression_type); + const CompressionParams& params); // Helpful functions to test Dataset op kernels. class DatasetOpsTestBase : public ::testing::Test { diff --git a/tensorflow/core/kernels/data/fixed_length_record_dataset_op_test.cc b/tensorflow/core/kernels/data/fixed_length_record_dataset_op_test.cc deleted file mode 100644 index 073b506863c..00000000000 --- a/tensorflow/core/kernels/data/fixed_length_record_dataset_op_test.cc +++ /dev/null @@ -1,656 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/core/kernels/data/text_line_dataset_op.h" - -#include "tensorflow/core/kernels/data/dataset_test_base.h" - -namespace tensorflow { -namespace data { -namespace { - -constexpr char kNodeName[] = "fixed_length_record_dataset"; -constexpr char kIteratorPrefix[] = "Iterator"; - -class FixedLengthRecordDatasetOpTest : public DatasetOpsTestBase { - protected: - // Create a new `TextLineDataset` op kernel. - Status CreateFixedLengthRecordDatasetOpKernel( - std::unique_ptr* text_line_dataset_op_kernel) { - NodeDef node_def = test::function::NDef( - kNodeName, "FixedLengthRecordDatasetV2", - {"filenames", "header_bytes", "record_bytes", "footer_bytes", - "buffer_size", "compression_type"}, - {}); - TF_RETURN_IF_ERROR(CreateOpKernel(node_def, text_line_dataset_op_kernel)); - return Status::OK(); - } - - // Create a new `FixedLengthRecordDataset` op kernel context - Status CreateFixedLengthRecordDatasetContext( - OpKernel* const op_kernel, - gtl::InlinedVector* const inputs, - std::unique_ptr* context) { - TF_RETURN_IF_ERROR(CheckOpKernelInput(*op_kernel, *inputs)); - TF_RETURN_IF_ERROR(CreateOpKernelContext(op_kernel, inputs, context)); - return Status::OK(); - } -}; - -struct TestCase { - std::vector filenames; - int64 header_bytes; - int64 record_bytes; - int64 footer_bytes; - int64 buffer_size; - CompressionType compression_type; - string header; - string footer; - std::vector expected_outputs; - DataTypeVector expected_output_dtypes; - std::vector expected_output_shapes; - int64 expected_cardinality; - std::vector breakpoints; -}; - -std::vector ToStringVector(const std::vector& str_tensors) { - std::vector str_vec; - for (const Tensor& tensor : str_tensors) { - for (int i = 0; i < tensor.NumElements(); ++i) { - str_vec.push_back(tensor.flat()(i)); - } - } - return str_vec; -} - -// Test case 1: multiple text files with ZLIB compression. -TestCase TestCase1() { - return { - /*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_1"), - absl::StrCat(testing::TmpDir(), "/text_line_ZLIB_2")}, - /*header_bytes*/ 5, - /*record_bytes*/ 3, - /*footer_bytes*/ 2, - /*buffer_size*/ 10, - /*compression_type*/ ZLIB, - /*header*/ "HHHHH", - /*footer*/ "FF", - /*expected_outputs*/ - {DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"111"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"222"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"333"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"444"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), {"555"})}, - /*expected_output_dtypes*/ {DT_STRING}, - /*expected_output_shapes*/ {PartialTensorShape({})}, - /*expected_cardinality*/ kUnknownCardinality, - /*breakpoints*/ {0, 2, 6}}; -} - -// Test case 2: multiple text files with GZIP compression. -TestCase TestCase2() { - return {/*filenames*/ {absl::StrCat(testing::TmpDir(), "/text_line_GZIP_1"), - absl::StrCat(testing::TmpDir(), "/text_line_GZIP_2")}, - /*header_bytes*/ 5, - /*record_bytes*/ 3, - /*footer_bytes*/ 2, - /*buffer_size*/ 10, - /*compression_type*/ GZIP, - /*header*/ "HHHHH", - /*footer*/ "FF", - /*expected_outputs*/ - {DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"aaa"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"bbb"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"ccc"})}, - /*expected_output_dtypes*/ {DT_STRING}, - /*expected_output_shapes*/ {PartialTensorShape({})}, - /*expected_cardinality*/ kUnknownCardinality, - /*breakpoints*/ {0, 2, 6}}; -} - -// Test case 3: a single text file without compression. -TestCase TestCase3() { - return {/*filenames*/ { - absl::StrCat(testing::TmpDir(), "/text_line_UNCOMPRESSED")}, - /*header_bytes*/ 5, - /*record_bytes*/ 3, - /*footer_bytes*/ 2, - /*buffer_size*/ 10, - /*compression_type*/ UNCOMPRESSED, - /*header*/ "HHHHH", - /*footer*/ "FF", - /*expected_outputs*/ - {DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"aa1"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"b2b"}), - DatasetOpsTestBase::CreateTensor(TensorShape({}), - {"3cc"})}, - /*expected_output_dtypes*/ {DT_STRING}, - /*expected_output_shapes*/ {PartialTensorShape({})}, - /*expected_cardinality*/ kUnknownCardinality, - /*breakpoints*/ {0, 2, 6}}; -} - -class ParameterizedFixedLengthRecordDatasetOpTest - : public FixedLengthRecordDatasetOpTest, - public ::testing::WithParamInterface {}; - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, GetNext) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - - std::unique_ptr iterator_ctx; - TF_ASSERT_OK( - CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); - std::unique_ptr iterator; - TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), - kIteratorPrefix, &iterator)); - bool end_of_sequence = false; - std::vector out_tensors; - while (!end_of_sequence) { - std::vector next; - TF_EXPECT_OK( - iterator->GetNext(iterator_ctx.get(), &next, &end_of_sequence)); - out_tensors.insert(out_tensors.end(), next.begin(), next.end()); - } - - TF_EXPECT_OK(ExpectEqual(out_tensors, test_case.expected_outputs, - /*compare_order*/ true)); -} - -TEST_F(FixedLengthRecordDatasetOpTest, DatasetNodeName) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = TestCase1(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - EXPECT_EQ(text_line_dataset->node_name(), kNodeName); -} - -TEST_F(FixedLengthRecordDatasetOpTest, DatasetTypeString) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = TestCase1(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - EXPECT_EQ(text_line_dataset->type_string(), - name_utils::OpName(FixedLengthRecordDatasetOp::kDatasetType)); -} - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, DatasetOutputDtypes) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - TF_EXPECT_OK(VerifyTypesMatch(text_line_dataset->output_dtypes(), - test_case.expected_output_dtypes)); -} - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, DatasetOutputShapes) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - TF_EXPECT_OK(VerifyShapesCompatible(text_line_dataset->output_shapes(), - test_case.expected_output_shapes)); -} - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, Cardinality) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - EXPECT_EQ(text_line_dataset->Cardinality(), test_case.expected_cardinality); -} - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, DatasetSave) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - - std::unique_ptr serialization_context; - TF_ASSERT_OK(CreateSerializationContext(&serialization_context)); - VariantTensorData data; - VariantTensorDataWriter writer(&data); - TF_ASSERT_OK(text_line_dataset->Save(serialization_context.get(), &writer)); - TF_ASSERT_OK(writer.Flush()); -} - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, IteratorOutputDtypes) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - - std::unique_ptr iterator_ctx; - TF_ASSERT_OK( - CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); - std::unique_ptr iterator; - TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), - kIteratorPrefix, &iterator)); - - TF_EXPECT_OK(VerifyTypesMatch(iterator->output_dtypes(), - test_case.expected_output_dtypes)); -} - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, IteratorOutputShapes) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - - std::unique_ptr iterator_ctx; - TF_ASSERT_OK( - CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); - std::unique_ptr iterator; - TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), - kIteratorPrefix, &iterator)); - - TF_EXPECT_OK(VerifyShapesCompatible(iterator->output_shapes(), - test_case.expected_output_shapes)); -} - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, IteratorOutputPrefix) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - - std::unique_ptr iterator_ctx; - TF_ASSERT_OK( - CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); - std::unique_ptr iterator; - TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), - kIteratorPrefix, &iterator)); - - EXPECT_EQ(iterator->prefix(), - name_utils::IteratorPrefix(FixedLengthRecordDatasetOp::kDatasetType, - kIteratorPrefix)); -} - -TEST_P(ParameterizedFixedLengthRecordDatasetOpTest, Roundtrip) { - int thread_num = 2, cpu_num = 2; - TestCase test_case = GetParam(); - TF_ASSERT_OK(InitThreadPool(thread_num)); - TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - TF_ASSERT_OK(CreateMultiTextFiles( - test_case.filenames, multi_texts, test_case.buffer_size, - test_case.buffer_size, test_case.compression_type)); - - std::unique_ptr text_line_dataset_kernel; - TF_ASSERT_OK( - CreateFixedLengthRecordDatasetOpKernel(&text_line_dataset_kernel)); - - int64 num_files = test_case.filenames.size(); - Tensor filenames = - CreateTensor(TensorShape({num_files}), test_case.filenames); - Tensor compression_type = CreateTensor( - TensorShape({}), {CompressionName(test_case.compression_type)}); - Tensor buffer_size = - CreateTensor(TensorShape({}), {test_case.buffer_size}); - gtl::InlinedVector inputs{TensorValue(&filenames), - TensorValue(&compression_type), - TensorValue(&buffer_size)}; - std::unique_ptr text_line_dataset_context; - TF_ASSERT_OK(CreateFixedLengthRecordDatasetContext( - text_line_dataset_kernel.get(), &inputs, &text_line_dataset_context)); - - DatasetBase* text_line_dataset; - TF_ASSERT_OK(CreateDataset(text_line_dataset_kernel.get(), - text_line_dataset_context.get(), - &text_line_dataset)); - core::ScopedUnref scoped_unref(text_line_dataset); - - std::unique_ptr iterator_ctx; - TF_ASSERT_OK( - CreateIteratorContext(text_line_dataset_context.get(), &iterator_ctx)); - std::unique_ptr iterator; - TF_ASSERT_OK(text_line_dataset->MakeIterator(iterator_ctx.get(), - kIteratorPrefix, &iterator)); - - std::unique_ptr serialization_ctx; - TF_ASSERT_OK(CreateSerializationContext(&serialization_ctx)); - - bool end_of_sequence = false; - std::vector out_tensors; - int cur_iteration = 0; - const std::vector& breakpoints = test_case.breakpoints; - for (int breakpoint : breakpoints) { - VariantTensorData data; - VariantTensorDataWriter writer(&data); - TF_EXPECT_OK(iterator->Save(serialization_ctx.get(), &writer)); - TF_EXPECT_OK(writer.Flush()); - VariantTensorDataReader reader(&data); - TF_EXPECT_OK(RestoreIterator(iterator_ctx.get(), &reader, kIteratorPrefix, - *text_line_dataset, &iterator)); - - while (cur_iteration <= breakpoint) { - std::vector next; - TF_EXPECT_OK( - iterator->GetNext(iterator_ctx.get(), &next, &end_of_sequence)); - out_tensors.insert(out_tensors.end(), next.begin(), next.end()); - cur_iteration++; - } - } - - TF_EXPECT_OK(ExpectEqual(out_tensors, test_case.expected_outputs, - /*compare_order*/ true)); -} - -INSTANTIATE_TEST_SUITE_P(FixedLengthRecordDatasetOpTest, - ParameterizedFixedLengthRecordDatasetOpTest, - ::testing::ValuesIn(std::vector( - {TestCase1(), TestCase2(), TestCase3()}))); - -} // namespace -} // namespace data -} // namespace tensorflow diff --git a/tensorflow/core/kernels/data/text_line_dataset_op_test.cc b/tensorflow/core/kernels/data/text_line_dataset_op_test.cc index a3b5c7d3847..19ec360f314 100644 --- a/tensorflow/core/kernels/data/text_line_dataset_op_test.cc +++ b/tensorflow/core/kernels/data/text_line_dataset_op_test.cc @@ -57,14 +57,26 @@ struct TestCase { std::vector breakpoints; }; -std::vector ToStringVector(const std::vector& str_tensors) { - std::vector str_vec; - for (const Tensor& tensor : str_tensors) { - for (int i = 0; i < tensor.NumElements(); ++i) { - str_vec.push_back(tensor.flat()(i)); +Status CreateTestFiles(const TestCase& test_case) { + if (test_case.filenames.size() != test_case.texts.size()) { + return tensorflow::errors::InvalidArgument( + "The number of files does not match with the contents"); + } + if (test_case.compression_type == CompressionType::UNCOMPRESSED) { + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_RETURN_IF_ERROR( + WriteDataToFile(test_case.filenames[i], test_case.texts[i].data())); + } + } else { + CompressionParams params; + params.compression_type = test_case.compression_type; + params.input_buffer_size = test_case.buffer_size; + params.output_buffer_size = test_case.buffer_size; + for (int i = 0; i < test_case.filenames.size(); ++i) { + TF_RETURN_IF_ERROR(WriteDataToFile(test_case.filenames[i], + test_case.texts[i].data(), params)); } } - return str_vec; } // Test case 1: multiple text files with ZLIB compression. @@ -156,14 +168,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, GetNext) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -213,14 +218,7 @@ TEST_F(TextLineDatasetOpTest, DatasetNodeName) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -253,14 +251,7 @@ TEST_F(TextLineDatasetOpTest, DatasetTypeString) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -294,14 +285,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetOutputDtypes) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -335,14 +319,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetOutputShapes) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -376,14 +353,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, Cardinality) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -416,14 +386,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, DatasetSave) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -462,14 +425,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputDtypes) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -511,14 +467,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputShapes) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -560,14 +509,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, IteratorOutputPrefix) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); @@ -610,14 +552,7 @@ TEST_P(ParameterizedTextLineDatasetOpTest, Roundtrip) { TF_ASSERT_OK(InitThreadPool(thread_num)); TF_ASSERT_OK(InitFunctionLibraryRuntime({}, cpu_num)); - std::vector multi_texts = ToStringVector(test_case.expected_outputs); - EXPECT_EQ(test_case.filenames.size(), test_case.texts.size()); - for (int i = 0; i < test_case.filenames.size(); ++i) { - TF_ASSERT_OK(WriteDataToFile(test_case.filenames[i], - test_case.texts[i].data(), - test_case.buffer_size, test_case.buffer_size, - test_case.compression_type)); - } + TF_ASSERT_OK(CreateTestFiles(test_case)); std::unique_ptr text_line_dataset_kernel; TF_ASSERT_OK(CreateTextLineDatasetOpKernel(&text_line_dataset_kernel)); From d722774a8a5e8e6d1d55a376cdbfd562835d5a37 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 15:49:27 -0700 Subject: [PATCH 238/332] NFC: Replace Module::getNamedFunction with lookupSymbol. This allows for removing the last direct reference to FuncOp from ModuleOp. PiperOrigin-RevId: 257498296 --- tensorflow/compiler/mlir/lite/flatbuffer_translate.cc | 4 ++-- tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc | 3 ++- tensorflow/compiler/mlir/lite/tf_tfl_translate.cc | 4 +++- tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc | 2 +- .../mlir/lite/transforms/lower_static_tensor_list.cc | 4 ++-- tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc | 8 ++++---- .../transforms/functional_control_flow_to_cfg.cc | 8 ++++---- .../compiler/mlir/tensorflow/translate/export_graphdef.cc | 7 ++++--- .../compiler/mlir/tensorflow/translate/import_graphdef.cc | 4 ++-- .../mlir/tensorflow/translate/tf_mlir_translate.cc | 1 + .../mlir/tensorflow/translate/translate_tf_dialect_op.cc | 3 ++- 11 files changed, 27 insertions(+), 21 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc index 34ee8814423..f189d159efa 100644 --- a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc +++ b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc @@ -247,7 +247,7 @@ static bool IsValidTFLiteMlirModule(ModuleOp module) { MLIRContext* context = module.getContext(); // Verify that module has a function named main. - FuncOp main_fn = module.getNamedFunction("main"); + FuncOp main_fn = module.lookupSymbol("main"); if (!main_fn) { return emitError(UnknownLoc::get(context), "should have a function named 'main'"), @@ -944,7 +944,7 @@ Optional Translator::TranslateInternal() { functions.reserve(std::distance(module_.begin(), module_.end())); int subgraph_idx = 0; - FuncOp main_fn = module_.getNamedFunction("main"); + FuncOp main_fn = module_.lookupSymbol("main"); subgraph_index_map_[main_fn.getName().str()] = subgraph_idx++; functions.push_back(main_fn); for (auto fn : module_.getOps()) { diff --git a/tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc b/tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc index c89e7602cc9..ff27ad76136 100644 --- a/tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc +++ b/tensorflow/compiler/mlir/lite/mlir_tflite_runner.cc @@ -31,6 +31,7 @@ limitations under the License. #include "llvm/Support/PrettyStackTrace.h" #include "llvm/Support/SMLoc.h" #include "llvm/Support/SourceMgr.h" +#include "mlir/IR/Function.h" // TF:local_config_mlir #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Module.h" // TF:local_config_mlir #include "mlir/Parser.h" // TF:local_config_mlir @@ -98,7 +99,7 @@ int main(int argc, char** argv) { if (!module) return 1; // TODO(jpienaar): Expand to support inputs. - mlir::FuncOp main = module->getNamedFunction("main"); + mlir::FuncOp main = module->lookupSymbol("main"); QCHECK(main) << "No 'main' function specified."; if (main.getType().getNumInputs() != 0) LOG(QFATAL) << "NYI: Only nullary functions supported."; diff --git a/tensorflow/compiler/mlir/lite/tf_tfl_translate.cc b/tensorflow/compiler/mlir/lite/tf_tfl_translate.cc index 5e8216dbc3b..9656abb1611 100644 --- a/tensorflow/compiler/mlir/lite/tf_tfl_translate.cc +++ b/tensorflow/compiler/mlir/lite/tf_tfl_translate.cc @@ -20,6 +20,7 @@ limitations under the License. #include "llvm/Support/SourceMgr.h" #include "llvm/Support/ToolOutputFile.h" #include "mlir/IR/Diagnostics.h" // TF:local_config_mlir +#include "mlir/IR/Function.h" // TF:local_config_mlir #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Module.h" // TF:local_config_mlir #include "mlir/Support/FileUtilities.h" // TF:local_config_mlir @@ -32,6 +33,7 @@ limitations under the License. #include "tensorflow/lite/schema/schema_generated.h" #include "tensorflow/stream_executor/lib/statusor.h" +using mlir::FuncOp; using mlir::MLIRContext; using mlir::ModuleOp; using stream_executor::port::StatusOr; @@ -83,7 +85,7 @@ static int PrintFunctionResultMapping(const std::string &result, std::cout << '\'' << subgraph_name << "' outputs:\n"; mlir::Operation *terminator = nullptr; if (subgraph->name()) { - if (auto fn = module.getNamedFunction(subgraph->name()->str())) + if (auto fn = module.lookupSymbol(subgraph->name()->str())) terminator = fn.back().getTerminator(); } i = 0; diff --git a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc index e13ca089a9d..afad51b7218 100644 --- a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc +++ b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc @@ -88,7 +88,7 @@ StatusOr LoadFromGraphdefOrMlirSource( } bool ShouldRunQuantizePasses(mlir::ModuleOp m) { - if (mlir::FuncOp main_fn = m.getNamedFunction("main")) { + if (mlir::FuncOp main_fn = m.lookupSymbol("main")) { return main_fn.getAttrOfType("tf.quantize") != mlir::Attribute(); } diff --git a/tensorflow/compiler/mlir/lite/transforms/lower_static_tensor_list.cc b/tensorflow/compiler/mlir/lite/transforms/lower_static_tensor_list.cc index 27175b05660..39e89f53423 100644 --- a/tensorflow/compiler/mlir/lite/transforms/lower_static_tensor_list.cc +++ b/tensorflow/compiler/mlir/lite/transforms/lower_static_tensor_list.cc @@ -276,8 +276,8 @@ LogicalResult LowerStaticTensorListPass::UpdateWhileFunctionType( auto *context = &getContext(); auto module = getModule(); - FuncOp cond_func = module.getNamedFunction(while_op->getCond()); - FuncOp body_func = module.getNamedFunction(while_op->getBody()); + FuncOp cond_func = module.lookupSymbol(while_op->getCond()); + FuncOp body_func = module.lookupSymbol(while_op->getBody()); if (cond_func) { // Change `cond_func`'s argument types to `unranked_argument_types`. diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc index 150eead08be..5838cd2c6b6 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc @@ -282,11 +282,11 @@ LogicalResult IfOp::verify() { if (!elseAttr) return emitOpError("requires else_branch attribute"); auto module = getParentOfType(); - auto thenFn = module.getNamedFunction(thenAttr.getValue()); + auto thenFn = module.lookupSymbol(thenAttr.getValue()); if (!thenFn) return emitOpError("then_branch refers to an undefined function : ") << thenAttr; - auto elseFn = module.getNamedFunction(elseAttr.getValue()); + auto elseFn = module.lookupSymbol(elseAttr.getValue()); if (!elseFn) return emitOpError("else_branch refers to an undefined function : ") << elseAttr; @@ -732,7 +732,7 @@ LogicalResult WhileOp::verify() { if (!condAttr) return emitOpError("requires cond attribute"); auto module = getParentOfType(); - auto condFn = module.getNamedFunction(condAttr.getValue()); + auto condFn = module.lookupSymbol(condAttr.getValue()); auto condFuncType = condFn.getType(); // Verify that the cond function has exactly one result. @@ -741,7 +741,7 @@ LogicalResult WhileOp::verify() { auto bodyAttr = getAttrOfType("body"); if (!bodyAttr) return emitOpError("requires body attribute"); - auto bodyFn = module.getNamedFunction(bodyAttr.getValue()); + auto bodyFn = module.lookupSymbol(bodyAttr.getValue()); auto bodyFuncType = bodyFn.getType(); SmallVector operands(getOperandTypes()); diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc b/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc index 5be13da57b0..af3e1e05ade 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/functional_control_flow_to_cfg.cc @@ -154,8 +154,8 @@ static LogicalResult LowerIfOp(IfOp op) { if (!cond_i1) return failure(); auto module = op_inst->getParentOfType(); - auto then_fn = module.getNamedFunction(op.getThen()); - auto else_fn = module.getNamedFunction(op.getElse()); + auto then_fn = module.lookupSymbol(op.getThen()); + auto else_fn = module.lookupSymbol(op.getElse()); // Split the basic block before the 'if'. The new dest will be our merge // point. @@ -211,8 +211,8 @@ static LogicalResult LowerWhileOp(WhileOp op) { OpBuilder builder(op_inst); auto module = op_inst->getParentOfType(); - auto cond_fn = module.getNamedFunction(op.getCond()); - auto body_fn = module.getNamedFunction(op.getBody()); + auto cond_fn = module.lookupSymbol(op.getCond()); + auto body_fn = module.lookupSymbol(op.getBody()); // Split the block containing the While op into two blocks. One containing // operations before the While op and other containing the rest. Create two diff --git a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc index b60bbe2c4b9..ce3c6332f8b 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc @@ -448,8 +448,9 @@ StatusOr> Exporter::Convert(const ExporterConfigs& confs, // definition library // TODO(prakalps): If two functions have cyclic dependence, this will // introduce an infinite loop. - auto func = function.getParentOfType().getNamedFunction( - op_name.ValueOrDie()); + auto func = + function.getParentOfType().lookupSymbol( + op_name.ValueOrDie()); if (func != nullptr) { TF_RETURN_IF_ERROR(ConvertLibFunction(confs, tf_dialect, func, flib)); TF_RETURN_IF_ERROR(graph->AddFunctionLibrary(*flib)); @@ -513,7 +514,7 @@ Status Exporter::ConvertLibFunction(const ExporterConfigs& configs, auto grad_string = mlir::TF::TensorFlowDialect::GetGradientAttrName(); if (auto attr = function.getAttrOfType(grad_string)) { auto grad_func = - function.getParentOfType().getNamedFunction( + function.getParentOfType().lookupSymbol( attr.getValue()); TF_RETURN_IF_ERROR( ConvertLibFunction(configs, tf_dialect, grad_func, flib)); diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc index 92cbfc70f62..a75ea3496d1 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc @@ -615,7 +615,7 @@ StatusOr Importer::ConvertFunctionCallName( const std::string& func_name) { TF_RETURN_IF_ERROR(ConvertLibFunction(func_name)); auto mlir_func_name = (*tf_name_to_mlir_name_)[func_name]; - auto func = module_.getNamedFunction(mlir_func_name); + auto func = module_.lookupSymbol(mlir_func_name); return builder_->getFunctionAttr(func); } @@ -721,7 +721,7 @@ Status Importer::ConvertLibFunction(const std::string& func_name) { if (!grad_func_name.empty()) { TF_RETURN_IF_ERROR(ConvertLibFunction(grad_func_name)); auto mlir_grad_func_name = (*tf_name_to_mlir_name_)[grad_func_name]; - auto grad_func = module_.getNamedFunction(mlir_grad_func_name); + auto grad_func = module_.lookupSymbol(mlir_grad_func_name); auto gradient_attr = builder_->getFunctionAttr(grad_func); auto grad_string = mlir::TF::TensorFlowDialect::GetGradientAttrName(); attributes.push_back(builder_->getNamedAttr(grad_string, gradient_attr)); diff --git a/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc b/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc index b0107e5ab3a..5c7b1e824fe 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/tf_mlir_translate.cc @@ -19,6 +19,7 @@ limitations under the License. #include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" #include "mlir/IR/Attributes.h" // TF:local_config_mlir +#include "mlir/IR/Function.h" // TF:local_config_mlir #include "mlir/IR/Identifier.h" // TF:local_config_mlir #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Module.h" // TF:local_config_mlir diff --git a/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc b/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc index 5d33138c626..9c02ce2278f 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/translate_tf_dialect_op.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "llvm/Support/ToolOutputFile.h" +#include "mlir/IR/Function.h" // TF:local_config_mlir #include "mlir/IR/Location.h" // TF:local_config_mlir #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Module.h" // TF:local_config_mlir @@ -23,7 +24,7 @@ limitations under the License. namespace mlir { static mlir::Operation* ExtractOnlyOp(mlir::ModuleOp module) { - mlir::FuncOp fn = module.getNamedFunction("main"); + mlir::FuncOp fn = module.lookupSymbol("main"); if (!fn) return nullptr; if (fn.getBlocks().size() != 1) return nullptr; From 00100e438dc8fdd861dd2da11f3df76b7789be1f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 15:54:18 -0700 Subject: [PATCH 239/332] Changed `.call` to `__call__` in the wrapper layers, removed specific `activity_regularizer` handling from `TimeDistributedLayer`, and added an `activity_regularizer` in the test to make sure it is still being passed through properly. PiperOrigin-RevId: 257499129 --- tensorflow/python/keras/layers/wrappers.py | 24 +++++++------------ .../python/keras/layers/wrappers_test.py | 16 +++++++------ 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/tensorflow/python/keras/layers/wrappers.py b/tensorflow/python/keras/layers/wrappers.py index 1c249aa87a0..553c56609d9 100644 --- a/tensorflow/python/keras/layers/wrappers.py +++ b/tensorflow/python/keras/layers/wrappers.py @@ -23,7 +23,6 @@ import copy from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import backend as K -from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.keras.engine.base_layer import Layer from tensorflow.python.keras.engine.input_spec import InputSpec from tensorflow.python.keras.layers.recurrent import _standardize_args @@ -225,7 +224,7 @@ class TimeDistributed(Wrapper): if input_shape[0] and not self._always_use_reshape: # batch size matters, use rnn-based implementation def step(x, _): - output = self.layer.call(x, **kwargs) + output = self.layer(x, **kwargs) return output, [] _, outputs, _ = K.rnn( @@ -252,20 +251,13 @@ class TimeDistributed(Wrapper): if generic_utils.has_arg(self.layer.call, 'mask') and mask is not None: inner_mask_shape = self._get_shape_tuple((-1,), mask, 2) kwargs['mask'] = K.reshape(mask, inner_mask_shape) - y = self.layer.call(inputs, **kwargs) + y = self.layer(inputs, **kwargs) # Shape: (num_samples, timesteps, ...) output_shape = self.compute_output_shape(input_shape).as_list() output_shape = self._get_shape_tuple( (-1, input_length), y, 1, output_shape[2:]) y = array_ops.reshape(y, output_shape) - # Apply activity regularizer if any: - if (hasattr(self.layer, 'activity_regularizer') and - self.layer.activity_regularizer is not None): - regularization_loss = self.layer.activity_regularizer(y) - base_layer_utils.check_graph_consistency( - regularization_loss, method='activity_regularizer') - self.add_loss(regularization_loss, inputs) return y def compute_mask(self, inputs, mask=None): @@ -646,13 +638,13 @@ class Bidirectional(Wrapper): forward_inputs, backward_inputs = inputs, inputs forward_state, backward_state = None, None - y = self.forward_layer.call(forward_inputs, - initial_state=forward_state, **kwargs) - y_rev = self.backward_layer.call(backward_inputs, - initial_state=backward_state, **kwargs) + y = self.forward_layer(forward_inputs, + initial_state=forward_state, **kwargs) + y_rev = self.backward_layer(backward_inputs, + initial_state=backward_state, **kwargs) else: - y = self.forward_layer.call(inputs, **kwargs) - y_rev = self.backward_layer.call(inputs, **kwargs) + y = self.forward_layer(inputs, **kwargs) + y_rev = self.backward_layer(inputs, **kwargs) if self.return_state: states = y[1:] + y_rev[1:] diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py index c11211807bd..7b9c741256f 100644 --- a/tensorflow/python/keras/layers/wrappers_test.py +++ b/tensorflow/python/keras/layers/wrappers_test.py @@ -154,11 +154,12 @@ class TimeDistributedTest(test.TestCase): model = keras.models.Sequential() model.add( keras.layers.TimeDistributed( - keras.layers.Dense(2, kernel_regularizer='l1'), + keras.layers.Dense(2, kernel_regularizer='l1', + activity_regularizer='l1'), input_shape=(3, 4))) model.add(keras.layers.Activation('relu')) model.compile(optimizer='rmsprop', loss='mse') - self.assertEqual(len(model.losses), 1) + self.assertEqual(len(model.losses), 2) def test_TimeDistributed_batchnorm(self): with self.cached_session(): @@ -626,11 +627,12 @@ class BidirectionalTest(test.TestCase, parameterized.TestCase): x_reachable_loss = x * x layer = keras.layers.Bidirectional( keras.layers.SimpleRNN( - 3, kernel_regularizer='l1', bias_regularizer='l1')) + 3, kernel_regularizer='l1', bias_regularizer='l1', + activity_regularizer='l1')) _ = layer(x) - assert len(layer.losses) == 4 + assert len(layer.losses) == 6 assert len(layer.get_losses_for(None)) == 4 - assert not layer.get_losses_for(x) + assert len(layer.get_losses_for(x)) == 2 # Create a random tensor that is not conditional on the inputs. with keras.backend.get_graph().as_default(): @@ -640,9 +642,9 @@ class BidirectionalTest(test.TestCase, parameterized.TestCase): layer.forward_layer.add_loss(const_tensor, inputs=None) layer.backward_layer.add_loss(x_reachable_loss, inputs=x) layer.backward_layer.add_loss(const_tensor, inputs=None) - assert len(layer.losses) == 8 + assert len(layer.losses) == 10 assert len(layer.get_losses_for(None)) == 6 - assert len(layer.get_losses_for(x)) == 2 + assert len(layer.get_losses_for(x)) == 4 def test_Bidirectional_with_constants(self): with self.cached_session(): From 49433223e2a789b31f514019e8a0bc6df0aa1474 Mon Sep 17 00:00:00 2001 From: Penporn Koanantakool Date: Wed, 10 Jul 2019 16:04:06 -0700 Subject: [PATCH 240/332] Increase tolerance for LinearOperatorLowRankUpdatetestNoDiagCannotUseCholesky complex64 in preparation for MatrixDiag V2 ops. (The randomized input matrix is seeded based on op ID, so the input matrix changes when we switch from V1 to V2 ops). PiperOrigin-RevId: 257501148 --- .../kernel_tests/linalg/linear_operator_low_rank_update_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py index 348574116f0..5c89607c1da 100644 --- a/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py +++ b/tensorflow/python/kernel_tests/linalg/linear_operator_low_rank_update_test.py @@ -239,6 +239,7 @@ class LinearOperatorLowRankUpdatetestNoDiagCannotUseCholesky( self._rtol[dtypes.float32] = 1e-4 self._atol[dtypes.float64] = 1e-9 self._rtol[dtypes.float64] = 1e-9 + self._atol[dtypes.complex64] = 1e-5 self._rtol[dtypes.complex64] = 2e-4 From f6e973eb9ffaac771c57fac24ab03cdb658343f0 Mon Sep 17 00:00:00 2001 From: Fei Hu Date: Wed, 10 Jul 2019 16:50:23 -0700 Subject: [PATCH 241/332] Add default values for CompressionParams --- tensorflow/core/kernels/data/dataset_test_base.cc | 4 +--- tensorflow/core/kernels/data/dataset_test_base.h | 6 +++--- tensorflow/core/kernels/data/text_line_dataset_op_test.cc | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/kernels/data/dataset_test_base.cc b/tensorflow/core/kernels/data/dataset_test_base.cc index dfc8733e250..8bfa0b18ba8 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.cc +++ b/tensorflow/core/kernels/data/dataset_test_base.cc @@ -52,9 +52,7 @@ io::ZlibCompressionOptions GetZlibCompressionOptions( } Status WriteDataToFile(const string& filename, const char* data) { - CompressionParams params; - params.compression_type = CompressionType::UNCOMPRESSED; - return WriteDataToFile(filename, data, params); + return WriteDataToFile(filename, data, CompressionParams()); } Status WriteDataToFile(const string& filename, const char* data, diff --git a/tensorflow/core/kernels/data/dataset_test_base.h b/tensorflow/core/kernels/data/dataset_test_base.h index 8caff593834..7162425d198 100644 --- a/tensorflow/core/kernels/data/dataset_test_base.h +++ b/tensorflow/core/kernels/data/dataset_test_base.h @@ -60,9 +60,9 @@ io::ZlibCompressionOptions GetZlibCompressionOptions( // `input_buffer_size` and `output_buffer_size` specify the input and output // buffer size when ZLIB and GZIP compression is used. struct CompressionParams { - CompressionType compression_type; - int32 input_buffer_size; - int32 output_buffer_size; + CompressionType compression_type = CompressionType::UNCOMPRESSED; + int32 input_buffer_size = 0; + int32 output_buffer_size = 0; }; // Writes the input data into the file without compression. diff --git a/tensorflow/core/kernels/data/text_line_dataset_op_test.cc b/tensorflow/core/kernels/data/text_line_dataset_op_test.cc index 19ec360f314..d5909c857e6 100644 --- a/tensorflow/core/kernels/data/text_line_dataset_op_test.cc +++ b/tensorflow/core/kernels/data/text_line_dataset_op_test.cc @@ -77,6 +77,7 @@ Status CreateTestFiles(const TestCase& test_case) { test_case.texts[i].data(), params)); } } + return Status::OK(); } // Test case 1: multiple text files with ZLIB compression. From 99517fc626935f782f66942f54f9d85ca0f7c086 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 16:22:05 -0700 Subject: [PATCH 242/332] [tracing] Traceme construction already check TraceMeRecorder::Active(level). MightTrace are checking different event_collector, and annotation setting etc, which is not relevant. Remove MightTrace make thing more clean. I expect no performance problem when tracing is off, when tracing is on, this definitely faster. PiperOrigin-RevId: 257504342 --- tensorflow/core/common_runtime/executor.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc index d643af669c2..065a6782811 100644 --- a/tensorflow/core/common_runtime/executor.cc +++ b/tensorflow/core/common_runtime/executor.cc @@ -1816,8 +1816,7 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) { if (completed) ScheduleFinish(); }; nodestats::SetOpStart(stats); - if (TF_PREDICT_FALSE( - MightTrace(item, event_collector_, trace_using_annotations_))) { + { profiler::TraceMe activity( [&] { return strings::StrCat( @@ -1827,8 +1826,6 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_nsec) { }, profiler::GetTFTraceMeLevel(op_kernel->IsExpensive())); device->ComputeAsync(async, &state->ctx, done); - } else { - device->ComputeAsync(async, &state->ctx, done); } } else { // Synchronous computes. From 2246b3830588cd19f09e4106e72fb8ed9d9b9e79 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 17:04:40 -0700 Subject: [PATCH 243/332] Added a function for nested_value_rowids, similar to nested_row_lengths. This will help when handling value_rowids natively. PiperOrigin-RevId: 257512332 --- tensorflow/python/ops/ragged/ragged_tensor.py | 42 ++++++++++++++++++- .../python/ops/ragged/ragged_tensor_test.py | 6 +++ .../golden/v1/tensorflow.-ragged-tensor.pbtxt | 4 ++ .../golden/v2/tensorflow.-ragged-tensor.pbtxt | 4 ++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/ragged/ragged_tensor.py b/tensorflow/python/ops/ragged/ragged_tensor.py index 99205128693..d06819cbf90 100644 --- a/tensorflow/python/ops/ragged/ragged_tensor.py +++ b/tensorflow/python/ops/ragged/ragged_tensor.py @@ -980,6 +980,44 @@ class RaggedTensor(composite_tensor.CompositeTensor): with ops.name_scope(name, "RaggedValueRowIds", [self]): return segment_id_ops.row_splits_to_segment_ids(self.row_splits) + def nested_value_rowids(self, name=None): + """Returns a tuple containing the value_rowids for all ragged dimensions. + + `rt.nested_value_rowids` is a tuple containing the `value_rowids` tensors + for + all ragged dimensions in `rt`, ordered from outermost to innermost. In + particular, `rt.nested_value_rowids = (rt.value_rowids(),) + value_ids` + where: + + * `value_ids = ()` if `rt.values` is a `Tensor`. + * `value_ids = rt.values.nested_value_rowids` otherwise. + + Args: + name: A name prefix for the returned tensors (optional). + + Returns: + A `tuple` of 1-D integer `Tensor`s. + + #### Example: + + ```python + >>> rt = ragged.constant([[[[3, 1, 4, 1], [], [5, 9, 2]], [], [[6], []]]]) + >>> for i, ids in enumerate(rt.nested_value_rowids()): + ... print('row ids for dimension %d: %s' % (i+1, ids)) + row ids for dimension 1: [0] + row ids for dimension 2: [0, 0, 0, 2, 2] + row ids for dimension 3: [0, 0, 0, 0, 2, 2, 2, 3] + ``` + + """ + with ops.name_scope(name, "RaggedNestedValueRowIds", [self]): + rt_nested_ids = [self.value_rowids()] + rt_values = self.values + while isinstance(rt_values, RaggedTensor): + rt_nested_ids.append(rt_values.value_rowids()) + rt_values = rt_values.values + return tuple(rt_nested_ids) + def nrows(self, out_type=None, name=None): """Returns the number of rows in this ragged tensor. @@ -1106,8 +1144,8 @@ class RaggedTensor(composite_tensor.CompositeTensor): def nested_row_lengths(self, name=None): """Returns a tuple containing the row_lengths for all ragged dimensions. - `rtnested_row_lengths()` is a tuple containing the `row_lengths` tensors for - all ragged dimensions in `rt`, ordered from outermost to innermost. + `rt.nested_row_lengths()` is a tuple containing the `row_lengths` tensors + for all ragged dimensions in `rt`, ordered from outermost to innermost. Args: name: A name prefix for the returned tensors (optional). diff --git a/tensorflow/python/ops/ragged/ragged_tensor_test.py b/tensorflow/python/ops/ragged/ragged_tensor_test.py index 60f04c4dc9b..eb8767b56e0 100644 --- a/tensorflow/python/ops/ragged/ragged_tensor_test.py +++ b/tensorflow/python/ops/ragged/ragged_tensor_test.py @@ -651,6 +651,9 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13]]) self.assertLen(rt.nested_row_splits, 1) self.assertAllEqual(rt.nested_row_splits[0], [0, 2, 2, 5, 6, 7]) + self.assertLen(rt.nested_value_rowids(), 1) + + self.assertAllEqual(rt.nested_value_rowids()[0], [0, 0, 2, 2, 2, 3, 4]) def testRaggedTensorAccessors_3d_with_ragged_rank_2(self): values = constant_op.constant(['a', 'b', 'c', 'd', 'e', 'f', 'g']) @@ -685,6 +688,9 @@ class RaggedTensorTest(test_util.TensorFlowTestCase, self.assertLen(rt.nested_row_splits, 2) self.assertAllEqual(rt.nested_row_splits[0], [0, 2, 3, 3, 5]) self.assertAllEqual(rt.nested_row_splits[1], [0, 2, 2, 5, 6, 7]) + self.assertLen(rt.nested_value_rowids(), 2) + self.assertAllEqual(rt.nested_value_rowids()[0], [0, 0, 1, 3, 3]) + self.assertAllEqual(rt.nested_value_rowids()[1], [0, 0, 2, 2, 2, 3, 4]) #============================================================================= # RaggedTensor.shape diff --git a/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt index 972e7d12daa..2f7918843dd 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.-ragged-tensor.pbtxt @@ -87,6 +87,10 @@ tf_class { name: "nested_row_lengths" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "nested_value_rowids" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "nrows" argspec: "args=[\'self\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt index 972e7d12daa..2f7918843dd 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.-ragged-tensor.pbtxt @@ -87,6 +87,10 @@ tf_class { name: "nested_row_lengths" argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "nested_value_rowids" + argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "nrows" argspec: "args=[\'self\', \'out_type\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " From 6e03f9430ac4110c89f6c53c0a777815ecc1eaec Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Wed, 10 Jul 2019 17:08:29 -0700 Subject: [PATCH 244/332] Use XLA_TEST_P instead of TEST_P in iota_test This address github PR 30518. PiperOrigin-RevId: 257513086 --- tensorflow/compiler/xla/tests/iota_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/xla/tests/iota_test.cc b/tensorflow/compiler/xla/tests/iota_test.cc index 0723dd4bbdf..d7d5de5b186 100644 --- a/tensorflow/compiler/xla/tests/iota_test.cc +++ b/tensorflow/compiler/xla/tests/iota_test.cc @@ -34,7 +34,7 @@ class IotaR1Test : public ClientLibraryTestBase, public ::testing::WithParamInterface> {}; -TEST_P(IotaR1Test, DoIt) { +XLA_TEST_P(IotaR1Test, DoIt) { const auto& spec = GetParam(); const auto element_type = std::get<0>(spec); const int64 num_elements = std::get<1>(spec); @@ -63,7 +63,7 @@ class IotaR2Test : public ClientLibraryTestBase, public ::testing::WithParamInterface< std::tuple> {}; -TEST_P(IotaR2Test, DoIt) { +XLA_TEST_P(IotaR2Test, DoIt) { const auto& spec = GetParam(); const auto element_type = std::get<0>(spec); const int64 num_elements = std::get<1>(spec); @@ -95,7 +95,7 @@ class IotaR3Test : public ClientLibraryTestBase, public ::testing::WithParamInterface< std::tuple> {}; -TEST_P(IotaR3Test, DoIt) { +XLA_TEST_P(IotaR3Test, DoIt) { const auto& spec = GetParam(); const auto element_type = std::get<0>(spec); const int64 num_elements = std::get<1>(spec); From 08370f31ad948a9508df5f9630611c28c699e5e6 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 10 Jul 2019 17:19:06 -0700 Subject: [PATCH 245/332] Temporarily add two bazel flags to test with an older version of bazel. This will be rolled back in at most 1 hour since submission PiperOrigin-RevId: 257515069 --- tensorflow/tools/ci_build/ci_sanity.sh | 2 ++ tensorflow/tools/pip_package/check_load_py_test.py | 2 ++ tensorflow/tools/pip_package/pip_smoke_test.py | 11 ++++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index b78281dfc23..f7ae00db99e 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -449,6 +449,8 @@ do_bazel_nobuild() { BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/delegates/gpu/..." BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/java/demo/app/..." BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/schema/..." + BAZEL_FLAGS="${BAZEL_FLAGS} --incompatible_package_name_is_a_function=false" + BAZEL_FLAGS="${BAZEL_FLAGS} --incompatible_remove_native_http_archive=false" BUILD_CMD="bazel build --nobuild ${BAZEL_FLAGS} -- ${BUILD_TARGET}" ${BUILD_CMD} diff --git a/tensorflow/tools/pip_package/check_load_py_test.py b/tensorflow/tools/pip_package/check_load_py_test.py index 8e9de562a30..10415d7feab 100644 --- a/tensorflow/tools/pip_package/check_load_py_test.py +++ b/tensorflow/tools/pip_package/check_load_py_test.py @@ -47,6 +47,8 @@ def main(): try: targets = subprocess.check_output([ 'bazel', 'query', + "--incompatible_package_name_is_a_function=false", + "--incompatible_remove_native_http_archive=false", 'kind(py_test, //tensorflow/contrib/... + ' '//tensorflow/python/... - ' '//tensorflow/contrib/tensorboard/...)']).strip() diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py index 9adbb36a74a..4c076a9891b 100644 --- a/tensorflow/tools/pip_package/pip_smoke_test.py +++ b/tensorflow/tools/pip_package/pip_smoke_test.py @@ -30,6 +30,10 @@ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))) PIP_PACKAGE_QUERY_EXPRESSION = ( "deps(//tensorflow/tools/pip_package:build_pip_package)") +RELEASE_PATCH_FLAGS = ( + " --incompatible_package_name_is_a_function=false" + " --incompatible_remove_native_http_archive=false") + # List of file paths containing BUILD files that should not be included for the # pip smoke test. BUILD_BLACKLIST = [ @@ -140,7 +144,7 @@ def main(): # pip_package_dependencies_list is the list of included files in pip packages pip_package_dependencies = subprocess.check_output( - ["bazel", "cquery", PIP_PACKAGE_QUERY_EXPRESSION]) + ["bazel", "cquery", RELEASE_PATCH_FLAGS, PIP_PACKAGE_QUERY_EXPRESSION]) if isinstance(pip_package_dependencies, bytes): pip_package_dependencies = pip_package_dependencies.decode("utf-8") pip_package_dependencies_list = pip_package_dependencies.strip().split("\n") @@ -152,7 +156,7 @@ def main(): # tf_py_test_dependencies is the list of dependencies for all python # tests in tensorflow tf_py_test_dependencies = subprocess.check_output( - ["bazel", "cquery", PY_TEST_QUERY_EXPRESSION]) + ["bazel", "cquery", RELEASE_PATCH_FLAGS, PY_TEST_QUERY_EXPRESSION]) if isinstance(tf_py_test_dependencies, bytes): tf_py_test_dependencies = tf_py_test_dependencies.decode("utf-8") tf_py_test_dependencies_list = tf_py_test_dependencies.strip().split("\n") @@ -193,7 +197,8 @@ def main(): print("Affected Tests:") rdep_query = ("rdeps(kind(py_test, %s), %s)" % (" + ".join(PYTHON_TARGETS), missing_dependency)) - affected_tests = subprocess.check_output(["bazel", "cquery", rdep_query]) + affected_tests = subprocess.check_output( + ["bazel", "cquery", RELEASE_PATCH_FLAGS, rdep_query]) affected_tests_list = affected_tests.split("\n")[:-2] print("\n".join(affected_tests_list)) From daac7c7101e52aa5f061ed7aaab82b1a4a1634dc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 10 Jul 2019 17:20:28 -0700 Subject: [PATCH 246/332] Update NNAPI delegate support for SUB, TANH PiperOrigin-RevId: 257515288 --- .../lite/delegates/nnapi/nnapi_delegate.cc | 54 ++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc index 28efbd19a2b..30f754156a4 100644 --- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc +++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc @@ -123,13 +123,12 @@ bool IsFloatOrUint8Operator(const TfLiteContext* context, // Check if the operation requires explict conversion from int8 to uint8 values. bool NeedInt8Conversion(const TfLiteContext* context, int builtin_code, const TfLiteNode* node) { + const int input_id = node->inputs->data[0]; + const TfLiteType input_type = context->tensors[input_id].type; switch (builtin_code) { case kTfLiteBuiltinConv2d: case kTfLiteBuiltinDepthwiseConv2d: - case kTfLiteBuiltinFullyConnected: - case kTfLiteBuiltinL2Normalization: { - const int input_id = node->inputs->data[0]; - const TfLiteType input_type = context->tensors[input_id].type; + case kTfLiteBuiltinFullyConnected: { if (input_type == kTfLiteInt8) { const int weights_id = node->inputs->data[1]; const auto& weights_tensor = context->tensors[weights_id]; @@ -141,6 +140,11 @@ bool NeedInt8Conversion(const TfLiteContext* context, int builtin_code, } return false; } + case kTfLiteBuiltinL2Normalization: + case kTfLiteBuiltinSub: + case kTfLiteBuiltinTanh: { + return input_type == kTfLiteInt8; + } default: return false; } @@ -1379,23 +1383,34 @@ class NNAPIDelegateKernel { break; case kTfLiteBuiltinTanh: // TODO(miaowang): add additional checks for the parameters. - if (version == 1 && - context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) { - // NNAPI only support float tanh. - return BasicMappingFn; + if (version == 1) { + const TfLiteType input_type = + context->tensors[node->inputs->data[0]].type; + if (IsFloat(input_type) || + (IsQuantized(input_type) && + android_sdk_version >= kMinSdkVersionForNNAPI12)) { + // NNAPI only support float tanh. + return BasicMappingFn; + } } break; case kTfLiteBuiltinSub: - if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI11 && - context->tensors[node->inputs->data[0]].type == kTfLiteFloat32) { - // NNAPI only support float sub. - return [](const NNAPIOpMappingArgs& mapping_args) - -> ANeuralNetworksOperationType { - auto builtin = reinterpret_cast( - mapping_args.node->builtin_data); - mapping_args.builder->AddScalarInt32Operand(builtin->activation); - return ANEURALNETWORKS_SUB; - }; + if (version == 1) { + const TfLiteType input_type = + context->tensors[node->inputs->data[0]].type; + if ((android_sdk_version >= kMinSdkVersionForNNAPI11 && + IsFloat(input_type)) || + (android_sdk_version >= kMinSdkVersionForNNAPI12 && + IsQuantized(input_type))) { + // NNAPI only support float sub. + return [](const NNAPIOpMappingArgs& mapping_args) + -> ANeuralNetworksOperationType { + auto builtin = reinterpret_cast( + mapping_args.node->builtin_data); + mapping_args.builder->AddScalarInt32Operand(builtin->activation); + return ANEURALNETWORKS_SUB; + }; + } } break; case kTfLiteBuiltinDiv: @@ -2355,7 +2370,8 @@ class NNAPIDelegateKernel { const auto input_index = node->inputs->data[input_pos]; if (need_int8_conversion && (input_pos == 0 || - reg->builtin_code == kTfLiteBuiltinFullyConnected)) { + reg->builtin_code == kTfLiteBuiltinFullyConnected || + reg->builtin_code == kTfLiteBuiltinSub)) { // Only selected inputs require int8 conversion. TF_LITE_ENSURE_STATUS(builder.AddTensorInput( input_index, hybrid_op, From 28310aed63f9fc849a55c163b523564db106c901 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Wed, 10 Jul 2019 17:29:08 -0700 Subject: [PATCH 247/332] Automated rollback of commit 08370f31ad948a9508df5f9630611c28c699e5e6 PiperOrigin-RevId: 257516611 --- tensorflow/tools/ci_build/ci_sanity.sh | 2 -- tensorflow/tools/pip_package/check_load_py_test.py | 2 -- tensorflow/tools/pip_package/pip_smoke_test.py | 11 +++-------- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index f7ae00db99e..b78281dfc23 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -449,8 +449,6 @@ do_bazel_nobuild() { BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/delegates/gpu/..." BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/java/demo/app/..." BUILD_TARGET="${BUILD_TARGET} -//tensorflow/lite/schema/..." - BAZEL_FLAGS="${BAZEL_FLAGS} --incompatible_package_name_is_a_function=false" - BAZEL_FLAGS="${BAZEL_FLAGS} --incompatible_remove_native_http_archive=false" BUILD_CMD="bazel build --nobuild ${BAZEL_FLAGS} -- ${BUILD_TARGET}" ${BUILD_CMD} diff --git a/tensorflow/tools/pip_package/check_load_py_test.py b/tensorflow/tools/pip_package/check_load_py_test.py index 10415d7feab..8e9de562a30 100644 --- a/tensorflow/tools/pip_package/check_load_py_test.py +++ b/tensorflow/tools/pip_package/check_load_py_test.py @@ -47,8 +47,6 @@ def main(): try: targets = subprocess.check_output([ 'bazel', 'query', - "--incompatible_package_name_is_a_function=false", - "--incompatible_remove_native_http_archive=false", 'kind(py_test, //tensorflow/contrib/... + ' '//tensorflow/python/... - ' '//tensorflow/contrib/tensorboard/...)']).strip() diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py index 4c076a9891b..9adbb36a74a 100644 --- a/tensorflow/tools/pip_package/pip_smoke_test.py +++ b/tensorflow/tools/pip_package/pip_smoke_test.py @@ -30,10 +30,6 @@ os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))) PIP_PACKAGE_QUERY_EXPRESSION = ( "deps(//tensorflow/tools/pip_package:build_pip_package)") -RELEASE_PATCH_FLAGS = ( - " --incompatible_package_name_is_a_function=false" - " --incompatible_remove_native_http_archive=false") - # List of file paths containing BUILD files that should not be included for the # pip smoke test. BUILD_BLACKLIST = [ @@ -144,7 +140,7 @@ def main(): # pip_package_dependencies_list is the list of included files in pip packages pip_package_dependencies = subprocess.check_output( - ["bazel", "cquery", RELEASE_PATCH_FLAGS, PIP_PACKAGE_QUERY_EXPRESSION]) + ["bazel", "cquery", PIP_PACKAGE_QUERY_EXPRESSION]) if isinstance(pip_package_dependencies, bytes): pip_package_dependencies = pip_package_dependencies.decode("utf-8") pip_package_dependencies_list = pip_package_dependencies.strip().split("\n") @@ -156,7 +152,7 @@ def main(): # tf_py_test_dependencies is the list of dependencies for all python # tests in tensorflow tf_py_test_dependencies = subprocess.check_output( - ["bazel", "cquery", RELEASE_PATCH_FLAGS, PY_TEST_QUERY_EXPRESSION]) + ["bazel", "cquery", PY_TEST_QUERY_EXPRESSION]) if isinstance(tf_py_test_dependencies, bytes): tf_py_test_dependencies = tf_py_test_dependencies.decode("utf-8") tf_py_test_dependencies_list = tf_py_test_dependencies.strip().split("\n") @@ -197,8 +193,7 @@ def main(): print("Affected Tests:") rdep_query = ("rdeps(kind(py_test, %s), %s)" % (" + ".join(PYTHON_TARGETS), missing_dependency)) - affected_tests = subprocess.check_output( - ["bazel", "cquery", RELEASE_PATCH_FLAGS, rdep_query]) + affected_tests = subprocess.check_output(["bazel", "cquery", rdep_query]) affected_tests_list = affected_tests.split("\n")[:-2] print("\n".join(affected_tests_list)) From 85d39341cd2a86758adb5f63f6047ad0d6475dfc Mon Sep 17 00:00:00 2001 From: Eugene Zhulenev Date: Wed, 10 Jul 2019 17:29:52 -0700 Subject: [PATCH 248/332] Configurable inlined function body placement strategy PiperOrigin-RevId: 257516726 --- .../jit/encapsulate_subgraphs_pass.cc | 2 - tensorflow/core/common_runtime/function.cc | 210 +++++++++++++++--- tensorflow/core/common_runtime/function.h | 60 ++++- .../core/common_runtime/function_test.cc | 122 +++++++++- .../core/common_runtime/graph_optimizer.cc | 3 +- .../common_runtime/lower_function_call_op.cc | 7 +- .../grappler/optimizers/function_optimizer.cc | 9 +- .../optimizers/function_optimizer_test.cc | 14 +- 8 files changed, 373 insertions(+), 54 deletions(-) diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc index 38d6bc4b5fd..6992a0165d4 100644 --- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc +++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc @@ -1087,8 +1087,6 @@ Status Encapsulator::MakePrunedGraphCopyAndInline( FunctionDefToBodyHelper(*fdef, node->attrs(), library, &fbody)); InlineFunctionBodyOptions inline_opts; - inline_opts.override_device = false; - TF_RETURN_IF_ERROR(InlineFunctionBody(*library, pruned_graph->get(), node, fbody.get(), inline_opts)); } diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc index 8e66e10cc3f..89f97ff165d 100644 --- a/tensorflow/core/common_runtime/function.cc +++ b/tensorflow/core/common_runtime/function.cc @@ -109,10 +109,6 @@ static Node* AddIdentity(StringPiece name, Graph* g, Endpoint input) { NodeDef ndef; ndef.set_name(g->NewName(absl::StrCat(kNodeLabel, "/", name))); ndef.set_op("Identity"); - // NOTE(skyewm): we explicitly set the device here to address a multi-GPU - // performance issue where this Identity would be placed alone on a GPU, - // causing unnecessary device traffic. See b/122483225 for details. - ndef.set_device(input.node->def().device()); ndef.add_input(input.name()); AddNodeAttr("T", BaseType(input.dtype()), &ndef); Status s; @@ -1393,12 +1389,14 @@ bool RemoveListArrayConverter(Graph* g) { } gtl::InlinedVector identity_nodes(n->num_inputs(), nullptr); - const auto no_op = [&](StringPiece name) { + const auto no_op = [&](StringPiece name) -> Node* { return AddNoOp(absl::StrCat(n->name(), "/", name), g); }; - const auto identity = [&](StringPiece name, Endpoint input) { - return AddIdentity(absl::StrCat(n->name(), "/", name), g, input); + const auto identity = [&](StringPiece name, Endpoint input) -> Node* { + Node* node = AddIdentity(absl::StrCat(n->name(), "/", name), g, input); + node->set_requested_device(input.node->def().device()); + return node; }; // Process input edges first. @@ -1495,6 +1493,152 @@ Status InstantiateFunctionCall(const NodeDef& call_def, namespace { +std::vector InputDevices(const Node& caller) { + std::vector input_devices(caller.in_edges().size()); + for (const Edge* edge : caller.in_edges()) { + if (edge->IsControlEdge()) continue; + const string& input_device = edge->src()->has_assigned_device_name() + ? edge->src()->assigned_device_name() + : edge->src()->requested_device(); + input_devices[edge->dst_input()] = input_device; + } + return input_devices; +} + +// Place input nodes on the same device as the correspinding caller input +// node. Do not specify any placement for all other nodes. +class DefaultFunctionBodyPlacer : public InlinedFunctionBodyPlacer { + public: + explicit DefaultFunctionBodyPlacer(const Node& caller) + : input_devices_(InputDevices(caller)) {} + + absl::optional InputNodeDevice(int input_index) const override { + return input_devices_[input_index]; + } + absl::optional OutputNodeDevice(int output_index) const override { + return absl::nullopt; + } + absl::optional ControlNodeDevice() const override { + return absl::nullopt; + } + absl::optional BodyNodeDevice(const NodeDef& ndef) const override { + return absl::nullopt; + } + + private: + const std::vector input_devices_; +}; + +// Place all nodes on the same device as caller node. +class SingleDeviceFunctionBodyPlacer : public InlinedFunctionBodyPlacer { + public: + explicit SingleDeviceFunctionBodyPlacer(const Node& caller) + : caller_device_(caller.def().device()) {} + + absl::optional InputNodeDevice(int input_index) const override { + return caller_device_; + } + absl::optional OutputNodeDevice(int output_index) const override { + return caller_device_; + } + absl::optional ControlNodeDevice() const override { + return caller_device_; + } + absl::optional BodyNodeDevice(const NodeDef& ndef) const override { + return caller_device_; + } + + private: + const string caller_device_; +}; + +// Place input nodes on the same device as the correspinding caller input +// node. Do not place output node. Place control nodes on the same device as +// caller node. For all function body nodes overrides job, replica and task +// parts of the device assignment to match function caller node. +class MultiDeviceFunctionBodyPlacer : public InlinedFunctionBodyPlacer { + public: + explicit MultiDeviceFunctionBodyPlacer(const Node& caller) + : caller_device_(caller.def().device()), + input_devices_(InputDevices(caller)) { + has_parsed_caller_device_ = + DeviceNameUtils::ParseFullName(caller_device_, &caller_parsed_device_); + } + + absl::optional InputNodeDevice(int input_index) const override { + return input_devices_[input_index]; + } + absl::optional OutputNodeDevice(int output_index) const override { + return absl::nullopt; + } + absl::optional ControlNodeDevice() const override { + return caller_device_; + } + absl::optional BodyNodeDevice(const NodeDef& ndef) const override { + // TODO(ezhulenev): If function would have been instantiated as a + // multi-device function and executed via FunctionLibraryRuntime, it could + // be potentially placed on any available device. However there are multiple + // tests relying on this assumption. Fix them, and remove this line. + if (ndef.device().empty()) return caller_device_; + + if (!has_parsed_caller_device_) return ndef.device(); + + DeviceNameUtils::ParsedName ndef_parsed_device; + if (!DeviceNameUtils::ParseFullName(ndef.device(), &ndef_parsed_device)) + return ndef.device(); + + if (caller_parsed_device_.has_job) { + ndef_parsed_device.has_job = caller_parsed_device_.has_job; + ndef_parsed_device.job = caller_parsed_device_.job; + } + + if (caller_parsed_device_.has_replica) { + ndef_parsed_device.has_replica = caller_parsed_device_.has_replica; + ndef_parsed_device.replica = caller_parsed_device_.replica; + } + + if (caller_parsed_device_.has_task) { + ndef_parsed_device.has_task = caller_parsed_device_.has_task; + ndef_parsed_device.task = caller_parsed_device_.task; + } + return DeviceNameUtils::ParsedNameToString(ndef_parsed_device); + } + + private: + string caller_device_; + bool has_parsed_caller_device_; + DeviceNameUtils::ParsedName caller_parsed_device_; + std::vector input_devices_; +}; + +} // namespace + +std::unique_ptr +InlinedFunctionBodyPlacer::DefaultPlacer(const Graph& graph, + const Node& caller) { + VLOG(3) << "Create default placer for inlined function body: " + << SummarizeNode(caller); + return absl::make_unique(caller); +} + +std::unique_ptr +InlinedFunctionBodyPlacer::SingleDevicePlacer(const Graph& graph, + const Node& caller) { + VLOG(3) << "Create single device placer for inlined function body: " + << SummarizeNode(caller); + return absl::make_unique(caller); +} + +std::unique_ptr +InlinedFunctionBodyPlacer::MultiDevicePlacer(const Graph& graph, + const Node& caller) { + VLOG(3) << "Create multi device placer for inlined function body: " + << SummarizeNode(caller); + return absl::make_unique(caller); +} + +namespace { + Status ValidateNoInline(const FunctionBody* fbody) { const auto attr = AttrSlice(&fbody->fdef.attr()); bool noinline = false; @@ -1547,13 +1691,12 @@ string InlineFunctionBodyOptions::DebugString() const { return absl::StrCat( "disable_inlining=", true_false(disable_inlining), ", ignore_noinline=", true_false(ignore_noinline), - ", override_device=", true_false(ignore_noinline), - ", initialize_empty_device=", true_false(initialize_empty_device), ", inline_impl_selection_group_functions=", true_false(inline_impl_selection_group_functions), ", keep_caller_node=", keep_caller_node_str(), ", output_control_src=", output_control_src == OutputControlSrc::kDataOutputs ? "DataOutputs" - : "ControlOutputs"); + : "ControlOutputs", + ", inlined_function_body_placer=", inlined_function_body_placer.name); } Status ValidateInlining(const Node* node, const FunctionBody* fbody, @@ -1711,6 +1854,11 @@ Status InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g, return errors::Internal("Inlining mismatch: ", validation.error_message()); } + // Placer is responsible for assigning devices for all nodes that we will add + // to the graph. + const std::unique_ptr placer = + options.inlined_function_body_placer.get(*g, *caller); + // We can't possibly introduce a duplicate control edge during function // inlining, so we skip this check in calls to the 'g->AddControlEdge(...)'. static constexpr bool kDoNotCheckDuplicates = true; @@ -1720,15 +1868,29 @@ Status InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g, // control nodes and inlined function inputs and outputs. // Add a NoOp node for function control inputs/outputs. - const auto no_op = [&](StringPiece name) { + const auto no_op = [&](StringPiece name) -> Node* { Node* node = AddNoOp(absl::StrCat(caller->name(), "/", name), g); - node->set_requested_device(caller->def().device()); + const absl::optional device = placer->ControlNodeDevice(); + if (device.has_value()) node->set_requested_device(*device); return node; }; - // Add an Identity node for function data inputs/outputs. - const auto identity = [&](StringPiece name, Endpoint input) { - return AddIdentity(absl::StrCat(caller->name(), "/", name), g, input); + // Add an Identity node for function input. + const auto input_identity = [&](StringPiece name, Endpoint input, + int index) -> Node* { + Node* node = AddIdentity(absl::StrCat(caller->name(), "/", name), g, input); + const absl::optional device = placer->InputNodeDevice(index); + if (device.has_value()) node->set_requested_device(*device); + return node; + }; + + // Add an Identity node for function output. + const auto output_identity = [&](StringPiece name, Endpoint input, + int index) -> Node* { + Node* node = AddIdentity(absl::StrCat(caller->name(), "/", name), g, input); + const absl::optional device = placer->OutputNodeDevice(index); + if (device.has_value()) node->set_requested_device(*device); + return node; }; // ------------------------------------------------------------------------ // @@ -1760,12 +1922,11 @@ Status InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g, for (Node* n : fbody->graph->op_nodes()) { NodeDef ndef = n->def(); - if (options.override_device) { - ndef.set_device(caller->def().device()); - } - if (options.initialize_empty_device && ndef.device().empty()) { - ndef.set_device(caller->def().device()); - } + // Maybe override requested node device assignment. + const absl::optional device = placer->BodyNodeDevice(ndef); + if (device.has_value()) ndef.set_device(*device); + + // Add inlined function name to inlined node debug information. PropagateDebugInfoToNode(fbody->fdef.signature().name(), {n}, &ndef); // Add the function node name as a prefix: @@ -1777,9 +1938,6 @@ Status InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g, Status added_node; Node* clone = g->AddNode(ndef, &added_node); - if (options.override_device && !caller->assigned_device_name().empty()) { - clone->set_assigned_device_name(caller->assigned_device_name()); - } TF_CHECK_OK(added_node); node_map[n->id()] = clone; @@ -1827,7 +1985,7 @@ Status InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g, // The added identity nodes depend on "input_control_node". for (std::size_t i = 0; i < fbody->arg_nodes.size(); ++i) { Node* arg = node_map[fbody->arg_nodes[i]->id()]; - Node* n = identity("input", inputs[i]); + Node* n = input_identity("input", inputs[i], i); if (input_control_node) { g->AddControlEdge(input_control_node, n, kDoNotCheckDuplicates); } @@ -1871,7 +2029,7 @@ Status InlineFunctionBody(const FunctionLibraryDefinition& flib_def, Graph* g, } } CHECK(data.node != nullptr); - Node* n = identity("output", data); + Node* n = output_identity("output", data, i); outputs[i] = n; for (const Edge* e : ret->in_edges()) { if (e->IsControlEdge()) { diff --git a/tensorflow/core/common_runtime/function.h b/tensorflow/core/common_runtime/function.h index e0bd3704214..715652088f1 100644 --- a/tensorflow/core/common_runtime/function.h +++ b/tensorflow/core/common_runtime/function.h @@ -19,6 +19,7 @@ limitations under the License. #include #include +#include "absl/types/optional.h" #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/common_runtime/device_mgr.h" #include "tensorflow/core/common_runtime/graph_optimizer.h" @@ -158,6 +159,50 @@ void ToGraphDef(const Graph* g, GraphDef* gdef, bool pretty = false); // TODO(zhifengc): Asks math expert to say the comment again. std::unique_ptr SymbolicGradient(const FunctionBody& f); +// Optionally override device assignment for nodes added to the graph for +// inlined functions: +// (1) Identity nodes added in place of function input arguments. +// (2) Identity nodes added in place of function return values. +// (3) Special NoOp nodes that enforce side-effects execution order. +// (4) All nodes inside function body specified in FunctionDef. +class InlinedFunctionBodyPlacer { + public: + virtual ~InlinedFunctionBodyPlacer() = default; + + virtual absl::optional InputNodeDevice(int input_index) const = 0; + virtual absl::optional OutputNodeDevice(int output_index) const = 0; + virtual absl::optional ControlNodeDevice() const = 0; + virtual absl::optional BodyNodeDevice(const NodeDef& ndef) const = 0; + + // Place input nodes on the same device as the corresponding caller input + // node. Do not specify any placement for all other nodes. + static std::unique_ptr DefaultPlacer( + const Graph& graph, const Node& caller); + + // Place all nodes on the same device as caller node. + static std::unique_ptr SingleDevicePlacer( + const Graph& graph, const Node& caller); + + // Place input nodes on the same device as the corresponding caller input + // node. Do not place output node. Place control nodes on the same device as + // caller node. For all function body nodes overrides job, replica and task + // parts of the device assignment to match function caller node. + static std::unique_ptr MultiDevicePlacer( + const Graph& graph, const Node& caller); + + using Factory = std::function( + const Graph&, const Node&)>; + + struct Config { + string name; + Factory get; + }; + + static Config Default() { return {"default", DefaultPlacer}; } + static Config SingleDevice() { return {"single_device", SingleDevicePlacer}; } + static Config MultiDevice() { return {"multi_device", MultiDevicePlacer}; } +}; + struct InlineFunctionBodyOptions { // All nodes that have incoming control edge *from* the function call node, // will be forwarded to the "output control node". There are two options for @@ -198,16 +243,6 @@ struct InlineFunctionBodyOptions { bool disable_inlining = false; // Ignore '_noinline' function attribute. bool ignore_noinline = false; - // If 'true' function inlining will override explicitly specified devices - // inside function body with the caller node device. - bool override_device = false; - // If 'true' function inlining will fill an empty device annotation inside - // function body with the caller node device. - // TODO(ezhulenev): Remove this flag. This is mostly legacy-compatibility - // mode. We should never explicitly define devices when we inline multi-device - // functions. However we do that in 'lower_function_call_op.cc' and - // 'function_optimizer' for now. - bool initialize_empty_device = false; // If 'true' function inlining will inline functions in implementation // selection group. Normally those functions should not be inlined; they will // be handled by Grappler. @@ -219,6 +254,11 @@ struct InlineFunctionBodyOptions { // Control returns were added to Tensorflow v2 with automatic control // dependencies tracking in Eager mode. OutputControlSource output_control_src = OutputControlSource::kDataOutputs; + // Inlined function body placer decides what requested device assignments + // should be added to the nodes added to the graph. See documentation above + // for available strategies. + InlinedFunctionBodyPlacer::Config inlined_function_body_placer = + InlinedFunctionBodyPlacer::Default(); // A human-readable debug string for this options. string DebugString() const; diff --git a/tensorflow/core/common_runtime/function_test.cc b/tensorflow/core/common_runtime/function_test.cc index 46691aa42f1..5bfd9f5eba3 100644 --- a/tensorflow/core/common_runtime/function_test.cc +++ b/tensorflow/core/common_runtime/function_test.cc @@ -1062,6 +1062,124 @@ TEST_F(FunctionLibraryRuntimeTest, ExpandInlineFunctionsAndKeepCallerNode) { } } +TEST_F(FunctionLibraryRuntimeTest, ExpandInlineFunctionsAndPlaceInlinedNodes) { + using test::function::NDef; + using KeepCallerNode = InlineFunctionBodyOptions::KeepCallerNode; + + const string arg_device = "/job:arg/replica:0/task:0/device:GPU"; + const string call_device = "/job:call/replica:0/task:1/device:GPU"; + const string body_device = "/job:body/replica:0/task:1/device:CPU"; + + const FunctionDef func = FDH::Create( + "AddFunc", {"i: float"}, {"o: float"}, {}, + {{{"ret"}, "Add", {"i", "i"}, {{"T", DT_FLOAT}}, {}, body_device}}, + /*ret_def=*/{{"o", "ret:z:0"}}); + Init({func}); + + // Construct a graph: + // a = Arg[dtype=DT_FLOAT, _device=arg_device] + // b = AddFunc[_device=call_device](a) + auto construct_graph = [&](std::unique_ptr* g) -> Status { + Scope s = Scope::NewRootScope(); + TF_RETURN_IF_ERROR(s.graph()->AddFunctionLibrary(fdef_lib_)); + auto a = ops::_Arg(s.WithOpName("a").WithDevice(arg_device), DT_FLOAT, 0); + auto b = test::function::Call(&s, "b", "AddFunc", {a}); + TF_RETURN_IF_ERROR(s.ToGraph(g->get())); + for (Node* node : (*g)->op_nodes()) { + if (node->name() == "b") node->set_requested_device(call_device); + } + return Status::OK(); + }; + + const string input_node = "Func/b/input/_0"; + const string output_node = "Func/b/output/_1"; + const string output_control_node = "Func/b/output_control_node/_2"; + + // Construct expected graph after function inlining. + auto expected_graph = [&](const std::vector& placed) -> GraphDef { + return test::function::GDef( + { + NDef("a", "_Arg", {}, {{"T", DT_FLOAT}, {"index", 0}}, placed[0]), + NDef(input_node, "Identity", {"a"}, {{"T", DT_FLOAT}}, placed[1]), + NDef("b/ret", "Add", {input_node, input_node}, {{"T", DT_FLOAT}}, + placed[2]), + NDef(output_node, "Identity", {"b/ret"}, {{"T", DT_FLOAT}}, + placed[3]), + NDef(output_control_node, "NoOp", {"^" + output_node}, {}, + placed[4]), + }, + {func}); + }; + + ExpandInlineFunctionsOptions opts; + opts.native_options.keep_caller_node = KeepCallerNode::kDoNotKeep; + + // Place only input nodes to match input device. + { + opts.native_options.inlined_function_body_placer = + InlinedFunctionBodyPlacer::Default(); + + auto g = absl::make_unique(OpRegistry::Global()); + TF_ASSERT_OK(construct_graph(&g)); + + ExpandInlineFunctions(flr0_, g.get(), opts); + GraphDef expected = expected_graph({/*a*/ arg_device, // + /*input*/ arg_device, // + /*body*/ body_device, // + /*output*/ "", // + /*control_output*/ ""} // + ); + + GraphDef actual; + g->ToGraphDef(&actual); + TF_EXPECT_GRAPH_EQ(expected, actual); + } + + // Place all nodes on the call node device. + { + opts.native_options.inlined_function_body_placer = + InlinedFunctionBodyPlacer::SingleDevice(); + + auto g = absl::make_unique(OpRegistry::Global()); + TF_ASSERT_OK(construct_graph(&g)); + + ExpandInlineFunctions(flr0_, g.get(), opts); + GraphDef expected = expected_graph({/*a*/ arg_device, // + /*input*/ call_device, // + /*body*/ call_device, // + /*output*/ call_device, // + /*control_output*/ call_device} // + ); + + GraphDef actual; + g->ToGraphDef(&actual); + TF_EXPECT_GRAPH_EQ(expected, actual); + } + + // Multi device function placement. + { + opts.native_options.inlined_function_body_placer = + InlinedFunctionBodyPlacer::MultiDevice(); + + auto g = absl::make_unique(OpRegistry::Global()); + TF_ASSERT_OK(construct_graph(&g)); + + const string merged_device = "/job:call/replica:0/task:1/device:CPU:*"; + + ExpandInlineFunctions(flr0_, g.get(), opts); + GraphDef expected = expected_graph({/*a*/ arg_device, // + /*input*/ arg_device, // + /*body*/ merged_device, // + /*output*/ "", // + /*control_output*/ call_device} // + ); + + GraphDef actual; + g->ToGraphDef(&actual); + TF_EXPECT_GRAPH_EQ(expected, actual); + } +} + TEST_F(FunctionLibraryRuntimeTest, PruneBody) { auto T = DT_INT32; FunctionDef stateful_func = FDH::Define( @@ -1285,8 +1403,8 @@ TEST_F(FunctionLibraryRuntimeTest, ControlDeps) { ASSERT_TRUE(g != nullptr); OptimizeGraph(flr0_, &g); - // NOTE: We can remove func0, func1, func2, func9 with a control edge n8->n5. - // But we don't have a pass doing that. + // NOTE: We can remove func0, func1, func2, func9 with a control edge + // n8->n5. But we don't have a pass doing that. { Scope s = Scope::NewRootScope(); auto x = ops::_Arg(s.WithOpName("x"), DT_FLOAT, 0); diff --git a/tensorflow/core/common_runtime/graph_optimizer.cc b/tensorflow/core/common_runtime/graph_optimizer.cc index 2171a9cb157..774a5067481 100644 --- a/tensorflow/core/common_runtime/graph_optimizer.cc +++ b/tensorflow/core/common_runtime/graph_optimizer.cc @@ -90,7 +90,8 @@ void GraphOptimizer::Optimize( } if (opts_.do_function_inlining()) { ExpandInlineFunctionsOptions expand_inline_opts; - expand_inline_opts.native_options.override_device = true; + expand_inline_opts.native_options.inlined_function_body_placer = + InlinedFunctionBodyPlacer::SingleDevice(); if (!inline_multi_device_functions) { // GraphOptimizer is running: // (1) After partitioning when executing with a Session API. diff --git a/tensorflow/core/common_runtime/lower_function_call_op.cc b/tensorflow/core/common_runtime/lower_function_call_op.cc index 4df335af783..87b024636fc 100644 --- a/tensorflow/core/common_runtime/lower_function_call_op.cc +++ b/tensorflow/core/common_runtime/lower_function_call_op.cc @@ -59,15 +59,16 @@ Status RewriteFunctionCallNode(Node* n, Graph* g, // belong to different devices. This type of functions was added in // Tensorflow 2.0 Eager mode, and it has control outputs to represent // side-effects that must always execute (see `control_ret` in FunctionDef). - inline_options.override_device = false; - inline_options.initialize_empty_device = true; inline_options.output_control_src = OutputControlSrc::kControlOutputs; + inline_options.inlined_function_body_placer = + InlinedFunctionBodyPlacer::MultiDevice(); } else { // Native function call (node.type_string() is the function name). These // functions are always executed on a single-device, which is the device of // the function call node. - inline_options.override_device = true; inline_options.output_control_src = OutputControlSrc::kDataOutputs; + inline_options.inlined_function_body_placer = + InlinedFunctionBodyPlacer::SingleDevice(); } const FunctionDef* fdef; diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc index 6d04c203d74..59a550ee793 100644 --- a/tensorflow/core/grappler/optimizers/function_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc @@ -1190,19 +1190,20 @@ Status InlineFunctionCalls(const GrapplerItem& item, // `PartitionedCall` is a TF-2.0 function call mechanism for multi-device // functions: - // a) Function can be multi-device, and we can't override device placements. + // a) Function can be multi-device. // b) Automatic control dependencies tracking guarantees that all function // side-effectful nodes will have a path to one of the control outputs. // Control outputs and control edges between side-effectful (stateful) // nodes are used to explicitly mark the nodes that must execute, and to // define their execution order. if (n->IsPartitionedCall() || force_inline_as_multi_device) { - inline_options.override_device = false; - inline_options.initialize_empty_device = true; inline_options.output_control_src = OutputControlSource::kControlOutputs; + inline_options.inlined_function_body_placer = + InlinedFunctionBodyPlacer::MultiDevice(); } else { - inline_options.override_device = true; inline_options.output_control_src = OutputControlSource::kDataOutputs; + inline_options.inlined_function_body_placer = + InlinedFunctionBodyPlacer::SingleDevice(); } if (fetch_nodes.contains(n->name())) { diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc index 14553997b19..5849bae857f 100644 --- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc @@ -520,14 +520,16 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionSimpleFunction) { {NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice), NDef("b", "Placeholder", {}, {{"dtype", DT_FLOAT}}, kDevice), - // Function body nodes are not placed, however function input nodes - // must copy device assignment from input arguments. + // Function body nodes copy only job/task/replica parts of device + // assignment, and function input nodes must copy full device + // assignment from input arguments. Optimized graph is not fully + // placed. NDef(input_x, "Identity", {"a"}, {{"T", DT_FLOAT}}, kDevice), NDef(input_y, "Identity", {"b"}, {{"T", DT_FLOAT}}, kDevice), - // TODO(ezhulenev): Currently inlined function body "implicitly placed" - // with a 'inline_options.initialize_empty_device' flag. + // NOTE(ezhulenev): Currently multi-device function inlining placer + // strategy will override all empty devices with function call device. NDef("c/mul", "Mul", {input_x, input_y}, {{"T", DT_FLOAT}}, kDevice), - NDef(output_z, "Identity", {"c/mul"}, {{"T", DT_FLOAT}}, kDevice), + NDef(output_z, "Identity", {"c/mul"}, {{"T", DT_FLOAT}}), NDef("d", "Identity", {output_z}, {{"T", DT_FLOAT}}, kDevice)}, // Function library. @@ -780,7 +782,7 @@ TEST_F(FunctionOptimizerTest, InlineIndirectFunctionWithDevicePlacement) { NDef(input_x, "Identity", {"a"}, {{"T", DT_FLOAT}}, cpu0), NDef(input_y, "Identity", {"b"}, {{"T", DT_FLOAT}}, cpu1), NDef("c/mul", "Mul", {input_x, input_y}, {{"T", DT_FLOAT}}, cpu1), - NDef(output_z, "Identity", {"c/mul"}, {{"T", DT_FLOAT}}, cpu1), + NDef(output_z, "Identity", {"c/mul"}, {{"T", DT_FLOAT}}, cpu0), NDef("d", "Identity", {output_z}, {{"T", DT_FLOAT}}, cpu0)}, // Function library. From 6d8e1bce63af9bd6e0ea9d82ddc3eb2c411a42cb Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Wed, 10 Jul 2019 19:01:51 -0700 Subject: [PATCH 249/332] Unref EagerContext when migrating from Unshaped to shaped remote handle We were leaking this reference. PiperOrigin-RevId: 257527704 --- .../core/distributed_runtime/eager/remote_tensor_handle_data.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc b/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc index 0b40da16232..747eaed1f44 100644 --- a/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc +++ b/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc @@ -137,6 +137,8 @@ UnshapedRemoteTensorHandleData::~UnshapedRemoteTensorHandleData() { if (delete_remote_tensor_) { DestoryRemoteTensorHandle(ctx_, eager_client_, context_id_, op_id_, output_num_); + } else { + ctx_->Unref(); } } From 618dd8bab488d6dc6198843502fbe56925f00e20 Mon Sep 17 00:00:00 2001 From: Eugene Zhulenev Date: Wed, 10 Jul 2019 20:11:27 -0700 Subject: [PATCH 250/332] Add input/output index to log messages in PFLR PiperOrigin-RevId: 257534475 --- .../core/common_runtime/process_function_library_runtime.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc index 42c5d59b400..6d8cd093531 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc @@ -566,13 +566,15 @@ Status ProcessFunctionLibraryRuntime::InstantiateMultiDevice( VLOG(1) << "Instantiating MultiDevice function \"" << function_name << "\" on default device \"" << options.target << "\""; if (VLOG_IS_ON(3)) { + int index = 0; VLOG(3) << "Requested input devices:"; for (const string& device : options.input_devices) { - VLOG(3) << " " << device; + VLOG(3) << " " << device << " for input at index " << index++; } + index = 0; VLOG(3) << "Requested output devices:"; for (const string& device : options.output_devices) { - VLOG(3) << " " << device; + VLOG(3) << " " << device << " for output at index " << index++; } } From e4a67c6d70344c75f2809cbe080de7069045f447 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Thu, 11 Jul 2019 01:07:49 -0700 Subject: [PATCH 251/332] Enable autograph-by-default in TPUStrategy.experimental_run_v2 because it doesn't support eager execution and always compiles its target to graph, unlike the more generic distribute.Strategy. PiperOrigin-RevId: 257562505 --- tensorflow/python/distribute/tpu_strategy.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/distribute/tpu_strategy.py b/tensorflow/python/distribute/tpu_strategy.py index 9c19e80a5d4..7aa99b9a8c4 100644 --- a/tensorflow/python/distribute/tpu_strategy.py +++ b/tensorflow/python/distribute/tpu_strategy.py @@ -105,10 +105,9 @@ class TPUStrategy(distribute_lib.Strategy): # This implementation runs a single step. It does not use infeed or outfeed. def experimental_run_v2(self, fn, args=(), kwargs=None): """See base class.""" - # tf.distribute supports Eager functions, so AutoGraph should not be applied - # when when the caller is also in Eager mode. - fn = autograph.tf_convert(fn, ag_ctx.control_status_ctx(), - convert_by_default=False) + # Note: the target function is converted to graph even when in Eager mode, + # so autograph is on by default here. + fn = autograph.tf_convert(fn, ag_ctx.control_status_ctx()) return self.extended.tpu_run(fn, args, kwargs) From 9f290c9bf9ac92b1d8d3828fdc07aab3a1694fcc Mon Sep 17 00:00:00 2001 From: Sergei Lebedev Date: Thu, 11 Jul 2019 01:55:10 -0700 Subject: [PATCH 252/332] ->Tensor conversion function is now expected to only raise {Type,Value}Error The precise contract for conversion functions is currently undocumented, but the implementation additionally allowed errors.UnimplementedError and errors.InvalidArgumentError to be raised. As far as I can tell, these are not required by any existing internal conversion function. PiperOrigin-RevId: 257567409 --- tensorflow/python/framework/ops.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 390115764d7..983a3a8f311 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -1165,8 +1165,7 @@ def internal_convert_to_tensor(value, try: ret = conversion_func( value, dtype=preferred_dtype, name=name, as_ref=as_ref) - except (TypeError, ValueError, errors.UnimplementedError, - errors.InvalidArgumentError): + except (TypeError, ValueError): # Could not coerce the conversion to use the preferred dtype. ret = None From 305394739324036ab6966b6f1227c5d356038560 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 02:02:18 -0700 Subject: [PATCH 253/332] Update GraphDef version to 93. PiperOrigin-RevId: 257568308 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index f55a1fcfea4..614f9016b24 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 92 // Updated: 2019/7/10 +#define TF_GRAPH_DEF_VERSION 93 // Updated: 2019/7/11 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From 108987cccd28f8f935f395f891baf06e6d893b8c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 02:02:18 -0700 Subject: [PATCH 254/332] compat: Update forward compatibility horizon to 2019-07-11 PiperOrigin-RevId: 257568311 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index b38483bbffb..1c1eafe4024 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -27,7 +27,7 @@ import datetime from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 7, 10) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 7, 11) @tf_export("compat.forward_compatible") From c07e9e0e17bd059f35a70b14fa0f48e83e6435d2 Mon Sep 17 00:00:00 2001 From: Ian Langmore Date: Thu, 11 Jul 2019 04:38:45 -0700 Subject: [PATCH 255/332] Change linear_operator_util `convert_immutable_to_tensor` to `convert_nonref_to_tensor` PiperOrigin-RevId: 257584511 --- .../python/ops/linalg/linear_operator_util.py | 150 ++++++++++-------- 1 file changed, 86 insertions(+), 64 deletions(-) diff --git a/tensorflow/python/ops/linalg/linear_operator_util.py b/tensorflow/python/ops/linalg/linear_operator_util.py index 1057c28a4bd..573d373ea93 100644 --- a/tensorflow/python/ops/linalg/linear_operator_util.py +++ b/tensorflow/python/ops/linalg/linear_operator_util.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.module import module from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops @@ -29,7 +30,6 @@ from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables as variables_module from tensorflow.python.ops.linalg import linalg_impl as linalg -from tensorflow.python.util import tf_inspect ################################################################################ @@ -37,43 +37,14 @@ from tensorflow.python.util import tf_inspect ################################################################################ -def convert_immutable_to_tensor(value, dtype=None, dtype_hint=None, name=None): - """Converts the given `value` to a `Tensor` only if input is immutable. +def convert_nonref_to_tensor(value, dtype=None, dtype_hint=None, name=None): + """Converts the given `value` to a `Tensor` if input is nonreference type. This function converts Python objects of various types to `Tensor` objects - except if the input is mutable. A mutable object is characterized by - `tensor_util.is_mutable` and is, roughly speaking, any object which is a - `tf.Variable` or known to depend on a `tf.Variable`. It accepts `Tensor` - objects, numpy arrays, Python lists, and Python scalars. This function does - not descend through structured input--it only verifies if the input is mutable - per `tensor_util.is_mutable`. For example: - - ```python - from tensorflow_probability.python.internal import tensor_util - - x = tf.Variable(0.) - y = tensor_util.convert_immutable_to_tensor(x) - x is y - # ==> True - - x = tf.constant(0.) - y = tensor_util.convert_immutable_to_tensor(x) - x is y - # ==> True - - x = np.array(0.) - y = tensor_util.convert_immutable_to_tensor(x) - x is y - # ==> False - tf.is_tensor(y) - # ==> True - ``` - - This function can be useful when composing a new operation in Python - (such as `my_func` in the example above). All standard Python op - constructors apply this function to each of their Tensor-valued - inputs, which allows those ops to accept numpy arrays, Python lists, - and scalars in addition to `Tensor` objects. + except if the input has nonreference semantics. Reference semantics are + characterized by `is_ref` and is any object which is a + `tf.Variable` or instance of `tf.Module`. This function accepts any input + which `tf.convert_to_tensor` would also. Note: This function diverges from default Numpy behavior for `float` and `string` types when `None` is present in a Python list or scalar. Rather @@ -81,13 +52,13 @@ def convert_immutable_to_tensor(value, dtype=None, dtype_hint=None, name=None): Args: value: An object whose type has a registered `Tensor` conversion function. - dtype: Optional element type for the returned tensor. If missing, the type - is inferred from the type of `value`. - dtype_hint: Optional element type for the returned tensor, used when dtype - is None. In some cases, a caller may not have a dtype in mind when - converting to a tensor, so dtype_hint can be used as a soft preference. - If the conversion to `dtype_hint` is not possible, this argument has no - effect. + dtype: Optional element type for the returned tensor. If missing, the + type is inferred from the type of `value`. + dtype_hint: Optional element type for the returned tensor, + used when dtype is None. In some cases, a caller may not have a + dtype in mind when converting to a tensor, so dtype_hint + can be used as a soft preference. If the conversion to + `dtype_hint` is not possible, this argument has no effect. name: Optional name to use if a new `Tensor` is created. Returns: @@ -97,42 +68,93 @@ def convert_immutable_to_tensor(value, dtype=None, dtype_hint=None, name=None): TypeError: If no conversion function is registered for `value` to `dtype`. RuntimeError: If a registered conversion function returns an invalid value. ValueError: If the `value` is a tensor not of given `dtype` in graph mode. + + + #### Examples: + + ```python + + x = tf.Variable(0.) + y = convert_nonref_to_tensor(x) + x is y + # ==> True + + x = tf.constant(0.) + y = convert_nonref_to_tensor(x) + x is y + # ==> True + + x = np.array(0.) + y = convert_nonref_to_tensor(x) + x is y + # ==> False + tf.is_tensor(y) + # ==> True + + x = tfp.util.DeferredTensor(lambda x: x, 13.37) + y = convert_nonref_to_tensor(x) + x is y + # ==> True + tf.is_tensor + # ==> False + tf.equal(y, 13.37) + # ==> True + ``` + """ # We explicitly do not use a tf.name_scope to avoid graph clutter. if value is None: return None - if is_mutable(value): - if not hasattr(value, "dtype"): - raise ValueError("Mutable type ({}) must implement `dtype` property " - "({}).".format(type(value).__name__, value)) - if not hasattr(value, "shape"): - raise ValueError("Mutable type ({}) must implement `shape` property " - "({}).".format(type(value).__name__, value)) - if dtype is not None and dtype.base_dtype != value.dtype.base_dtype: - raise TypeError("Mutable type must be of dtype '{}' but is '{}'.".format( - dtype.base_dtype.name, value.dtype.base_dtype.name)) + if is_ref(value): + if dtype is None: + return value + dtype_base = base_dtype(dtype) + value_dtype_base = base_dtype(value.dtype) + if dtype_base != value_dtype_base: + raise TypeError('Mutable type must be of dtype "{}" but is "{}".'.format( + dtype_name(dtype_base), dtype_name(value_dtype_base))) return value return ops.convert_to_tensor( value, dtype=dtype, dtype_hint=dtype_hint, name=name) -def is_mutable(x): - """Evaluates if the object is known to have `tf.Variable` ancestors. +def base_dtype(dtype): + """Returns a non-reference `dtype` based on this `dtype`.""" + dtype = dtypes.as_dtype(dtype) + if hasattr(dtype, "base_dtype"): + return dtype.base_dtype + return dtype - An object is deemed mutable if it is a `tf.Variable` instance or has a - properties `variables` or `trainable_variables` one of which is non-empty (as - might be the case for a subclasses of `tf.Module` or a Keras layer). + +def dtype_name(dtype): + """Returns the string name for this `dtype`.""" + dtype = dtypes.as_dtype(dtype) + if hasattr(dtype, "name"): + return dtype.name + if hasattr(dtype, "__name__"): + return dtype.__name__ + return str(dtype) + + +def is_ref(x): + """Evaluates if the object has reference semantics. + + An object is deemed "reference" if it is a `tf.Variable` instance or is + derived from a `tf.Module` with `dtype` and `shape` properties. Args: - x: Python object which may or may not have a `tf.Variable` ancestor. + x: Any object. Returns: - is_mutable: Python `bool` indicating input is mutable or is known to depend - on mutable objects. + is_ref: Python `bool` indicating input is has nonreference semantics, i.e., + is a `tf.Variable` or a `tf.Module` with `dtype` and `shape` properties. """ - return ((tf_inspect.isclass(variables_module.Variable) and - isinstance(x, variables_module.Variable)) or - getattr(x, "variables", ()) or getattr(x, "trainable_variables", ())) + return ( + # Note: we check that tf.Variable is a class because we might be using a + # different backend other than TF. + isinstance(x, variables_module.Variable) or + (isinstance(x, module.Module) and hasattr(x, "dtype") and + hasattr(x, "shape"))) ################################################################################ From 6e2c7d25bfd45f699a4abd614ae2dfe9cc8fe2bb Mon Sep 17 00:00:00 2001 From: Adrian Kuegel Date: Thu, 11 Jul 2019 05:26:46 -0700 Subject: [PATCH 256/332] Prevent fusion_merger pass to create too big fusions. Everywhere else we already check that the fusions we create will not become too big. We need to check it in fusion_merger as well. PiperOrigin-RevId: 257589636 --- tensorflow/compiler/xla/service/gpu/BUILD | 2 + .../compiler/xla/service/gpu/fusion_merger.cc | 16 ++++++- .../xla/service/gpu/fusion_merger_test.cc | 48 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 062210f1f75..92c147c1c71 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -870,12 +870,14 @@ tf_cc_test( srcs = ["fusion_merger_test.cc"], deps = [ ":fusion_merger", + ":gpu_fusible", ":instruction_fusion", "//tensorflow/compiler/xla:test_helpers", "//tensorflow/compiler/xla/service:hlo_matchers", "//tensorflow/compiler/xla/service:hlo_parser", "//tensorflow/compiler/xla/tests:hlo_test_base", "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "@com_google_absl//absl/types:span", ], ) diff --git a/tensorflow/compiler/xla/service/gpu/fusion_merger.cc b/tensorflow/compiler/xla/service/gpu/fusion_merger.cc index 6e15ec5f939..f707a87d79e 100644 --- a/tensorflow/compiler/xla/service/gpu/fusion_merger.cc +++ b/tensorflow/compiler/xla/service/gpu/fusion_merger.cc @@ -151,6 +151,7 @@ class FusionInstructionMerger { int num_fail_expensive_fused_instruction_ = 0; int num_fail_net_bytes_transferred_ratio_ = 0; int num_fail_inefficient_fusion_emitter_ = 0; + int num_fail_fusion_too_large_ = 0; TF_DISALLOW_COPY_AND_ASSIGN(FusionInstructionMerger); }; @@ -172,7 +173,8 @@ Status FusionInstructionMerger::Run() { << " expensive_instruction: " << num_fail_expensive_fused_instruction_ << " net_bytes_transferred: " << num_fail_net_bytes_transferred_ratio_ << " inefficient_fusion_emitter: " - << num_fail_inefficient_fusion_emitter_ << " }"; + << num_fail_inefficient_fusion_emitter_ + << " fusion_too_large: " << num_fail_fusion_too_large_ << " }"; return Status::OK(); } @@ -258,6 +260,18 @@ Status FusionInstructionMerger::HandleFusion(HloInstruction* fusion) { return Status::OK(); } + // Skip 'fusion' instruction if merging it into at least one of the users + // would make the fusion too big. + if (absl::c_any_of(fusion->users(), [fusion](const HloInstruction* user) { + return FusionWouldBeTooLarge(*fusion, *user); + })) { + VLOG(3) << "Not merging " << fusion->name() + << ": Contains one or more users where fusing would cause " + "the fusion to have too many parameters."; + ++num_fail_fusion_too_large_; + return Status::OK(); + } + // Merge fused instructions from 'fusion' into each user. std::vector users = fusion->users(); for (HloInstruction* user : users) { diff --git a/tensorflow/compiler/xla/service/gpu/fusion_merger_test.cc b/tensorflow/compiler/xla/service/gpu/fusion_merger_test.cc index 45f23fcdc56..50ed7448790 100644 --- a/tensorflow/compiler/xla/service/gpu/fusion_merger_test.cc +++ b/tensorflow/compiler/xla/service/gpu/fusion_merger_test.cc @@ -15,6 +15,10 @@ limitations under the License. #include "tensorflow/compiler/xla/service/gpu/fusion_merger.h" +#include + +#include "absl/types/span.h" +#include "tensorflow/compiler/xla/service/gpu/gpu_fusible.h" #include "tensorflow/compiler/xla/service/gpu/instruction_fusion.h" #include "tensorflow/compiler/xla/service/hlo_matchers.h" #include "tensorflow/compiler/xla/service/hlo_parser.h" @@ -263,6 +267,50 @@ TEST_F(FusionMergerTest, WillNotMergeReduceUnfriendlyLayouts) { EXPECT_FALSE(FusionMerger().Run(module.get()).ValueOrDie()); } +// Check that we limit the number of operands to fusions we create. +TEST_F(FusionMergerTest, AvoidsLargeFusion) { + constexpr int64 kNumParams = kMaxOperandsAndOutputsPerFusion + 1; + + // Compute + // p0 + p1 + p2 + ... + pn, + // Use so many parameters that they do not fit into one fusion. + auto module = CreateNewVerifiedModule(); + HloComputation::Builder b(TestName()); + Shape shape = ShapeUtil::MakeShape(F32, {10, 100}); + + std::vector entry_params; + + for (int64 i = 0; i < kNumParams; ++i) { + entry_params.push_back( + b.AddInstruction(HloInstruction::CreateParameter(i, shape, "p"))); + } + auto make_fusion = [&](absl::Span params) { + // Build a fusion computation for calculating the sum of all parameters. + HloComputation::Builder sub_builder("subcomp"); + HloInstruction* sum = nullptr; + for (int64 i = 0; i < params.size(); ++i) { + auto p = sub_builder.AddInstruction( + HloInstruction::CreateParameter(i, shape, "p")); + if (sum == nullptr) { + sum = p; + } else { + sum = sub_builder.AddInstruction( + HloInstruction::CreateBinary(shape, HloOpcode::kAdd, sum, p)); + } + } + HloComputation* subcomp = + module->AddEmbeddedComputation(sub_builder.Build()); + return HloInstruction::CreateFusion( + shape, HloInstruction::FusionKind::kLoop, params, subcomp); + }; + auto fusion = b.AddInstruction( + make_fusion(absl::MakeSpan(entry_params) + .subspan(0, kMaxOperandsAndOutputsPerFusion))); + b.AddInstruction(make_fusion({entry_params.back(), fusion})); + module->AddEntryComputation(b.Build()); + EXPECT_FALSE(FusionMerger().Run(module.get()).ValueOrDie()); +} + // TODO(b/119692968): Remove this test once fusion emitter is fixed. TEST_F(FusionMergerTest, WillNotMergeIfFusionEmitterIsInefficient) { auto module = ParseAndReturnVerifiedModule(R"( From 557759d885d3b918f84e4852d53634ea2c4849b8 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 11 Jul 2019 05:42:27 -0700 Subject: [PATCH 257/332] [XLA:Python] Add support for pooling se::Event objects. On GPU, events are expensive to create and apparently can and should be reused. PiperOrigin-RevId: 257591195 --- tensorflow/compiler/xla/python/BUILD | 16 ++++ tensorflow/compiler/xla/python/event_pool.cc | 52 +++++++++++ tensorflow/compiler/xla/python/event_pool.h | 76 ++++++++++++++++ .../compiler/xla/python/local_client.cc | 86 +++++++++++-------- tensorflow/compiler/xla/python/local_client.h | 10 ++- .../xla/python/shared_device_buffer.cc | 19 ++-- .../xla/python/shared_device_buffer.h | 14 ++- 7 files changed, 213 insertions(+), 60 deletions(-) create mode 100644 tensorflow/compiler/xla/python/event_pool.cc create mode 100644 tensorflow/compiler/xla/python/event_pool.h diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD index 093152b975f..81a61b8a5e5 100644 --- a/tensorflow/compiler/xla/python/BUILD +++ b/tensorflow/compiler/xla/python/BUILD @@ -104,11 +104,26 @@ cc_library( ], ) +cc_library( + name = "event_pool", + srcs = ["event_pool.cc"], + hdrs = ["event_pool.h"], + deps = [ + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", + "//tensorflow/core:stream_executor", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/synchronization", + ], +) + cc_library( name = "shared_device_buffer", srcs = ["shared_device_buffer.cc"], hdrs = ["shared_device_buffer.h"], deps = [ + ":event_pool", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla/service:shaped_buffer", "//tensorflow/compiler/xla/service:transfer_manager", @@ -149,6 +164,7 @@ cc_library( ], features = ["-use_header_modules"], deps = [ + ":event_pool", ":shared_device_buffer", ":types", ":worker_thread", diff --git a/tensorflow/compiler/xla/python/event_pool.cc b/tensorflow/compiler/xla/python/event_pool.cc new file mode 100644 index 00000000000..4edb41fd41f --- /dev/null +++ b/tensorflow/compiler/xla/python/event_pool.cc @@ -0,0 +1,52 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/python/event_pool.h" + +#include "absl/memory/memory.h" +#include "tensorflow/compiler/xla/status_macros.h" + +namespace xla { + +EventPool::Handle::~Handle() { + if (pool_ && event_) { + absl::MutexLock lock(&pool_->mu_); + pool_->free_events_.push(std::move(event_)); + } +} + +EventPool::EventPool(bool allow_reuse) : allow_reuse_(allow_reuse) {} + +StatusOr EventPool::ThenAllocateAndRecordEvent( + se::Stream* stream) { + Handle event; + + if (allow_reuse_) { + event.pool_ = this; + absl::MutexLock lock(&mu_); + if (!free_events_.empty()) { + event.event_ = std::move(free_events_.top()); + free_events_.pop(); + } + } + if (!event.event_) { + event.event_ = absl::make_unique(stream->parent()); + TF_RET_CHECK(event.event_->Init()) << "Event initialization failed"; + } + stream->ThenRecordEvent(event.event_.get()); + return event; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/python/event_pool.h b/tensorflow/compiler/xla/python/event_pool.h new file mode 100644 index 00000000000..56787acd87e --- /dev/null +++ b/tensorflow/compiler/xla/python/event_pool.h @@ -0,0 +1,76 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_PYTHON_EVENT_POOL_H_ +#define TENSORFLOW_COMPILER_XLA_PYTHON_EVENT_POOL_H_ + +#include +#include + +#include "absl/synchronization/mutex.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/platform/stream_executor.h" + +namespace xla { + +class EventPool { + public: + class Handle { + public: + Handle() = default; + ~Handle(); + + Handle(const Handle&) = delete; + Handle(Handle&&) = default; + Handle& operator=(const Handle&) = delete; + Handle& operator=(Handle&&) = default; + + se::Event* event() const { return event_.get(); } + + private: + friend class EventPool; + + EventPool* pool_ = nullptr; + std::unique_ptr event_; + }; + + // Initializes a new EventPool. If `allow_reuse` is true, then events will be + // returned to the pool when their handles are deleted and made available to + // subsequent allocations. Reuse only works on the GPU platform. + explicit EventPool(bool allow_reuse); + + // Allocates a new (or reused) event from the pool, and records the event on + // `stream`. + // + // Reuse is only possible on GPU. Event allocation and recording are coupled + // in a single operation because on GPU it is recording an event that makes it + // a "new" event. According to the CUDA documentation it is safe to call + // cudaEventRecord even if that event may still be in use on the device; APIs + // such as cudaStreamWaitEvent capture the state of the event at the time of + // the host-side call and are not affected by a later host-side + // cudaEventRecord. + StatusOr ThenAllocateAndRecordEvent(se::Stream* stream); + + private: + const bool allow_reuse_; + + absl::Mutex mu_; + std::stack> free_events_ GUARDED_BY(mu_); +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_PYTHON_EVENT_POOL_H_ diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index c34c01425b9..8d3704efba9 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -110,10 +110,12 @@ Status RegisterCpuCustomCallTarget(const std::string& fn_name, } Device::Device(se::StreamExecutor* executor, bool use_multiple_streams, - bool synchronous_deallocation, bool asynchronous) + bool synchronous_deallocation, bool asynchronous, + bool allow_event_reuse) : use_multiple_streams_(use_multiple_streams), synchronous_deallocation_(synchronous_deallocation), - asynchronous_(asynchronous) { + asynchronous_(asynchronous), + event_pool_(allow_event_reuse) { compute_stream_ = std::make_shared(executor); compute_stream_->Init(); if (use_multiple_streams) { @@ -237,11 +239,12 @@ StatusOr> PyLocalClient::Get( options.set_platform(platform); TF_ASSIGN_OR_RETURN(LocalClient * client, ClientLibrary::GetOrCreateLocalClient(options)); + bool gpu_platform = platform_name == "gpu"; std::unique_ptr allocator; if (allocator_config.kind == AllocatorConfig::Kind::kBFC || - (platform_name == "gpu" && + (gpu_platform && allocator_config.kind == AllocatorConfig::Kind::kDefault)) { - if (platform_name != "gpu") { + if (!gpu_platform) { return Unimplemented("BFCAllocator only available for GPU."); } TF_ASSIGN_OR_RETURN( @@ -258,9 +261,9 @@ StatusOr> PyLocalClient::Get( for (int i = 0; i < client->device_count(); ++i) { se::StreamExecutor* executor = client->backend().stream_executor(i).ValueOrDie(); - devices.push_back(absl::make_unique(executor, use_multiple_streams, - synchronous_deallocation, - asynchronous)); + devices.push_back(absl::make_unique( + executor, use_multiple_streams, synchronous_deallocation, asynchronous, + /*allow_event_reuse=*/gpu_platform)); } return std::make_shared(platform_name, client, std::move(devices), @@ -305,7 +308,7 @@ StatusOr PyLocalClient::TransferFromOutfeed( static StatusOr> TransferHostToDeviceAsync( const PythonBufferTree& tree, int device_ordinal, - std::shared_ptr client, const Device& device) { + std::shared_ptr client, Device* device) { se::DeviceMemoryAllocator* allocator = client->allocator(); TransferManager* transfer_manager = client->client()->backend().transfer_manager(); @@ -315,7 +318,7 @@ static StatusOr> TransferHostToDeviceAsync( transfer_manager->AllocateScopedShapedBuffer( shape, allocator, device_ordinal)); TF_RETURN_IF_ERROR(transfer_manager->WriteTupleIndexTablesAsync( - device.host_to_device_stream(), buffer)); + device->host_to_device_stream(), buffer)); auto it = tree.leaves.begin(); for (const ShapeUtil::IndexedShape& indexed_shape : @@ -326,28 +329,30 @@ static StatusOr> TransferHostToDeviceAsync( transfer_manager->HostShapeToDeviceShape(indexed_shape.shape), client->client()->platform(), device_ordinal); leaf.buffers().CopySubtreeFrom(buffer.buffers(), indexed_shape.index, {}); - if (device.use_multiple_streams() && + if (device->use_multiple_streams() && !transfer_manager->CanShapedBufferBeAccessedNow( - device.host_to_device_stream()->parent(), leaf)) { - device.host_to_device_stream()->ThenWaitFor(device.compute_stream()); + device->host_to_device_stream()->parent(), leaf)) { + device->host_to_device_stream()->ThenWaitFor(device->compute_stream()); } TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralToDeviceAsync( - device.host_to_device_stream(), *it, leaf)); + device->host_to_device_stream(), *it, leaf)); ++it; } std::shared_ptr definition_event; - if (device.use_multiple_streams()) { - TF_ASSIGN_OR_RETURN(definition_event, - BufferDefinitionEvent::Create( - device.host_to_device_stream()->parent())); - definition_event->RecordOnStream(device.host_to_device_stream()); + if (device->use_multiple_streams()) { + definition_event = std::make_shared(); + TF_ASSIGN_OR_RETURN(EventPool::Handle event, + device->event_pool().ThenAllocateAndRecordEvent( + device->host_to_device_stream())); + definition_event->SetDefinitionEvent(std::move(event), + device->host_to_device_stream()); } std::shared_ptr device_buffer = SharedDeviceBuffer::FromScopedShapedBuffer(std::move(buffer), definition_event); - if (device.synchronous_deallocation()) { - device.ThenReleaseOnWorkerThread(device.host_to_device_stream(), - device_buffer); + if (device->synchronous_deallocation()) { + device->ThenReleaseOnWorkerThread(device->host_to_device_stream(), + device_buffer); } return absl::make_unique(shape, std::move(device_buffer), std::move(client)); @@ -372,12 +377,13 @@ StatusOr> PyLocalBuffer::FromPython( VLOG(1) << "PyLocalBuffer::FromPython: shape: " << tree.shape.ToString() << " device ordinal: " << device_ordinal; - const Device& device = client->device(device_ordinal); + Device* device = &client->device(device_ordinal); TF_ASSIGN_OR_RETURN(std::unique_ptr buffer, TransferHostToDeviceAsync(tree, device_ordinal, std::move(client), device)); - device.ThenRelease(device.host_to_device_stream(), std::move(py_buffer_ref)); + device->ThenRelease(device->host_to_device_stream(), + std::move(py_buffer_ref)); return buffer; } @@ -402,12 +408,10 @@ StatusOr> PyLocalBuffer::FromPython( se::DeviceMemoryAllocator* allocator = client->allocator(); TransferManager* transfer_manager = client->client()->backend().transfer_manager(); - const Device& device = client->device(device_ordinal); + Device& device = client->device(device_ordinal); std::shared_ptr definition_event; if (device.use_multiple_streams()) { - TF_ASSIGN_OR_RETURN(definition_event, - BufferDefinitionEvent::Create( - device.host_to_device_stream()->parent())); + definition_event = std::make_shared(); } TF_ASSIGN_OR_RETURN( std::shared_ptr tuple_buffer, @@ -428,7 +432,11 @@ StatusOr> PyLocalBuffer::FromPython( TF_RETURN_IF_ERROR(transfer_manager->WriteRootTupleIndexTable( device.host_to_device_stream(), shaped_buffer)); if (definition_event) { - definition_event->RecordOnStream(device.host_to_device_stream()); + TF_ASSIGN_OR_RETURN(EventPool::Handle event, + device.event_pool().ThenAllocateAndRecordEvent( + device.host_to_device_stream())); + definition_event->SetDefinitionEvent(std::move(event), + device.host_to_device_stream()); } if (device.synchronous_deallocation()) { @@ -608,10 +616,12 @@ StatusOr> PyLocalBuffer::CopyToDevice( std::shared_ptr definition_event; if (dst_device.use_multiple_streams()) { - TF_ASSIGN_OR_RETURN( - definition_event, - BufferDefinitionEvent::Create(src_device_to_device_stream->parent())); - definition_event->RecordOnStream(src_device_to_device_stream); + definition_event = std::make_shared(); + TF_ASSIGN_OR_RETURN(EventPool::Handle event, + src_device.event_pool().ThenAllocateAndRecordEvent( + src_device_to_device_stream)); + definition_event->SetDefinitionEvent(std::move(event), + src_device_to_device_stream); } std::shared_ptr dst_device_buffer = @@ -697,7 +707,7 @@ StatusOr> PyLocalExecutable::ExecuteHelper( << " buffer: " << argument_buffers.back().ToString(); } - const Device& device = client_->device(device_ordinal); + Device& device = client_->device(device_ordinal); // The choice of where we wait in "synchronous" mode is arbitrary; the reason // for the wait is pacing to avoid problems such as memory fragmentation, not // for correctness. @@ -730,10 +740,12 @@ StatusOr> PyLocalExecutable::ExecuteHelper( std::shared_ptr definition_event; if (device.use_multiple_streams()) { - TF_ASSIGN_OR_RETURN( - definition_event, - BufferDefinitionEvent::Create(device.compute_stream()->parent())); - definition_event->RecordOnStream(device.compute_stream()); + definition_event = std::make_shared(); + TF_ASSIGN_OR_RETURN(EventPool::Handle event, + device.event_pool().ThenAllocateAndRecordEvent( + device.compute_stream())); + definition_event->SetDefinitionEvent(std::move(event), + device.compute_stream()); } Shape on_host_shape = result_buffer.ValueOrDie().on_host_shape(); std::shared_ptr out_buffer = diff --git a/tensorflow/compiler/xla/python/local_client.h b/tensorflow/compiler/xla/python/local_client.h index 38315d82e2c..ba9227e2c2f 100644 --- a/tensorflow/compiler/xla/python/local_client.h +++ b/tensorflow/compiler/xla/python/local_client.h @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/executable_build_options.h" #include "tensorflow/compiler/xla/client/local_client.h" #include "tensorflow/compiler/xla/client/xla_computation.h" +#include "tensorflow/compiler/xla/python/event_pool.h" #include "tensorflow/compiler/xla/python/python_ref_manager.h" #include "tensorflow/compiler/xla/python/shared_device_buffer.h" #include "tensorflow/compiler/xla/python/worker_thread.h" @@ -35,6 +36,7 @@ limitations under the License. #include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/status.h" #include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/core/lib/core/status.h" namespace xla { @@ -61,12 +63,16 @@ class Device { // If asynchronous is false, the host will synchronize to the device after // each execution or transfer. This is intended for debugging only. Device(se::StreamExecutor* executor, bool use_multiple_streams, - bool synchronous_deallocation, bool asynchronous); + bool synchronous_deallocation, bool asynchronous, + bool allow_event_reuse); virtual ~Device(); bool use_multiple_streams() const { return use_multiple_streams_; } bool synchronous_deallocation() const { return synchronous_deallocation_; } bool asynchronous() const { return asynchronous_; } + + EventPool& event_pool() { return event_pool_; } + se::Stream* compute_stream() const { return compute_stream_.get(); } se::Stream* host_to_device_stream() const { return host_to_device_stream_.get(); @@ -139,6 +145,8 @@ class Device { bool use_multiple_streams_; bool synchronous_deallocation_; bool asynchronous_; + + EventPool event_pool_; std::shared_ptr compute_stream_; std::shared_ptr host_to_device_stream_; std::shared_ptr device_to_host_stream_; diff --git a/tensorflow/compiler/xla/python/shared_device_buffer.cc b/tensorflow/compiler/xla/python/shared_device_buffer.cc index fd85145eddc..40fc8f1cc65 100644 --- a/tensorflow/compiler/xla/python/shared_device_buffer.cc +++ b/tensorflow/compiler/xla/python/shared_device_buffer.cc @@ -21,21 +21,12 @@ limitations under the License. namespace xla { -/*static*/ StatusOr> -BufferDefinitionEvent::Create(se::StreamExecutor* executor) { - auto event = std::make_shared(executor); - TF_RET_CHECK(event->event_.Init()) - << "Buffer definition event initialization failed"; - return event; -} - -BufferDefinitionEvent::BufferDefinitionEvent(se::StreamExecutor* executor) - : event_(executor) {} - -void BufferDefinitionEvent::RecordOnStream(se::Stream* stream) { +void BufferDefinitionEvent::SetDefinitionEvent(EventPool::Handle event, + se::Stream* stream) { absl::MutexLock lock(&mu_); + CHECK(!event_.event()); + event_ = std::move(event); CHECK(streams_defined_on_.empty()); - stream->ThenRecordEvent(&event_); streams_defined_on_.push_back(stream); } @@ -50,7 +41,7 @@ void BufferDefinitionEvent::WaitForEventOnStream(se::Stream* stream) { return; } - stream->ThenWaitFor(&event_); + stream->ThenWaitFor(event_.event()); streams_defined_on_.push_back(stream); } diff --git a/tensorflow/compiler/xla/python/shared_device_buffer.h b/tensorflow/compiler/xla/python/shared_device_buffer.h index 469237488b4..ea4aa67bff9 100644 --- a/tensorflow/compiler/xla/python/shared_device_buffer.h +++ b/tensorflow/compiler/xla/python/shared_device_buffer.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_PYTHON_SHARED_DEVICE_BUFFER_H_ #include "absl/container/flat_hash_set.h" +#include "tensorflow/compiler/xla/python/event_pool.h" #include "tensorflow/compiler/xla/service/shaped_buffer.h" #include "tensorflow/compiler/xla/service/transfer_manager.h" #include "tensorflow/compiler/xla/shape.h" @@ -50,14 +51,11 @@ namespace xla { // same stream causes no additional waiting. class BufferDefinitionEvent { public: - // Creates a new definition event whose event has not yet been triggered. - static StatusOr> Create( - se::StreamExecutor* executor); + BufferDefinitionEvent() = default; - explicit BufferDefinitionEvent(se::StreamExecutor* executor); - - // Records the definition event on the tail of 'stream'. - void RecordOnStream(se::Stream* stream); + // Sets the definition event of the buffer to 'event', which is recorded + // on 'stream'. Must be called at most once. + void SetDefinitionEvent(EventPool::Handle event, se::Stream* stream); // Adds synchronization events to 'stream' that wait for this event to be // defined on 'stream'. Does nothing if the event is already known to have @@ -68,7 +66,7 @@ class BufferDefinitionEvent { // An event that is triggered when the content of one or more buffers is // ready. If this event is nullptr, it is assumed that the buffer's content is // always defined. - se::Event event_; + EventPool::Handle event_; absl::Mutex mu_; From 78ee8186db860185c7b73a9f70bf3a59530675dd Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 11 Jul 2019 06:06:56 -0700 Subject: [PATCH 258/332] [XLA:Python] Remove some special cases in the XLA Python bindings that exist to work around the fact that the Host device didn't support events. Enable "multi stream" mode everywhere and delete the code for "single stream" mode. This has the side effect of using more threading than we did previously on CPU, in particular the "host to device" and "device to host" transfers are done in separate streams (threads). It's not clear that's a bad thing, and more commonality between backends is nice. [SE:Host] Implement the se::Event API for the host platform, as a minimal wrapper around absl::Notification. PiperOrigin-RevId: 257594142 --- .../compiler/xla/python/local_client.cc | 126 ++++++++---------- tensorflow/compiler/xla/python/local_client.h | 22 +-- tensorflow/stream_executor/host/BUILD | 1 + .../stream_executor/host/host_gpu_executor.cc | 57 ++++++++ .../stream_executor/host/host_gpu_executor.h | 29 +--- 5 files changed, 123 insertions(+), 112 deletions(-) diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index 8d3704efba9..4b313a4181e 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -30,10 +30,9 @@ limitations under the License. // Multi-stream execution: // ----------------------- // -// On certain platforms (e.g., TPU), we use a multistream execution design, -// where different Streams are used for host-to-device transfers, -// device-to-host transfers, and compute. This allows us to overlap transfers on -// and off the device with computation. +// We use a multistream execution design, where different Streams are used for +// host-to-device transfers, device-to-host transfers, and compute. This allows +// us to overlap transfers on and off the device with computation. // // Synchronization between streams occurs via BufferDefinitionEvents that // describe when the contents of a logical buffer are known to be valid on @@ -109,32 +108,24 @@ Status RegisterCpuCustomCallTarget(const std::string& fn_name, return Status::OK(); } -Device::Device(se::StreamExecutor* executor, bool use_multiple_streams, - bool synchronous_deallocation, bool asynchronous, - bool allow_event_reuse) - : use_multiple_streams_(use_multiple_streams), - synchronous_deallocation_(synchronous_deallocation), +Device::Device(se::StreamExecutor* executor, bool synchronous_deallocation, + bool asynchronous, bool allow_event_reuse) + : synchronous_deallocation_(synchronous_deallocation), asynchronous_(asynchronous), event_pool_(allow_event_reuse) { - compute_stream_ = std::make_shared(executor); + compute_stream_ = absl::make_unique(executor); + host_to_device_stream_ = absl::make_unique(executor); + device_to_host_stream_ = absl::make_unique(executor); + callback_stream_ = absl::make_unique(executor); compute_stream_->Init(); - if (use_multiple_streams) { - host_to_device_stream_ = std::make_shared(executor); - device_to_host_stream_ = std::make_shared(executor); - callback_stream_ = std::make_shared(executor); - host_to_device_stream_->Init(); - device_to_host_stream_->Init(); - callback_stream_->Init(); - device_to_device_streams_.reserve(kNumDeviceToDeviceStreams); - for (int i = 0; i < kNumDeviceToDeviceStreams; ++i) { - auto stream = std::make_shared(executor); - stream->Init(); - device_to_device_streams_.push_back(std::move(stream)); - } - } else { - callback_stream_ = host_to_device_stream_ = device_to_host_stream_ = - compute_stream_; - device_to_device_streams_.push_back(compute_stream_); + host_to_device_stream_->Init(); + device_to_host_stream_->Init(); + callback_stream_->Init(); + device_to_device_streams_.reserve(kNumDeviceToDeviceStreams); + for (int i = 0; i < kNumDeviceToDeviceStreams; ++i) { + auto stream = absl::make_unique(executor); + stream->Init(); + device_to_device_streams_.push_back(std::move(stream)); } worker_thread_ = absl::make_unique(tensorflow::Env::Default(), "py_xla_execute"); @@ -256,13 +247,12 @@ StatusOr> PyLocalClient::Get( std::vector> devices; devices.reserve(client->device_count()); - bool use_multiple_streams = (platform_name != "cpu"); - bool synchronous_deallocation = !use_multiple_streams; + bool synchronous_deallocation = platform_name == "cpu"; for (int i = 0; i < client->device_count(); ++i) { se::StreamExecutor* executor = client->backend().stream_executor(i).ValueOrDie(); devices.push_back(absl::make_unique( - executor, use_multiple_streams, synchronous_deallocation, asynchronous, + executor, synchronous_deallocation, asynchronous, /*allow_event_reuse=*/gpu_platform)); } return std::make_shared(platform_name, client, @@ -329,8 +319,7 @@ static StatusOr> TransferHostToDeviceAsync( transfer_manager->HostShapeToDeviceShape(indexed_shape.shape), client->client()->platform(), device_ordinal); leaf.buffers().CopySubtreeFrom(buffer.buffers(), indexed_shape.index, {}); - if (device->use_multiple_streams() && - !transfer_manager->CanShapedBufferBeAccessedNow( + if (!transfer_manager->CanShapedBufferBeAccessedNow( device->host_to_device_stream()->parent(), leaf)) { device->host_to_device_stream()->ThenWaitFor(device->compute_stream()); } @@ -338,15 +327,14 @@ static StatusOr> TransferHostToDeviceAsync( device->host_to_device_stream(), *it, leaf)); ++it; } - std::shared_ptr definition_event; - if (device->use_multiple_streams()) { - definition_event = std::make_shared(); - TF_ASSIGN_OR_RETURN(EventPool::Handle event, - device->event_pool().ThenAllocateAndRecordEvent( - device->host_to_device_stream())); - definition_event->SetDefinitionEvent(std::move(event), - device->host_to_device_stream()); - } + + auto definition_event = std::make_shared(); + TF_ASSIGN_OR_RETURN(EventPool::Handle event, + device->event_pool().ThenAllocateAndRecordEvent( + device->host_to_device_stream())); + definition_event->SetDefinitionEvent(std::move(event), + device->host_to_device_stream()); + std::shared_ptr device_buffer = SharedDeviceBuffer::FromScopedShapedBuffer(std::move(buffer), definition_event); @@ -409,10 +397,8 @@ StatusOr> PyLocalBuffer::FromPython( TransferManager* transfer_manager = client->client()->backend().transfer_manager(); Device& device = client->device(device_ordinal); - std::shared_ptr definition_event; - if (device.use_multiple_streams()) { - definition_event = std::make_shared(); - } + + auto definition_event = std::make_shared(); TF_ASSIGN_OR_RETURN( std::shared_ptr tuple_buffer, SharedDeviceBuffer::MakeTuple(device_buffers, transfer_manager, allocator, @@ -423,21 +409,19 @@ StatusOr> PyLocalBuffer::FromPython( // TODO(phawkins): extend TransferManager so we do not need to form a full // ShapedBuffer just to write the root tuple index table. TF_ASSIGN_OR_RETURN(ShapedBuffer shaped_buffer, buffer->AsShapedBuffer()); - if (device.use_multiple_streams() && - !transfer_manager->CanShapedBufferBeAccessedNow( + if (!transfer_manager->CanShapedBufferBeAccessedNow( device.host_to_device_stream()->parent(), shaped_buffer)) { // Wait for the compute stream so that memory allocations are synchronized. device.host_to_device_stream()->ThenWaitFor(device.compute_stream()); } TF_RETURN_IF_ERROR(transfer_manager->WriteRootTupleIndexTable( device.host_to_device_stream(), shaped_buffer)); - if (definition_event) { - TF_ASSIGN_OR_RETURN(EventPool::Handle event, - device.event_pool().ThenAllocateAndRecordEvent( - device.host_to_device_stream())); - definition_event->SetDefinitionEvent(std::move(event), - device.host_to_device_stream()); - } + + TF_ASSIGN_OR_RETURN(EventPool::Handle event, + device.event_pool().ThenAllocateAndRecordEvent( + device.host_to_device_stream())); + definition_event->SetDefinitionEvent(std::move(event), + device.host_to_device_stream()); if (device.synchronous_deallocation()) { device.ThenReleaseOnWorkerThread(device.host_to_device_stream(), @@ -574,8 +558,7 @@ StatusOr> PyLocalBuffer::CopyToDevice( ScopedShapedBuffer dst_buffer, transfer_manager->AllocateScopedShapedBuffer( on_host_shape_, client_->allocator(), dst_device_ordinal)); - if (dst_device.use_multiple_streams() && - !transfer_manager->CanShapedBufferBeAccessedNow( + if (!transfer_manager->CanShapedBufferBeAccessedNow( dst_device.compute_stream()->parent(), dst_buffer)) { src_device_to_device_stream->ThenWaitFor(dst_device.compute_stream()); } @@ -614,15 +597,12 @@ StatusOr> PyLocalBuffer::CopyToDevice( dst_device.host_to_device_stream()); } - std::shared_ptr definition_event; - if (dst_device.use_multiple_streams()) { - definition_event = std::make_shared(); - TF_ASSIGN_OR_RETURN(EventPool::Handle event, - src_device.event_pool().ThenAllocateAndRecordEvent( - src_device_to_device_stream)); - definition_event->SetDefinitionEvent(std::move(event), - src_device_to_device_stream); - } + auto definition_event = std::make_shared(); + TF_ASSIGN_OR_RETURN(EventPool::Handle event, + src_device.event_pool().ThenAllocateAndRecordEvent( + src_device_to_device_stream)); + definition_event->SetDefinitionEvent(std::move(event), + src_device_to_device_stream); std::shared_ptr dst_device_buffer = SharedDeviceBuffer::FromScopedShapedBuffer(std::move(dst_buffer), @@ -738,15 +718,13 @@ StatusOr> PyLocalExecutable::ExecuteHelper( return result_buffer.status(); } - std::shared_ptr definition_event; - if (device.use_multiple_streams()) { - definition_event = std::make_shared(); - TF_ASSIGN_OR_RETURN(EventPool::Handle event, - device.event_pool().ThenAllocateAndRecordEvent( - device.compute_stream())); - definition_event->SetDefinitionEvent(std::move(event), - device.compute_stream()); - } + auto definition_event = std::make_shared(); + TF_ASSIGN_OR_RETURN( + EventPool::Handle event, + device.event_pool().ThenAllocateAndRecordEvent(device.compute_stream())); + definition_event->SetDefinitionEvent(std::move(event), + device.compute_stream()); + Shape on_host_shape = result_buffer.ValueOrDie().on_host_shape(); std::shared_ptr out_buffer = SharedDeviceBuffer::FromScopedShapedBuffer( diff --git a/tensorflow/compiler/xla/python/local_client.h b/tensorflow/compiler/xla/python/local_client.h index ba9227e2c2f..1cf1b39226e 100644 --- a/tensorflow/compiler/xla/python/local_client.h +++ b/tensorflow/compiler/xla/python/local_client.h @@ -50,11 +50,6 @@ Status RegisterCpuCustomCallTarget(const std::string& fn_name, // can perform computation and transfers. class Device { public: - // If use_multiple_streams is true, we allocate separate streams for compute - // and transfers. If it is false, we share a single stream for compute and - // transfers. The CPU device does not support multiple streams, and this is - // a workaround until it does. - // // If synchronous_deallocation is true, the host must not free buffers until // compute/transfers that use those buffers have completed. For example, this // typically is the case for the "platform" where compute/transfers are @@ -62,12 +57,10 @@ class Device { // // If asynchronous is false, the host will synchronize to the device after // each execution or transfer. This is intended for debugging only. - Device(se::StreamExecutor* executor, bool use_multiple_streams, - bool synchronous_deallocation, bool asynchronous, - bool allow_event_reuse); + Device(se::StreamExecutor* executor, bool synchronous_deallocation, + bool asynchronous, bool allow_event_reuse); virtual ~Device(); - bool use_multiple_streams() const { return use_multiple_streams_; } bool synchronous_deallocation() const { return synchronous_deallocation_; } bool asynchronous() const { return asynchronous_; } @@ -142,15 +135,14 @@ class Device { private: Status SynchronizeAllActivity(); - bool use_multiple_streams_; bool synchronous_deallocation_; bool asynchronous_; EventPool event_pool_; - std::shared_ptr compute_stream_; - std::shared_ptr host_to_device_stream_; - std::shared_ptr device_to_host_stream_; - std::vector> device_to_device_streams_; + std::unique_ptr compute_stream_; + std::unique_ptr host_to_device_stream_; + std::unique_ptr device_to_host_stream_; + std::vector> device_to_device_streams_; // Number of device-to-device streams to create in the multistream case. static constexpr int kNumDeviceToDeviceStreams = 4; @@ -161,7 +153,7 @@ class Device { // Callback stream is used for running short host-side callbacks after device // side events, without preventing the device-side stream from doing useful // work. - std::shared_ptr callback_stream_; + std::unique_ptr callback_stream_; std::unique_ptr worker_thread_; }; diff --git a/tensorflow/stream_executor/host/BUILD b/tensorflow/stream_executor/host/BUILD index 80beaa4e4f7..c8f833f8777 100644 --- a/tensorflow/stream_executor/host/BUILD +++ b/tensorflow/stream_executor/host/BUILD @@ -111,6 +111,7 @@ cc_library( "//tensorflow/stream_executor:stream_executor_pimpl", "//tensorflow/stream_executor:timer", "//tensorflow/stream_executor/lib", + "@com_google_absl//absl/synchronization", ], alwayslink = True, ) diff --git a/tensorflow/stream_executor/host/host_gpu_executor.cc b/tensorflow/stream_executor/host/host_gpu_executor.cc index ad25dd453da..f17e188cbc4 100644 --- a/tensorflow/stream_executor/host/host_gpu_executor.cc +++ b/tensorflow/stream_executor/host/host_gpu_executor.cc @@ -19,12 +19,14 @@ limitations under the License. #include +#include "absl/synchronization/notification.h" #include "tensorflow/core/platform/profile_utils/cpu_utils.h" #include "tensorflow/stream_executor/host/host_platform_id.h" #include "tensorflow/stream_executor/host/host_stream.h" #include "tensorflow/stream_executor/host/host_timer.h" #include "tensorflow/stream_executor/lib/statusor.h" #include "tensorflow/stream_executor/plugin_registry.h" +#include "tensorflow/stream_executor/stream_executor_internal.h" namespace stream_executor { namespace host { @@ -167,6 +169,61 @@ bool HostExecutor::CreateStreamDependency(Stream *dependent, Stream *other) { return true; } +class HostEvent : public internal::EventInterface { + public: + HostEvent() : notification_(std::make_shared()) {} + + std::shared_ptr ¬ification() { return notification_; } + + private: + // We use a std::shared_ptr here because the client may delete the HostEvent + // object while there are still RecordEvent and WaitForEvent callbacks pending + // on a stream. + std::shared_ptr notification_; +}; + +std::unique_ptr +HostExecutor::CreateEventImplementation() { + return std::unique_ptr(new HostEvent()); +} + +static HostEvent *AsHostEvent(Event *event) { + DCHECK(event != nullptr); + return static_cast(event->implementation()); +} + +port::Status HostExecutor::AllocateEvent(Event * /*event*/) { + return port::Status::OK(); +} + +port::Status HostExecutor::DeallocateEvent(Event * /*event*/) { + return port::Status::OK(); +} + +port::Status HostExecutor::RecordEvent(Stream *stream, Event *event) { + std::shared_ptr notification = + AsHostEvent(event)->notification(); + AsHostStream(stream)->EnqueueTask([notification]() { + CHECK(!notification->HasBeenNotified()); + notification->Notify(); + }); + return port::Status::OK(); +} + +port::Status HostExecutor::WaitForEvent(Stream *stream, Event *event) { + std::shared_ptr notification = + AsHostEvent(event)->notification(); + AsHostStream(stream)->EnqueueTask( + [notification]() { notification->WaitForNotification(); }); + return port::Status::OK(); +} + +Event::Status HostExecutor::PollForEventStatus(Event *event) { + absl::Notification ¬ification = *AsHostEvent(event)->notification(); + return notification.HasBeenNotified() ? Event::Status::kComplete + : Event::Status::kPending; +} + bool HostExecutor::StartTimer(Stream *stream, Timer *timer) { dynamic_cast(timer->implementation())->Start(stream); return true; diff --git a/tensorflow/stream_executor/host/host_gpu_executor.h b/tensorflow/stream_executor/host/host_gpu_executor.h index a1dbb9f40e9..dfe43e1f43a 100644 --- a/tensorflow/stream_executor/host/host_gpu_executor.h +++ b/tensorflow/stream_executor/host/host_gpu_executor.h @@ -106,25 +106,11 @@ class HostExecutor : public internal::StreamExecutorInterface { bool HostCallback(Stream *stream, std::function callback) override; - port::Status AllocateEvent(Event *event) override { - return port::Status(port::error::UNIMPLEMENTED, ""); - } - - port::Status DeallocateEvent(Event *event) override { - return port::Status(port::error::UNIMPLEMENTED, ""); - } - - port::Status RecordEvent(Stream *stream, Event *event) override { - return port::Status(port::error::UNIMPLEMENTED, ""); - } - - port::Status WaitForEvent(Stream *stream, Event *event) override { - return port::Status(port::error::UNIMPLEMENTED, ""); - } - - Event::Status PollForEventStatus(Event *event) override { - return Event::Status::kError; - } + port::Status AllocateEvent(Event *event) override; + port::Status DeallocateEvent(Event *event) override; + port::Status RecordEvent(Stream *stream, Event *event) override; + port::Status WaitForEvent(Stream *stream, Event *event) override; + Event::Status PollForEventStatus(Event *event) override; bool AllocateStream(Stream *stream) override; void DeallocateStream(Stream *stream) override; @@ -190,10 +176,7 @@ class HostExecutor : public internal::StreamExecutorInterface { rng::RngSupport *CreateRng() override; std::unique_ptr CreateEventImplementation() - override { - LOG(WARNING) << "Events not currently supported by HostExecutor."; - return nullptr; - } + override; std::unique_ptr CreateKernelImplementation() override { From b2b1c7181fe1ed116883ef677dbd3ada403b32fa Mon Sep 17 00:00:00 2001 From: Sergei Lebedev Date: Thu, 11 Jul 2019 06:09:44 -0700 Subject: [PATCH 259/332] Use self->status in EagerTensor_numpy and EagerTensor_getbuffer There is no reason to create a new status on each call. PiperOrigin-RevId: 257594441 --- tensorflow/python/eager/pywrap_tensor.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc index 3c129fe80bd..ab512e9e630 100644 --- a/tensorflow/python/eager/pywrap_tensor.cc +++ b/tensorflow/python/eager/pywrap_tensor.cc @@ -657,10 +657,11 @@ static PyObject* EagerTensor_copy_to_device(EagerTensor* self, PyObject* args, // other. // Note that if `self` is not on CPU, we raise an Exception. static PyObject* EagerTensor_numpy(EagerTensor* self) { - auto status = tensorflow::make_safe(TF_NewStatus()); - auto* py_array = TFE_TensorHandleToNumpy(self->handle, status.get()); - if (MaybeRaiseExceptionFromTFStatus(status.get(), PyExc_ValueError)) { + auto* py_array = TFE_TensorHandleToNumpy(self->handle, self->status); + if (MaybeRaiseExceptionFromTFStatus(self->status, PyExc_ValueError)) { Py_XDECREF(py_array); + // Cleanup self->status before returning. + TF_SetStatus(self->status, TF_OK, ""); return nullptr; } else { return PyArray_Return(reinterpret_cast(py_array)); @@ -745,13 +746,14 @@ static int EagerTensor_getbuffer(EagerTensor* self, Py_buffer* view, return -1; } - auto status = tensorflow::make_safe(TF_NewStatus()); // TensorHandleToNumpy is zero-copy for everything but DT_RESOURCE and // DT_STRING so the following is only slightly slower than a NumPy-free // implementation. auto py_array = tensorflow::make_safe( - TFE_TensorHandleToNumpy(self->handle, status.get())); - if (MaybeRaiseExceptionFromTFStatus(status.get(), PyExc_BufferError)) { + TFE_TensorHandleToNumpy(self->handle, self->status)); + if (MaybeRaiseExceptionFromTFStatus(self->status, PyExc_BufferError)) { + // Cleanup self->status before returning. + TF_SetStatus(self->status, TF_OK, ""); return -1; } if (PyObject_GetBuffer(py_array.get(), view, flags) < 0) { From f4e03226b6eee1e8576a7758d6bf640c3c18a9ec Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Thu, 11 Jul 2019 06:18:20 -0700 Subject: [PATCH 260/332] Typo fix in tf.where() docstring. PiperOrigin-RevId: 257595324 --- tensorflow/python/ops/array_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 041ae2d8526..0a65d19c014 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -3588,7 +3588,7 @@ def where(condition, x=None, y=None, name=None): If both non-None, `x` and `y` must have the same shape. The `condition` tensor must be a scalar if `x` and `y` are scalar. - If `x` and `y` are vectors of higher rank, then `condition` must be either a + If `x` and `y` are tensors of higher rank, then `condition` must be either a vector with size matching the first dimension of `x`, or must have the same shape as `x`. From 46be09906137d785649637dbc28021ec5325e76b Mon Sep 17 00:00:00 2001 From: Sergei Lebedev Date: Thu, 11 Jul 2019 06:32:23 -0700 Subject: [PATCH 261/332] Added benchmarks for tf.convert_to_tensor and tf.constant performance issues PiperOrigin-RevId: 257596912 --- tensorflow/python/eager/benchmarks_test.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py index 1799969e720..e773e0dfcd2 100644 --- a/tensorflow/python/eager/benchmarks_test.py +++ b/tensorflow/python/eager/benchmarks_test.py @@ -935,6 +935,26 @@ class MicroBenchmarks(test.Benchmark): self._run(fn, 10000) + def benchmark_convert_3x_list_to_tensor(self): + xs = [1, 2, 3] + self._run(lambda: ops.convert_to_tensor(xs), 1000) + + def benchmark_convert_3x_array_to_tensor(self): + xs = np.array([1, 2, 3], dtype=np.int32) + self._run(lambda: ops.convert_to_tensor(xs), 1000) + + def benchmark_constant_40x2_list_to_tensor(self): + xs = [[0] * 2] * 40 + self._run(lambda: constant_op.constant(xs), 1000) + + def benchmark_constant_40x2_array_to_tensor(self): + xs = np.array([[0] * 2] * 40, dtype=np.int32) + self._run(lambda: constant_op.constant(xs), 1000) + + def benchmark_constant_40x_list_of_2x_arrays_to_tensor(self): + xs = [np.array([0] * 2, dtype=np.int32)] * 40 + self._run(lambda: constant_op.constant(xs), 1000) + def _benchmarkFunctionWithResourceInputs(self, num_resources, num_iters): @def_function.function def add_all(*args): From eaafddf7e64391c79a0cf3b31f50c33275e38a40 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 11 Jul 2019 06:38:24 -0700 Subject: [PATCH 262/332] [XLA:Python] Simplify the Device::ThenRelease() helper API. Consolidate on the variant that calls destructors from a worker thread; of the two uses of the non-threaded version, one was buggy since it might have freed GPU buffers, and for the other the benefit seemed minimal. Also remove overly complicated and confusing code that tried to control which thread released a reference; the only thing that matters is that on GPU references to GPU memory aren't released from a device callback. Change PythonRefManager so the RAII class it returns is non-copyable but is wrapped in a std::shared_ptr<>. This makes its copying semantics easier to reason about. PiperOrigin-RevId: 257597653 --- .../compiler/xla/python/local_client.cc | 12 +++--- tensorflow/compiler/xla/python/local_client.h | 40 +++++-------------- .../compiler/xla/python/python_ref_manager.cc | 6 +-- .../compiler/xla/python/python_ref_manager.h | 7 ++-- 4 files changed, 21 insertions(+), 44 deletions(-) diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index 4b313a4181e..b8b5eea59fe 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -339,8 +339,7 @@ static StatusOr> TransferHostToDeviceAsync( SharedDeviceBuffer::FromScopedShapedBuffer(std::move(buffer), definition_event); if (device->synchronous_deallocation()) { - device->ThenReleaseOnWorkerThread(device->host_to_device_stream(), - device_buffer); + device->ThenRelease(device->host_to_device_stream(), device_buffer); } return absl::make_unique(shape, std::move(device_buffer), std::move(client)); @@ -359,6 +358,7 @@ StatusOr> PyLocalBuffer::FromPython( // remain live until the transfer is complete. auto py_buffer_ref = client->py_ref_manager().ManageReferences(absl::MakeSpan(tree.arrays)); + tree.arrays.clear(); // We are done manipulating Python objects; release the GIL. py::gil_scoped_release gil_release; @@ -424,8 +424,7 @@ StatusOr> PyLocalBuffer::FromPython( device.host_to_device_stream()); if (device.synchronous_deallocation()) { - device.ThenReleaseOnWorkerThread(device.host_to_device_stream(), - std::move(tuple_buffer)); + device.ThenRelease(device.host_to_device_stream(), std::move(tuple_buffer)); } return buffer; } @@ -732,10 +731,9 @@ StatusOr> PyLocalExecutable::ExecuteHelper( if (device.synchronous_deallocation()) { device_buffers.push_back(out_buffer); - device.ThenReleaseOnWorkerThread(device.compute_stream(), - std::move(device_buffers)); + device.ThenRelease(device.compute_stream(), std::move(device_buffers)); } - device.ThenReleaseOnWorkerThread(device.compute_stream(), executable_); + device.ThenRelease(device.compute_stream(), executable_); return absl::make_unique(on_host_shape, std::move(out_buffer), client_); } diff --git a/tensorflow/compiler/xla/python/local_client.h b/tensorflow/compiler/xla/python/local_client.h index 1cf1b39226e..caed279aed2 100644 --- a/tensorflow/compiler/xla/python/local_client.h +++ b/tensorflow/compiler/xla/python/local_client.h @@ -94,42 +94,20 @@ class Device { void ThenExecuteOnWorkerThread(se::Stream* stream, std::function callback) const; - // Helper for releasing values from a callback at the tail of a stream. - // This is only permitted if object's destructor will not free any device - // objects, since the callback may be called from a device thread pool on - // GPU. + // Helpers for releasing values on a worker thread at the tail of a stream on + // a worker thread. Copies `object`, and destroys the copy when the tail of + // the stream is reached. The destruction happens either in the caller's + // thread or on the worker thread (depending on thread schedules), not a + // device callback, so it is safe if the destructor frees device resource + // (e.g., GPU objects). + // TODO(phawkins): use move-capture when we can use C++14 features. template void ThenRelease(se::Stream* stream, T object) const { if (callback_stream_.get() != stream) { callback_stream_->ThenWaitFor(stream); } - callback_stream_->ThenDoHostCallback( - std::bind([](T& object) { /* releases object */ }, std::move(object))); - } - - // Helpers for releasing values on a worker thread at the tail of a stream on - // a worker thread. - template - void ThenReleaseOnWorkerThread(se::Stream* stream, - std::shared_ptr object) const { - // We use a non-smart pointer here because we want to ensure that the worker - // thread is the only callee of the shared_ptr destructor, and if we passed - // object by lambda capture we have a race where the worker thread might - // run and release its reference first. - auto* ref = new std::shared_ptr(std::move(object)); - if (callback_stream_.get() != stream) { - callback_stream_->ThenWaitFor(stream); - } - ThenExecuteOnWorkerThread(callback_stream_.get(), [ref]() { delete ref; }); - } - template - void ThenReleaseOnWorkerThread(se::Stream* stream, - std::vector> object) const { - auto* ref = new std::vector>(std::move(object)); - if (callback_stream_.get() != stream) { - callback_stream_->ThenWaitFor(stream); - } - ThenExecuteOnWorkerThread(callback_stream_.get(), [ref]() { delete ref; }); + ThenExecuteOnWorkerThread(callback_stream_.get(), + [object]() { /* releases object */ }); } private: diff --git a/tensorflow/compiler/xla/python/python_ref_manager.cc b/tensorflow/compiler/xla/python/python_ref_manager.cc index 4ca6be7d09d..1e9cc58d090 100644 --- a/tensorflow/compiler/xla/python/python_ref_manager.cc +++ b/tensorflow/compiler/xla/python/python_ref_manager.cc @@ -37,9 +37,9 @@ PythonRefManager::ManagedPyObjects::~ManagedPyObjects() { } } -PythonRefManager::ManagedPyObjects PythonRefManager::ManageReferences( - absl::Span objects) { - return ManagedPyObjects(this, objects); +std::shared_ptr +PythonRefManager::ManageReferences(absl::Span objects) { + return std::make_shared(this, objects); } void PythonRefManager::CollectGarbage() { diff --git a/tensorflow/compiler/xla/python/python_ref_manager.h b/tensorflow/compiler/xla/python/python_ref_manager.h index 655f16a9921..8be19336a89 100644 --- a/tensorflow/compiler/xla/python/python_ref_manager.h +++ b/tensorflow/compiler/xla/python/python_ref_manager.h @@ -48,9 +48,9 @@ class PythonRefManager { ~ManagedPyObjects(); - ManagedPyObjects(const ManagedPyObjects& other) = default; + ManagedPyObjects(const ManagedPyObjects& other) = delete; ManagedPyObjects(ManagedPyObjects&& other) = default; - ManagedPyObjects& operator=(const ManagedPyObjects& other) = default; + ManagedPyObjects& operator=(const ManagedPyObjects& other) = delete; ManagedPyObjects& operator=(ManagedPyObjects&& other) = default; private: @@ -61,7 +61,8 @@ class PythonRefManager { // Creates a managed std::shared_ptr to an object. When the shared_ptr is // destroyed, the reference to 'object' will be added to python_garbage_, // and collected next time CollectGarbage() is called. - ManagedPyObjects ManageReferences(absl::Span objects); + std::shared_ptr ManageReferences( + absl::Span objects); // Releases the contents of python_garbage_. Requires that the GIL is held. // The client calls this method during API entry points where the GIL is held From 1fd5537b74cd359191c8a5d2773d5dfa81882689 Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Thu, 11 Jul 2019 06:38:34 -0700 Subject: [PATCH 263/332] Bug fix for nest.pack_sequence_as handling of TypeSpecs. PiperOrigin-RevId: 257597675 --- tensorflow/python/util/nest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py index 3e02efffa9a..d43720f0ed8 100644 --- a/tensorflow/python/util/nest.py +++ b/tensorflow/python/util/nest.py @@ -142,8 +142,6 @@ def _sequence_like(instance, args): elif _is_type_spec(instance): # Pack a CompositeTensor's components according to a TypeSpec. assert len(args) == 1 - if args[0] and _is_type_spec(args[0][0]): - raise ValueError("Can not pack TypeSpec into a TypeSpec.") return instance._from_components(args[0]) # pylint: disable=protected-access elif isinstance(instance, _six.moves.range): return _sequence_like(list(instance), args) From ea06373c38efbd847267ddc6e90b76c0fc3a2ace Mon Sep 17 00:00:00 2001 From: Doe Hyun Yoon Date: Thu, 11 Jul 2019 07:21:11 -0700 Subject: [PATCH 264/332] Handle non-loop merge input in VirtualScheduler to correctly. The way VirtualScheduler currently checks node ready is to count how many input nodes are ready; it's possible that a certain pattern (e.g., Switch - Merge) can break this assumption; both outputs of Switch is triggered, and then Merge becomes ready twice. If a Merge node is executed more than once, we skip triggering its output node. PiperOrigin-RevId: 257603202 --- .../core/grappler/costs/virtual_scheduler.cc | 17 +- .../grappler/costs/virtual_scheduler_test.cc | 164 +++++++++++++----- 2 files changed, 135 insertions(+), 46 deletions(-) diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc index 1a38df27dc5..44e94b83c7a 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler.cc +++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc @@ -807,6 +807,12 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) { // Update graph_costs_ and per-op costs. const NodeDef* node = ready_nodes_->GetCurrNode(); auto& node_state = node_map_[node]; + // TODO(dyoon, andiryxu): Consider to revisit node execution w.r.t. Switch and + // Merge -- it can create a loop which may include loop-carried dependency, + // diverge-merge, and other complex execution patterns. + bool previously_executed_merge = + IsMerge(*node) && (node_state.time_finished != Costs::Duration::max()); + // If there is annotation in the graph about execution times, we use that // number, otherwise, we assume the node is executed once. node_state.execution_count = node->attr().count(kExecutionCount) == 0 @@ -876,8 +882,15 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) { << ", scheduled: " << node_state.time_scheduled.count() << ", finished: " << node_state.time_finished.count(); - // Checks outputs, and adds ready nodes to queue. - AddOutputNodesToReadyQueue(node, curr_time); + if (previously_executed_merge) { + // Skip AddOutputNodesToReadyQueue; this is due to Switch-Merge. + VLOG(1) << "node [ " << node->name() << ", " << node->op() << " ] " + << "is executed more than once. " + << "Skip scheduling its output nodes."; + } else { + // Checks outputs, and adds ready nodes to queue. + AddOutputNodesToReadyQueue(node, curr_time); + } // Increment num_outputs_executed of the input nodes and maybe update memory. for (const auto& input_port : node_state.inputs) { diff --git a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc index 21ca9ca4fc6..588bfce5e90 100644 --- a/tensorflow/core/grappler/costs/virtual_scheduler_test.cc +++ b/tensorflow/core/grappler/costs/virtual_scheduler_test.cc @@ -625,10 +625,10 @@ class TestVirtualScheduler : public VirtualScheduler { public: TestVirtualScheduler(const bool use_static_shapes, const bool use_aggressive_shape_inference, - Cluster* cluster) + ReadyNodeManager* ready_node_manager, Cluster* cluster) : VirtualScheduler( use_static_shapes, use_aggressive_shape_inference, cluster, - &ready_node_manager_, + ready_node_manager, absl::make_unique(cluster->GetDevices())) { enable_mem_usage_tracking(); } @@ -638,9 +638,6 @@ class TestVirtualScheduler : public VirtualScheduler { FRIEND_TEST(VirtualSchedulerTest, ComplexDependency); FRIEND_TEST(VirtualSchedulerTest, Variable); FRIEND_TEST(VirtualSchedulerTest, InterDeviceTransfer); - - protected: - FirstReadyManager ready_node_manager_; }; class VirtualSchedulerTest : public ::testing::Test { @@ -659,7 +656,8 @@ class VirtualSchedulerTest : public ::testing::Test { cluster_ = absl::make_unique(devices); scheduler_ = absl::make_unique( /*use_static_shapes=*/true, - /*use_aggressive_shape_inference=*/true, cluster_.get()); + /*use_aggressive_shape_inference=*/true, &first_ready_manager_, + cluster_.get()); } DeviceProperties GetDummyCPUDevice() { @@ -689,12 +687,10 @@ class VirtualSchedulerTest : public ::testing::Test { auto c0 = ops::Conv2D(s.WithOpName("c0"), x, f, strides, "SAME"); auto c1 = ops::Conv2D(s.WithOpName("c1"), y, f, strides, "SAME"); auto c2 = ops::Conv2D(s.WithOpName("c2"), z, f, strides, "SAME"); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_conv2d_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"c0", "c1"}; dependency_["c0"] = {"x", "f"}; @@ -710,12 +706,11 @@ class VirtualSchedulerTest : public ::testing::Test { {kernel_, kernel_, depth_in_, depth_out_}, DT_FLOAT); std::vector strides = {1, 1, 1, 1}; auto y = ops::Conv2D(s.WithOpName("y"), x, f, strides, "SAME"); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_conv2d_var_graph"; - grappler_item_->graph = def; + grappler_item_->fetch = {"y"}; dependency_["y"] = {"x", "f"}; @@ -740,12 +735,9 @@ class VirtualSchedulerTest : public ::testing::Test { auto abcd = ops::MatMul(s.WithOpName("abcd"), abc, d); auto abcde = ops::MatMul(s.WithOpName("abcde"), abcd, e); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_matmul_sequence_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"abcde"}; dependency_["ab"] = {"a", "b"}; @@ -763,12 +755,10 @@ class VirtualSchedulerTest : public ::testing::Test { auto w = ops::RandomUniform(s.WithOpName("w"), {10, 10, 10, 10}, DT_FLOAT); OutputList input_tensors = {x, y, z, w}; auto out = ops::AddN(s.WithOpName("out"), input_tensors); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_addn_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"out"}; dependency_["out"] = {"x", "y", "z", "w"}; @@ -780,12 +770,10 @@ class VirtualSchedulerTest : public ::testing::Test { auto unnecessary = ops::Placeholder(s.WithOpName("unnecessary"), DT_FLOAT); auto x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); - grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_extra_placeholders"; - grappler_item_->graph = def; grappler_item_->fetch = {"x"}; // Grappler Item Builder puts all placeholder nodes into the feed @@ -804,17 +792,62 @@ class VirtualSchedulerTest : public ::testing::Test { } auto out = ops::NoOp(s.WithControlDependencies(input_tensors).WithOpName("out")); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + grappler_item_->id = "test_control_dependency_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"out"}; dependency_["out"] = input_noop_names; } + void CreateGrapplerItemWithAddFromOneTensor() { + Scope s = Scope::NewRootScope().WithDevice(kCPU0); + auto x = tensorflow::ops::RandomUniform( + s.WithOpName("x"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); + + auto y = tensorflow::ops::Add(s.WithOpName("y"), x, x); + Output fetch = ops::Identity(s.WithOpName("fetch"), y); + + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + + grappler_item_->id = "test_add_from_one_tensor"; + grappler_item_->fetch = {"fetch"}; + + dependency_["fetch"] = {"y"}; + dependency_["y"] = {"x"}; + } + + void CreateGrapplerItemWithSwitchMergeInput() { + // sw = Switch(x, pred) + // a = Add(S:1, b) + // m = Merge(sw:0, a) + // y = Add(m, z) + + Scope s = Scope::NewRootScope().WithDevice(kCPU0); + auto x = ops::RandomUniform( + s.WithOpName("x"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); + auto pred = ops::Const(s.WithOpName("pred"), false, {}); + auto sw = ops::Switch(s.WithOpName("switch"), x, pred); + auto b = ops::RandomUniform( + s.WithOpName("b"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); + auto a = ops::Add(s.WithOpName("a"), sw.output_true, b); + auto m = ops::Merge(s.WithOpName("m"), {sw.output_false, a.z}); + auto z = ops::RandomUniform( + s.WithOpName("z"), {batch_size_, width_, height_, depth_in_}, DT_FLOAT); + auto y = ops::Add(s.WithOpName("y"), m.output, z); + + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); + + grappler_item_->id = "test_add_merge_switch"; + grappler_item_->fetch = {"y"}; + + dependency_["y"] = {"m", "z"}; + } + // FusedBN [an op with multiple outputs] with multiple consumers (including // control dependency). void CreateGrapplerItemWithBatchNorm() { @@ -846,12 +879,10 @@ class VirtualSchedulerTest : public ::testing::Test { }; auto z4 = ops::NoOp(s.WithControlDependencies(batch_var).WithOpName("z4")); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); - grappler_item_.reset(new GrapplerItem); grappler_item_->id = "test_complex_dependency_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"z1", "z2", "z3", "z4"}; dependency_["bn"] = {"x", "scale", "offset", "mean", "var"}; @@ -975,7 +1006,8 @@ versions { } )EOF"; - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -1032,7 +1064,7 @@ versions { } )EOF"; - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -1428,7 +1460,7 @@ versions { } )EOF"; - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -2095,7 +2127,7 @@ versions { producer: 27 })EOF"; - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); CHECK(protobuf::TextFormat::ParseFromString(gdef_ascii, &grappler_item_->graph)); grappler_item_->id = "test_graph"; @@ -2136,12 +2168,9 @@ versions { .WithControlDependencies(y) .WithDevice(kCPU1)); - GraphDef def; - TF_CHECK_OK(s.ToGraphDef(&def)); - - grappler_item_.reset(new GrapplerItem); + grappler_item_ = absl::make_unique(); + TF_CHECK_OK(s.ToGraphDef(&grappler_item_->graph)); grappler_item_->id = "test_conv2d_graph"; - grappler_item_->graph = def; grappler_item_->fetch = {"y1", "y2", "batch_mean1", "batch_var1", "control_dep"}; @@ -2280,6 +2309,8 @@ versions { // cluster_ and scheduler_ are initialized in the c'tor. std::unique_ptr cluster_; std::unique_ptr scheduler_; + FirstReadyManager first_ready_manager_; + CompositeNodeManager composite_node_manager_; // grappler_item_ will be initialized differently for each test case. std::unique_ptr grappler_item_; @@ -2933,6 +2964,51 @@ TEST_F(VirtualSchedulerTest, GraphWihtOnlyRecv) { EXPECT_GT(ops_executed.count("Recv"), 0); } +TEST_F(VirtualSchedulerTest, AddMergeSwitch) { + // Override scheduler_ with CompositeNodeNamager. + scheduler_ = absl::make_unique( + /*use_static_shapes=*/true, + /*use_aggressive_shape_inference=*/true, &composite_node_manager_, + cluster_.get()); + CreateGrapplerItemWithSwitchMergeInput(); + InitScheduler(); + + // pred --+ z --+ + // | | + // V V + // x -> Switch --------> Merge ---> Add --> y + // | ^ + // | | + // +-----> Add -----+ + // ^ + // | + // b --------------+ + + // Run the scheduler. The current VirtualScheduler, w/o annotation, triggers + // both outputs of Switch; then Merge (as long as one input is ready, it's z + // is ready, if we just use num_inputs_ready counter, the final Add becomes + // ready. possible to skipt scheduling z. (Need to use CompositeNodeManager + // to test this case). + auto ops_executed = RunScheduler(""); + + EXPECT_GT(ops_executed.count("z"), 0); +} + +TEST_F(VirtualSchedulerTest, AddFromOneTensor) { + CreateGrapplerItemWithAddFromOneTensor(); + InitScheduler(); + + // x -+----> Add --> y + // | ^ + // | | + // +-------+ + + // Run the scheduler. + auto ops_executed = RunScheduler(""); + EXPECT_GT(ops_executed.count("y"), 0); + EXPECT_GT(ops_executed.count("x"), 0); +} + } // namespace } // end namespace grappler } // end namespace tensorflow From a7b9015ec9780dc46b614785bff4e244811b2c9d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 07:42:56 -0700 Subject: [PATCH 265/332] Support additional conversions. PiperOrigin-RevId: 257605915 --- tensorflow/lite/delegates/gpu/api.cc | 6 ++---- tensorflow/lite/delegates/gpu/api.h | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/api.cc b/tensorflow/lite/delegates/gpu/api.cc index fb11cbc8ecf..ac649658c34 100644 --- a/tensorflow/lite/delegates/gpu/api.cc +++ b/tensorflow/lite/delegates/gpu/api.cc @@ -62,11 +62,9 @@ ObjectType GetType(const TensorObject& object) { return absl::visit(ObjectTypeGetter{}, object); } -bool IsValid(const TensorObject& object) { - return absl::visit(ObjectValidityChecker{DataType::UNKNOWN}, object); -} +bool IsValid(const TensorObjectDef& def) { return IsValid(def.object_def); } -bool IsCompatible(const TensorObjectDef& def, const TensorObject& object) { +bool IsValid(const TensorObjectDef& def, const TensorObject& object) { return GetType(object) == def.object_def.object_type && absl::visit(ObjectValidityChecker{def.object_def.data_type}, object); } diff --git a/tensorflow/lite/delegates/gpu/api.h b/tensorflow/lite/delegates/gpu/api.h index ca0707b6935..00e80956c43 100644 --- a/tensorflow/lite/delegates/gpu/api.h +++ b/tensorflow/lite/delegates/gpu/api.h @@ -168,17 +168,17 @@ struct TensorObjectDef { } }; +// @return true if tensor object def is defined. +bool IsValid(const TensorObjectDef& def); + using TensorObject = absl::variant; // @return true if object is set and corresponding values are defined. -bool IsValid(const TensorObject& object); +bool IsValid(const TensorObjectDef& def, const TensorObject& object); ObjectType GetType(const TensorObject& object); -// @return true if object representation corresponds to the given definition. -bool IsCompatible(const TensorObjectDef& def, const TensorObject& object); - // @return true if corresponding object is set for the given type bool IsObjectPresent(ObjectType type, const TensorObject& obj); From 658cbd4275380d73b5a3271388d28b4159754620 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 11 Jul 2019 07:51:43 -0700 Subject: [PATCH 266/332] [XLA:Python] Generalize stream pacing mechanism. Currently on CPU and GPU there is no limit to how many operations the host can enqueue on the device stream. On GPU this doesn't usually cause a problem because the allocator is logically synchronized to the tail of the compute stream, and so we can free and reuse memory for operations enqueued on the stream. On CPU, the allocator is logically synchronized to the head of the compute stream, which means that the allocator cannot reuse buffers between operations enqueued on the stream. This means that the memory usage is proportional to the number of enqueued operations, which can rapidly blow up. Add a semaphore class and use it to set a moderate limit on the depth of the queue (32). The existing "synchronous" mode, used on TPU at present, is a special case of this support where the queue depth is 1. This may help with https://github.com/google/jax/issues/928 . PiperOrigin-RevId: 257606960 --- tensorflow/compiler/xla/python/BUILD | 24 +++++ .../compiler/xla/python/local_client.cc | 90 ++++++++++++------- tensorflow/compiler/xla/python/local_client.h | 35 +++++--- tensorflow/compiler/xla/python/semaphore.cc | 74 +++++++++++++++ tensorflow/compiler/xla/python/semaphore.h | 67 ++++++++++++++ .../compiler/xla/python/semaphore_test.cc | 74 +++++++++++++++ 6 files changed, 318 insertions(+), 46 deletions(-) create mode 100644 tensorflow/compiler/xla/python/semaphore.cc create mode 100644 tensorflow/compiler/xla/python/semaphore.h create mode 100644 tensorflow/compiler/xla/python/semaphore_test.cc diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD index 81a61b8a5e5..ca36d382395 100644 --- a/tensorflow/compiler/xla/python/BUILD +++ b/tensorflow/compiler/xla/python/BUILD @@ -118,6 +118,29 @@ cc_library( ], ) +cc_library( + name = "semaphore", + srcs = ["semaphore.cc"], + hdrs = ["semaphore.h"], + deps = [ + "//tensorflow/compiler/xla:types", + "//tensorflow/core:lib", + "@com_google_absl//absl/synchronization", + ], +) + +tf_cc_test( + name = "semaphore_test", + srcs = ["semaphore_test.cc"], + deps = [ + ":semaphore", + "//tensorflow/compiler/xla:test", + "//tensorflow/core:lib", + "//tensorflow/core:test_main", + "@com_google_absl//absl/synchronization", + ], +) + cc_library( name = "shared_device_buffer", srcs = ["shared_device_buffer.cc"], @@ -165,6 +188,7 @@ cc_library( features = ["-use_header_modules"], deps = [ ":event_pool", + ":semaphore", ":shared_device_buffer", ":types", ":worker_thread", diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index b8b5eea59fe..49eb2341392 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -18,15 +18,33 @@ limitations under the License. // Asynchronous execution: // ----------------------- // -// If 'asynchronous' is set when constructing the client, computations and -// host-to-device transfers do not block the host waiting for the operation to -// complete but instead return control to the host immediately. This allows -// Python logic to overlap with device-side computation. +// Computations and host-to-device transfers do not need to block the host +// waiting for the operation to complete but instead return control to the host +// immediately. This allows Python logic to overlap with device-side +// computation. // // For a good user experience, we must be careful only to enqueue operations // that are unlikely to fail; as a rule error checking must be done eagerly // before returning control to the client. // +// The degree to which the client can enqueue operations ahead of the client +// is limited by a semaphore. There are at two modes: asynchronous, where we +// allow the client to enqueue up to 32 executions ahead of the device, and +// synchronous, where we limit the client to having one enqueued operation at +// a time. The value of 32 is arbitrary. +// +// Even in asynchronous mode, it is important that we do not permit +// unbounded queue-ahead. Firstly it is problematic when the user does something +// like the following in Python: +// %timeit run_computation() +// To the timeit logic, op() appears to be extremely cheap since it is deferring +// all of its real work and not blocking, and so the %timeit will run op() many +// (e.g., 10000) times to get better timing resolution, even though in reality +// it may be expensive. Secondly, on CPU the allocator is synchronized with the +// head of the compute stream, and we allocate buffers for all of the enqueued +// programs without any reuse (unlike GPU). This means that the memory usage +// is proportional to the queue size. +// // Multi-stream execution: // ----------------------- // @@ -111,8 +129,8 @@ Status RegisterCpuCustomCallTarget(const std::string& fn_name, Device::Device(se::StreamExecutor* executor, bool synchronous_deallocation, bool asynchronous, bool allow_event_reuse) : synchronous_deallocation_(synchronous_deallocation), - asynchronous_(asynchronous), - event_pool_(allow_event_reuse) { + event_pool_(allow_event_reuse), + compute_semaphore_(/*capacity=*/asynchronous ? 32 : 1) { compute_stream_ = absl::make_unique(executor); host_to_device_stream_ = absl::make_unique(executor); device_to_host_stream_ = absl::make_unique(executor); @@ -127,8 +145,10 @@ Device::Device(se::StreamExecutor* executor, bool synchronous_deallocation, stream->Init(); device_to_device_streams_.push_back(std::move(stream)); } - worker_thread_ = absl::make_unique(tensorflow::Env::Default(), - "py_xla_execute"); + execute_thread_ = absl::make_unique(tensorflow::Env::Default(), + "py_xla_execute"); + callback_thread_ = absl::make_unique(tensorflow::Env::Default(), + "py_xla_callback"); } Device::~Device() { @@ -164,10 +184,11 @@ Status Device::ThenMemcpyDeviceToDevice(se::Stream* src_stream, return Status::OK(); } -void Device::ThenExecuteOnWorkerThread(se::Stream* stream, - std::function callback) const { - stream->ThenDoHostCallback( - [this, callback]() { worker_thread_->Schedule(std::move(callback)); }); +void Device::ThenExecuteOnCallbackThread(se::Stream* stream, + std::function callback) const { + stream->ThenDoHostCallback([this, callback]() mutable { + callback_thread_->Schedule(std::move(callback)); + }); } se::Stream* Device::GetDeviceToDeviceStream() { @@ -255,15 +276,14 @@ StatusOr> PyLocalClient::Get( executor, synchronous_deallocation, asynchronous, /*allow_event_reuse=*/gpu_platform)); } - return std::make_shared(platform_name, client, - std::move(devices), - std::move(allocator), asynchronous); + return std::make_shared( + platform_name, client, std::move(devices), std::move(allocator)); } PyLocalClient::PyLocalClient( std::string platform_name, LocalClient* client, std::vector> devices, - std::unique_ptr allocator, bool asynchronous) + std::unique_ptr allocator) : platform_name_(std::move(platform_name)), client_(client), devices_(std::move(devices)), @@ -686,21 +706,21 @@ StatusOr> PyLocalExecutable::ExecuteHelper( << " buffer: " << argument_buffers.back().ToString(); } - Device& device = client_->device(device_ordinal); - // The choice of where we wait in "synchronous" mode is arbitrary; the reason - // for the wait is pacing to avoid problems such as memory fragmentation, not - // for correctness. - if (!device.asynchronous()) { - TF_RETURN_IF_ERROR(device.compute_stream()->BlockHostUntilDone()); - } + Device* device = &client_->device(device_ordinal); + // The choice of where we wait is arbitrary; the reason for the wait is pacing + // to avoid problems such as memory fragmentation and running ahead too far, + // not for correctness. Placing it before the executable launch allows the + // inputs for the next executable to be fetched even if the launch is delayed. + auto compute_reservation = std::make_shared( + device->compute_semaphore().ScopedAcquire(1)); for (BufferDefinitionEvent* event : events) { - event->WaitForEventOnStream(device.compute_stream()); + event->WaitForEventOnStream(device->compute_stream()); } ExecutableRunOptions options; - options.set_stream(device.compute_stream()); - options.set_host_to_device_stream(device.host_to_device_stream()); + options.set_stream(device->compute_stream()); + options.set_host_to_device_stream(device->host_to_device_stream()); options.set_allocator(client_->allocator()); options.set_intra_op_thread_pool( client_->client()->backend().eigen_intra_op_thread_pool_device()); @@ -718,22 +738,24 @@ StatusOr> PyLocalExecutable::ExecuteHelper( } auto definition_event = std::make_shared(); - TF_ASSIGN_OR_RETURN( - EventPool::Handle event, - device.event_pool().ThenAllocateAndRecordEvent(device.compute_stream())); + TF_ASSIGN_OR_RETURN(EventPool::Handle event, + device->event_pool().ThenAllocateAndRecordEvent( + device->compute_stream())); definition_event->SetDefinitionEvent(std::move(event), - device.compute_stream()); + device->compute_stream()); Shape on_host_shape = result_buffer.ValueOrDie().on_host_shape(); std::shared_ptr out_buffer = SharedDeviceBuffer::FromScopedShapedBuffer( std::move(result_buffer.ValueOrDie()), definition_event); - if (device.synchronous_deallocation()) { + if (device->synchronous_deallocation()) { device_buffers.push_back(out_buffer); - device.ThenRelease(device.compute_stream(), std::move(device_buffers)); + device->ThenRelease(device->compute_stream(), std::move(device_buffers)); } - device.ThenRelease(device.compute_stream(), executable_); + + device->ThenRelease(device->compute_stream(), + std::make_pair(executable_, compute_reservation)); return absl::make_unique(on_host_shape, std::move(out_buffer), client_); } @@ -782,7 +804,7 @@ PyLocalExecutable::ExecutePerReplica( for (int replica = 0; replica < num_replicas(); ++replica) { const int device_ordinal = device_assignment_(replica, 0); const Device& device = client_->device(device_ordinal); - device.worker_thread()->Schedule([&, replica] { + device.execute_thread()->Schedule([&, replica] { results[replica] = ExecuteHelper(argument_handles[replica], replica, run_id); diff --git a/tensorflow/compiler/xla/python/local_client.h b/tensorflow/compiler/xla/python/local_client.h index caed279aed2..3ff6fa4bb10 100644 --- a/tensorflow/compiler/xla/python/local_client.h +++ b/tensorflow/compiler/xla/python/local_client.h @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/xla_computation.h" #include "tensorflow/compiler/xla/python/event_pool.h" #include "tensorflow/compiler/xla/python/python_ref_manager.h" +#include "tensorflow/compiler/xla/python/semaphore.h" #include "tensorflow/compiler/xla/python/shared_device_buffer.h" #include "tensorflow/compiler/xla/python/worker_thread.h" #include "tensorflow/compiler/xla/service/computation_placer.h" @@ -62,7 +63,6 @@ class Device { virtual ~Device(); bool synchronous_deallocation() const { return synchronous_deallocation_; } - bool asynchronous() const { return asynchronous_; } EventPool& event_pool() { return event_pool_; } @@ -84,15 +84,14 @@ class Device { se::DeviceMemoryBase src_buffer, se::DeviceMemoryBase dst_buffer); - // A worker thread, used for replicated computation launches and callbacks. - WorkerThread* worker_thread() const { return worker_thread_.get(); } + WorkerThread* execute_thread() const { return execute_thread_.get(); } - // Enqueues a host callback on 'stream', to be executed by worker_thread_. + // Enqueues a host callback on 'stream', to be executed by callback_thread_. // ThenDoHostCallback is often constrained in what it can do, in particular, // on GPU the callback runs on a thread belonging to the GPU runtime and // cannot perform GPU operations itself. - void ThenExecuteOnWorkerThread(se::Stream* stream, - std::function callback) const; + void ThenExecuteOnCallbackThread(se::Stream* stream, + std::function callback) const; // Helpers for releasing values on a worker thread at the tail of a stream on // a worker thread. Copies `object`, and destroys the copy when the tail of @@ -106,17 +105,23 @@ class Device { if (callback_stream_.get() != stream) { callback_stream_->ThenWaitFor(stream); } - ThenExecuteOnWorkerThread(callback_stream_.get(), - [object]() { /* releases object */ }); + ThenExecuteOnCallbackThread(callback_stream_.get(), + [object]() { /* releases object */ }); } + Semaphore& compute_semaphore() { return compute_semaphore_; } + private: Status SynchronizeAllActivity(); bool synchronous_deallocation_; - bool asynchronous_; EventPool event_pool_; + + // Semaphore used to limit how many programs can be enqueued on the compute + // stream by the host ahead of the device. + Semaphore compute_semaphore_; + std::unique_ptr compute_stream_; std::unique_ptr host_to_device_stream_; std::unique_ptr device_to_host_stream_; @@ -133,7 +138,14 @@ class Device { // work. std::unique_ptr callback_stream_; - std::unique_ptr worker_thread_; + // A worker thread, used for replicated computation launches. + std::unique_ptr execute_thread_; + + // A worker thread, used for callbacks. It is necessary that this be a + // different thread to the execute thread because we acquire the compute + // semaphore during calls to Execute but release it from a callback and if + // they are the same thread we might deadlock. + std::unique_ptr callback_thread_; }; struct AllocatorConfig { @@ -168,8 +180,7 @@ class PyLocalClient { // `allocator` may null, in which case the platform default allocator is used. explicit PyLocalClient(std::string platform_name, LocalClient* client, std::vector> devices, - std::unique_ptr allocator, - bool asynchronous); + std::unique_ptr allocator); virtual ~PyLocalClient() = default; Status TransferToInfeed(const LiteralSlice& literal, int device_ordinal); diff --git a/tensorflow/compiler/xla/python/semaphore.cc b/tensorflow/compiler/xla/python/semaphore.cc new file mode 100644 index 00000000000..5926618bddc --- /dev/null +++ b/tensorflow/compiler/xla/python/semaphore.cc @@ -0,0 +1,74 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/python/semaphore.h" + +#include "tensorflow/core/platform/logging.h" + +namespace xla { + +Semaphore::Semaphore(int64 capacity) : value_(capacity) { + CHECK_GE(capacity, 0); +} + +bool Semaphore::CanAcquire(CanAcquireArgs* args) { + return args->semaphore->value_ >= args->amount; +} + +void Semaphore::Acquire(int64 amount) { + CHECK_GE(amount, 0); + + CanAcquireArgs args; + args.semaphore = this; + args.amount = amount; + + mu_.LockWhen(absl::Condition(&CanAcquire, &args)); + value_ -= amount; + mu_.Unlock(); +} + +void Semaphore::Release(int64 amount) { + CHECK_GE(amount, 0); + absl::MutexLock lock(&mu_); + value_ += amount; +} + +Semaphore::ScopedReservation::~ScopedReservation() { + if (semaphore_) { + semaphore_->Release(amount_); + } +} + +Semaphore::ScopedReservation::ScopedReservation( + ScopedReservation&& other) noexcept { + semaphore_ = other.semaphore_; + amount_ = other.amount_; + other.semaphore_ = nullptr; +} + +Semaphore::ScopedReservation& Semaphore::ScopedReservation::operator=( + ScopedReservation&& other) noexcept { + semaphore_ = other.semaphore_; + amount_ = other.amount_; + other.semaphore_ = nullptr; + return *this; +} + +Semaphore::ScopedReservation Semaphore::ScopedAcquire(int64 amount) { + Acquire(amount); + return ScopedReservation(this, amount); +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/python/semaphore.h b/tensorflow/compiler/xla/python/semaphore.h new file mode 100644 index 00000000000..4afd44f4cc0 --- /dev/null +++ b/tensorflow/compiler/xla/python/semaphore.h @@ -0,0 +1,67 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_PYTHON_SEMAPHORE_H_ +#define TENSORFLOW_COMPILER_XLA_PYTHON_SEMAPHORE_H_ + +#include "absl/synchronization/mutex.h" +#include "tensorflow/compiler/xla/types.h" + +namespace xla { + +class Semaphore { + public: + explicit Semaphore(int64 capacity); + + // Acquires `amount` units. Blocks until `amount` units are available. + void Acquire(int64 amount); + + // Returns `amount` units to the semaphore. + void Release(int64 amount); + + class ScopedReservation { + public: + ScopedReservation(Semaphore* semaphore, int64 amount) + : semaphore_(semaphore), amount_(amount) {} + ~ScopedReservation(); + + ScopedReservation(const ScopedReservation&) = delete; + ScopedReservation(ScopedReservation&& other) noexcept; + ScopedReservation& operator=(const ScopedReservation&) = delete; + ScopedReservation& operator=(ScopedReservation&& other) noexcept; + + private: + Semaphore* semaphore_; + int64 amount_; + }; + // RAII version of Acquire. Releases the reservation when the + // ScopedReservation is destroyed. + ScopedReservation ScopedAcquire(int64 amount); + + private: + struct CanAcquireArgs { + Semaphore* semaphore; + int64 amount; + }; + static bool CanAcquire(CanAcquireArgs* args) + EXCLUSIVE_LOCKS_REQUIRED(args->semaphore->mu_); + + absl::Mutex mu_; + int64 value_ GUARDED_BY(mu_); +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_PYTHON_SEMAPHORE_H_ diff --git a/tensorflow/compiler/xla/python/semaphore_test.cc b/tensorflow/compiler/xla/python/semaphore_test.cc new file mode 100644 index 00000000000..5ef59618b8b --- /dev/null +++ b/tensorflow/compiler/xla/python/semaphore_test.cc @@ -0,0 +1,74 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/python/semaphore.h" + +#include "absl/synchronization/notification.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/env.h" + +namespace xla { +namespace { + +TEST(SemaphoreTest, UnthreadedTests) { + Semaphore semaphore(2); + semaphore.Acquire(1); + semaphore.Release(1); + + semaphore.Acquire(2); + semaphore.Release(2); + + semaphore.Acquire(1); + semaphore.Acquire(1); + semaphore.Release(1); + semaphore.Acquire(1); + semaphore.Release(1); + semaphore.Acquire(1); + semaphore.Release(2); + + { + auto a = semaphore.ScopedAcquire(1); + { auto b = semaphore.ScopedAcquire(1); } + { auto c = semaphore.ScopedAcquire(1); } + } +} + +TEST(SemaphoreTest, ConcurrentTest) { + tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "test", 2); + Semaphore semaphore(2); + semaphore.Acquire(1); + + absl::Notification a_done; + pool.Schedule([&]() { + semaphore.Acquire(2); + semaphore.Release(2); + a_done.Notify(); + }); + + absl::Notification b_done; + pool.Schedule([&]() { + semaphore.Acquire(1); + semaphore.Release(1); + b_done.Notify(); + }); + b_done.WaitForNotification(); + EXPECT_FALSE(a_done.HasBeenNotified()); + semaphore.Release(1); + a_done.WaitForNotification(); +} + +} // namespace +} // namespace xla From d1a7879a2590b05191da86b4989e502e70e9d0e3 Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Thu, 11 Jul 2019 08:14:32 -0700 Subject: [PATCH 267/332] In NestedStructureCoder: fixed bug where a TensorSpec with name=None would get deserialized with name=''. PiperOrigin-RevId: 257610414 --- .../python/saved_model/nested_structure_coder.py | 3 ++- .../saved_model/nested_structure_coder_test.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/saved_model/nested_structure_coder.py b/tensorflow/python/saved_model/nested_structure_coder.py index 59a2687edaf..66b02b119d1 100644 --- a/tensorflow/python/saved_model/nested_structure_coder.py +++ b/tensorflow/python/saved_model/nested_structure_coder.py @@ -423,6 +423,7 @@ class _TensorSpecCodec(object): return value.HasField("tensor_spec_value") def do_decode(self, value, decode_fn): + name = value.tensor_spec_value.name return tensor_spec.TensorSpec( shape=decode_fn( struct_pb2.StructuredValue( @@ -430,7 +431,7 @@ class _TensorSpecCodec(object): dtype=decode_fn( struct_pb2.StructuredValue( tensor_dtype_value=value.tensor_spec_value.dtype)), - name=value.tensor_spec_value.name) + name=(name if name else None)) StructureCoder.register_codec(_TensorSpecCodec()) diff --git a/tensorflow/python/saved_model/nested_structure_coder_test.py b/tensorflow/python/saved_model/nested_structure_coder_test.py index 1538fbf1271..16c56b1ddbf 100644 --- a/tensorflow/python/saved_model/nested_structure_coder_test.py +++ b/tensorflow/python/saved_model/nested_structure_coder_test.py @@ -171,6 +171,22 @@ class NestedStructureTest(test.TestCase): decoded = self._coder.decode_proto(encoded) self.assertEqual(structure, decoded) + def testEncodeDecodeTensorSpecWithNoName(self): + structure = [tensor_spec.TensorSpec([1, 2, 3], dtypes.int64)] + self.assertTrue(self._coder.can_encode(structure)) + encoded = self._coder.encode_structure(structure) + expected = struct_pb2.StructuredValue() + expected_list = expected.list_value + expected_tensor_spec = expected_list.values.add().tensor_spec_value + expected_tensor_spec.shape.dim.add().size = 1 + expected_tensor_spec.shape.dim.add().size = 2 + expected_tensor_spec.shape.dim.add().size = 3 + expected_tensor_spec.name = "" + expected_tensor_spec.dtype = dtypes.int64.as_datatype_enum + self.assertEqual(expected, encoded) + decoded = self._coder.decode_proto(encoded) + self.assertEqual(structure, decoded) + def testNotEncodable(self): class NotEncodable(object): From 4bd3d188fdcae463d557101315eb0cedd827237c Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 11 Jul 2019 08:30:50 -0700 Subject: [PATCH 268/332] [XLA:Python] Split Device into its own file. Refactoring only, NFC. Also move RegisterCpuCustomCallTarget into the main Python binding module (xla.cc) since it is unrelated to anything else in local_client.*. PiperOrigin-RevId: 257612668 --- tensorflow/compiler/xla/python/BUILD | 25 +++- tensorflow/compiler/xla/python/device.cc | 100 +++++++++++++ tensorflow/compiler/xla/python/device.h | 134 ++++++++++++++++++ .../compiler/xla/python/local_client.cc | 92 ------------ tensorflow/compiler/xla/python/local_client.h | 113 +-------------- tensorflow/compiler/xla/python/xla.cc | 18 +++ 6 files changed, 274 insertions(+), 208 deletions(-) create mode 100644 tensorflow/compiler/xla/python/device.cc create mode 100644 tensorflow/compiler/xla/python/device.h diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD index ca36d382395..a6a1bd1830e 100644 --- a/tensorflow/compiler/xla/python/BUILD +++ b/tensorflow/compiler/xla/python/BUILD @@ -170,6 +170,23 @@ tf_cc_test( ], ) +cc_library( + name = "device", + srcs = ["device.cc"], + hdrs = ["device.h"], + deps = [ + ":event_pool", + ":semaphore", + ":worker_thread", + "//tensorflow/compiler/xla:status", + "//tensorflow/compiler/xla:util", + "//tensorflow/core:lib", + "//tensorflow/core:stream_executor", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/synchronization", + ], +) + cc_library( name = "local_client", srcs = [ @@ -187,11 +204,9 @@ cc_library( ], features = ["-use_header_modules"], deps = [ - ":event_pool", - ":semaphore", + ":device", ":shared_device_buffer", ":types", - ":worker_thread", "//tensorflow/compiler/xla:executable_run_options", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:literal_util", @@ -205,7 +220,6 @@ cc_library( "//tensorflow/compiler/xla/client:local_client", "//tensorflow/compiler/xla/client:xla_computation", "//tensorflow/compiler/xla/service:computation_placer", - "//tensorflow/compiler/xla/service:custom_call_target_registry", "//tensorflow/compiler/xla/service:platform_util", "//tensorflow/compiler/xla/service:shaped_buffer", "//tensorflow/core:bfc_allocator", @@ -239,11 +253,11 @@ tf_pybind_extension( ":local_client", ":shared_device_buffer", ":types", - ":worker_thread", ":xrt", "@com_google_absl//absl/base", "@com_google_absl//absl/hash", "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", @@ -266,6 +280,7 @@ tf_pybind_extension( "//tensorflow/compiler/xla/client/lib:self_adjoint_eig", "//tensorflow/compiler/xla/client/lib:svd", "//tensorflow/compiler/xla/service:computation_placer", + "//tensorflow/compiler/xla/service:custom_call_target_registry", "//tensorflow/compiler/xla/service:hlo", "//tensorflow/compiler/xla/service:hlo_parser", "//tensorflow/compiler/xla/service:hlo_graph_dumper", diff --git a/tensorflow/compiler/xla/python/device.cc b/tensorflow/compiler/xla/python/device.cc new file mode 100644 index 00000000000..73df698a274 --- /dev/null +++ b/tensorflow/compiler/xla/python/device.cc @@ -0,0 +1,100 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/python/device.h" + +#include +#include + +#include "absl/memory/memory.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { + +Device::Device(se::StreamExecutor* executor, bool synchronous_deallocation, + bool asynchronous, bool allow_event_reuse) + : synchronous_deallocation_(synchronous_deallocation), + event_pool_(allow_event_reuse), + compute_semaphore_(/*capacity=*/asynchronous ? 32 : 1) { + compute_stream_ = absl::make_unique(executor); + host_to_device_stream_ = absl::make_unique(executor); + device_to_host_stream_ = absl::make_unique(executor); + callback_stream_ = absl::make_unique(executor); + compute_stream_->Init(); + host_to_device_stream_->Init(); + device_to_host_stream_->Init(); + callback_stream_->Init(); + device_to_device_streams_.reserve(kNumDeviceToDeviceStreams); + for (int i = 0; i < kNumDeviceToDeviceStreams; ++i) { + auto stream = absl::make_unique(executor); + stream->Init(); + device_to_device_streams_.push_back(std::move(stream)); + } + execute_thread_ = absl::make_unique(tensorflow::Env::Default(), + "py_xla_execute"); + callback_thread_ = absl::make_unique(tensorflow::Env::Default(), + "py_xla_callback"); +} + +Device::~Device() { + Status status = SynchronizeAllActivity(); + if (!status.ok()) { + LOG(ERROR) << "Error when closing device: " << status; + } +} + +Status Device::SynchronizeAllActivity() { + Status status; + // TODO(phawkins): in theory the call to SynchronizeAllActivity below should + // suffice. However on the Host platform SynchronizeAllActivity is a dummy + // implementation that doesn't actually block. To make sure activity has + // stopped, also block on the compute stream. If SynchronizeAllActivity is + // fixed, we could remove the BlockHostUntilDone call. + status.Update(compute_stream_->BlockHostUntilDone()); + bool ok = compute_stream_->parent()->SynchronizeAllActivity(); + if (!ok) { + status.Update(Unknown("SynchronizeAllActivity failed.")); + } + return status; +} + +Status Device::ThenMemcpyDeviceToDevice(se::Stream* src_stream, + se::Stream* dst_stream, + se::DeviceMemoryBase src_buffer, + se::DeviceMemoryBase dst_buffer) { + // The default implementation simply calls ThenMemcpyD2D, and assumes that + // the buffer addresses identify the devices. This does not work + // on all platforms; this method is virtual so it can be overridden. + src_stream->ThenMemcpyD2D(&dst_buffer, src_buffer, dst_buffer.size()); + return Status::OK(); +} + +void Device::ThenExecuteOnCallbackThread(se::Stream* stream, + std::function callback) const { + stream->ThenDoHostCallback([this, callback]() mutable { + callback_thread_->Schedule(std::move(callback)); + }); +} + +se::Stream* Device::GetDeviceToDeviceStream() { + absl::MutexLock lock(&mu_); + int i = next_device_to_device_stream_; + next_device_to_device_stream_ = + (next_device_to_device_stream_ + 1) % device_to_device_streams_.size(); + return device_to_device_streams_.at(i).get(); +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/python/device.h b/tensorflow/compiler/xla/python/device.h new file mode 100644 index 00000000000..f40c5df7c61 --- /dev/null +++ b/tensorflow/compiler/xla/python/device.h @@ -0,0 +1,134 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_PYTHON_DEVICE_H_ +#define TENSORFLOW_COMPILER_XLA_PYTHON_DEVICE_H_ + +#include +#include + +#include "absl/synchronization/mutex.h" +#include "tensorflow/compiler/xla/python/event_pool.h" +#include "tensorflow/compiler/xla/python/semaphore.h" +#include "tensorflow/compiler/xla/python/worker_thread.h" +#include "tensorflow/compiler/xla/status.h" +#include "tensorflow/core/platform/stream_executor.h" + +namespace xla { + +// Class that encapsulates state relating to a device (e.g., a GPU) on which we +// can perform computation and transfers. +class Device { + public: + // If synchronous_deallocation is true, the host must not free buffers until + // compute/transfers that use those buffers have completed. For example, this + // typically is the case for the "platform" where compute/transfers are + // operations that take place on another thread. + // + // If asynchronous is false, the host will synchronize to the device after + // each execution or transfer. This is intended for debugging only. + Device(se::StreamExecutor* executor, bool synchronous_deallocation, + bool asynchronous, bool allow_event_reuse); + virtual ~Device(); + + bool synchronous_deallocation() const { return synchronous_deallocation_; } + + EventPool& event_pool() { return event_pool_; } + + se::Stream* compute_stream() const { return compute_stream_.get(); } + se::Stream* host_to_device_stream() const { + return host_to_device_stream_.get(); + } + se::Stream* device_to_host_stream() const { + return device_to_host_stream_.get(); + } + + // Returns a device to device stream. Allocates streams in a round-robin + // fashion amongst the available streams. + se::Stream* GetDeviceToDeviceStream(); + + // Enqueues a copy of `src_buffer` to `dst_buffer` onto `src_stream`. + virtual Status ThenMemcpyDeviceToDevice(se::Stream* src_stream, + se::Stream* dst_stream, + se::DeviceMemoryBase src_buffer, + se::DeviceMemoryBase dst_buffer); + + WorkerThread* execute_thread() const { return execute_thread_.get(); } + + // Enqueues a host callback on 'stream', to be executed by callback_thread_. + // ThenDoHostCallback is often constrained in what it can do, in particular, + // on GPU the callback runs on a thread belonging to the GPU runtime and + // cannot perform GPU operations itself. + void ThenExecuteOnCallbackThread(se::Stream* stream, + std::function callback) const; + + // Helpers for releasing values on a worker thread at the tail of a stream on + // a worker thread. Copies `object`, and destroys the copy when the tail of + // the stream is reached. The destruction happens either in the caller's + // thread or on the worker thread (depending on thread schedules), not a + // device callback, so it is safe if the destructor frees device resource + // (e.g., GPU objects). + // TODO(phawkins): use move-capture when we can use C++14 features. + template + void ThenRelease(se::Stream* stream, T object) const { + if (callback_stream_.get() != stream) { + callback_stream_->ThenWaitFor(stream); + } + ThenExecuteOnCallbackThread(callback_stream_.get(), + [object]() { /* releases object */ }); + } + + Semaphore& compute_semaphore() { return compute_semaphore_; } + + private: + Status SynchronizeAllActivity(); + + bool synchronous_deallocation_; + + EventPool event_pool_; + + // Semaphore used to limit how many programs can be enqueued on the compute + // stream by the host ahead of the device. + Semaphore compute_semaphore_; + + std::unique_ptr compute_stream_; + std::unique_ptr host_to_device_stream_; + std::unique_ptr device_to_host_stream_; + std::vector> device_to_device_streams_; + + // Number of device-to-device streams to create in the multistream case. + static constexpr int kNumDeviceToDeviceStreams = 4; + + absl::Mutex mu_; + int next_device_to_device_stream_ GUARDED_BY(mu_) = 0; + + // Callback stream is used for running short host-side callbacks after device + // side events, without preventing the device-side stream from doing useful + // work. + std::unique_ptr callback_stream_; + + // A worker thread, used for replicated computation launches. + std::unique_ptr execute_thread_; + + // A worker thread, used for callbacks. It is necessary that this be a + // different thread to the execute thread because we acquire the compute + // semaphore during calls to Execute but release it from a callback and if + // they are the same thread we might deadlock. + std::unique_ptr callback_thread_; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_PYTHON_DEVICE_H_ diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index 49eb2341392..e78e64ac225 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -83,9 +83,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "absl/strings/str_format.h" -#include "absl/synchronization/blocking_counter.h" #include "absl/synchronization/mutex.h" -#include "absl/synchronization/notification.h" #include "absl/time/time.h" #include "include/pybind11/pybind11.h" #include "tensorflow/compiler/xla/client/client_library.h" @@ -95,7 +93,6 @@ limitations under the License. #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/python/shared_device_buffer.h" #include "tensorflow/compiler/xla/python/types.h" -#include "tensorflow/compiler/xla/service/custom_call_target_registry.h" #include "tensorflow/compiler/xla/service/platform_util.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/util.h" @@ -110,95 +107,6 @@ namespace xla { namespace py = pybind11; -// Registers a 'fn_capsule' as a CPU custom call target. -// 'fn_capsule' is a void* pointer encapsulated in a PyCapsule object, with name -// "xla._CPU_CUSTOM_CALL_TARGET". -Status RegisterCpuCustomCallTarget(const std::string& fn_name, - py::capsule capsule) { - static const char* const kName = "xla._CPU_CUSTOM_CALL_TARGET"; - if (absl::string_view(capsule.name()) != kName) { - return InvalidArgument( - "Argument to RegisterCpuCustomCallTargetRegistry was not a " - "xla._CPU_CUSTOM_CALL_TARGET capsule."); - } - CustomCallTargetRegistry::Global()->Register( - fn_name, static_cast(capsule), "Host"); - return Status::OK(); -} - -Device::Device(se::StreamExecutor* executor, bool synchronous_deallocation, - bool asynchronous, bool allow_event_reuse) - : synchronous_deallocation_(synchronous_deallocation), - event_pool_(allow_event_reuse), - compute_semaphore_(/*capacity=*/asynchronous ? 32 : 1) { - compute_stream_ = absl::make_unique(executor); - host_to_device_stream_ = absl::make_unique(executor); - device_to_host_stream_ = absl::make_unique(executor); - callback_stream_ = absl::make_unique(executor); - compute_stream_->Init(); - host_to_device_stream_->Init(); - device_to_host_stream_->Init(); - callback_stream_->Init(); - device_to_device_streams_.reserve(kNumDeviceToDeviceStreams); - for (int i = 0; i < kNumDeviceToDeviceStreams; ++i) { - auto stream = absl::make_unique(executor); - stream->Init(); - device_to_device_streams_.push_back(std::move(stream)); - } - execute_thread_ = absl::make_unique(tensorflow::Env::Default(), - "py_xla_execute"); - callback_thread_ = absl::make_unique(tensorflow::Env::Default(), - "py_xla_callback"); -} - -Device::~Device() { - Status status = SynchronizeAllActivity(); - if (!status.ok()) { - LOG(ERROR) << "Error when closing device: " << status; - } -} - -Status Device::SynchronizeAllActivity() { - Status status; - // TODO(phawkins): in theory the call to SynchronizeAllActivity below should - // suffice. However on the Host platform SynchronizeAllActivity is a dummy - // implementation that doesn't actually block. To make sure activity has - // stopped, also block on the compute stream. If SynchronizeAllActivity is - // fixed, we could remove the BlockHostUntilDone call. - status.Update(compute_stream_->BlockHostUntilDone()); - bool ok = compute_stream_->parent()->SynchronizeAllActivity(); - if (!ok) { - status.Update(Unknown("SynchronizeAllActivity failed.")); - } - return status; -} - -Status Device::ThenMemcpyDeviceToDevice(se::Stream* src_stream, - se::Stream* dst_stream, - se::DeviceMemoryBase src_buffer, - se::DeviceMemoryBase dst_buffer) { - // The default implementation simply calls ThenMemcpyD2D, and assumes that - // the buffer addresses identify the devices. This does not work - // on all platforms; this method is virtual so it can be overridden. - src_stream->ThenMemcpyD2D(&dst_buffer, src_buffer, dst_buffer.size()); - return Status::OK(); -} - -void Device::ThenExecuteOnCallbackThread(se::Stream* stream, - std::function callback) const { - stream->ThenDoHostCallback([this, callback]() mutable { - callback_thread_->Schedule(std::move(callback)); - }); -} - -se::Stream* Device::GetDeviceToDeviceStream() { - absl::MutexLock lock(&mu_); - int i = next_device_to_device_stream_; - next_device_to_device_stream_ = - (next_device_to_device_stream_ + 1) % device_to_device_streams_.size(); - return device_to_device_streams_.at(i).get(); -} - static StatusOr> CreateBFCAllocator( se::Platform* platform, LocalClient* client, double memory_fraction, bool preallocate) { diff --git a/tensorflow/compiler/xla/python/local_client.h b/tensorflow/compiler/xla/python/local_client.h index 3ff6fa4bb10..daa7d42d1d0 100644 --- a/tensorflow/compiler/xla/python/local_client.h +++ b/tensorflow/compiler/xla/python/local_client.h @@ -16,7 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_PYTHON_LOCAL_CLIENT_H_ #define TENSORFLOW_COMPILER_XLA_PYTHON_LOCAL_CLIENT_H_ -#include +#include #include #include @@ -27,11 +27,9 @@ limitations under the License. #include "tensorflow/compiler/xla/client/executable_build_options.h" #include "tensorflow/compiler/xla/client/local_client.h" #include "tensorflow/compiler/xla/client/xla_computation.h" -#include "tensorflow/compiler/xla/python/event_pool.h" +#include "tensorflow/compiler/xla/python/device.h" #include "tensorflow/compiler/xla/python/python_ref_manager.h" -#include "tensorflow/compiler/xla/python/semaphore.h" #include "tensorflow/compiler/xla/python/shared_device_buffer.h" -#include "tensorflow/compiler/xla/python/worker_thread.h" #include "tensorflow/compiler/xla/service/computation_placer.h" #include "tensorflow/compiler/xla/service/shaped_buffer.h" #include "tensorflow/compiler/xla/shape.h" @@ -41,113 +39,6 @@ limitations under the License. namespace xla { -// Registers a 'fn_capsule' as a CPU custom call target. -// 'fn_capsule' is a void* pointer encapsulated in a PyCapsule object, with name -// "xla._CPU_CUSTOM_CALL_TARGET". -Status RegisterCpuCustomCallTarget(const std::string& fn_name, - pybind11::capsule capsule); - -// Class that encapsulates state relating to a device (e.g., a GPU) on which we -// can perform computation and transfers. -class Device { - public: - // If synchronous_deallocation is true, the host must not free buffers until - // compute/transfers that use those buffers have completed. For example, this - // typically is the case for the "platform" where compute/transfers are - // operations that take place on another thread. - // - // If asynchronous is false, the host will synchronize to the device after - // each execution or transfer. This is intended for debugging only. - Device(se::StreamExecutor* executor, bool synchronous_deallocation, - bool asynchronous, bool allow_event_reuse); - virtual ~Device(); - - bool synchronous_deallocation() const { return synchronous_deallocation_; } - - EventPool& event_pool() { return event_pool_; } - - se::Stream* compute_stream() const { return compute_stream_.get(); } - se::Stream* host_to_device_stream() const { - return host_to_device_stream_.get(); - } - se::Stream* device_to_host_stream() const { - return device_to_host_stream_.get(); - } - - // Returns a device to device stream. Allocates streams in a round-robin - // fashion amongst the available streams. - se::Stream* GetDeviceToDeviceStream(); - - // Enqueues a copy of `src_buffer` to `dst_buffer` onto `src_stream`. - virtual Status ThenMemcpyDeviceToDevice(se::Stream* src_stream, - se::Stream* dst_stream, - se::DeviceMemoryBase src_buffer, - se::DeviceMemoryBase dst_buffer); - - WorkerThread* execute_thread() const { return execute_thread_.get(); } - - // Enqueues a host callback on 'stream', to be executed by callback_thread_. - // ThenDoHostCallback is often constrained in what it can do, in particular, - // on GPU the callback runs on a thread belonging to the GPU runtime and - // cannot perform GPU operations itself. - void ThenExecuteOnCallbackThread(se::Stream* stream, - std::function callback) const; - - // Helpers for releasing values on a worker thread at the tail of a stream on - // a worker thread. Copies `object`, and destroys the copy when the tail of - // the stream is reached. The destruction happens either in the caller's - // thread or on the worker thread (depending on thread schedules), not a - // device callback, so it is safe if the destructor frees device resource - // (e.g., GPU objects). - // TODO(phawkins): use move-capture when we can use C++14 features. - template - void ThenRelease(se::Stream* stream, T object) const { - if (callback_stream_.get() != stream) { - callback_stream_->ThenWaitFor(stream); - } - ThenExecuteOnCallbackThread(callback_stream_.get(), - [object]() { /* releases object */ }); - } - - Semaphore& compute_semaphore() { return compute_semaphore_; } - - private: - Status SynchronizeAllActivity(); - - bool synchronous_deallocation_; - - EventPool event_pool_; - - // Semaphore used to limit how many programs can be enqueued on the compute - // stream by the host ahead of the device. - Semaphore compute_semaphore_; - - std::unique_ptr compute_stream_; - std::unique_ptr host_to_device_stream_; - std::unique_ptr device_to_host_stream_; - std::vector> device_to_device_streams_; - - // Number of device-to-device streams to create in the multistream case. - static constexpr int kNumDeviceToDeviceStreams = 4; - - absl::Mutex mu_; - int next_device_to_device_stream_ GUARDED_BY(mu_) = 0; - - // Callback stream is used for running short host-side callbacks after device - // side events, without preventing the device-side stream from doing useful - // work. - std::unique_ptr callback_stream_; - - // A worker thread, used for replicated computation launches. - std::unique_ptr execute_thread_; - - // A worker thread, used for callbacks. It is necessary that this be a - // different thread to the execute thread because we acquire the compute - // semaphore during calls to Execute but release it from a callback and if - // they are the same thread we might deadlock. - std::unique_ptr callback_thread_; -}; - struct AllocatorConfig { enum class Kind { kDefault, // Client picks the best option for the platform. diff --git a/tensorflow/compiler/xla/python/xla.cc b/tensorflow/compiler/xla/python/xla.cc index da4c847efb8..172e24f801e 100644 --- a/tensorflow/compiler/xla/python/xla.cc +++ b/tensorflow/compiler/xla/python/xla.cc @@ -19,6 +19,7 @@ limitations under the License. #include "absl/base/casts.h" #include "absl/hash/hash.h" +#include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "absl/types/optional.h" #include "absl/types/span.h" @@ -36,6 +37,7 @@ limitations under the License. #include "tensorflow/compiler/xla/python/local_client.h" #include "tensorflow/compiler/xla/python/types.h" #include "tensorflow/compiler/xla/python/xrt.h" +#include "tensorflow/compiler/xla/service/custom_call_target_registry.h" #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_module.h" @@ -106,6 +108,22 @@ StatusOr GetComputationHloDotGraph( RenderedGraphFormat::kDot); } +// Registers a 'fn_capsule' as a CPU custom call target. +// 'fn_capsule' is a void* pointer encapsulated in a PyCapsule object, with name +// "xla._CPU_CUSTOM_CALL_TARGET". +Status RegisterCpuCustomCallTarget(const std::string& fn_name, + py::capsule capsule) { + static const char* const kName = "xla._CPU_CUSTOM_CALL_TARGET"; + if (absl::string_view(capsule.name()) != kName) { + return InvalidArgument( + "Argument to RegisterCpuCustomCallTargetRegistry was not a " + "xla._CPU_CUSTOM_CALL_TARGET capsule."); + } + CustomCallTargetRegistry::Global()->Register( + fn_name, static_cast(capsule), "Host"); + return Status::OK(); +} + } // namespace PYBIND11_MODULE(xla_extension, m) { From b9efe65ca25b7a308fa274c202884507a8221397 Mon Sep 17 00:00:00 2001 From: Sergei Lebedev Date: Thu, 11 Jul 2019 08:44:52 -0700 Subject: [PATCH 269/332] Disallowed returning None from a ->Tensor conversion function PiperOrigin-RevId: 257614950 --- tensorflow/python/framework/ops.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 983a3a8f311..070ce32a95a 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -1157,6 +1157,9 @@ def internal_convert_to_tensor(value, (dtype.name, value.dtype.name, value)) return value + if preferred_dtype is not None: + preferred_dtype = dtypes.as_dtype(preferred_dtype) + for base_type, conversion_func in tensor_conversion_registry.get(type(value)): # If dtype is None but preferred_dtype is not None, we try to # cast to preferred_dtype first. @@ -1167,15 +1170,13 @@ def internal_convert_to_tensor(value, value, dtype=preferred_dtype, name=name, as_ref=as_ref) except (TypeError, ValueError): # Could not coerce the conversion to use the preferred dtype. - ret = None - - if ret is not None and ret is not NotImplemented: - if (ret.dtype.base_dtype != - dtypes.as_dtype(preferred_dtype).base_dtype): + pass + else: + if (ret is not NotImplemented and + ret.dtype.base_dtype != preferred_dtype.base_dtype): raise TypeError("convert_to_tensor did not convert to " "the preferred dtype: %s vs %s " % - (ret.dtype.base_dtype, - dtypes.as_dtype(preferred_dtype).base_dtype)) + (ret.dtype.base_dtype, preferred_dtype.base_dtype)) if ret is None: ret = conversion_func(value, dtype=dtype, name=name, as_ref=as_ref) From 85dd97886361127ff5add22986594dc943bf6c42 Mon Sep 17 00:00:00 2001 From: Alex Itkes <38556752+alexitkes@users.noreply.github.com> Date: Fri, 7 Sep 2018 22:38:25 +0300 Subject: [PATCH 270/332] Fix docstring for DynamicRNNEstimator The 'optimizer' argument was referred to as 'optimizer_type'. Also a quote sign was forgotten. --- .../learn/python/learn/estimators/dynamic_rnn_estimator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py index a703dc66e92..35352b32ec8 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/dynamic_rnn_estimator.py @@ -625,14 +625,14 @@ class DynamicRnnEstimator(estimator.Estimator): optimizer: The type of optimizer to use. Either a subclass of `Optimizer`, an instance of an `Optimizer`, a callback that returns an optimizer, or a string. Strings must be one of 'Adagrad', 'Adam', - 'Ftrl', 'Momentum', 'RMSProp' or 'SGD. See `layers.optimize_loss` for + 'Ftrl', 'Momentum', 'RMSProp' or 'SGD'. See `layers.optimize_loss` for more details. learning_rate: Learning rate. This argument has no effect if `optimizer` is an instance of an `Optimizer`. predict_probabilities: A boolean indicating whether to predict probabilities for all classes. Used only if `problem_type` is `ProblemType.CLASSIFICATION` - momentum: Momentum value. Only used if `optimizer_type` is 'Momentum'. + momentum: Momentum value. Only used if `optimizer` is 'Momentum'. gradient_clipping_norm: Parameter used for gradient clipping. If `None`, then no clipping is performed. dropout_keep_probabilities: a list of dropout probabilities or `None`. From 3e799d065a6b44263a77185da2813461ff1a6501 Mon Sep 17 00:00:00 2001 From: Prakalp Srivastava Date: Thu, 11 Jul 2019 09:32:42 -0700 Subject: [PATCH 271/332] Handle uint8/16/32/64 as special types in TF dialect. If the return of a function is an unsigned tensor type, it was being modeled as standard int type, losing information that the return is unsigned. PiperOrigin-RevId: 257623093 --- .../mlir/lite/flatbuffer_translate.cc | 2 + tensorflow/compiler/mlir/lite/ir/tfl_ops.td | 24 ++-- tensorflow/compiler/mlir/lite/tests/ops.mlir | 2 +- .../compiler/mlir/tensorflow/ir/tf_types.def | 4 + .../graphdef2mlir/graph-uint8-return.pbtxt | 111 ++++++++++++++++++ .../mlir/tensorflow/utils/convert_type.cc | 4 - 6 files changed, 132 insertions(+), 15 deletions(-) create mode 100644 tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-uint8-return.pbtxt diff --git a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc index f189d159efa..0862c5fb881 100644 --- a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc +++ b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc @@ -167,6 +167,8 @@ static StatusOr GetTFLiteType(Type type, return tflite::TensorType_STRING; case mlir::TF::TensorFlowTypes::COMPLEX64: return tflite::TensorType_COMPLEX64; + case mlir::TF::TensorFlowTypes::UINT8: + return tflite::TensorType_UINT8; case mlir::StandardTypes::Integer: { const auto& itype = type.cast(); switch (itype.getWidth()) { diff --git a/tensorflow/compiler/mlir/lite/ir/tfl_ops.td b/tensorflow/compiler/mlir/lite/ir/tfl_ops.td index b9972cebdce..34f588993d5 100644 --- a/tensorflow/compiler/mlir/lite/ir/tfl_ops.td +++ b/tensorflow/compiler/mlir/lite/ir/tfl_ops.td @@ -50,6 +50,13 @@ def TFL_Str : Type()">, "TFLite string type">, BuildableType<"getType()">; +//===----------------------------------------------------------------------===// +// TFLite dialect uint8 type - uses the TF uint8 type as implementation +//===----------------------------------------------------------------------===// +def TFL_Uint8 : Type()">, + "TFLite uint8 type">, + BuildableType<"getType()">; + //===----------------------------------------------------------------------===// // Activation function enum definitions. //===----------------------------------------------------------------------===// @@ -438,13 +445,13 @@ def TFL_ConcatenationOp : TFL_Op<"concatenation", }]; let arguments = ( - ins Variadic>:$values, + ins Variadic>:$values, I32Attr:$axis, TFL_AFAttr:$fused_activation_function ); let results = (outs - TensorOf<[F32, I64, I32, I16, I8, TFL_QI8]>:$output + TensorOf<[F32, I64, I32, I16, I8, TFL_QI8, TFL_Uint8]>:$output ); let hasOptions = 1; @@ -559,9 +566,8 @@ def TFL_LessEqualOp : TFL_Op<"less_equal", [Broadcastable, NoSideEffect]> { }]; let arguments = ( - // TODO(haoliang): missing Uint8 - ins TensorOf<[F32, I32, I64, I8]>:$lhs, - TensorOf<[F32, I32, I64, I8]>:$rhs); + ins TensorOf<[F32, I32, I64, I8, TFL_Uint8]>:$lhs, + TensorOf<[F32, I32, I64, I8, TFL_Uint8]>:$rhs); let results = (outs TFL_BoolTensor:$output); @@ -671,10 +677,9 @@ def TFL_EqualOp: TFL_Op<"equal", [Commutative, Broadcastable, }]; let arguments = ( - // TODO: missing Uint8 ins - TensorOf<[I1, F32, I32, I64, I8]>:$x, - TensorOf<[I1, F32, I32, I64, I8]>:$y + TensorOf<[I1, F32, I32, I64, I8, TFL_Uint8]>:$x, + TensorOf<[I1, F32, I32, I64, I8, TFL_Uint8]>:$y ); let results = (outs TFL_BoolTensor:$output); @@ -1072,8 +1077,7 @@ def TFL_MeanOp : TFL_Op<"mean", [NoSideEffect]> { }]; let arguments = (ins - // TODO: missing uint8 - TensorOf<[F32, I8, I32, I64]>:$input, + TensorOf<[F32, I8, I32, I64, TFL_Uint8]>:$input, TensorOf<[I32, I64]>:$axis, BoolAttr:$keep_dims ); diff --git a/tensorflow/compiler/mlir/lite/tests/ops.mlir b/tensorflow/compiler/mlir/lite/tests/ops.mlir index 1eb3f5f48d6..88b137efadb 100644 --- a/tensorflow/compiler/mlir/lite/tests/ops.mlir +++ b/tensorflow/compiler/mlir/lite/tests/ops.mlir @@ -817,7 +817,7 @@ func @testConcatInvalidOutputElementalType(%arg0: tensor<2xi32>, %arg1: tensor<2 // ----- func @testConcatInvalidStorageType(%arg0: tensor<2x!quant.uniform>, %arg1: tensor<2x!quant.uniform>) -> tensor<2x2x!quant.uniform> { - // expected-error @+1 {{'tfl.concatenation' op operand #0 must be tensor of 32-bit float or 64-bit integer or 32-bit integer or 16-bit integer or 8-bit integer or quantized type with 8 bits storage type values}} + // expected-error @+1 {{'tfl.concatenation' op operand #0 must be tensor of 32-bit float or 64-bit integer or 32-bit integer or 16-bit integer or 8-bit integer or quantized type with 8 bits storage type or TFLite uint8 type values}} %0 = "tfl.concatenation"(%arg0, %arg1) {axis = 0 : i32, fused_activation_function = "NONE"} : (tensor<2x!quant.uniform>, tensor<2x!quant.uniform>) -> tensor<2x2x!quant.uniform> return %0 : tensor<2x2x!quant.uniform> } diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_types.def b/tensorflow/compiler/mlir/tensorflow/ir/tf_types.def index 20a58722edf..9f1154b84f1 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_types.def +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_types.def @@ -19,6 +19,10 @@ limitations under the License. #ifdef HANDLE_TF_TYPE // class, enumerant, name +HANDLE_TF_TYPE(Uint8, UINT8, "uint8") +HANDLE_TF_TYPE(Uint16, UINT16, "uint16") +HANDLE_TF_TYPE(Uint32, UINT32, "uint32") +HANDLE_TF_TYPE(Uint64, UINT64, "uint64") HANDLE_TF_TYPE(Qint8, QINT8, "qint8") HANDLE_TF_TYPE(Qint16, QINT16, "qint16") HANDLE_TF_TYPE(Qint32, QINT32, "qint32") diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-uint8-return.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-uint8-return.pbtxt new file mode 100644 index 00000000000..32b816f5e39 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/graph-uint8-return.pbtxt @@ -0,0 +1,111 @@ +# RUN: tf-mlir-translate -graphdef-to-mlir -mlir-print-debuginfo %s -o - | FileCheck %s + +node { + name: "PartitionedCall" + op: "PartitionedCall" + attr { + key: "Tin" + value { + list { + } + } + } + attr { + key: "Tout" + value { + list { + type: DT_UINT8 + } + } + } + attr { + key: "_gradient_op_type" + value { + s: "PartitionedCall-15" + } + } + attr { + key: "config" + value { + s: "" + } + } + attr { + key: "config_proto" + value { + s: "\n\007\n\003GPU\020\000\n\007\n\003CPU\020\0012\002J\0008\001" + } + } + attr { + key: "executor_type" + value { + s: "" + } + } + attr { + key: "f" + value { + func { + name: "__inference_uint_const_14" + } + } + } +} +library { + function { + signature { + name: "__inference_uint_const_14" + output_arg { + name: "identity" + type: DT_UINT8 + } + } + node_def { + name: "Const" + op: "Const" + attr { + key: "dtype" + value { + type: DT_UINT8 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_UINT8 + tensor_shape { + } + int_val: 5 + } + } + } + } + node_def { + name: "Identity" + op: "Identity" + input: "Const:output:0" + attr { + key: "T" + value { + type: DT_UINT8 + } + } + } + ret { + key: "identity" + value: "Identity:output:0" + } + } +} +versions { + producer: 29 + min_consumer: 12 +} + +# CHECK: func @main +# CHECK: "_tf.PartitionedCall"() +# CHECK-SAME: Tout = ["tfdtype$DT_UINT8"] +# CHECK-SAME: f = @[[FUNCTION:[A-Za-z0-9_]*]] +# CHECK: func @[[FUNCTION]]() -> tensor +# CHECK: return {{%[0-9]*#[0-9]*}} : tensor diff --git a/tensorflow/compiler/mlir/tensorflow/utils/convert_type.cc b/tensorflow/compiler/mlir/tensorflow/utils/convert_type.cc index 3006cc5bc3f..0aa6b460b73 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/convert_type.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/convert_type.cc @@ -46,19 +46,15 @@ Status ConvertDataType(const DataType& dtype, Builder builder, Type* type) { *type = builder.getIntegerType(1); return Status::OK(); case DT_INT8: - case DT_UINT8: *type = builder.getIntegerType(8); return Status::OK(); case DT_INT16: - case DT_UINT16: *type = builder.getIntegerType(16); return Status::OK(); case DT_INT32: - case DT_UINT32: *type = builder.getIntegerType(32); return Status::OK(); case DT_INT64: - case DT_UINT64: *type = builder.getIntegerType(64); return Status::OK(); case DT_BFLOAT16: From 47fda4a3ad846e434788dc42535d67429ebe3fb0 Mon Sep 17 00:00:00 2001 From: Ihor Indyk Date: Thu, 11 Jul 2019 09:44:15 -0700 Subject: [PATCH 272/332] [tf.data] Increase the number of `iterator.get_next()` calls to 10000 (from 1000) in `map_and_interleave` autotuning benchmark to decrease variance of the output. PiperOrigin-RevId: 257624991 --- .../python/data/experimental/benchmarks/autotune_benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py index 6a42c1cb6e5..fdd4f4d11b7 100644 --- a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py +++ b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py @@ -189,14 +189,14 @@ class AutotuneBenchmark(test.Benchmark): with session.Session() as sess: for _ in range(5): sess.run(get_next) - for _ in range(1000): + for _ in range(10000): start = time.time() sess.run(get_next) end = time.time() deltas.append(end - start) self.report_benchmark( - iters=1000, + iters=10000, wall_time=np.median(deltas), name="map_and_interleave" + ("_autotune" if autotune else "")) return np.median(deltas) From d1bf811d7a8542e535ea36b5c602490847d7fe61 Mon Sep 17 00:00:00 2001 From: Gaurav Jain Date: Thu, 11 Jul 2019 10:45:25 -0700 Subject: [PATCH 273/332] Move ObjectIdentity to python/utils PiperOrigin-RevId: 257637727 --- tensorflow/python/keras/engine/base_layer.py | 2 +- tensorflow/python/keras/layers/recurrent_test.py | 2 +- tensorflow/python/keras/layers/wrappers_test.py | 2 +- tensorflow/python/saved_model/BUILD | 1 - tensorflow/python/saved_model/save.py | 2 +- tensorflow/python/training/tracking/BUILD | 8 -------- tensorflow/python/training/tracking/graph_view.py | 2 +- tensorflow/python/training/tracking/layer_utils.py | 2 +- tensorflow/python/training/tracking/util.py | 2 +- .../python/{training/tracking => util}/object_identity.py | 0 10 files changed, 7 insertions(+), 16 deletions(-) rename tensorflow/python/{training/tracking => util}/object_identity.py (100%) diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index 04c9a387e24..a80a4c2a072 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -67,11 +67,11 @@ from tensorflow.python.ops import variables as tf_variables from tensorflow.python.training.tracking import base as trackable from tensorflow.python.training.tracking import data_structures from tensorflow.python.training.tracking import layer_utils as trackable_layer_utils -from tensorflow.python.training.tracking import object_identity from tensorflow.python.training.tracking import tracking from tensorflow.python.util import compat from tensorflow.python.util import deprecation from tensorflow.python.util import nest +from tensorflow.python.util import object_identity from tensorflow.python.util import serialization from tensorflow.python.util import tf_decorator from tensorflow.python.util import tf_inspect diff --git a/tensorflow/python/keras/layers/recurrent_test.py b/tensorflow/python/keras/layers/recurrent_test.py index 55233120a09..f3ebf8a83c8 100644 --- a/tensorflow/python/keras/layers/recurrent_test.py +++ b/tensorflow/python/keras/layers/recurrent_test.py @@ -46,9 +46,9 @@ from tensorflow.python.ops import special_math_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables as variables_lib from tensorflow.python.platform import test -from tensorflow.python.training.tracking import object_identity from tensorflow.python.training.tracking import util as trackable_util from tensorflow.python.util import nest +from tensorflow.python.util import object_identity # Used for nested input/output/state RNN test. NestedInput = collections.namedtuple('NestedInput', ['t1', 't2']) diff --git a/tensorflow/python/keras/layers/wrappers_test.py b/tensorflow/python/keras/layers/wrappers_test.py index 7b9c741256f..8fe13f4546f 100644 --- a/tensorflow/python/keras/layers/wrappers_test.py +++ b/tensorflow/python/keras/layers/wrappers_test.py @@ -32,8 +32,8 @@ from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.keras.layers.rnn_cell_wrapper_v2 import ResidualWrapper from tensorflow.python.ops.array_ops import concat from tensorflow.python.platform import test -from tensorflow.python.training.tracking import object_identity from tensorflow.python.training.tracking import util as trackable_util +from tensorflow.python.util import object_identity class _RNNCellWithConstants(keras.layers.Layer): diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD index 7201bb48d1e..29ce69ce9a3 100644 --- a/tensorflow/python/saved_model/BUILD +++ b/tensorflow/python/saved_model/BUILD @@ -311,7 +311,6 @@ py_library( "//tensorflow/python/training/tracking", "//tensorflow/python/training/tracking:base", "//tensorflow/python/training/tracking:graph_view", - "//tensorflow/python/training/tracking:object_identity", "//tensorflow/python/training/tracking:util", ], ) diff --git a/tensorflow/python/saved_model/save.py b/tensorflow/python/saved_model/save.py index e82334b4923..f357ed0728d 100644 --- a/tensorflow/python/saved_model/save.py +++ b/tensorflow/python/saved_model/save.py @@ -51,10 +51,10 @@ from tensorflow.python.saved_model import utils_impl from tensorflow.python.training.saving import functional_saver from tensorflow.python.training.tracking import base from tensorflow.python.training.tracking import graph_view -from tensorflow.python.training.tracking import object_identity from tensorflow.python.training.tracking import tracking from tensorflow.python.training.tracking import util from tensorflow.python.util import compat +from tensorflow.python.util import object_identity from tensorflow.python.util.tf_export import tf_export _UNCOPIABLE_DTYPES = frozenset((dtypes.resource, dtypes.variant)) diff --git a/tensorflow/python/training/tracking/BUILD b/tensorflow/python/training/tracking/BUILD index 6fd1e7826ec..ce8e9af3328 100644 --- a/tensorflow/python/training/tracking/BUILD +++ b/tensorflow/python/training/tracking/BUILD @@ -103,19 +103,12 @@ tf_py_test( ], ) -py_library( - name = "object_identity", - srcs = ["object_identity.py"], - srcs_version = "PY2AND3", -) - py_library( name = "graph_view", srcs = ["graph_view.py"], srcs_version = "PY2AND3", deps = [ ":base", - ":object_identity", ":tracking", "//tensorflow/core:protos_all_py", "//tensorflow/python:constant_op", @@ -134,7 +127,6 @@ py_library( ":base", ":data_structures", ":graph_view", - ":object_identity", ":tracking", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", diff --git a/tensorflow/python/training/tracking/graph_view.py b/tensorflow/python/training/tracking/graph_view.py index ba238787018..54b22fa07f9 100644 --- a/tensorflow/python/training/tracking/graph_view.py +++ b/tensorflow/python/training/tracking/graph_view.py @@ -28,8 +28,8 @@ from tensorflow.python.training import optimizer as optimizer_v1 from tensorflow.python.training.saving import saveable_object as saveable_object_lib from tensorflow.python.training.saving import saveable_object_util from tensorflow.python.training.tracking import base -from tensorflow.python.training.tracking import object_identity from tensorflow.python.training.tracking import tracking +from tensorflow.python.util import object_identity _ESCAPE_CHAR = "." # For avoiding conflicts with user-specified names. diff --git a/tensorflow/python/training/tracking/layer_utils.py b/tensorflow/python/training/tracking/layer_utils.py index 75c766e7fc3..b83b0f84f91 100644 --- a/tensorflow/python/training/tracking/layer_utils.py +++ b/tensorflow/python/training/tracking/layer_utils.py @@ -21,7 +21,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.training.tracking import object_identity +from tensorflow.python.util import object_identity def is_layer(obj): diff --git a/tensorflow/python/training/tracking/util.py b/tensorflow/python/training/tracking/util.py index a22bef263da..c4c2ccdbf29 100644 --- a/tensorflow/python/training/tracking/util.py +++ b/tensorflow/python/training/tracking/util.py @@ -49,11 +49,11 @@ from tensorflow.python.training.saving import saveable_object_util from tensorflow.python.training.tracking import base from tensorflow.python.training.tracking import data_structures from tensorflow.python.training.tracking import graph_view as graph_view_lib -from tensorflow.python.training.tracking import object_identity from tensorflow.python.training.tracking import tracking from tensorflow.python.util import compat from tensorflow.python.util import deprecation from tensorflow.python.util import lazy_loader +from tensorflow.python.util import object_identity from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export diff --git a/tensorflow/python/training/tracking/object_identity.py b/tensorflow/python/util/object_identity.py similarity index 100% rename from tensorflow/python/training/tracking/object_identity.py rename to tensorflow/python/util/object_identity.py From 3526f05b16ed8ab00f4287b62b8b49589fbf7971 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 10:47:44 -0700 Subject: [PATCH 274/332] Add detection_responder which allows each platform to process the person detection output in its own way. For example, sparkfun_edge lights up the yellow LED for no person and the green LED for person, and toggles the blue LED on each run. PiperOrigin-RevId: 257638242 --- .../micro/examples/micro_vision/BUILD | 26 +++++++++ .../micro/examples/micro_vision/Makefile.inc | 13 +++++ .../micro_vision/detection_responder.cc | 25 +++++++++ .../micro_vision/detection_responder.h | 34 ++++++++++++ .../micro_vision/detection_responder_test.cc | 34 ++++++++++++ .../micro/examples/micro_vision/main.cc | 7 ++- .../sparkfun_edge/detection_responder.cc | 54 +++++++++++++++++++ 7 files changed, 189 insertions(+), 4 deletions(-) create mode 100644 tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h create mode 100644 tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder_test.cc create mode 100644 tensorflow/lite/experimental/micro/examples/micro_vision/sparkfun_edge/detection_responder.cc diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/BUILD b/tensorflow/lite/experimental/micro/examples/micro_vision/BUILD index 479a178ea44..6555e03fdb4 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_vision/BUILD +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/BUILD @@ -74,12 +74,38 @@ tflite_micro_cc_test( ], ) +cc_library( + name = "detection_responder", + srcs = [ + "detection_responder.cc", + ], + hdrs = [ + "detection_responder.h", + ], + deps = [ + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + ], +) + +tflite_micro_cc_test( + name = "detection_responder_test", + srcs = [ + "detection_responder_test.cc", + ], + deps = [ + ":detection_responder", + "//tensorflow/lite/experimental/micro/testing:micro_test", + ], +) + cc_binary( name = "micro_vision", srcs = [ "main.cc", ], deps = [ + ":detection_responder", ":image_provider", ":model_settings", ":person_detect_model_data", diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/Makefile.inc b/tensorflow/lite/experimental/micro/examples/micro_vision/Makefile.inc index 69d3ea479f4..af792629e08 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_vision/Makefile.inc +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/Makefile.inc @@ -28,12 +28,21 @@ IMAGE_PROVIDER_TEST_HDRS := \ tensorflow/lite/experimental/micro/examples/micro_vision/image_provider.h \ tensorflow/lite/experimental/micro/examples/micro_vision/model_settings.h +DETECTION_RESPONDER_TEST_SRCS := \ +tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.cc \ +tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder_test.cc + +DETECTION_RESPONDER_TEST_HDRS := \ +tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h + MICRO_VISION_SRCS := \ +tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.cc \ tensorflow/lite/experimental/micro/examples/micro_vision/image_provider.cc \ tensorflow/lite/experimental/micro/examples/micro_vision/main.cc \ $(MICRO_VISION_MODEL_SRCS) MICRO_VISION_HDRS := \ +tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h \ tensorflow/lite/experimental/micro/examples/micro_vision/image_provider.h \ $(MICRO_VISION_MODEL_HDRS) @@ -48,6 +57,10 @@ $(MICRO_VISION_TEST_SRCS),$(MICRO_VISION_TEST_HDRS))) $(eval $(call microlite_test,image_provider_test,\ $(IMAGE_PROVIDER_TEST_SRCS),$(IMAGE_PROVIDER_TEST_HDRS))) +# Tests the detection responder module. +$(eval $(call microlite_test,detection_responder_test,\ +$(DETECTION_RESPONDER_TEST_SRCS),$(DETECTION_RESPONDER_TEST_HDRS))) + # Builds a standalone object recognition binary. $(eval $(call microlite_test,micro_vision,\ $(MICRO_VISION_SRCS),$(MICRO_VISION_HDRS))) diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.cc b/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.cc new file mode 100644 index 00000000000..e2ac98fecab --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.cc @@ -0,0 +1,25 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h" + +// This dummy implementation writes person and no person scores to the error +// console. Real applications will want to take some custom action instead, and +// should implement their own versions of this function. +void RespondToDetection(tflite::ErrorReporter* error_reporter, + uint8_t person_score, uint8_t no_person_score) { + error_reporter->Report("person score:%d no person score %d", person_score, + no_person_score); +} diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h b/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h new file mode 100644 index 00000000000..a1aca63cf3c --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h @@ -0,0 +1,34 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Provides an interface to take an action based on the output from the person +// detection model. + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_VISION_DETECTION_RESPONDER_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_VISION_DETECTION_RESPONDER_H_ + +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" + +// Called every time the results of a person detection run are available. The +// `person_score` has the numerical confidence that the captured image contains +// a person, and `no_person_score` has the numerical confidence that the image +// does not contain a person. Typically if person_score > no person score, the +// image is considered to contain a person. This threshold may be adjusted for +// particular applications. +void RespondToDetection(tflite::ErrorReporter* error_reporter, + uint8_t person_score, uint8_t no_person_score); + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_MICRO_VISION_DETECTION_RESPONDER_H_ diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder_test.cc b/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder_test.cc new file mode 100644 index 00000000000..ec25533e82c --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder_test.cc @@ -0,0 +1,34 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h" + +#include "tensorflow/lite/experimental/micro/testing/micro_test.h" +#include "tensorflow/lite/experimental/micro/testing/test_utils.h" + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(TestCallability) { + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + // This will have external side-effects (like printing to the debug console + // or lighting an LED) that are hard to observe, so the most we can do is + // make sure the call doesn't crash. + RespondToDetection(error_reporter, 100, 200); + RespondToDetection(error_reporter, 200, 100); +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/main.cc b/tensorflow/lite/experimental/micro/examples/micro_vision/main.cc index ab736d4bc14..f1b3d897e39 100644 --- a/tensorflow/lite/experimental/micro/examples/micro_vision/main.cc +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/main.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h" #include "tensorflow/lite/experimental/micro/examples/micro_vision/image_provider.h" #include "tensorflow/lite/experimental/micro/examples/micro_vision/model_settings.h" #include "tensorflow/lite/experimental/micro/examples/micro_vision/person_detect_model_data.h" @@ -69,12 +70,10 @@ int main(int argc, char* argv[]) { TfLiteTensor* output = interpreter.output(0); - // Log the person score and no person score. + // Process the inference results. uint8_t person_score = output->data.uint8[kPersonIndex]; uint8_t no_person_score = output->data.uint8[kNotAPersonIndex]; - error_reporter->Report( - "person data. person score: %d, no person score: %d\n", person_score, - no_person_score); + RespondToDetection(error_reporter, person_score, no_person_score); } return 0; diff --git a/tensorflow/lite/experimental/micro/examples/micro_vision/sparkfun_edge/detection_responder.cc b/tensorflow/lite/experimental/micro/examples/micro_vision/sparkfun_edge/detection_responder.cc new file mode 100644 index 00000000000..43425b76e68 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/micro_vision/sparkfun_edge/detection_responder.cc @@ -0,0 +1,54 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/micro_vision/detection_responder.h" + +#include "am_bsp.h" // NOLINT + +// This implementation will light up LEDs on the board in response to the +// inference results. +void RespondToDetection(tflite::ErrorReporter* error_reporter, + uint8_t person_score, uint8_t no_person_score) { + static bool is_initialized = false; + if (!is_initialized) { + // Setup LED's as outputs. Leave red LED alone since that's an error + // indicator for sparkfun_edge in image_provider. + am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_BLUE, g_AM_HAL_GPIO_OUTPUT_12); + am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_GREEN, g_AM_HAL_GPIO_OUTPUT_12); + am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_YELLOW, g_AM_HAL_GPIO_OUTPUT_12); + is_initialized = true; + } + + // Toggle the blue LED every time an inference is performed. + static int count = 0; + if (++count & 1) { + am_hal_gpio_output_set(AM_BSP_GPIO_LED_BLUE); + } else { + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_BLUE); + } + + // Turn on the green LED if a person was detected. Turn on the yellow LED + // otherwise. + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_YELLOW); + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_GREEN); + if (person_score > no_person_score) { + am_hal_gpio_output_set(AM_BSP_GPIO_LED_GREEN); + } else { + am_hal_gpio_output_set(AM_BSP_GPIO_LED_YELLOW); + } + + error_reporter->Report("person score:%d no person score %d", person_score, + no_person_score); +} From 44bc571630f6629e3a820c2eeab6d26f9847462f Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 11 Jul 2019 11:02:07 -0700 Subject: [PATCH 275/332] Create a single, static default runner for KernelAndDevice objects. This change replaces an std::function<> instance variable that has the same value for every KernelAndDevice object with a static local variable. PiperOrigin-RevId: 257641403 --- .../common_runtime/eager/kernel_and_device.cc | 16 ++++++++++++++-- .../common_runtime/eager/kernel_and_device.h | 12 +++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc index a9d0774fa0f..77271d257fd 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc +++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc @@ -45,6 +45,18 @@ limitations under the License. namespace tensorflow { +std::function)>* KernelAndDevice::get_runner() + const { + if (runner_) { + return runner_; + } else { + static auto* default_runner = + new std::function)>( + [](std::function f) { f(); }); + return default_runner; + } +} + KernelAndDeviceFunc::~KernelAndDeviceFunc() { if (handle_ != kInvalidHandle) { Status status = pflr_->ReleaseHandle(handle_); @@ -274,7 +286,7 @@ Status KernelAndDeviceOp::Run(ScopedStepContainer* step_container, params.stats_collector = step_stats_collector.get(); params.graph_collector = graph_collector; } - params.runner = runner_ != nullptr ? runner_ : &default_runner_; + params.runner = get_runner(); params.step_container = step_container; params.collective_executor = @@ -360,7 +372,7 @@ Status KernelAndDeviceFunc::Run( step_stats_collector.reset(new StepStatsCollector(step_stats)); } opts.stats_collector = step_stats_collector.get(); - opts.runner = (runner_ == nullptr) ? &default_runner_ : runner_; + opts.runner = get_runner(); Notification done; Status status; diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.h b/tensorflow/core/common_runtime/eager/kernel_and_device.h index 876594e27b1..edec07dec86 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device.h +++ b/tensorflow/core/common_runtime/eager/kernel_and_device.h @@ -70,9 +70,8 @@ class KernelAndDevice : public core::RefCounted { : device_(flr == nullptr ? nullptr : flr->device()), host_cpu_device_(host_cpu_device), flr_(flr), - runner_(runner), - default_runner_([](std::function f) { f(); }), - collective_executor_(std::move(collective_executor)) {} + collective_executor_(std::move(collective_executor)), + runner_(runner) {} // Not thread safe. virtual ~KernelAndDevice() {} @@ -114,6 +113,8 @@ class KernelAndDevice : public core::RefCounted { virtual const string& name() const = 0; protected: + std::function)>* get_runner() const; + // TODO(apassos) Consider a shared cancellation manager. Note that this // cancellation manager is not useful to actually cancel anything, and is // provided here only for the few kernels which can't handle one being @@ -122,9 +123,10 @@ class KernelAndDevice : public core::RefCounted { Device* const device_; // can be null Device* const host_cpu_device_; // non-null FunctionLibraryRuntime* const flr_; // can be null - std::function)>* const runner_; - std::function)> default_runner_; const std::unique_ptr collective_executor_; + + private: + std::function)>* const runner_; // can be null }; // Represents an op kernel and the device it will be run on. From 7f4052cfe5a6f26ffd3cdd9b2c31e5cf6e1733eb Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Thu, 11 Jul 2019 11:13:42 -0700 Subject: [PATCH 276/332] Remove dependence on //tools/target_cpu:haswell from the tf_library macro //tools/target_cpu:haswell isn't present in open source. PiperOrigin-RevId: 257644093 --- tensorflow/compiler/aot/tfcompile.bzl | 9 ++------- tensorflow/tensorflow.bzl | 3 +++ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl index d9f871dc2e5..79b4654d677 100644 --- a/tensorflow/compiler/aot/tfcompile.bzl +++ b/tensorflow/compiler/aot/tfcompile.bzl @@ -22,6 +22,7 @@ load( "tf_cc_test", "tf_copts", ) +load("//tensorflow:tensorflow.bzl", "tfcompile_extra_flags") def tf_library( name, @@ -180,13 +181,7 @@ def tf_library( # `find` on such an object. need_xla_data_proto = flags and flags.find("--gen_program_shape") != -1 - # Pass --target_cpu=haswell to tfcompile if compiling for Haswell (bazel - # build --cpu=haswell). We put it at the beginning of the flags list so - # that tfcompile_flags can override if if desired. - flags = select({ - "//tools/target_cpu:haswell": "--target_cpu=haswell ", - "//conditions:default": "", - }) + flags + flags = tfcompile_extra_flags() + flags if enable_xla_hlo_profiling: profiling_flag = "--xla_hlo_profile" diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 3d90bbed05a..5d9aba8637a 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -2492,3 +2492,6 @@ def if_mlir(if_true, if_false = []): "//conditions:default": if_false, "//tensorflow:with_mlir_support": if_true, }) + +def tfcompile_extra_flags(): + return "" From 25da861019d8caabf5cb7c543d34430848f71b91 Mon Sep 17 00:00:00 2001 From: Ashwin Murthy Date: Thu, 11 Jul 2019 11:14:19 -0700 Subject: [PATCH 277/332] The change in tensorflow/compiler/mlir/lite/BUILD adds a new filegroup for tf_tfl_translate.cc PiperOrigin-RevId: 257644235 --- tensorflow/compiler/mlir/lite/BUILD | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/lite/BUILD b/tensorflow/compiler/mlir/lite/BUILD index c93a02c588a..ba2afffb019 100644 --- a/tensorflow/compiler/mlir/lite/BUILD +++ b/tensorflow/compiler/mlir/lite/BUILD @@ -462,9 +462,16 @@ cc_library( alwayslink = 1, ) +filegroup( + name = "tf_tfl_translate_main", + srcs = [ + "tf_tfl_translate.cc", + ], +) + tf_cc_binary( name = "tf_tfl_translate", - srcs = ["tf_tfl_translate.cc"], + srcs = [":tf_tfl_translate_main"], deps = [ ":flatbuffer_translate_lib", ":tensorflow_lite", From d251e8c923ff680b50916ca3d71403408ee78712 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 11:24:27 -0700 Subject: [PATCH 278/332] Change pfor logic to work in nested contexts inside an xla compile call. PiperOrigin-RevId: 257646455 --- .../ops/parallel_for/control_flow_ops.py | 20 ++++++++++--- .../parallel_for/xla_control_flow_ops_test.py | 28 +++++++++++++++++-- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/ops/parallel_for/control_flow_ops.py b/tensorflow/python/ops/parallel_for/control_flow_ops.py index 787d03fc9c0..7c569560d43 100644 --- a/tensorflow/python/ops/parallel_for/control_flow_ops.py +++ b/tensorflow/python/ops/parallel_for/control_flow_ops.py @@ -113,6 +113,21 @@ def _flatten_first_two_dims(x): PFOR_CONFIG_ARG = "pfor_config" +def _is_under_xla_context(): + """Check if we are currently inside an XLA compile context.""" + g = ops.get_default_graph() + while g is not None: + control_flow_context = g._get_control_flow_context() # pylint: disable=protected-access + while control_flow_context is not None: + if control_flow_context.IsXLAContext(): + return True + else: + control_flow_context = control_flow_context.outer_context + # If g is a FuncGraph, get its outer_graph. + g = getattr(g, "outer_graph", None) + return False + + def pfor(loop_fn, iters, parallel_iterations=None): """Equivalent to running `loop_fn` `iters` times and stacking the outputs. @@ -162,13 +177,10 @@ def pfor(loop_fn, iters, parallel_iterations=None): """ def f(): return _pfor_impl(loop_fn, iters, parallel_iterations=parallel_iterations) - control_flow_context = ops.get_default_graph()._get_control_flow_context() # pylint: disable=protected-access # Note that we wrap into a tf.function if in eager execution mode or under # XLA compilation. The latter is so that we don't compile operations like # tf.placeholder that are created by the loop body. - if (context.executing_eagerly() or - (control_flow_context is not None and - control_flow_context.IsXLAContext())): + if context.executing_eagerly() or _is_under_xla_context(): f = function.defun(f) return f() diff --git a/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py index 6d2bad23b94..0b1678823f9 100644 --- a/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py +++ b/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py @@ -22,6 +22,7 @@ from __future__ import print_function from tensorflow.python.compiler.xla import xla from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.parallel_for import control_flow_ops as pfor_control_flow_ops from tensorflow.python.ops.parallel_for.test_util import PForTestCase @@ -39,10 +40,31 @@ class PForTest(PForTestCase): def vectorized_compute(x): return pfor_control_flow_ops.vectorized_map(compute, x) - result = xla.compile(vectorized_compute, - inputs=[array_ops.ones((10, 5, 3))]) + result = xla.compile( + vectorized_compute, inputs=[array_ops.ones((10, 5, 3))]) self.run_and_assert_equal(result, array_ops.ones((10, 1, 3))) + def test_xla_while_loop(self): -if __name__ == "__main__": + def compute(x): + return math_ops.reduce_mean(x, axis=0, keepdims=True) + + def vectorized_compute(x, i): + inp = array_ops.gather(x, i) + output = pfor_control_flow_ops.vectorized_map(compute, inp) + output.set_shape([5, 1]) + return output + + def while_compute(x): + return control_flow_ops.while_loop_v2( + lambda i, _: i < 10, + lambda i, y: (i + 1, y + vectorized_compute(x, i)), + (0, array_ops.zeros([5, 1])))[1] + + result = xla.compile(while_compute, inputs=[array_ops.ones((10, 5, 3))]) + expected = array_ops.ones([5, 1]) * 10 + self.run_and_assert_equal(expected, result) + + +if __name__ == '__main__': test.main() From d12c671f014d24e1ee960b2b0d9d42b9c29fedae Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 11:27:39 -0700 Subject: [PATCH 279/332] Segmentation Fault 11 thrown by tensorflow::HistogramFixedWidthFunctor::Compute in histogram_op.cc Occasionally tensorflow::HistogramFixedWidthFunctor::Compute triggers a segmentation fault 11. It happens when the argument i of index_to_bin() is negative, which is caused by casting a big (x-a)/step int64 value that exceeds int32 range to int32. Switching the order of cwiseMin and cast fixes the problem. PiperOrigin-RevId: 257647164 --- tensorflow/core/kernels/histogram_op.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc index 75f896b407d..7b66c879fbf 100644 --- a/tensorflow/core/kernels/histogram_op.cc +++ b/tensorflow/core/kernels/histogram_op.cc @@ -48,18 +48,22 @@ struct HistogramFixedWidthFunctor { const double step = static_cast(value_range(1) - value_range(0)) / static_cast(nbins); + const double nbins_minus_1 = static_cast(nbins - 1); // The calculation is done by finding the slot of each value in `values`. // With [a, b]: // step = (b - a) / nbins // (x - a) / step // , then the entries are mapped to output. + + // Bug fix: Switch the order of cwiseMin and int32-casting to avoid + // producing a negative index when casting an big int64 number to int32 index_to_bin.device(d) = ((values.cwiseMax(value_range(0)) - values.constant(value_range(0))) .template cast() / step) - .template cast() - .cwiseMin(nbins - 1); + .cwiseMin(nbins_minus_1) + .template cast(); out.setZero(); for (int32 i = 0; i < index_to_bin.size(); i++) { From 77095978f486c5c34565036915b4ea676735b9ba Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Thu, 11 Jul 2019 11:35:16 -0700 Subject: [PATCH 280/332] [Grappler] Move indicator of removed node in MutableGraphView::Mutation out of MutableNodeViewDiff. PiperOrigin-RevId: 257648888 --- tensorflow/core/grappler/utils/graph_view.cc | 94 +++++++++---------- tensorflow/core/grappler/utils/graph_view.h | 1 + .../core/grappler/utils/graph_view_internal.h | 12 +-- .../utils/graph_view_internal_test.cc | 24 ----- 4 files changed, 49 insertions(+), 82 deletions(-) diff --git a/tensorflow/core/grappler/utils/graph_view.cc b/tensorflow/core/grappler/utils/graph_view.cc index 0f210529840..0dccee582ee 100644 --- a/tensorflow/core/grappler/utils/graph_view.cc +++ b/tensorflow/core/grappler/utils/graph_view.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow/core/grappler/utils/graph_view.h" +#include + #include "absl/container/flat_hash_set.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -22,6 +24,7 @@ limitations under the License. #include "tensorflow/core/graph/tensor_id.h" #include "tensorflow/core/grappler/op_types.h" #include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/grappler/utils/graph_view_internal.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/gtl/map_util.h" @@ -264,21 +267,24 @@ void Mutation::AddMutation( node->update_index_ = updated_nodes_.size(); updated_nodes_.emplace_back(graph_view_, node->node_index_); mutate_fn(&updated_nodes_.back()); - } else { + } else if (!removed_nodes_[node->node_index_]) { auto& diff = updated_nodes_[node->update_index_]; - if (!diff.removed) { - mutate_fn(&diff); - } + mutate_fn(&diff); } } void Mutation::RemoveNode(MutableNodeView* node) { - AddMutation(node, [](MutableNodeViewDiff* diff) { - // Clear existing MutableNodeViewDiff as when node is removed no change to - // its internal state matter. - internal::Reset(diff); - internal::SetRemoved(diff, true); - }); + auto& update_index = node->update_index_; + if (update_index != internal::kMissingIndex) { + if (update_index < updated_nodes_.size() - 1) { + graph_view_->nodes_[updated_nodes_.back().node_index].update_index_ = + update_index; + std::swap(updated_nodes_[update_index], updated_nodes_.back()); + } + updated_nodes_.pop_back(); + update_index = internal::kMissingIndex; + } + removed_nodes_[node->node_index_] = true; } void Mutation::UpdateNodeName(MutableNodeView* node, absl::string_view name) { @@ -408,6 +414,7 @@ void Mutation::RemoveNodeAttr(const MutationNewNode& node, void Mutation::ResetInternal() { std::vector().swap(updated_nodes_); + std::vector(graph_view_->NumNodes()).swap(removed_nodes_); std::vector().swap(new_nodes_); } @@ -469,6 +476,7 @@ MutableGraphView::MutableGraphView(GraphDef* graph, Status* status) return; } AddFaninsInternal(&fanins); + mutation_.ResetInternal(); *status = Status::OK(); } @@ -591,16 +599,23 @@ Status MutableGraphView::GetNodeNamesAndPartitionUpdatedNodes( std::vector* renamed_nodes, std::vector* inplace_nodes, std::vector* empty_diff_node_indices) { + // For all nodes to be removed and renamed, mark their original names as + // missing and put associated node index in graph. for (const auto& diff : mutation_.updated_nodes_) { - // For all nodes to be removed and renamed, mark their original names as - // missing and put associated node index in graph. - if (diff.removed || diff.update_name) { + if (diff.update_name) { const int index = diff.node_index; const string& node_name = nodes_[index].GetName(); node_names->emplace(node_name, index); } } + for (int i = 0; i < mutation_.removed_nodes_.size(); ++i) { + if (mutation_.removed_nodes_[i]) { + const string& node_name = nodes_[i].GetName(); + node_names->emplace(node_name, i); + } + } + auto name_conflict = [](const absl::string_view node_name) { return errors::InvalidArgument(kMutableGraphViewApplyError, "multiple nodes with the name: '", node_name, @@ -617,8 +632,6 @@ Status MutableGraphView::GetNodeNamesAndPartitionUpdatedNodes( if (internal::IsEmpty(&diff)) { empty_diff_node_indices->emplace_back(diff.node_index); continue; - } else if (diff.removed) { - continue; } // Get name of updated node after potential mutation. const string& node_name = @@ -699,16 +712,15 @@ Status MutableGraphView::RemovedOrMissingNodeFanoutsWellFormed( // Check all fanouts of a single port. MutableNodeView* fanout_view = regular_fanout.node_view(); if (fanout_view->update_index_ == internal::kMissingIndex) { - if (!overwritten_nodes[fanout_view->node_index_]) { + if (mutation_.removed_nodes_[fanout_view->node_index_]) { + // Fanout node will be removed, this can be ignored. + continue; + } else if (!overwritten_nodes[fanout_view->node_index_]) { // Fanout is not updated or removed/overwritten. return bad_fanout(fanout_view->GetName(), node_name_state.first); } } else { auto& diff = mutation_.updated_nodes_[fanout_view->update_index_]; - if (diff.removed) { - // Fanout node will be removed, this can be ignored. - continue; - } const int last_index = fanout_view->NumRegularFanins() - diff.num_regular_inputs_to_remove - 1; if (regular_fanout.index() > last_index) { @@ -726,16 +738,15 @@ Status MutableGraphView::RemovedOrMissingNodeFanoutsWellFormed( for (const auto& controlled_fanout : node_view.GetControlledFanouts()) { MutableNodeView* fanout_view = controlled_fanout.node_view(); if (fanout_view->update_index_ == internal::kMissingIndex) { - if (!overwritten_nodes[fanout_view->node_index_]) { + if (mutation_.removed_nodes_[fanout_view->node_index_]) { + // Fanout node will be removed, this can be ignored. + continue; + } else if (!overwritten_nodes[fanout_view->node_index_]) { // Fanout is not updated or removed/overwritten. return bad_fanout(fanout_view->GetName(), node_name_state.first); } } else { auto& diff = mutation_.updated_nodes_[fanout_view->update_index_]; - if (diff.removed) { - // Fanout node will be removed, this can be ignored. - continue; - } // Check if controlling fanin is removed. if (diff.controlling_inputs_to_remove.find( controlled_fanout.fanin_index_) == @@ -789,7 +800,7 @@ Status MutableGraphView::CheckNodeNamesAndFanins( Status MutableGraphView::CheckKernelRegisteredForNodes() { Status s; for (auto& diff : mutation_.updated_nodes_) { - if (internal::IsEmpty(&diff) || diff.removed) { + if (internal::IsEmpty(&diff)) { continue; } @@ -903,10 +914,8 @@ void MutableGraphView::FixRenamedNodes( nodes_[renamed.overwritten_node_index_]; ReplaceNodeFanouts(&renamed_node, &node_to_overwrite); node_index_by_name_.erase(node_to_overwrite.GetName()); - if (node_to_overwrite.update_index_ != internal::kMissingIndex && - mutation_.updated_nodes_[node_to_overwrite.update_index_].removed) { - (*overwritten_name_removed_nodes)[node_to_overwrite.update_index_] = - true; + if (mutation_.removed_nodes_[node_to_overwrite.node_index_]) { + (*overwritten_name_removed_nodes)[node_to_overwrite.node_index_] = true; } } else { // No existing fanouts. @@ -939,15 +948,7 @@ void MutableGraphView::AddNewNodes( node_def->mutable_device()->swap(*new_node.node.mutable_device()); node_def->mutable_input()->Clear(); node_def->mutable_attr()->swap(*new_node.node.mutable_attr()); - if (node_view.update_index_ != internal::kMissingIndex) { - // The only case for this to occur is if a node is explicitly marked for - // removal. In that case, unlink it from it's associated - // MutableNodeViewDiff. - mutation_.updated_nodes_[node_view.update_index_].node_index = - internal::kMissingIndex; - mutation_.updated_nodes_[node_view.update_index_].removed = false; - node_view.update_index_ = internal::kMissingIndex; - } + mutation_.removed_nodes_[node_index] = false; } else { // New node. auto* new_node_def = graph_->add_node(); @@ -1169,8 +1170,7 @@ inline void MutableGraphView::AddControllingFaninInternal( void MutableGraphView::ApplyNodeUpdates() { for (auto& diff : mutation_.updated_nodes_) { - if (diff.removed || diff.node_index == internal::kMissingIndex || - internal::IsEmpty(&diff)) { + if (internal::IsEmpty(&diff)) { continue; } MutableNodeView& node_view = nodes_[diff.node_index]; @@ -1299,12 +1299,11 @@ void MutableGraphView::RemoveNodesInternal( std::vector node_indices_to_remove; node_indices_to_remove.reserve(mutation_.updated_nodes_.size() + overwritten_nodes.size()); - for (int i = 0; i < mutation_.updated_nodes_.size(); ++i) { - const auto& diff = mutation_.updated_nodes_[i]; - if (diff.removed) { - auto& node = nodes_[diff.node_index]; + for (int i = 0; i < mutation_.removed_nodes_.size(); ++i) { + if (mutation_.removed_nodes_[i]) { + auto& node = nodes_[i]; RemoveAllFaninFanoutInternal(&node); - node_indices_to_remove.push_back(diff.node_index); + node_indices_to_remove.push_back(i); if (!overwritten_name_removed_nodes[i]) { node_index_by_name_.erase(node.GetName()); } @@ -1645,8 +1644,7 @@ Status MutableGraphView::ApplyMutationInternal() { // Node name and associated fanouts. absl::flat_hash_map renamed_fanouts; // Removed nodes where name was overwritten by a renamed node. - std::vector overwritten_name_removed_nodes( - mutation_.updated_nodes_.size()); + std::vector overwritten_name_removed_nodes(nodes_.size()); // Fix renaming of existing nodes by swapping fanouts and rehashing names. // This will also overwrite removed or unmodified nodes. FixRenamedNodes(&renamed_nodes, &renamed_fanouts, diff --git a/tensorflow/core/grappler/utils/graph_view.h b/tensorflow/core/grappler/utils/graph_view.h index d5186cf08e3..456c68a30e9 100644 --- a/tensorflow/core/grappler/utils/graph_view.h +++ b/tensorflow/core/grappler/utils/graph_view.h @@ -359,6 +359,7 @@ class Mutation { MutableGraphView* graph_view_ = nullptr; int mutation_counter_ = 0; std::vector updated_nodes_; + std::vector removed_nodes_; using MutationNewNodeHolder = internal::NewNode; std::vector new_nodes_; diff --git a/tensorflow/core/grappler/utils/graph_view_internal.h b/tensorflow/core/grappler/utils/graph_view_internal.h index b1756a465fe..837c05ecdbd 100644 --- a/tensorflow/core/grappler/utils/graph_view_internal.h +++ b/tensorflow/core/grappler/utils/graph_view_internal.h @@ -364,7 +364,6 @@ struct NodeViewDiff { GraphViewT* graph_view; int node_index; - bool removed = false; string name; bool update_name = false; string op; @@ -393,12 +392,6 @@ struct NodeViewDiff { AttrValueMap processed_attrs; }; -// Sets node for removal via diff. -template -inline void SetRemoved(NodeViewDiff* diff, bool removed) { - diff->removed = removed; -} - // Updates node name. If `name` is the same as the name in the original node, // the field will be cleared in the diff. template @@ -629,8 +622,8 @@ template inline bool IsEmpty(NodeViewDiff* diff) { ResizeByTrimmingEndForValue(&diff->regular_inputs_to_remove, false); ResizeByTrimmingEndForValue(&diff->regular_inputs_to_add, EmptyTensorId()); - return !diff->removed && !diff->update_name && !diff->update_op && - !diff->update_device && diff->regular_inputs_to_add.empty() && + return !diff->update_name && !diff->update_op && !diff->update_device && + diff->regular_inputs_to_add.empty() && diff->regular_inputs_to_update.empty() && diff->regular_inputs_to_remove.empty() && diff->controlling_inputs_to_add.empty() && @@ -641,7 +634,6 @@ inline bool IsEmpty(NodeViewDiff* diff) { // Resets and clears existing diff. template inline void Reset(NodeViewDiff* diff) { - diff->removed = false; diff->name.clear(); diff->update_name = false; diff->op.clear(); diff --git a/tensorflow/core/grappler/utils/graph_view_internal_test.cc b/tensorflow/core/grappler/utils/graph_view_internal_test.cc index cb959aea16b..865badc71d6 100644 --- a/tensorflow/core/grappler/utils/graph_view_internal_test.cc +++ b/tensorflow/core/grappler/utils/graph_view_internal_test.cc @@ -52,30 +52,6 @@ absl::flat_hash_map GetUpdatedNodeNames( using MutableNodeViewDiff = NodeViewDiff; -TEST(MutableNodeViewDiffTest, SetRemoved) { - GraphDef graph = SimpleTestGraphForMutation(); - - Status s; - MutableGraphView graph_view(&graph, &s); - TF_ASSERT_OK(s); - auto updated_node_names = GetUpdatedNodeNames(&graph_view); - - MutableNodeView* d_node = graph_view.GetNode("d"); - ASSERT_NE(d_node, nullptr); - - MutableNodeViewDiff diff(&graph_view, d_node->node_index()); - EXPECT_TRUE(IsEmpty(&diff)); - EXPECT_TRUE(IsWellFormed(&diff, updated_node_names)); - - SetRemoved(&diff, true); - EXPECT_FALSE(IsEmpty(&diff)); - EXPECT_TRUE(IsWellFormed(&diff, updated_node_names)); - - SetRemoved(&diff, false); - EXPECT_TRUE(IsEmpty(&diff)); - EXPECT_TRUE(IsWellFormed(&diff, updated_node_names)); -} - TEST(MutableNodeViewDiffTest, UpdateName) { GraphDef graph = SimpleTestGraphForMutation(); From 30ecc601c90dd4d57f0e9cb381dbcc5eda811890 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 11:41:04 -0700 Subject: [PATCH 281/332] Rename FunctionAttr to SymbolRefAttr. This allows for the attribute to hold symbolic references to other operations than FuncOp. This also allows for removing the dependence on FuncOp from the base Builder. PiperOrigin-RevId: 257650017 --- .../compiler/mlir/lite/utils/quantization_driver.cc | 1 + tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc | 8 ++++---- tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h | 12 ++++++++---- .../mlir/tensorflow/translate/export_graphdef.cc | 2 +- .../mlir/tensorflow/translate/import_graphdef.cc | 11 ++++++----- .../compiler/mlir/tensorflow/utils/export_utils.cc | 8 ++++---- tensorflow/compiler/mlir/xla/ir/xla_ops.td | 6 +++--- 7 files changed, 27 insertions(+), 21 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/utils/quantization_driver.cc b/tensorflow/compiler/mlir/lite/utils/quantization_driver.cc index fa99cf38c8e..d78aa92f36c 100644 --- a/tensorflow/compiler/mlir/lite/utils/quantization_driver.cc +++ b/tensorflow/compiler/mlir/lite/utils/quantization_driver.cc @@ -25,6 +25,7 @@ limitations under the License. #include "mlir/Dialect/QuantOps/QuantTypes.h" // TF:local_config_mlir #include "mlir/IR/Attributes.h" // TF:local_config_mlir #include "mlir/IR/Builders.h" // TF:local_config_mlir +#include "mlir/IR/Function.h" // TF:local_config_mlir #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir #include "mlir/IR/Matchers.h" // TF:local_config_mlir #include "mlir/IR/Operation.h" // TF:local_config_mlir diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc index 5838cd2c6b6..c664983bc1b 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc @@ -275,10 +275,10 @@ static LogicalResult Verify(FusedBatchNormOp op) { //===----------------------------------------------------------------------===// LogicalResult IfOp::verify() { - auto thenAttr = getAttrOfType("then_branch"); + auto thenAttr = getAttrOfType("then_branch"); if (!thenAttr) return emitOpError("requires then_branch attribute"); - auto elseAttr = getAttrOfType("else_branch"); + auto elseAttr = getAttrOfType("else_branch"); if (!elseAttr) return emitOpError("requires else_branch attribute"); auto module = getParentOfType(); @@ -728,7 +728,7 @@ void TruncateDivOp::getCanonicalizationPatterns( //===----------------------------------------------------------------------===// LogicalResult WhileOp::verify() { - auto condAttr = getAttrOfType("cond"); + auto condAttr = getAttrOfType("cond"); if (!condAttr) return emitOpError("requires cond attribute"); auto module = getParentOfType(); @@ -739,7 +739,7 @@ LogicalResult WhileOp::verify() { if (condFuncType.getNumResults() != 1) return emitOpError("requires cond function to have exactly one result"); - auto bodyAttr = getAttrOfType("body"); + auto bodyAttr = getAttrOfType("body"); if (!bodyAttr) return emitOpError("requires body attribute"); auto bodyFn = module.lookupSymbol(bodyAttr.getValue()); auto bodyFuncType = bodyFn.getType(); diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h index c9c65a94ea9..dbd37e6ee54 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h @@ -119,11 +119,11 @@ class IfOp : public Op::Impl, // TODO(b/132271680): This is not following Google naming style StringRef getThen() { - return getAttrOfType("then_branch").getValue(); + return getAttrOfType("then_branch").getValue(); } StringRef getElse() { - return getAttrOfType("else_branch").getValue(); + return getAttrOfType("else_branch").getValue(); } LogicalResult verify(); @@ -157,8 +157,12 @@ class WhileOp : public Op("cond").getValue(); } - StringRef getBody() { return getAttrOfType("body").getValue(); } + StringRef getCond() { + return getAttrOfType("cond").getValue(); + } + StringRef getBody() { + return getAttrOfType("body").getValue(); + } LogicalResult verify(); }; diff --git a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc index ce3c6332f8b..22c1a4dd70c 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/export_graphdef.cc @@ -512,7 +512,7 @@ Status Exporter::ConvertLibFunction(const ExporterConfigs& configs, // Checks for gradient attribute. If present converts the gradient function // and populates the GradientDef. auto grad_string = mlir::TF::TensorFlowDialect::GetGradientAttrName(); - if (auto attr = function.getAttrOfType(grad_string)) { + if (auto attr = function.getAttrOfType(grad_string)) { auto grad_func = function.getParentOfType().lookupSymbol( attr.getValue()); diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc index a75ea3496d1..73fe028f366 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_graphdef.cc @@ -30,6 +30,7 @@ limitations under the License. #include "llvm/Support/raw_ostream.h" #include "mlir/IR/Attributes.h" // TF:local_config_mlir #include "mlir/IR/Builders.h" // TF:local_config_mlir +#include "mlir/IR/Function.h" // TF:local_config_mlir #include "mlir/IR/Identifier.h" // TF:local_config_mlir #include "mlir/IR/Location.h" // TF:local_config_mlir #include "mlir/IR/MLIRContext.h" // TF:local_config_mlir @@ -158,8 +159,8 @@ class Importer { return ::tensorflow::ConvertTensorProto(value, builder_.get()); } - // Converts func name in graphdef to mlir::FunctionAttribute. - StatusOr ConvertFunctionCallName( + // Converts func name in graphdef to mlir::SymbolRefAttribute. + StatusOr ConvertFunctionCallName( const std::string& func_name); // Converts the given non-function-call AttrValue to an MLIR Attribute. @@ -611,12 +612,12 @@ Status Importer::ConvertFunctionCallAttribute( return Status::OK(); } -StatusOr Importer::ConvertFunctionCallName( +StatusOr Importer::ConvertFunctionCallName( const std::string& func_name) { TF_RETURN_IF_ERROR(ConvertLibFunction(func_name)); auto mlir_func_name = (*tf_name_to_mlir_name_)[func_name]; auto func = module_.lookupSymbol(mlir_func_name); - return builder_->getFunctionAttr(func); + return builder_->getSymbolRefAttr(func); } StatusOr Importer::ConvertAttributeValue( @@ -722,7 +723,7 @@ Status Importer::ConvertLibFunction(const std::string& func_name) { TF_RETURN_IF_ERROR(ConvertLibFunction(grad_func_name)); auto mlir_grad_func_name = (*tf_name_to_mlir_name_)[grad_func_name]; auto grad_func = module_.lookupSymbol(mlir_grad_func_name); - auto gradient_attr = builder_->getFunctionAttr(grad_func); + auto gradient_attr = builder_->getSymbolRefAttr(grad_func); auto grad_string = mlir::TF::TensorFlowDialect::GetGradientAttrName(); attributes.push_back(builder_->getNamedAttr(grad_string, gradient_attr)); } diff --git a/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc b/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc index 1067aabdf81..a2f803c0858 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc @@ -115,7 +115,7 @@ Status ConvertAttribute(const mlir::UnitAttr& attr, AttrValue* value) { return Status::OK(); } -Status ConvertAttribute(const mlir::FunctionAttr& attr, AttrValue* value) { +Status ConvertAttribute(const mlir::SymbolRefAttr& attr, AttrValue* value) { value->mutable_func()->set_name(attr.getValue()); return Status::OK(); } @@ -149,7 +149,7 @@ Status ConvertAttribute(const mlir::ArrayAttr& attr, AttrValue* value) { TensorProto tensor; TF_RETURN_IF_ERROR(ConvertToTensorProto(attr, &tensor)); *list->add_tensor() = tensor; - } else if (auto attr = a.dyn_cast()) { + } else if (auto attr = a.dyn_cast()) { AttrValue attrVal; TF_RETURN_IF_ERROR(ConvertAttribute(attr, &attrVal)); *list->add_func() = attrVal.func(); @@ -218,8 +218,8 @@ Status ConvertAttributes(const llvm::ArrayRef attrs, } AttrValue value; switch (attr.getKind()) { - case mlir::StandardAttributes::Function: { - auto func_attr = attr.cast(); + case mlir::StandardAttributes::SymbolRef: { + auto func_attr = attr.cast(); value.mutable_func()->set_name(func_attr.getValue()); func_call_attrs[string(name)] = value; continue; diff --git a/tensorflow/compiler/mlir/xla/ir/xla_ops.td b/tensorflow/compiler/mlir/xla/ir/xla_ops.td index 9c57c8e1543..a05dd9b3d1d 100644 --- a/tensorflow/compiler/mlir/xla/ir/xla_ops.td +++ b/tensorflow/compiler/mlir/xla/ir/xla_ops.td @@ -298,8 +298,8 @@ def XLA_WhileOp: XLA_Op<"while", [NoSideEffect, SameOperandsAndResultType]> { let arguments = (ins Variadic:$val, - FunctionAttr:$cond, - FunctionAttr:$body + SymbolRefAttr:$cond, + SymbolRefAttr:$body ); let results = (outs Variadic:$res); @@ -320,7 +320,7 @@ def XLA_ReduceOp: XLA_Op<"reduce", [NoSideEffect]> { let arguments = (ins Variadic:$operands_and_init, - FunctionAttr:$computation, + SymbolRefAttr:$computation, ElementsAttr:$dimensions ); From 74e01e8cd5dcc5c72101bc63af38d6c1c8d9c066 Mon Sep 17 00:00:00 2001 From: Pavithra Vijay Date: Thu, 11 Jul 2019 11:44:21 -0700 Subject: [PATCH 282/332] Fix issue - in multi-output model if class weight is specified just for some outputs the weights get incorrectly mapped to placeholders PiperOrigin-RevId: 257650645 --- tensorflow/python/keras/engine/training.py | 35 +- .../python/keras/metrics_correctness_test.py | 512 ++++++++++++++---- 2 files changed, 414 insertions(+), 133 deletions(-) diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index 44b938d1235..584ec1a8a0e 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -1415,14 +1415,19 @@ class Model(network.Network): def _update_sample_weight_modes(self, sample_weights=None): """Updates sample weight modes based on training/eval inputs. + Sample weight placeholders will be created for all or no outputs + based on whether sample_weight is provided for any output. + If model contains `_sample_weight_modes` we check if the input `sample_weights` corresponds to the sample weight modes. - 1. If sample weight mode for output i is 'temporal', we do not - change it as the `temporal` mode has been set by the user. - 2. Set sample weight mode to be 'samplewise' for output i if sample - weight mode was not set before and sample weight inputs are given. + 1. Set sample weight mode to be 'temporal' for output i, if `compile` + sample_weight_mode was set to `temporal` and sample weight inputs + are given for one or more outputs. + 2. Set sample weight mode to be 'samplewise' for output i, if `compile` + sample_weight_mode was not set and sample weight inputs are given for + one or more outputs. 3. Reset sample weight mode to None for output i if sample weight mode - was set to 'samplewise' but there is no sample weight input. + was set but there is no sample weight input. Args: sample_weights: List of sample weights of the same length as model outputs @@ -1430,21 +1435,11 @@ class Model(network.Network): """ if not self._is_compiled: return - if not sample_weights: - sample_weights = [None] * len(self._training_endpoints) - for endpoint, sample_weight in zip(self._training_endpoints, - sample_weights): - if endpoint.sample_weight_mode == 'temporal': - # If sample weight mode for endpoint is 'temporal', do nothing. - continue - if endpoint.sample_weight_mode is None and sample_weight is not None: - # Set sample weight mode to be 'samplewise' for output i if sample - # weight mode was not set before and sample weight inputs are given. - endpoint.sample_weight_mode = 'samplewise' - elif (endpoint.sample_weight_mode == 'samplewise' and - sample_weight is None): - # Reset sample weight mode to None for output i if sample weight mode - # was set to 'samplewise' but there is no sample weight input. + if sample_weights and any([s is not None for s in sample_weights]): + for endpoint in self._training_endpoints: + endpoint.sample_weight_mode = self.sample_weight_mode or 'samplewise' + else: + for endpoint in self._training_endpoints: endpoint.sample_weight_mode = None def _recompile_weights_loss_and_weighted_metrics(self): diff --git a/tensorflow/python/keras/metrics_correctness_test.py b/tensorflow/python/keras/metrics_correctness_test.py index 4f761bfec11..29a89203dae 100644 --- a/tensorflow/python/keras/metrics_correctness_test.py +++ b/tensorflow/python/keras/metrics_correctness_test.py @@ -45,13 +45,19 @@ def get_multi_io_model(): return testing_utils.get_multi_io_model(branch_a, branch_b) -def custom_generator_multi_io(): +def custom_generator_multi_io(sample_weights=None): batch_size = 2 num_samples = 4 inputs = np.asarray([[1.], [2.], [3.], [4.]]) - targets = np.asarray([[2.], [4.], [6.], [8.]]) - w1 = np.asarray([2., 3., 4., 5.]) - w2 = np.asarray([3.5, 2.5, 1.5, 0.5]) + targets_1 = np.asarray([[2.], [4.], [6.], [8.]]) + targets_2 = np.asarray([[1.], [2.], [3.], [4.]]) + if sample_weights: + assert len(sample_weights) == 2 + w1 = sample_weights[0] + w2 = sample_weights[1] + else: + w1 = None + w2 = None i = 0 while True: batch_index = i * batch_size % num_samples @@ -59,8 +65,14 @@ def custom_generator_multi_io(): start = batch_index end = start + batch_size x = [inputs[start:end], inputs[start:end]] - y = [targets[start:end], targets[start:end]] - w = [w1[start:end], w2[start:end]] + y = [targets_1[start:end], targets_2[start:end]] + if sample_weights: + w = [ + None if w1 is None else w1[start:end], + None if w2 is None else w2[start:end] + ] + else: + w = None yield x, y, w @@ -80,37 +92,18 @@ class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase): run_eagerly=testing_utils.should_run_eagerly()) return model - def _custom_generator(self): - batch_size = 2 - num_samples = 4 - inputs = np.asarray([[1.], [2.], [3.], [4.]]) - targets = np.asarray([[2.], [4.], [6.], [8.]]) - w1 = np.asarray([2., 3., 4., 5.]) - w2 = np.asarray([3.5, 2.5, 1.5, 0.5]) - i = 0 - while True: - batch_index = i * batch_size % num_samples - i += 1 - start = batch_index - end = start + batch_size - x = [inputs[start:end], inputs[start:end]] - y = [targets[start:end], targets[start:end]] - w = [w1[start:end], w2[start:end]] - yield x, y, w - def setUp(self): super(TestMetricsCorrectnessMultiIO, self).setUp() self.x = np.asarray([[1.], [2.], [3.], [4.]]) - self.y = np.asarray([[2.], [4.], [6.], [8.]]) - self.weights_1 = np.asarray([2., 3., 4., 5.]) - self.weights_2 = np.asarray([3.5, 2.5, 1.5, 0.5]) + self.y1 = np.asarray([[2.], [4.], [6.], [8.]]) + self.y2 = np.asarray([[1.], [2.], [3.], [4.]]) + self.sample_weight_1 = np.asarray([2., 3., 4., 5.]) + self.sample_weight_2 = np.asarray([3.5, 2.5, 1.5, 0.5]) + self.class_weight_1 = {2: 2, 4: 3, 6: 4, 8: 5} + self.class_weight_2 = {1: 3.5, 2: 2.5, 3: 1.5, 4: 0.5} - # y_true = [[2.], [4.], [6.], [8.]], y_pred = [[3.], [6.], [9.], [12.]] - - # Metric `output_1`, `output_2`: - # Total = ((3 - 2)^2 + (6 - 4)^2) + ((9 - 6)^2 + (12 - 8)^2) = 30, - # Count = 2 + 2 - # Result = 7.5 + # y_true_1 = [[2.], [4.], [6.], [8.]], y_pred = [[3.], [6.], [9.], [12.]] + # y_true_2 = [[1.], [2.], [3.], [4.]], y_pred = [[3.], [6.], [9.], [12.]] # Weighted metric `output_1`: # Total = ((3 - 2)^2 * 2 + (6 - 4)^2 * 3) + @@ -120,68 +113,164 @@ class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase): # Result = 9.2857141 # Weighted metric `output_2`: - # Total = ((3 - 2)^2 * 3.5 + (6 - 4)^2 * 2.5) + - # ((9 - 6)^2 * 1.5 + (12 - 8)^2 * 0.5) - # = 35 + # Total = ((3 - 1)^2 * 3.5 + (6 - 2)^2 * 2.5) + + # ((9 - 3)^2 * 1.5 + (12 - 4)^2 * 0.5) + # = 140 # Count = (3.5 + 2.5) + (1.5 + 0.5) - # Result = 4.375 + # Result = 17.5 - # Loss `output_1`: + # Loss `output_1` with weights: # Total = ((3 - 2)^2 * 2 + (6 - 4)^2 * 3) + # ((9 - 6)^2 * 4 + (12 - 8)^2 * 5) # = 130 # Count = 2 + 2 # Result = 32.5 - # Loss `output_2`: - # Total = ((3 - 2)^2 * 3.5 + (6 - 4)^2 * 2.5) + - # ((9 - 6)^2 * 1.5 + (12 - 8)^2 * 0.5) - # = 35 + # Loss `output_1` without weights/Metric `output_1`: + # Total = ((3 - 2)^2 + (6 - 4)^2) + ((9 - 6)^2 + (12 - 8)^2) = 30 # Count = 2 + 2 - # Result = 8.75 + # Result = 7.5 - # Total loss = 32.5 + 8.75 = 41.25 + # Loss `output_2` with weights: + # Total = ((3 - 1)^2 * 3.5 + (6 - 2)^2 * 2.5) + + # ((9 - 3)^2 * 1.5 + (12 - 4)^2 * 0.5) + # = 140 + # Count = 2 + 2 + # Result = 35 - wmse = 'mean_squared_error_2' + # Loss `output_2` without weights/Metric `output_2`: + # Total = ((3 - 1)^2 + (6 - 2)^2) + ((9 - 3)^2 + (12 - 4)^2) = 120 + # Count = 2 + 2 + # Result = 30 + + # Total loss with weights = 32.5 + 35 = 67.5 + # Total loss without weights = 7.5 + 30 = 37.5 + + self.wmse = 'mean_squared_error_2' if not tf2.enabled(): - wmse = 'weighted_' + wmse + self.wmse = 'weighted_' + self.wmse + self.expected_fit_result_with_weights = { + 'output_1_mean_squared_error': [7.5, 7.5], + 'output_2_mean_squared_error': [30, 30], + 'output_1_' + self.wmse: [9.286, 9.286], + 'output_2_' + self.wmse: [17.5, 17.5], + 'loss': [67.5, 67.5], + 'output_1_loss': [32.5, 32.5], + 'output_2_loss': [35, 35], + } + + self.expected_fit_result_with_weights_output_2 = { + 'output_1_mean_squared_error': [7.5, 7.5], + 'output_2_mean_squared_error': [30, 30], + 'output_1_' + self.wmse: [7.5, 7.5], + 'output_2_' + self.wmse: [17.5, 17.5], + 'loss': [42.5, 42.5], + 'output_1_loss': [7.5, 7.5], + 'output_2_loss': [35, 35], + } + self.expected_fit_result = { 'output_1_mean_squared_error': [7.5, 7.5], - 'output_2_mean_squared_error': [7.5, 7.5], - 'output_1_' + wmse: [9.286, 9.286], - 'output_2_' + wmse: [4.375, 4.375], - 'loss': [41.25, 41.25], - 'output_1_loss': [32.5, 32.5], - 'output_2_loss': [8.75, 8.75], + 'output_2_mean_squared_error': [30, 30], + 'output_1_' + self.wmse: [7.5, 7.5], + 'output_2_' + self.wmse: [30, 30], + 'loss': [37.5, 37.5], + 'output_1_loss': [7.5, 7.5], + 'output_2_loss': [30, 30], } # In the order: 'loss', 'output_1_loss', 'output_2_loss', # 'output_1_mean_squared_error', 'output_1_mean_squared_error_2', # 'output_2_mean_squared_error', 'output_2_mean_squared_error_2' - self.expected_batch_result = [41.25, 32.5, 8.75, 7.5, 9.286, 7.5, 4.375] + self.expected_batch_result_with_weights = [ + 67.5, 32.5, 35, 7.5, 9.286, 30, 17.5 + ] + self.expected_batch_result_with_weights_output_2 = [ + 42.5, 7.5, 35, 7.5, 7.5, 30, 17.5 + ] + self.expected_batch_result = [37.5, 7.5, 30, 7.5, 7.5, 30, 30] def test_fit(self): model = self._get_compiled_multi_io_model() - history = model.fit([self.x, self.x], [self.y, self.y], - sample_weight={ - 'output_1': self.weights_1, - 'output_2': self.weights_2, - }, + history = model.fit([self.x, self.x], [self.y1, self.y2], batch_size=2, epochs=2, shuffle=False) for key, value in self.expected_fit_result.items(): self.assertAllClose(history.history[key], value, 1e-3) + def test_fit_with_sample_weight(self): + model = self._get_compiled_multi_io_model() + history = model.fit([self.x, self.x], [self.y1, self.y2], + sample_weight={ + 'output_1': self.sample_weight_1, + 'output_2': self.sample_weight_2, + }, + batch_size=2, + epochs=2, + shuffle=False) + for key, value in self.expected_fit_result_with_weights.items(): + self.assertAllClose(history.history[key], value, 1e-3) + + # Set weights for one output (use batch size). + history = model.fit([self.x, self.x], [self.y1, self.y2], + sample_weight={'output_2': self.sample_weight_2}, + batch_size=2, + epochs=2, + shuffle=False) + + for key, value in self.expected_fit_result_with_weights_output_2.items(): + self.assertAllClose(history.history[key], value, 1e-3) + + def test_fit_with_class_weight(self): + model = self._get_compiled_multi_io_model() + history = model.fit([self.x, self.x], [self.y1, self.y2], + class_weight={ + 'output_1': self.class_weight_1, + 'output_2': self.class_weight_2, + }, + batch_size=2, + epochs=2, + shuffle=False) + for key, value in self.expected_fit_result_with_weights.items(): + self.assertAllClose(history.history[key], value, 1e-3) + + # Set weights for one output. + history = model.fit([self.x, self.x], [self.y1, self.y2], + class_weight={'output_2': self.class_weight_2}, + batch_size=2, + epochs=2, + shuffle=False) + + for key, value in self.expected_fit_result_with_weights_output_2.items(): + self.assertAllClose(history.history[key], value, 1e-3) + def test_eval(self): model = self._get_compiled_multi_io_model() - eval_result = model.evaluate([self.x, self.x], [self.y, self.y], + eval_result = model.evaluate([self.x, self.x], [self.y1, self.y2], + batch_size=2) + self.assertAllClose(eval_result, self.expected_batch_result, 1e-3) + + def test_eval_with_sample_weight(self): + model = self._get_compiled_multi_io_model() + eval_result = model.evaluate([self.x, self.x], [self.y1, self.y2], batch_size=2, sample_weight={ - 'output_1': self.weights_1, - 'output_2': self.weights_2, + 'output_1': self.sample_weight_1, + 'output_2': self.sample_weight_2, }) - self.assertAllClose(eval_result, self.expected_batch_result, 1e-3) + self.assertAllClose(eval_result, self.expected_batch_result_with_weights, + 1e-3) + + # Set weights for one output. + model = self._get_compiled_multi_io_model() + eval_result = model.evaluate([self.x, self.x], [self.y1, self.y2], + batch_size=2, + sample_weight={ + 'output_2': self.sample_weight_2, + }) + self.assertAllClose(eval_result, + self.expected_batch_result_with_weights_output_2, 1e-3) # Verify that metric value is same with arbitrary weights and batch size. x = np.random.random((50, 1)) @@ -194,22 +283,65 @@ class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase): def test_train_on_batch(self): model = self._get_compiled_multi_io_model() - result = model.train_on_batch([self.x, self.x], [self.y, self.y], - sample_weight={ - 'output_1': self.weights_1, - 'output_2': self.weights_2, - }) + result = model.train_on_batch([self.x, self.x], [self.y1, self.y2]) self.assertAllClose(result, self.expected_batch_result, 1e-3) + def test_train_on_batch_with_sample_weight(self): + model = self._get_compiled_multi_io_model() + result = model.train_on_batch([self.x, self.x], [self.y1, self.y2], + sample_weight={ + 'output_1': self.sample_weight_1, + 'output_2': self.sample_weight_2, + }) + self.assertAllClose(result, self.expected_batch_result_with_weights, 1e-3) + + # Set weights for one output. + result = model.train_on_batch([self.x, self.x], [self.y1, self.y2], + sample_weight={ + 'output_2': self.sample_weight_2, + }) + self.assertAllClose(result, + self.expected_batch_result_with_weights_output_2, 1e-3) + + def test_train_on_batch_with_class_weight(self): + model = self._get_compiled_multi_io_model() + result = model.train_on_batch([self.x, self.x], [self.y1, self.y2], + class_weight={ + 'output_1': self.class_weight_1, + 'output_2': self.class_weight_2, + }) + self.assertAllClose(result, self.expected_batch_result_with_weights, 1e-3) + + # Set weights for one output. + result = model.train_on_batch([self.x, self.x], [self.y1, self.y2], + class_weight={ + 'output_2': self.class_weight_2, + }) + self.assertAllClose(result, + self.expected_batch_result_with_weights_output_2, 1e-3) + def test_test_on_batch(self): model = self._get_compiled_multi_io_model() - result = model.test_on_batch([self.x, self.x], [self.y, self.y], - sample_weight={ - 'output_1': self.weights_1, - 'output_2': self.weights_2, - }) + result = model.test_on_batch([self.x, self.x], [self.y1, self.y2]) self.assertAllClose(result, self.expected_batch_result, 1e-3) + def test_test_on_batch_with_sample_weight(self): + model = self._get_compiled_multi_io_model() + result = model.test_on_batch([self.x, self.x], [self.y1, self.y2], + sample_weight={ + 'output_1': self.sample_weight_1, + 'output_2': self.sample_weight_2, + }) + self.assertAllClose(result, self.expected_batch_result_with_weights, 1e-3) + + # Set weights for one output. + result = model.test_on_batch([self.x, self.x], [self.y1, self.y2], + sample_weight={ + 'output_2': self.sample_weight_2, + }) + self.assertAllClose(result, + self.expected_batch_result_with_weights_output_2, 1e-3) + def test_fit_generator(self): model = self._get_compiled_multi_io_model() history = model.fit_generator( @@ -217,11 +349,67 @@ class TestMetricsCorrectnessMultiIO(keras_parameterized.TestCase): for key, value in self.expected_fit_result.items(): self.assertAllClose(history.history[key], value, 1e-3) + def test_fit_generator_with_sample_weight(self): + model = self._get_compiled_multi_io_model() + history = model.fit_generator( + custom_generator_multi_io( + sample_weights=[self.sample_weight_1, self.sample_weight_2]), + steps_per_epoch=2, + epochs=2) + for key, value in self.expected_fit_result_with_weights.items(): + self.assertAllClose(history.history[key], value, 1e-3) + + # Set weights for one output. + history = model.fit_generator( + custom_generator_multi_io(sample_weights=[None, self.sample_weight_2]), + steps_per_epoch=2, + epochs=2) + for key, value in self.expected_fit_result_with_weights_output_2.items(): + self.assertAllClose(history.history[key], value, 1e-3) + + def test_fit_generator_with_class_weight(self): + model = self._get_compiled_multi_io_model() + history = model.fit_generator( + custom_generator_multi_io(), + class_weight={ + 'output_1': self.class_weight_1, + 'output_2': self.class_weight_2, + }, + steps_per_epoch=2, + epochs=2) + for key, value in self.expected_fit_result_with_weights.items(): + self.assertAllClose(history.history[key], value, 1e-3) + + # Set weights for one output. + history = model.fit_generator( + custom_generator_multi_io(), + class_weight={'output_2': self.class_weight_2}, + steps_per_epoch=2, + epochs=2) + for key, value in self.expected_fit_result_with_weights_output_2.items(): + self.assertAllClose(history.history[key], value, 1e-3) + def test_eval_generator(self): model = self._get_compiled_multi_io_model() eval_result = model.evaluate_generator(custom_generator_multi_io(), steps=2) self.assertAllClose(eval_result, self.expected_batch_result, 1e-3) + def test_eval_generator_with_sample_weight(self): + model = self._get_compiled_multi_io_model() + eval_result = model.evaluate_generator( + custom_generator_multi_io( + sample_weights=[self.sample_weight_1, self.sample_weight_2]), + steps=2) + self.assertAllClose(eval_result, self.expected_batch_result_with_weights, + 1e-3) + + # Set weights for one output. + eval_result = model.evaluate_generator( + custom_generator_multi_io(sample_weights=[None, self.sample_weight_2]), + steps=2) + self.assertAllClose(eval_result, + self.expected_batch_result_with_weights_output_2, 1e-3) + @keras_parameterized.run_with_all_model_types @keras_parameterized.run_all_keras_modes @@ -242,25 +430,27 @@ class TestMetricsCorrectnessSingleIO(keras_parameterized.TestCase): run_eagerly=testing_utils.should_run_eagerly()) return model - def _custom_generator(self): + def _custom_generator(self, sample_weight=None): batch_size = 2 num_samples = 4 x = np.asarray([[1.], [2.], [3.], [4.]]) y = np.asarray([[2.], [4.], [6.], [8.]]) - w = np.asarray([2., 3., 4., 5.]) + w = sample_weight i = 0 + while True: batch_index = i * batch_size % num_samples i += 1 start = batch_index end = start + batch_size - yield x[start:end], y[start:end], w[start:end] + yield x[start:end], y[start:end], None if w is None else w[start:end] def setUp(self): super(TestMetricsCorrectnessSingleIO, self).setUp() self.x = np.asarray([[1.], [2.], [3.], [4.]]) self.y = np.asarray([[2.], [4.], [6.], [8.]]) - self.weights = np.asarray([2., 3., 4., 5.]) + self.sample_weight = np.asarray([2., 3., 4., 5.]) + self.class_weight = {2: 2, 4: 3, 6: 4, 8: 5} # y_true = [[2.], [4.], [6.], [8.]], y_pred = [[3.], [6.], [9.], [12.]] @@ -276,43 +466,88 @@ class TestMetricsCorrectnessSingleIO(keras_parameterized.TestCase): # Count = (2 + 3) + (4 + 5) # Result = 9.2857141 - # Total loss: + # Total loss with weights: # Total = ((3 - 2)^2 * 2 + (6 - 4)^2 * 3) + # ((9 - 6)^2 * 4 + (12 - 8)^2 * 5) # = 130, # Count = 2 + 2 # Result = 32.5 + # Total loss without weights: + # Total = ((3 - 2)^2 + (6 - 4)^2) + + # ((9 - 6)^2 + (12 - 8)^2) + # = 30, + # Count = 2 + 2 + # Result = 7.5 + wmse = 'mean_squared_error_2' if not tf2.enabled(): wmse = 'weighted_' + wmse - self.expected_fit_result = { + + self.expected_fit_result_with_weights = { 'mean_squared_error': [7.5, 7.5], wmse: [9.286, 9.286], 'loss': [32.5, 32.5] } + self.expected_fit_result = { + 'mean_squared_error': [7.5, 7.5], + wmse: [7.5, 7.5], + 'loss': [7.5, 7.5] + } + # In the order: 'loss', 'mean_squared_error', 'mean_squared_error_2' - self.expected_batch_result = [32.5, 7.5, 9.286] + self.expected_batch_result_with_weights = [32.5, 7.5, 9.286] + self.expected_batch_result = [7.5, 7.5, 7.5] def test_fit(self): model = self._get_model() + history = model.fit( self.x, self.y, - sample_weight=self.weights, batch_size=2, epochs=2, shuffle=False) for key, value in self.expected_fit_result.items(): self.assertAllClose(history.history[key], value, 1e-3) + def test_fit_with_sample_weight(self): + model = self._get_model() + history = model.fit( + self.x, + self.y, + sample_weight=self.sample_weight, + batch_size=2, + epochs=2, + shuffle=False) + for key, value in self.expected_fit_result_with_weights.items(): + self.assertAllClose(history.history[key], value, 1e-3) + + def test_fit_with_class_weight(self): + model = self._get_model() + history = model.fit( + self.x, + self.y, + class_weight=self.class_weight, + batch_size=2, + epochs=2, + shuffle=False) + for key, value in self.expected_fit_result_with_weights.items(): + self.assertAllClose(history.history[key], value, 1e-3) + def test_eval(self): model = self._get_model() - eval_result = model.evaluate( - self.x, self.y, batch_size=2, sample_weight=self.weights) + eval_result = model.evaluate(self.x, self.y, batch_size=2) self.assertAllClose(eval_result, self.expected_batch_result, 1e-3) + def test_eval_with_sample_weight(self): + model = self._get_model() + eval_result = model.evaluate( + self.x, self.y, batch_size=2, sample_weight=self.sample_weight) + self.assertAllClose(eval_result, self.expected_batch_result_with_weights, + 1e-3) + # Verify that metric value is same with arbitrary weights and batch size. x = np.random.random((50, 1)) y = np.random.random((50, 1)) @@ -323,14 +558,32 @@ class TestMetricsCorrectnessSingleIO(keras_parameterized.TestCase): def test_train_on_batch(self): model = self._get_model() - result = model.train_on_batch(self.x, self.y, sample_weight=self.weights) + result = model.train_on_batch(self.x, self.y) self.assertAllClose(result, self.expected_batch_result, 1e-3) + def test_train_on_batch_with_sample_weight(self): + model = self._get_model() + result = model.train_on_batch( + self.x, self.y, sample_weight=self.sample_weight) + self.assertAllClose(result, self.expected_batch_result_with_weights, 1e-3) + + def test_train_on_batch_with_class_weight(self): + model = self._get_model() + result = model.train_on_batch( + self.x, self.y, class_weight=self.class_weight) + self.assertAllClose(result, self.expected_batch_result_with_weights, 1e-3) + def test_test_on_batch(self): model = self._get_model() - result = model.test_on_batch(self.x, self.y, sample_weight=self.weights) + result = model.test_on_batch(self.x, self.y) self.assertAllClose(result, self.expected_batch_result, 1e-3) + def test_test_on_batch_with_sample_weight(self): + model = self._get_model() + result = model.test_on_batch( + self.x, self.y, sample_weight=self.sample_weight) + self.assertAllClose(result, self.expected_batch_result_with_weights, 1e-3) + def test_fit_generator(self): model = self._get_model() history = model.fit_generator( @@ -338,11 +591,37 @@ class TestMetricsCorrectnessSingleIO(keras_parameterized.TestCase): for key, value in self.expected_fit_result.items(): self.assertAllClose(history.history[key], value, 1e-3) + def test_fit_generator_with_sample_weight(self): + model = self._get_model() + history = model.fit_generator( + self._custom_generator(sample_weight=self.sample_weight), + steps_per_epoch=2, + epochs=2) + for key, value in self.expected_fit_result_with_weights.items(): + self.assertAllClose(history.history[key], value, 1e-3) + + def test_fit_generator_with_class_weight(self): + model = self._get_model() + history = model.fit_generator( + self._custom_generator(), + steps_per_epoch=2, + epochs=2, + class_weight=self.class_weight) + for key, value in self.expected_fit_result_with_weights.items(): + self.assertAllClose(history.history[key], value, 1e-3) + def test_eval_generator(self): model = self._get_model() eval_result = model.evaluate_generator(self._custom_generator(), steps=2) self.assertAllClose(eval_result, self.expected_batch_result, 1e-3) + def test_eval_generator_with_sample_weight(self): + model = self._get_model() + eval_result = model.evaluate_generator( + self._custom_generator(sample_weight=self.sample_weight), steps=2) + self.assertAllClose(eval_result, self.expected_batch_result_with_weights, + 1e-3) + @keras_parameterized.run_with_all_model_types(exclude_models=['sequential']) @keras_parameterized.run_all_keras_modes @@ -364,9 +643,10 @@ class TestOutputLossMetrics(keras_parameterized.TestCase): def setUp(self): super(TestOutputLossMetrics, self).setUp() self.x = np.asarray([[1.], [2.], [3.], [4.]]) - self.y = np.asarray([[2.], [4.], [6.], [8.]]) - self.weights_1 = np.asarray([2., 3., 4., 5.]) - self.weights_2 = np.asarray([3.5, 2.5, 1.5, 0.5]) + self.y1 = np.asarray([[2.], [4.], [6.], [8.]]) + self.y2 = np.asarray([[1.], [2.], [3.], [4.]]) + self.sample_weight_1 = np.asarray([2., 3., 4., 5.]) + self.sample_weight_2 = np.asarray([3.5, 2.5, 1.5, 0.5]) # y_true = [[2.], [4.], [6.], [8.]], y_pred = [[3.], [6.], [9.], [12.]] @@ -380,11 +660,11 @@ class TestOutputLossMetrics(keras_parameterized.TestCase): # Loss `output_2`: # Per-sample weighted losses - # Batch 1 = [(3 - 2)^2 * 3.5, (6 - 4)^2 * 2.5)] = [3.5, 10] - # Batch 2 = [(9 - 6)^2 * 1.5, (12 - 8)^2 * 0.5)] = [13.5, 8] + # Batch 1 = [(3 - 1)^2 * 3.5, (6 - 2)^2 * 2.5)] = [14, 40] + # Batch 2 = [(9 - 3)^2 * 1.5, (12 - 4)^2 * 0.5)] = [54, 32] - # Result (reduction=SUM) = ((3.5 + 10) + (13.5 + 8))/2 = 17.5 - # Result (reduction=SUM_OVER_BATCH_SIZE/AUTO/NONE) = 35 / 4 = 8.75 + # Result (reduction=SUM) = ((14 + 40) + (54 + 32))/2 = 70 + # Result (reduction=SUM_OVER_BATCH_SIZE/AUTO/NONE) = 140 / 4 = 35 # When reduction is 'NONE' loss value that is passed to the optimizer will # be vector loss but what is reported is a scalar, which is an average of @@ -393,18 +673,18 @@ class TestOutputLossMetrics(keras_parameterized.TestCase): # Total loss = Output_loss_1 + Output_loss_2 sum_over_batch_size_fit_result = { - 'loss': [41.25, 41.25], + 'loss': [67.5, 67.5], 'output_1_loss': [32.5, 32.5], - 'output_2_loss': [8.75, 8.75], + 'output_2_loss': [35, 35], } self.expected_fit_result = { loss_reduction.ReductionV2.NONE: sum_over_batch_size_fit_result, loss_reduction.ReductionV2.SUM: { - 'loss': [82.5, 82.5], + 'loss': [135, 135], 'output_1_loss': [65, 65], - 'output_2_loss': [17.5, 17.5], + 'output_2_loss': [70, 70], }, loss_reduction.ReductionV2.AUTO: sum_over_batch_size_fit_result, @@ -414,19 +694,19 @@ class TestOutputLossMetrics(keras_parameterized.TestCase): # In the order: 'loss', 'output_1_loss', 'output_2_loss', self.expected_batch_result = { - loss_reduction.ReductionV2.NONE: [41.25, 32.5, 8.75], - loss_reduction.ReductionV2.SUM: [82.5, 65, 17.5], - loss_reduction.ReductionV2.AUTO: [41.25, 32.5, 8.75], - loss_reduction.ReductionV2.SUM_OVER_BATCH_SIZE: [41.25, 32.5, 8.75], + loss_reduction.ReductionV2.NONE: [67.5, 32.5, 35], + loss_reduction.ReductionV2.SUM: [135, 65, 70], + loss_reduction.ReductionV2.AUTO: [67.5, 32.5, 35], + loss_reduction.ReductionV2.SUM_OVER_BATCH_SIZE: [67.5, 32.5, 35], } def test_fit(self, reduction): model = self._get_compiled_multi_io_model( loss=losses.MeanSquaredError(reduction=reduction)) - history = model.fit([self.x, self.x], [self.y, self.y], + history = model.fit([self.x, self.x], [self.y1, self.y2], sample_weight={ - 'output_1': self.weights_1, - 'output_2': self.weights_2, + 'output_1': self.sample_weight_1, + 'output_2': self.sample_weight_2, }, batch_size=2, epochs=2, @@ -437,21 +717,21 @@ class TestOutputLossMetrics(keras_parameterized.TestCase): def test_eval(self, reduction): model = self._get_compiled_multi_io_model( loss=losses.MeanSquaredError(reduction=reduction)) - eval_result = model.evaluate([self.x, self.x], [self.y, self.y], + eval_result = model.evaluate([self.x, self.x], [self.y1, self.y2], batch_size=2, sample_weight={ - 'output_1': self.weights_1, - 'output_2': self.weights_2, + 'output_1': self.sample_weight_1, + 'output_2': self.sample_weight_2, }) self.assertAllClose(eval_result, self.expected_batch_result[reduction]) def test_train_on_batch(self, reduction): model = self._get_compiled_multi_io_model( loss=losses.MeanSquaredError(reduction=reduction)) - result = model.train_on_batch([self.x, self.x], [self.y, self.y], + result = model.train_on_batch([self.x, self.x], [self.y1, self.y2], sample_weight={ - 'output_1': self.weights_1, - 'output_2': self.weights_2, + 'output_1': self.sample_weight_1, + 'output_2': self.sample_weight_2, }) expected_values = self.expected_batch_result[reduction] @@ -463,10 +743,10 @@ class TestOutputLossMetrics(keras_parameterized.TestCase): def test_test_on_batch(self, reduction): model = self._get_compiled_multi_io_model( loss=losses.MeanSquaredError(reduction=reduction)) - result = model.test_on_batch([self.x, self.x], [self.y, self.y], + result = model.test_on_batch([self.x, self.x], [self.y1, self.y2], sample_weight={ - 'output_1': self.weights_1, - 'output_2': self.weights_2, + 'output_1': self.sample_weight_1, + 'output_2': self.sample_weight_2, }) expected_values = self.expected_batch_result[reduction] if reduction == loss_reduction.ReductionV2.SUM: @@ -478,14 +758,20 @@ class TestOutputLossMetrics(keras_parameterized.TestCase): model = self._get_compiled_multi_io_model( loss=losses.MeanSquaredError(reduction=reduction)) history = model.fit_generator( - custom_generator_multi_io(), steps_per_epoch=2, epochs=2) + custom_generator_multi_io( + sample_weights=[self.sample_weight_1, self.sample_weight_2]), + steps_per_epoch=2, + epochs=2) for key, value in self.expected_fit_result[reduction].items(): self.assertAllClose(history.history[key], value) def test_eval_generator(self, reduction): model = self._get_compiled_multi_io_model( loss=losses.MeanSquaredError(reduction=reduction)) - eval_result = model.evaluate_generator(custom_generator_multi_io(), steps=2) + eval_result = model.evaluate_generator( + custom_generator_multi_io( + sample_weights=[self.sample_weight_1, self.sample_weight_2]), + steps=2) self.assertAllClose(eval_result, self.expected_batch_result[reduction]) From a1a7abbaed92e4d36941c7288a4c9b0e2bb3cf7a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 11:44:53 -0700 Subject: [PATCH 283/332] [tracing] Demangle the kernel name which can be mangled c++ function name from cudnn. PiperOrigin-RevId: 257650763 --- tensorflow/core/platform/default/device_tracer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/platform/default/device_tracer.cc b/tensorflow/core/platform/default/device_tracer.cc index 38cdb65c566..04e6282edbe 100644 --- a/tensorflow/core/platform/default/device_tracer.cc +++ b/tensorflow/core/platform/default/device_tracer.cc @@ -33,6 +33,7 @@ limitations under the License. #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/lib/strings/stringprintf.h" +#include "tensorflow/core/platform/abi.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mem.h" @@ -586,7 +587,7 @@ class CudaEventCollector { auto elapsed_us = GetElapsedTimeUs(record.start_event, record.stop_event); auto stats = absl::make_unique(); - std::string node_name = record.kernel_name; + std::string node_name = port::MaybeAbiDemangle(record.kernel_name); // Sometimes CUPTI returns invalid characters. See b/129892466. if (!IsAscii(node_name)) { node_name = ""; From 51bd59a9f4b339cb23cbc8ec3d3df45eb63ed61e Mon Sep 17 00:00:00 2001 From: Guangda Lai Date: Thu, 11 Jul 2019 11:56:52 -0700 Subject: [PATCH 284/332] Simplify python API implementation: merge GraphConverter and TrtGraphConverter since GraphConverter is not used elsewhere. PiperOrigin-RevId: 257653003 --- .../python/compiler/tensorrt/trt_convert.py | 674 ++++++++---------- 1 file changed, 300 insertions(+), 374 deletions(-) diff --git a/tensorflow/python/compiler/tensorrt/trt_convert.py b/tensorflow/python/compiler/tensorrt/trt_convert.py index d25b1477886..b3befd69849 100644 --- a/tensorflow/python/compiler/tensorrt/trt_convert.py +++ b/tensorflow/python/compiler/tensorrt/trt_convert.py @@ -87,366 +87,6 @@ def _to_string(s): return s -class GraphConverter(object): - """Base class for offline converters to optimize SavedModels/GraphDefs. - - A `GraphConverter` object encapsulates the environment to convert (optimize) a - TensorFlow SavedModel or GraphDef. - - To create a custom GraphConverter: - - ```python - class MyGraphConverter(GraphConverter): - ... - - def get_rewriter_config(self): - my_rewriter_config = ... - return my_rewriter_config - ``` - - Then to run the conversion without quantization calibration: - - ```python - my_converter = MyGraphConverter(input_saved_model_dir="my_dir") - converted_graph_def = my_converter.convert() - my_converter.save(output_saved_model_dir) # Optional - ``` - - To run the conversion with quantization calibration: - - ```python - my_converter = MyGraphConverter(input_saved_model_dir="my_dir") - my_converter.convert() - - # Run calibration 10 times. - converted_graph_def = my_converter.calibrate( - fetch_names=['output:0'], - num_runs=10, - feed_dict_fn=lambda: {'input:0': my_next_data()}) - - my_converter.save(output_saved_model_dir) # Optional - ``` - """ - - # TODO(laigd): clean up the parameters. - def __init__(self, - input_saved_model_dir=None, - input_saved_model_tags=None, - input_saved_model_signature_key=None, - input_graph_def=None, - nodes_blacklist=None, - session_config=None): - """Initialize the converter. - - Args: - input_saved_model_dir: the directory to load the SavedModel which contains - the input graph to transforms. Used only when input_graph_def is None. - input_saved_model_tags: list of tags to load the SavedModel. - input_saved_model_signature_key: the key of the signature to optimize the - graph for. - input_graph_def: a GraphDef object containing a model to be transformed. - If set to None, the graph will be read from the SavedModel loaded from - input_saved_model_dir. - nodes_blacklist: list of node names to prevent the converter from - touching. - session_config: the ConfigProto used to create a Session. It's also used - as a template to create a RewriterConfig for conversion. If not - specified, a default ConfigProto will be used. - - Raises: - ValueError: if the combination of the parameters is invalid. - """ - if input_graph_def and input_saved_model_dir: - raise ValueError( - "Can only specify one of input_graph_def and input_saved_model_dir") - if not input_graph_def and not input_saved_model_dir: - raise ValueError("Must specify one of input_graph_def and " - "input_saved_model_dir") - - self._input_graph_def = input_graph_def - self._nodes_blacklist = nodes_blacklist - - self._input_saved_model_dir = input_saved_model_dir - self._converted = False - self._grappler_meta_graph_def = None - - self._input_saved_model_tags = ( - input_saved_model_tags or [tag_constants.SERVING]) - self._input_saved_model_signature_key = ( - input_saved_model_signature_key or - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY) - self._session_config = session_config or config_pb2.ConfigProto() - - # For calibration usage. - self._calibration_graph = None - self._calibration_sess = None - self._calibration_data_collected = False - - def get_rewriter_config(self): - """Returns a RewriterConfig proto for TRT transformation. - - Returns: - A RewriterConfig proto which will be used to run the conversion using - Grappler. - """ - raise NotImplementedError("get_rewriter_config") - - def _run_conversion(self): - """Run Grappler's OptimizeGraph() tool to convert the graph.""" - # Create custom ConfigProto for Grappler. - grappler_session_config = config_pb2.ConfigProto() - grappler_session_config.CopyFrom(self._session_config) - custom_rewriter_config = self.get_rewriter_config() - grappler_session_config.graph_options.rewrite_options.CopyFrom( - custom_rewriter_config) - - # Run Grappler. - self._converted_graph_def = tf_optimizer.OptimizeGraph( - grappler_session_config, - self._grappler_meta_graph_def, - graph_id=b"tf_graph") - self._converted = True - - def _add_nodes_blacklist(self): - if self._nodes_blacklist: - collection_def = self._grappler_meta_graph_def.collection_def["train_op"] - blacklist = collection_def.node_list.value - for i in self._nodes_blacklist: - if isinstance(i, ops.Tensor): - blacklist.append(_to_bytes(i.name)) - else: - blacklist.append(_to_bytes(i)) - - def _convert_graph_def(self): - """Convert the input GraphDef.""" - graph = ops.Graph() - with graph.as_default(): - importer.import_graph_def(self._input_graph_def, name="") - self._grappler_meta_graph_def = saver.export_meta_graph( - graph_def=graph.as_graph_def(add_shapes=True), graph=graph) - self._add_nodes_blacklist() - - self._run_conversion() - - def _collections_to_keep(self, collection_keys): - # TODO(laigd): currently we use the collection key to filter out - # collections that depend on variable ops, but this may miss some - # other user-defined collections. A better way would be to use - # CollectionDef::NodeList for the filtering. - collections_to_remove = ( - ops.GraphKeys._VARIABLE_COLLECTIONS + [ - ops.GraphKeys.TRAIN_OP, ops.GraphKeys.WHILE_CONTEXT, - ops.GraphKeys.COND_CONTEXT - ]) - return [key for key in collection_keys if key not in collections_to_remove] - - def _convert_saved_model(self): - """Convert the input SavedModel.""" - graph = ops.Graph() - with session.Session(graph=graph, config=self._session_config) as sess: - input_meta_graph_def = loader.load(sess, self._input_saved_model_tags, - self._input_saved_model_dir) - input_signature_def = input_meta_graph_def.signature_def[ - self._input_saved_model_signature_key] - - def _gather_names(tensor_info): - """Get the node names from a TensorInfo.""" - return set([tensor_info[key].name.split(":")[0] for key in tensor_info]) - - # Get input and outputs from all SignatureDef. - output_node_names = _gather_names(input_signature_def.inputs).union( - _gather_names(input_signature_def.outputs)) - - # Preserve nodes in collection - for collection_key in self._collections_to_keep( - input_meta_graph_def.collection_def): - for op in sess.graph.get_collection(collection_key): - if isinstance(op, ops.Operation): - output_node_names.add(op.name.split(":")[0]) - - # Freeze the variables in the SavedModel graph and copy the frozen - # graph over. - frozen_graph_def = graph_util.convert_variables_to_constants( - sess, sess.graph.as_graph_def(add_shapes=True), - list(output_node_names)) - self._grappler_meta_graph_def = meta_graph_pb2.MetaGraphDef() - self._grappler_meta_graph_def.graph_def.CopyFrom(frozen_graph_def) - - # Copy the collections that are not variables. - for collection_key in self._collections_to_keep( - input_meta_graph_def.collection_def): - self._grappler_meta_graph_def.collection_def[collection_key].CopyFrom( - input_meta_graph_def.collection_def[collection_key]) - - self._add_nodes_blacklist() - - # Copy other information. - self._grappler_meta_graph_def.meta_info_def.CopyFrom( - input_meta_graph_def.meta_info_def) - self._grappler_meta_graph_def.signature_def[ - self._input_saved_model_signature_key].CopyFrom(input_signature_def) - # TODO(laigd): maybe add back AssetFileDef. - - self._run_conversion() - - def convert(self): - """Run the conversion. - - Returns: - The converted GraphDef for TF 1.x, or the converted ConcreteFunction in TF - 2.0+. - """ - assert not self._converted - if self._input_graph_def: - self._convert_graph_def() - else: - self._convert_saved_model() - return self._converted_graph_def - - def calibrate(self, - fetch_names, - num_runs, - feed_dict_fn=None, - input_map_fn=None): - """Run the calibration and return the calibrated GraphDef. - - Args: - fetch_names: a list of output tensor name to fetch during calibration. - num_runs: number of runs of the graph during calibration. - feed_dict_fn: a function that returns a dictionary mapping input names (as - strings) in the GraphDef to be calibrated to values (e.g. Python list, - numpy arrays, etc). One and only one of `feed_dict_fn` and - `input_map_fn` should be specified. - input_map_fn: a function that returns a dictionary mapping input names (as - strings) in the GraphDef to be calibrated to Tensor objects. The values - of the named input tensors in the GraphDef to be calibrated will be - re-mapped to the respective `Tensor` values during calibration. One and - only one of `feed_dict_fn` and `input_map_fn` should be specified. - - Raises: - ValueError: if the input combination is invalid. - RuntimeError: if this method is called in eager mode. - - Returns: - The GraphDef after the calibration. - """ - assert self._converted - assert not self._calibration_sess - - if context.executing_eagerly(): - raise RuntimeError("Calibration for TF 2.0 is not supported yet.") - - if (feed_dict_fn and input_map_fn) or (not feed_dict_fn and - not input_map_fn): - raise ValueError( - "Should specify one and only one of feed_dict_fn and input_map_fn.") - - self._calibration_graph = ops.Graph() - with self._calibration_graph.as_default(): - fetches = importer.import_graph_def( - self._converted_graph_def, - input_map=input_map_fn() if input_map_fn else None, - return_elements=fetch_names, - name="") - self._calibration_sess = session.Session( - graph=self._calibration_graph, config=self._session_config) - - for _ in range(num_runs): - self._calibration_sess.run( - fetches, feed_dict=feed_dict_fn() if feed_dict_fn else None) - - self.finalize_calibration() - return self._converted_graph_def - - def finalize_calibration(self): - """Clean up calibration resources and finalize the calibration. - - Implementations need to close self._calibration_sess before returning. - """ - raise NotImplementedError("finalize_calibration") - - def save(self, output_saved_model_dir): - """Save the converted graph as a SavedModel. - - Args: - output_saved_model_dir: construct a SavedModel using the converted - GraphDef and save it to the specified directory. This option only works - when the input graph is loaded from a SavedModel, i.e. when - input_saved_model_dir is specified and input_graph_def is None in - __init__(). - - Raises: - ValueError: if the input to the converter is a GraphDef instead of a - SavedModel. - """ - assert self._converted - if self._input_graph_def: - raise ValueError( - "Not able to save to a SavedModel since input is a GraphDef") - - def _restore_collections(dest_graph, src_meta_graph_def, collection_keys): - """Restores collections that we need to keep.""" - scope = "" - for key in collection_keys: - collection_def = src_meta_graph_def.collection_def[key] - kind = collection_def.WhichOneof("kind") - if kind is None: - tf_logging.error( - "Cannot identify data type for collection %s. Skipping.", key) - continue - from_proto = ops.get_from_proto_function(key) - if from_proto and kind == "bytes_list": - proto_type = ops.get_collection_proto_type(key) - # It is assumed that there are no Variables Keys in collections - for value in collection_def.bytes_list.value: - proto = proto_type() - proto.ParseFromString(value) - try: - new_value = from_proto(proto, import_scope=scope) - except: - continue - dest_graph.add_to_collection(key, new_value) - else: - field = getattr(collection_def, kind) - if kind == "node_list": - for value in field.value: - name = ops.prepend_name_scope(value, scope) - # Since the graph has been optimized, the node may no longer - # exists - try: - col_op = dest_graph.as_graph_element(name) - except (TypeError, ValueError, KeyError) as e: - continue - dest_graph.add_to_collection(key, col_op) - elif kind == "int64_list": - # NOTE(opensource): This force conversion is to work around the - # fact that Python2 distinguishes between int and long, while - # Python3 has only int. - for value in field.value: - dest_graph.add_to_collection(key, int(value)) - else: - for value in field.value: - dest_graph.add_to_collection(key, - ops.prepend_name_scope(value, scope)) - - # Write the transformed graphdef as SavedModel. - saved_model_builder = builder.SavedModelBuilder(output_saved_model_dir) - with ops.Graph().as_default(): - importer.import_graph_def(self._converted_graph_def, name="") - _restore_collections( - ops.get_default_graph(), self._grappler_meta_graph_def, - self._collections_to_keep( - self._grappler_meta_graph_def.collection_def)) - # We don't use any specific converter here. - with session.Session(config=self._session_config) as sess: - saved_model_builder.add_meta_graph_and_variables( - sess, - self._input_saved_model_tags, - signature_def_map=self._grappler_meta_graph_def.signature_def) - # Ignore other meta graphs from the input SavedModel. - saved_model_builder.save() - - class TrtPrecisionMode(object): FP32 = "FP32" FP16 = "FP16" @@ -643,10 +283,38 @@ def get_tensorrt_rewriter_config( return rewriter_config_with_trt -class TrtGraphConverter(GraphConverter): - """A GraphConverter for TRT transformation.""" +class TrtGraphConverter(object): + """A converter for TF-TRT transformation for TF 1.x GraphDef/SavedModels. + + To run the conversion without quantization calibration (e.g. for FP32/FP16 + precision modes): + + ```python + converter = TrtGraphConverter( + input_saved_model_dir="my_dir", + precision_mode=TrtPrecisionMode.FP16) + converted_graph_def = converter.convert() + converter.save(output_saved_model_dir) + ``` + + To run the conversion with quantization calibration: + + ```python + converter = TrtGraphConverter( + input_saved_model_dir="my_dir", + precision_mode=TrtPrecisionMode.INT8) + converter.convert() + + # Run calibration 10 times. + converted_graph_def = converter.calibrate( + fetch_names=['output:0'], + num_runs=10, + feed_dict_fn=lambda: {'input:0': my_next_data()}) + + converter.save(output_saved_model_dir) + ``` + """ - # TODO(laigd): use TrtConversionParams here. def __init__(self, input_saved_model_dir=None, input_saved_model_tags=None, @@ -707,15 +375,32 @@ class TrtGraphConverter(GraphConverter): Raises: ValueError: if the combination of the parameters is invalid. """ - super(TrtGraphConverter, self).__init__( - input_saved_model_dir=input_saved_model_dir, - input_saved_model_tags=input_saved_model_tags, - input_saved_model_signature_key=input_saved_model_signature_key, - input_graph_def=input_graph_def, - nodes_blacklist=nodes_blacklist, - session_config=session_config) + if input_graph_def and input_saved_model_dir: + raise ValueError( + "Can only specify one of input_graph_def and input_saved_model_dir") + if not input_graph_def and not input_saved_model_dir: + raise ValueError("Must specify one of input_graph_def and " + "input_saved_model_dir") _check_trt_version_compatibility() + self._input_graph_def = input_graph_def + self._nodes_blacklist = nodes_blacklist + + self._input_saved_model_dir = input_saved_model_dir + self._converted = False + self._grappler_meta_graph_def = None + + self._input_saved_model_tags = ( + input_saved_model_tags or [tag_constants.SERVING]) + self._input_saved_model_signature_key = ( + input_saved_model_signature_key or + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY) + self._session_config = session_config or config_pb2.ConfigProto() + + # For calibration usage. + self._calibration_graph = None + self._calibration_sess = None + self._calibration_data_collected = False self._need_calibration = ( precision_mode == TrtPrecisionMode.INT8 and use_calibration) if self._need_calibration and not is_dynamic_op: @@ -750,11 +435,176 @@ class TrtGraphConverter(GraphConverter): max_batch_size=max_batch_size) _check_conversion_params(self._conversion_params) - def get_rewriter_config(self): - return get_tensorrt_rewriter_config( + def _run_conversion(self): + """Run Grappler's OptimizeGraph() tool to convert the graph.""" + # Create custom ConfigProto for Grappler. + grappler_session_config = config_pb2.ConfigProto() + grappler_session_config.CopyFrom(self._session_config) + custom_rewriter_config = get_tensorrt_rewriter_config( conversion_params=self._conversion_params) + grappler_session_config.graph_options.rewrite_options.CopyFrom( + custom_rewriter_config) + + # Run Grappler. + self._converted_graph_def = tf_optimizer.OptimizeGraph( + grappler_session_config, + self._grappler_meta_graph_def, + graph_id=b"tf_graph") + self._converted = True + + def _add_nodes_blacklist(self): + if self._nodes_blacklist: + collection_def = self._grappler_meta_graph_def.collection_def["train_op"] + blacklist = collection_def.node_list.value + for i in self._nodes_blacklist: + if isinstance(i, ops.Tensor): + blacklist.append(_to_bytes(i.name)) + else: + blacklist.append(_to_bytes(i)) + + def _convert_graph_def(self): + """Convert the input GraphDef.""" + graph = ops.Graph() + with graph.as_default(): + importer.import_graph_def(self._input_graph_def, name="") + self._grappler_meta_graph_def = saver.export_meta_graph( + graph_def=graph.as_graph_def(add_shapes=True), graph=graph) + self._add_nodes_blacklist() + + self._run_conversion() + + def _collections_to_keep(self, collection_keys): + # TODO(laigd): currently we use the collection key to filter out + # collections that depend on variable ops, but this may miss some + # other user-defined collections. A better way would be to use + # CollectionDef::NodeList for the filtering. + collections_to_remove = ( + ops.GraphKeys._VARIABLE_COLLECTIONS + [ + ops.GraphKeys.TRAIN_OP, ops.GraphKeys.WHILE_CONTEXT, + ops.GraphKeys.COND_CONTEXT + ]) + return [key for key in collection_keys if key not in collections_to_remove] + + def _convert_saved_model(self): + """Convert the input SavedModel.""" + graph = ops.Graph() + with session.Session(graph=graph, config=self._session_config) as sess: + input_meta_graph_def = loader.load(sess, self._input_saved_model_tags, + self._input_saved_model_dir) + input_signature_def = input_meta_graph_def.signature_def[ + self._input_saved_model_signature_key] + + def _gather_names(tensor_info): + """Get the node names from a TensorInfo.""" + return set([tensor_info[key].name.split(":")[0] for key in tensor_info]) + + # Get input and outputs from all SignatureDef. + output_node_names = _gather_names(input_signature_def.inputs).union( + _gather_names(input_signature_def.outputs)) + + # Preserve nodes in collection + for collection_key in self._collections_to_keep( + input_meta_graph_def.collection_def): + for op in sess.graph.get_collection(collection_key): + if isinstance(op, ops.Operation): + output_node_names.add(op.name.split(":")[0]) + + # Freeze the variables in the SavedModel graph and copy the frozen + # graph over. + frozen_graph_def = graph_util.convert_variables_to_constants( + sess, sess.graph.as_graph_def(add_shapes=True), + list(output_node_names)) + self._grappler_meta_graph_def = meta_graph_pb2.MetaGraphDef() + self._grappler_meta_graph_def.graph_def.CopyFrom(frozen_graph_def) + + # Copy the collections that are not variables. + for collection_key in self._collections_to_keep( + input_meta_graph_def.collection_def): + self._grappler_meta_graph_def.collection_def[collection_key].CopyFrom( + input_meta_graph_def.collection_def[collection_key]) + + self._add_nodes_blacklist() + + # Copy other information. + self._grappler_meta_graph_def.meta_info_def.CopyFrom( + input_meta_graph_def.meta_info_def) + self._grappler_meta_graph_def.signature_def[ + self._input_saved_model_signature_key].CopyFrom(input_signature_def) + # TODO(laigd): maybe add back AssetFileDef. + + self._run_conversion() + + def convert(self): + """Run the conversion. + + Returns: + The converted GraphDef for TF 1.x, or the converted ConcreteFunction in TF + 2.0+. + """ + assert not self._converted + if self._input_graph_def: + self._convert_graph_def() + else: + self._convert_saved_model() + return self._converted_graph_def + + def calibrate(self, + fetch_names, + num_runs, + feed_dict_fn=None, + input_map_fn=None): + """Run the calibration and return the calibrated GraphDef. + + Args: + fetch_names: a list of output tensor name to fetch during calibration. + num_runs: number of runs of the graph during calibration. + feed_dict_fn: a function that returns a dictionary mapping input names (as + strings) in the GraphDef to be calibrated to values (e.g. Python list, + numpy arrays, etc). One and only one of `feed_dict_fn` and + `input_map_fn` should be specified. + input_map_fn: a function that returns a dictionary mapping input names (as + strings) in the GraphDef to be calibrated to Tensor objects. The values + of the named input tensors in the GraphDef to be calibrated will be + re-mapped to the respective `Tensor` values during calibration. One and + only one of `feed_dict_fn` and `input_map_fn` should be specified. + + Raises: + ValueError: if the input combination is invalid. + RuntimeError: if this method is called in eager mode. + + Returns: + The GraphDef after the calibration. + """ + assert self._converted + assert not self._calibration_sess + + if context.executing_eagerly(): + raise RuntimeError("Calibration for TF 2.0 is not supported yet.") + + if (feed_dict_fn and input_map_fn) or (not feed_dict_fn and + not input_map_fn): + raise ValueError( + "Should specify one and only one of feed_dict_fn and input_map_fn.") + + self._calibration_graph = ops.Graph() + with self._calibration_graph.as_default(): + fetches = importer.import_graph_def( + self._converted_graph_def, + input_map=input_map_fn() if input_map_fn else None, + return_elements=fetch_names, + name="") + self._calibration_sess = session.Session( + graph=self._calibration_graph, config=self._session_config) + + for _ in range(num_runs): + self._calibration_sess.run( + fetches, feed_dict=feed_dict_fn() if feed_dict_fn else None) + + self.finalize_calibration() + return self._converted_graph_def def finalize_calibration(self): + """Clean up calibration resources and finalize the calibration.""" assert self._need_calibration assert self._converted assert not self._calibration_data_collected @@ -792,11 +642,87 @@ class TrtGraphConverter(GraphConverter): self._calibration_sess.close() def save(self, output_saved_model_dir): - """Save the converted graph as a SavedModel.""" + """Save the converted graph as a SavedModel. + + Args: + output_saved_model_dir: construct a SavedModel using the converted + GraphDef and save it to the specified directory. This option only works + when the input graph is loaded from a SavedModel, i.e. when + input_saved_model_dir is specified and input_graph_def is None in + __init__(). + + Raises: + ValueError: if the input to the converter is a GraphDef instead of a + SavedModel. + """ + assert self._converted if self._need_calibration: assert self._calibration_data_collected + if self._input_graph_def: + raise ValueError( + "Not able to save to a SavedModel since input is a GraphDef") - super(TrtGraphConverter, self).save(output_saved_model_dir) + def _restore_collections(dest_graph, src_meta_graph_def, collection_keys): + """Restores collections that we need to keep.""" + scope = "" + for key in collection_keys: + collection_def = src_meta_graph_def.collection_def[key] + kind = collection_def.WhichOneof("kind") + if kind is None: + tf_logging.error( + "Cannot identify data type for collection %s. Skipping.", key) + continue + from_proto = ops.get_from_proto_function(key) + if from_proto and kind == "bytes_list": + proto_type = ops.get_collection_proto_type(key) + # It is assumed that there are no Variables Keys in collections + for value in collection_def.bytes_list.value: + proto = proto_type() + proto.ParseFromString(value) + try: + new_value = from_proto(proto, import_scope=scope) + except: + continue + dest_graph.add_to_collection(key, new_value) + else: + field = getattr(collection_def, kind) + if kind == "node_list": + for value in field.value: + name = ops.prepend_name_scope(value, scope) + # Since the graph has been optimized, the node may no longer + # exists + try: + col_op = dest_graph.as_graph_element(name) + except (TypeError, ValueError, KeyError) as e: + continue + dest_graph.add_to_collection(key, col_op) + elif kind == "int64_list": + # NOTE(opensource): This force conversion is to work around the + # fact that Python2 distinguishes between int and long, while + # Python3 has only int. + for value in field.value: + dest_graph.add_to_collection(key, int(value)) + else: + for value in field.value: + dest_graph.add_to_collection(key, + ops.prepend_name_scope(value, scope)) + + # Write the transformed graphdef as SavedModel. + saved_model_builder = builder.SavedModelBuilder(output_saved_model_dir) + with ops.Graph().as_default(): + importer.import_graph_def(self._converted_graph_def, name="") + _restore_collections( + ops.get_default_graph(), self._grappler_meta_graph_def, + self._collections_to_keep( + self._grappler_meta_graph_def.collection_def)) + # We don't use any specific converter here. + with session.Session(config=self._session_config) as sess: + saved_model_builder.add_meta_graph_and_variables( + sess, + self._input_saved_model_tags, + signature_def_map=self._grappler_meta_graph_def.signature_def) + # Ignore other meta graphs from the input SavedModel. + saved_model_builder.save() def _get_resource_handle(name, device): From 765f71b808f957dc4d4d1b8965f76626f757f0bb Mon Sep 17 00:00:00 2001 From: Guangda Lai Date: Thu, 11 Jul 2019 11:57:38 -0700 Subject: [PATCH 285/332] Make TF-TRT python tests 2.0 compatible by adding v2 only tests and decorating all existing tests with run_v1_only. PiperOrigin-RevId: 257653149 --- .../test/tf_trt_integration_test_base.py | 345 ++++++++++++++---- 1 file changed, 267 insertions(+), 78 deletions(-) diff --git a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py index 268eee5e8b5..6b72cbec9bd 100644 --- a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py +++ b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py @@ -19,8 +19,11 @@ from __future__ import division from __future__ import print_function from collections import namedtuple +import errno +import gc import itertools import os +import shutil import tempfile import warnings import numpy as np @@ -31,6 +34,7 @@ from tensorflow.core.framework import graph_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import rewriter_config_pb2 from tensorflow.python.compiler.tensorrt import trt_convert +from tensorflow.python.eager import def_function from tensorflow.python.framework import graph_io from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_spec @@ -39,12 +43,16 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.saved_model import builder +from tensorflow.python.saved_model import load from tensorflow.python.saved_model import loader +from tensorflow.python.saved_model import save from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model import signature_def_utils from tensorflow.python.saved_model import tag_constants from tensorflow.python.saved_model import utils from tensorflow.python.tools import saved_model_utils +from tensorflow.python.training.tracking import tracking +from tensorflow.python.util import nest TfTrtIntegrationTestParams = namedtuple( "TfTrtIntegrationTestParams", @@ -73,6 +81,8 @@ RunParams = namedtuple( "dynamic_engine", "use_calibration", "test_name", + # Is this test for TF 2.0? + "is_v2", ]) FP32 = "FP32" @@ -286,18 +296,9 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): def _GetFeedDict(self, inputs_data): return {name: data for name, data in zip(self._GetFeedNames(), inputs_data)} - def _RunGraph(self, - run_params, - saved_model_dir, - inputs_data, - config, - graph_state, - num_runs=2): - """Run given graphdef multiple times.""" + def _RunGraphV1(self, saved_model_dir, inputs_data, config, num_runs=2): + """Run given graphdef multiple times using TF 1.x runtime.""" params = self._GetParamsCached() - for data in inputs_data: - assert len(params.input_specs) == len(data) - fetches = self._GetFetchNames() g = ops.Graph() with g.as_default(): @@ -321,10 +322,64 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): vals.append(val) return vals - def _CreateConverter(self, saved_model_dir, session_config, + def _RunGraphV2(self, saved_model_dir, inputs_data, graph_state, num_runs=2): + """Run given graphdef multiple times using TF 2.0 runtime.""" + params = self._GetParamsCached() + root = load.load(saved_model_dir) + func = root.signatures[ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + results = [] + for expected_shapes, current_input_data in zip(params.expected_output_dims, + inputs_data): + val = None + for _ in range(num_runs): + feed_dict = { + params.input_specs[i].name: current_input_data[i] + for i in range(len(params.input_specs)) + } + new_val = func(**feed_dict) + assert isinstance(new_val, dict) + # The key of the output map is always like output_i. + new_val = [new_val[key] for key in sorted(new_val)] + # Each element is an eager Tensor, and accessing individual elements is + # very expensive, so we convert them to a numpy array first. + new_val = [v.numpy() for v in new_val] + self.assertEqual(len(expected_shapes), len(new_val)) + for expected_shape, actual_val in zip(expected_shapes, new_val): + self.assertEqual(list(expected_shape), list(actual_val.shape)) + if val is not None: + self.assertAllClose(val, new_val, atol=1.e-06, rtol=1.e-06) + val = new_val + results.append(val) + + return results + + def _RunGraph(self, + run_params, + saved_model_dir, + inputs_data, + config, + graph_state, + num_runs=2): + params = self._GetParamsCached() + for data in inputs_data: + assert len(params.input_specs) == len(data) + + if run_params.is_v2: + results = self._RunGraphV2(saved_model_dir, inputs_data, graph_state, + num_runs) + gc.collect() # Force GC to destroy the TRT engine cache. + return results + return self._RunGraphV1(saved_model_dir, inputs_data, config, num_runs) + + def _CreateConverter(self, run_params, saved_model_dir, session_config, conversion_params): """Return a TrtGraphConverter.""" - converter = trt_convert.TrtGraphConverter( + if run_params.is_v2: + return trt_convert.TrtGraphConverterV2( + input_saved_model_dir=saved_model_dir, + conversion_params=conversion_params) + return trt_convert.TrtGraphConverter( input_saved_model_dir=saved_model_dir, session_config=session_config, max_batch_size=conversion_params.max_batch_size, @@ -335,7 +390,6 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): maximum_cached_engines=conversion_params.maximum_cached_engines, use_calibration=conversion_params.use_calibration, use_function_backup=conversion_params.use_function_backup) - return converter def _GetCalibratedInferGraph(self, run_params, saved_model_dir, inputs_data): """Return trt converted graphdef in INT8 mode.""" @@ -353,8 +407,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): session_config = self._GetConfigProto(run_params, GraphState.CALIBRATE) logging.info("Running calibration graph, config:\n%s", str(session_config)) - converter = self._CreateConverter(saved_model_dir, session_config, - conversion_params) + converter = self._CreateConverter(run_params, saved_model_dir, + session_config, conversion_params) int8_gdef = converter.convert() self._VerifyGraphDef(run_params, saved_model_dir, int8_gdef, GraphState.CALIBRATE) @@ -363,7 +417,8 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): fetch_names=self._GetFetchNames(), num_runs=5, feed_dict_fn=lambda: self._GetFeedDict(inputs_data[0])) - trt_saved_model_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + trt_saved_model_dir = self._GetSavedModelDir(run_params, + GraphState.CALIBRATE) converter.save(trt_saved_model_dir) return trt_saved_model_dir @@ -375,27 +430,34 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): session_config = self._GetConfigProto(run_params, GraphState.INFERENCE) logging.info("Creating TRT graph for inference, config\n%s", str(session_config)) - converter = self._CreateConverter(saved_model_dir, session_config, - conversion_params) + converter = self._CreateConverter(run_params, saved_model_dir, + session_config, conversion_params) converter.convert() - trt_saved_model_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + trt_saved_model_dir = self._GetSavedModelDir(run_params, + GraphState.INFERENCE) converter.save(trt_saved_model_dir) return trt_saved_model_dir - def _WriteGraph(self, run_params, gdef, graph_state): + def _GetGraphStateLabel(self, graph_state): if graph_state == GraphState.ORIGINAL: - label = "Original" + return "Original" elif graph_state == GraphState.CALIBRATE: - label = "CalibEngine" + return "CalibEngine" elif graph_state == GraphState.INFERENCE: - label = "InferEngine" + return "InferEngine" + else: + return "UnknownState" + + def _WriteGraph(self, run_params, gdef, graph_state): + temp_dir = os.getenv("TRT_TEST_TMPDIR") + if not temp_dir: + return + graph_name = ( - self.__class__.__name__ + "_" + run_params.test_name + "_" + label + - ".pbtxt") - temp_dir = os.getenv("TRT_TEST_TMPDIR", self.get_temp_dir()) - if temp_dir: - logging.info("Writing graph to %s/%s", temp_dir, graph_name) - graph_io.write_graph(gdef, temp_dir, graph_name) + self.__class__.__name__ + "_" + run_params.test_name + "_" + + self._GetGraphStateLabel(graph_state) + ".pbtxt") + logging.info("Writing graph to %s/%s", temp_dir, graph_name) + graph_io.write_graph(gdef, temp_dir, graph_name) def _VerifyConnections(self, expected_engines, original_gdef, converted_gdef): old_to_new_node_map = { @@ -467,19 +529,28 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): msg="\nexpected:\n%s\nvs actual:\n%s" % (sorted(expected_input_map.items()), sorted(actual_input_map.items()))) - def _GetGraphDef(self, gdef_or_saved_model_dir): + def _GetGraphDef(self, run_params, gdef_or_saved_model_dir): if isinstance(gdef_or_saved_model_dir, str): + if run_params.is_v2: + root = load.load(gdef_or_saved_model_dir) + func = root.signatures[ + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + gdef = func.graph.as_graph_def() + # Manually unref the loaded saved model and force GC to destroy the TRT + # engine cache after load(). There is currently a reference cycle in 2.0 + # which prevents auto deletion of the resource. + # TODO(laigd): fix this. + del func + del root + gc.collect() + return gdef return saved_model_utils.get_meta_graph_def( gdef_or_saved_model_dir, tag_constants.SERVING).graph_def assert isinstance(gdef_or_saved_model_dir, graph_pb2.GraphDef) return gdef_or_saved_model_dir - def _VerifyGraphDef(self, run_params, original_gdef_or_saved_model_dir, - gdef_or_saved_model_dir_to_verify, graph_state): - original_gdef = self._GetGraphDef(original_gdef_or_saved_model_dir) - gdef_to_verify = self._GetGraphDef(gdef_or_saved_model_dir_to_verify) - self._WriteGraph(run_params, gdef_to_verify, graph_state) - + def _VerifyGraphDefV1(self, run_params, original_gdef, gdef_to_verify, + graph_state): expected_engines = self.ExpectedEnginesToBuild(run_params) num_engines = 0 functions = [f.signature.name for f in gdef_to_verify.library.function] @@ -521,7 +592,69 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): self._VerifyConnections(expected_engines, original_gdef, gdef_to_verify) # TODO(aaroey): consider verifying the corresponding TF function. - def _MakeSavedModel(self, run_params): + def _VerifyGraphDefV2(self, run_params, original_gdef, gdef_to_verify, + graph_state): + if graph_state == GraphState.ORIGINAL: + return + expected_engines = self.ExpectedEnginesToBuild(run_params) + all_op_names = [node.name for node in gdef_to_verify.node] + trt_op_names = [ + node.name for node in gdef_to_verify.node if node.op == "TRTEngineOp" + ] + for func in gdef_to_verify.library.function: + for node in func.node_def: + all_op_names.append(node.name) + if node.op == "TRTEngineOp": + trt_op_names.append(node.name) + # Remove the function name prefix. + def _Canonicalize(names): + return set([self._ToString(name.split("/")[-1]) for name in names]) + + all_op_names = _Canonicalize(all_op_names) + trt_op_names = _Canonicalize(trt_op_names) + + if isinstance(expected_engines, dict): + # For simplicity we don't verify the connections inside the engine in + # 2.0, but we still make sure that the converted ops are gone from the + # graph. + unexpected_names = set(nest.flatten(expected_engines.values())) + self.assertEmpty( + [name for name in unexpected_names if name in all_op_names]) + expected_engines = set(expected_engines.keys()) + + self.assertEqual(set(expected_engines), trt_op_names) + + def _VerifyGraphDef(self, run_params, original_gdef_or_saved_model_dir, + gdef_or_saved_model_dir_to_verify, graph_state): + original_gdef = self._GetGraphDef(run_params, + original_gdef_or_saved_model_dir) + gdef_to_verify = self._GetGraphDef(run_params, + gdef_or_saved_model_dir_to_verify) + self._WriteGraph(run_params, gdef_to_verify, graph_state) + if run_params.is_v2: + self._VerifyGraphDefV2(run_params, original_gdef, gdef_to_verify, + graph_state) + else: + self._VerifyGraphDefV1(run_params, original_gdef, gdef_to_verify, + graph_state) + + def _GetSavedModelDir(self, run_params, graph_state): + test_tmpdir = os.getenv("TRT_TEST_TMPDIR") + if test_tmpdir: + saved_model_dir = os.path.join( + test_tmpdir, self.__class__.__name__ + "_" + run_params.test_name + + "_" + self._GetGraphStateLabel(graph_state)) + try: + # For TF 1.x we need to make sure the output directory doesn't exist + # before exporting the saved model. + shutil.rmtree(saved_model_dir) + except OSError as e: + if e.errno != errno.ENOENT: + raise + return saved_model_dir + return tempfile.mkdtemp(dir=self.get_temp_dir()) + + def _MakeSavedModelV1(self, run_params): """Write the saved model as an input for testing.""" params = self._GetParamsCached() g = ops.Graph() @@ -534,15 +667,13 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): outputs = params.graph_fn(*inputs) if not isinstance(outputs, list) and not isinstance(outputs, tuple): outputs = [outputs] - for spec, output in zip(params.output_specs, outputs): - assert spec.name == output.name.split(":")[0] signature_def = signature_def_utils.build_signature_def( inputs={inp.op.name: utils.build_tensor_info(inp) for inp in inputs}, outputs={out.op.name: utils.build_tensor_info(out) for out in outputs}, method_name=signature_constants.PREDICT_METHOD_NAME) - saved_model_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + saved_model_dir = self._GetSavedModelDir(run_params, GraphState.ORIGINAL) saved_model_builder = builder.SavedModelBuilder(saved_model_dir) with self.session( graph=g, config=self._GetConfigProto(run_params, @@ -556,6 +687,22 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): saved_model_builder.save() return saved_model_dir + def _MakeSavedModelV2(self, run_params): + params = self._GetParamsCached() + root = tracking.AutoTrackable() + root.run = def_function.function( + params.graph_fn, input_signature=params.input_specs) + saved_model_dir = self._GetSavedModelDir(run_params, GraphState.ORIGINAL) + logging.info("Saving input SavedModel to %s", saved_model_dir) + save.save(root, saved_model_dir, + {signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: root.run}) + return saved_model_dir + + def _MakeSavedModel(self, run_params): + if run_params.is_v2: + return self._MakeSavedModelV2(run_params) + return self._MakeSavedModelV1(run_params) + def RunTest(self, run_params): if not self.ShouldRunTest(run_params): return @@ -577,9 +724,12 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): # continuous natural numbers: # seq = np.arange(np.prod(np_shape)) # seq.resize(np_shape) - # inputs_data.append(scale * seq.astype(np_dtype)) - current_input_data.append( - (scale * np.random.random_sample(np_shape)).astype(np_dtype)) + # current_inputs_data.append(scale * seq.astype(np_dtype)) + data = (scale * np.random.random_sample(np_shape)).astype(np_dtype) + if run_params.is_v2: + with ops.device("/GPU:0"): + data = ops.convert_to_tensor(data) + current_input_data.append(data) inputs_data.append(current_input_data) # Verify original graph. @@ -626,55 +776,94 @@ class TfTrtIntegrationTestBase(test_util.TensorFlowTestCase): pass -def _AddTests(test_class): - """Adds test methods to TfTrtIntegrationTestBase.""" +def _GetTestConfigsV1(): + """Returns the config combinations to run the test.""" + convert_online, convert_offline = True, False + dynamic_engine, static_engine = True, False + use_calibration, no_calibration = True, False - def _GetTestConfigs(): - """Returns the config combinations to run the test.""" - convert_online, convert_offline = True, False - dynamic_engine, static_engine = True, False - use_calibration, no_calibration = True, False + # Add all possible test cases and let the derived test class to decide + # whether to run specific ones with ShouldRunTest(). + # + # Note: INT8 without calibration behaves like FP32/FP16. + opts = list( + itertools.product([FP32, FP16, INT8], [convert_online, convert_offline], + [dynamic_engine, static_engine], [no_calibration])) + # We always run calibration with offline tool. + # TODO(aaroey): static calibration engine is not supported yet. + opts.append((INT8, convert_offline, dynamic_engine, use_calibration)) + return opts - # Add all possible test cases and let the derived test class to decide - # whether to run specific ones with ShouldRunTest(). - # - # Note: INT8 without calibration behaves like FP32/FP16. - opts = list( - itertools.product([FP32, FP16, INT8], [convert_online, convert_offline], - [dynamic_engine, static_engine], [no_calibration])) - # We always run calibration with offline tool. - # TODO(aaroey): static calibration engine is not supported yet. - opts.append((INT8, convert_offline, dynamic_engine, use_calibration)) - return opts - def _GetTest(run_params): - """Gets a single test method based on the parameters.""" +def _GetTestConfigsV2(): + """Returns the config combinations to run the test.""" + convert_offline = False + # TODO(laigd): add support for static_engine. + dynamic_engine = True + # TODO(laigd): add support for calibration. + no_calibration = False - @test_util.deprecated_graph_mode_only - def _Test(self): - logging.info( - "Running TFv1 test %s with parameters: convert_online=%s, " - "precision_mode=%s, dynamic_engine=%s", - "testTfTrt_" + run_params.test_name, run_params.convert_online, - run_params.precision_mode, run_params.dynamic_engine) - self.RunTest(run_params) + # Add all possible test cases and let the derived test class to decide + # whether to run specific ones with ShouldRunTest(). + # + # Note: + # - In TF2.0 the conversion always produce dynamic engine, and we don't test + # the offline mode here. + # - For simplicity we don't test online conversion which requires setting the + # Grappler config in default eager context. + # - INT8 without calibration behaves like FP32/FP16. + opts = list( + itertools.product([FP32, FP16, INT8], [convert_offline], [dynamic_engine], + [no_calibration])) + # We always run calibration with offline tool. + # TODO(aaroey): INT8+calibration is not supported yet in V2. + # opts.append((INT8, convert_offline, dynamic_engine, use_calibration)) + return opts - return _Test - opts = _GetTestConfigs() +def _GetTest(run_params): + """Gets a single test method based on the parameters.""" + + def _Test(self): + logging.info( + "Running test %s with parameters: convert_online=%s, " + "precision_mode=%s, dynamic_engine=%s", run_params.test_name, + run_params.convert_online, run_params.precision_mode, + run_params.dynamic_engine) + self.RunTest(run_params) + + return _Test + + +def _AddTestsFor(test_class, is_v2): + """Adds test methods to TfTrtIntegrationTestBase for specific TF version.""" + opts = _GetTestConfigsV2() if is_v2 else _GetTestConfigsV1() for (precision_mode, convert_online, dynamic_engine, use_calibration) in opts: conversion = "OnlineConversion" if convert_online else "OfflineConversion" engine_type = "DynamicEngine" if dynamic_engine else "StaticEngine" calibration_type = "UseCalibration" if use_calibration else "NoCalibration" - test_name = "%s_%s_%s_%s" % (conversion, engine_type, precision_mode, - calibration_type) + test_name = "%s_%s_%s_%s_%s" % ("testTfTrtV2" if is_v2 else "testTfTrt", + conversion, engine_type, precision_mode, + calibration_type) run_params = RunParams( convert_online=convert_online, precision_mode=precision_mode, dynamic_engine=dynamic_engine, test_name=test_name, - use_calibration=use_calibration) - setattr(test_class, "testTfTrt_" + test_name, _GetTest(run_params)) + use_calibration=use_calibration, + is_v2=is_v2) + if is_v2: + setattr(test_class, test_name, + test_util.run_v2_only(_GetTest(run_params))) + else: + setattr(test_class, test_name, + test_util.run_v1_only("", _GetTest(run_params))) + + +def _AddTests(test_class): + """Adds test methods to TfTrtIntegrationTestBase.""" + _AddTestsFor(test_class, is_v2=False) + _AddTestsFor(test_class, is_v2=True) if is_tensorrt_enabled(): From 844aa275fd31fb6f3ac5c607d4ce9b143143d55a Mon Sep 17 00:00:00 2001 From: Penporn Koanantakool Date: Thu, 11 Jul 2019 12:15:39 -0700 Subject: [PATCH 286/332] Fix clang compilation errors in TF-MKL variant. PiperOrigin-RevId: 257656880 --- tensorflow/core/common_runtime/mkl_cpu_allocator.h | 1 + tensorflow/core/kernels/BUILD | 3 ++- tensorflow/core/kernels/gather_nd_op_cpu_impl.h | 5 ++++- tensorflow/core/kernels/mkl_conv_ops.h | 2 -- tensorflow/core/kernels/mkl_pooling_ops_common.h | 3 ++- tensorflow/core/kernels/mkl_qmatmul_op.cc | 3 ++- tensorflow/core/util/mkl_util.h | 6 +++--- 7 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h index b467e7b311e..5ec76f59671 100644 --- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h +++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h @@ -253,6 +253,7 @@ class MklCPUAllocator : public Allocator { auto l_stats = large_size_allocator_->GetStats(); // Combine statistics from small-size and large-size allocator. + mutex_lock l(mutex_); stats_.num_allocs = l_stats->num_allocs + s_stats->num_allocs; stats_.bytes_in_use = l_stats->bytes_in_use + s_stats->bytes_in_use; stats_.peak_bytes_in_use = diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 571c6aea635..572afde42d9 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -34,7 +34,6 @@ load( ) load( "//third_party/mkl:build_defs.bzl", - "if_mkl", "if_mkl_ml", "mkl_deps", ) @@ -7465,8 +7464,10 @@ tf_mkl_kernel_library( ], deps = [ ":bounds_check", + ":fill_functor", ":matmul_op", ":ops_util", + "//third_party/eigen3", "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:lib", diff --git a/tensorflow/core/kernels/gather_nd_op_cpu_impl.h b/tensorflow/core/kernels/gather_nd_op_cpu_impl.h index c3d2f701398..f587e969f67 100644 --- a/tensorflow/core/kernels/gather_nd_op_cpu_impl.h +++ b/tensorflow/core/kernels/gather_nd_op_cpu_impl.h @@ -107,13 +107,16 @@ struct GatherNdSlice { Eigen::Tensor::Dimensions reshape_dims{{ 1 }}; Eigen::array broadcast_dims{{ batch_size }}; #else +#if !defined(INTEL_MKL) || !defined(ENABLE_MKL) Eigen::IndexList > reshape_dims; +#endif // defined(INTEL_MKL) && defined(ENABLE_MKL) Eigen::IndexList broadcast_dims; broadcast_dims.set(0, batch_size); -#endif +#endif // !defined(EIGEN_HAS_INDEX_LIST) generator::GatherNdSliceGenerator gather_nd_generator( slice_size, Tindices, Tparams, Tout, &error_loc); +// TODO(b/137289929): Parallelize this with ParallelFor and remove OpenMP call. #if defined(INTEL_MKL) && defined(ENABLE_MKL) // Eigen implementation below is not highly performant. gather_nd_generator // does not seem to be called in parallel, leading to very poor performance. diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h index a0758ab6340..e9be11a4ded 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.h +++ b/tensorflow/core/kernels/mkl_conv_ops.h @@ -562,8 +562,6 @@ class MklConvBackpropCommonOp : public OpKernel { OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_)); int stride_n = GetTensorDim(strides_, data_format_, 'N'); int stride_c = GetTensorDim(strides_, data_format_, 'C'); - const int64 stride_h = GetTensorDim(strides_, data_format_, 'H'); - const int64 stride_w = GetTensorDim(strides_, data_format_, 'W'); OP_REQUIRES( context, (stride_n == 1 && stride_c == 1), errors::InvalidArgument("Current implementation does not yet support " diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.h b/tensorflow/core/kernels/mkl_pooling_ops_common.h index ec440a0aedf..a24bd71d53c 100644 --- a/tensorflow/core/kernels/mkl_pooling_ops_common.h +++ b/tensorflow/core/kernels/mkl_pooling_ops_common.h @@ -464,7 +464,8 @@ class MklPoolingOpBase : public OpKernel { // We may not get this attribute for this node if it does not go through // graph rewrite pass. So we do not check for error while retrieving this // attribute value. - context->GetAttr("workspace_enabled", &this->workspace_enabled_); + TF_CHECK_OK( + context->GetAttr("workspace_enabled", &this->workspace_enabled_)); } void Compute(OpKernelContext* context) override = 0; diff --git a/tensorflow/core/kernels/mkl_qmatmul_op.cc b/tensorflow/core/kernels/mkl_qmatmul_op.cc index eb4d519ef72..fc571602b35 100644 --- a/tensorflow/core/kernels/mkl_qmatmul_op.cc +++ b/tensorflow/core/kernels/mkl_qmatmul_op.cc @@ -222,10 +222,10 @@ class MklDnnMatMulFwdPrimitive : public MklPrimitive { bias_mem(nullptr), dst_mem(nullptr), fwd_desc(nullptr), + fwd_pd(nullptr), src_md(nullptr), weight_md(nullptr), bias_md(nullptr), - fwd_pd(nullptr), matmul_fwd(nullptr), fwd_stream(nullptr) {} }; @@ -719,6 +719,7 @@ class MklDnnQuantizedMatMulOp : public OpKernel { context->CtxFailure( errors::InvalidArgument("Quantization mode must be" "either MIN_FIRST or SCALED.")); + return nullptr; } } } diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h index 1b62dad8878..36ae80a6d2a 100644 --- a/tensorflow/core/util/mkl_util.h +++ b/tensorflow/core/util/mkl_util.h @@ -525,8 +525,8 @@ inline Tensor ConvertMklToTF(OpKernelContext* context, const Tensor& mkl_tensor, TensorShape output_shape = mkl_shape.GetTfShape(); // Allocate output tensor. - context->allocate_temp(DataTypeToEnum::v(), output_shape, - &output_tensor); + TF_CHECK_OK(context->allocate_temp(DataTypeToEnum::v(), output_shape, + &output_tensor)); auto cpu_engine = engine(engine::cpu, 0); MklDnnData input(&cpu_engine); @@ -576,7 +576,7 @@ inline const Tensor& MklGetInput(OpKernelContext* ctext, int n) { inline void GetMklInputList(OpKernelContext* ctext, StringPiece name, OpInputList* input_tensors) { CHECK_NOTNULL(input_tensors); - ctext->input_list(name, input_tensors); + TF_CHECK_OK(ctext->input_list(name, input_tensors)); } inline void GetMklShapeList(OpKernelContext* ctext, StringPiece name, From 21a1d47666b38e60882f3e6d2b8654cfc0202ed6 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 11 Jul 2019 12:36:36 -0700 Subject: [PATCH 287/332] [XLA:Python] Inline TransferLiteralToHostAsync into its only caller. No functional changes. PiperOrigin-RevId: 257660458 --- .../compiler/xla/python/local_client.cc | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index e78e64ac225..4f8a1b0e713 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -224,9 +224,27 @@ StatusOr PyLocalClient::TransferFromOutfeed( return LiteralToPython(std::make_shared(std::move(literal))); } -static StatusOr> TransferHostToDeviceAsync( - const PythonBufferTree& tree, int device_ordinal, - std::shared_ptr client, Device* device) { +/* static */ +StatusOr> PyLocalBuffer::FromPython( + const py::object& argument, std::shared_ptr client, + int device_ordinal) { + tensorflow::profiler::TraceMe traceme("PyLocalBuffer::FromPython"); + TF_ASSIGN_OR_RETURN(PythonBufferTree tree, GetPythonBufferTree(argument)); + + client->py_ref_manager().CollectGarbage(); + + // Take a reference to the buffer to ensure that the inputs in host memory + // remain live until the transfer is complete. + auto py_buffer_ref = + client->py_ref_manager().ManageReferences(absl::MakeSpan(tree.arrays)); + tree.arrays.clear(); + + // We are done manipulating Python objects; release the GIL. + py::gil_scoped_release gil_release; + VLOG(1) << "PyLocalBuffer::FromPython: shape: " << tree.shape.ToString() + << " device ordinal: " << device_ordinal; + + Device* device = &client->device(device_ordinal); se::DeviceMemoryAllocator* allocator = client->allocator(); TransferManager* transfer_manager = client->client()->backend().transfer_manager(); @@ -269,38 +287,10 @@ static StatusOr> TransferHostToDeviceAsync( if (device->synchronous_deallocation()) { device->ThenRelease(device->host_to_device_stream(), device_buffer); } - return absl::make_unique(shape, std::move(device_buffer), - std::move(client)); -} - -/* static */ -StatusOr> PyLocalBuffer::FromPython( - const py::object& argument, std::shared_ptr client, - int device_ordinal) { - tensorflow::profiler::TraceMe traceme("PyLocalBuffer::FromPython"); - TF_ASSIGN_OR_RETURN(PythonBufferTree tree, GetPythonBufferTree(argument)); - - client->py_ref_manager().CollectGarbage(); - - // Take a reference to the buffer to ensure that the inputs in host memory - // remain live until the transfer is complete. - auto py_buffer_ref = - client->py_ref_manager().ManageReferences(absl::MakeSpan(tree.arrays)); - tree.arrays.clear(); - - // We are done manipulating Python objects; release the GIL. - py::gil_scoped_release gil_release; - VLOG(1) << "PyLocalBuffer::FromPython: shape: " << tree.shape.ToString() - << " device ordinal: " << device_ordinal; - - Device* device = &client->device(device_ordinal); - TF_ASSIGN_OR_RETURN(std::unique_ptr buffer, - TransferHostToDeviceAsync(tree, device_ordinal, - std::move(client), device)); - device->ThenRelease(device->host_to_device_stream(), std::move(py_buffer_ref)); - return buffer; + return absl::make_unique(shape, std::move(device_buffer), + std::move(client)); } /* static */ StatusOr> PyLocalBuffer::MakeTuple( From 454dc1666d1a4fb1d9e8292619af5b5c8fd236a5 Mon Sep 17 00:00:00 2001 From: William Chargin Date: Thu, 11 Jul 2019 12:45:05 -0700 Subject: [PATCH 288/332] Update tensorboard nightly dependency to 1.15.x Now that TensorBoard 1.14.0 has been released, our nightlies have moved up to 1.15.x alphas. PiperOrigin-RevId: 257662026 --- tensorflow/tools/pip_package/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 103c8724edb..6661f328416 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -95,7 +95,7 @@ else: if 'tf_nightly' in project_name: for i, pkg in enumerate(REQUIRED_PACKAGES): if 'tensorboard' in pkg: - REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.14.0a0, < 1.15.0a0' + REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.15.0a0, < 1.16.0a0' elif 'tensorflow_estimator' in pkg and '2.0' in project_name: REQUIRED_PACKAGES[i] = 'tensorflow-estimator-2.0-preview' elif 'tensorflow_estimator' in pkg: From 75ef048dcaeee6eeb67ce0c91fe703732f8d1a98 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Thu, 11 Jul 2019 13:11:27 -0700 Subject: [PATCH 289/332] Disable some keras tests on MacOS PiperOrigin-RevId: 257666993 --- tensorflow/python/keras/BUILD | 40 ++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index c0c4ef10841..1e554b8a6e8 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -606,6 +606,9 @@ tf_py_test( "//third_party/py/numpy", "//tensorflow/python:client_testlib", ], + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + ], ) tf_py_test( @@ -815,6 +818,9 @@ tf_py_test( "@absl_py//absl/testing:parameterized", "//tensorflow/python:client_testlib", ], + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + ], ) tf_py_test( @@ -1293,6 +1299,9 @@ cuda_py_test( "//third_party/py/numpy", "//tensorflow/python:client_testlib", ], + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + ], xla_enable_strict_auto_jit = True, ) @@ -1394,7 +1403,10 @@ tf_py_test( "//tensorflow/python:client_testlib", ], shard_count = 2, - tags = ["notsan"], + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + "notsan", + ], ) tf_py_test( @@ -1406,6 +1418,9 @@ tf_py_test( "@absl_py//absl/testing:parameterized", "//tensorflow/python:client_testlib", ], + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + ], ) tf_py_test( @@ -1421,6 +1436,7 @@ tf_py_test( shard_count = 20, tags = [ "no_rocm", + "nomac", # TODO(mihaimaruseac): b/127695564 "notsan", ], ) @@ -1436,6 +1452,9 @@ tf_py_test( "//tensorflow/python:client_testlib", ], shard_count = 4, + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + ], ) tf_py_test( @@ -1451,7 +1470,10 @@ tf_py_test( "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python:client_testlib", ], - tags = ["no_rocm"], + tags = [ + "no_rocm", + "nomac", # TODO(mihaimaruseac): b/127695564 + ], ) tf_py_test( @@ -1484,6 +1506,7 @@ tf_py_test( "//tensorflow/python/feature_column:feature_column_py", ], tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 "notsan", ], ) @@ -1500,6 +1523,7 @@ tf_py_test( ], tags = [ "no_rocm", + "nomac", # TODO(mihaimaruseac): b/127695564 "notsan", ], ) @@ -1561,6 +1585,7 @@ tf_py_test( shard_count = 8, tags = [ "no-internal-py3", + "nomac", # TODO(mihaimaruseac): b/127695564 ], ) @@ -1575,7 +1600,10 @@ tf_py_test( "//tensorflow/python:client_testlib", ], shard_count = 8, - tags = ["no_rocm"], + tags = [ + "no_rocm", + "nomac", # TODO(mihaimaruseac): b/127695564 + ], ) tf_py_test( @@ -1589,6 +1617,9 @@ tf_py_test( "//tensorflow/python:client_testlib", ], shard_count = 8, + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + ], ) tf_py_test( @@ -1614,6 +1645,9 @@ tf_py_test( "//third_party/py/numpy", "//tensorflow/python:client_testlib", ], + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + ], ) tf_py_test( From a11b26549f7eebaa1347d2859f8462dbd32e74b9 Mon Sep 17 00:00:00 2001 From: Nupur Garg Date: Thu, 11 Jul 2019 13:21:31 -0700 Subject: [PATCH 290/332] Add support for control flow v2 to TFLite. PiperOrigin-RevId: 257668719 --- .../lite/python/graphdef_to_tfl_flatbuffer.cc | 2 + tensorflow/lite/python/BUILD | 1 + tensorflow/lite/python/lite.py | 4 +- tensorflow/lite/python/util.py | 31 +---- tensorflow/lite/python/util_test.py | 8 +- .../python/framework/convert_to_constants.py | 119 +++++++++++++----- 6 files changed, 106 insertions(+), 59 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/python/graphdef_to_tfl_flatbuffer.cc b/tensorflow/compiler/mlir/lite/python/graphdef_to_tfl_flatbuffer.cc index 68521016f51..cc03445bd1c 100644 --- a/tensorflow/compiler/mlir/lite/python/graphdef_to_tfl_flatbuffer.cc +++ b/tensorflow/compiler/mlir/lite/python/graphdef_to_tfl_flatbuffer.cc @@ -43,6 +43,8 @@ DataType ConvertIODataTypeToDataType(toco::IODataType dtype) { return DT_INT64; case toco::IODataType::STRING: return DT_STRING; + case toco::IODataType::BOOL: + return DT_BOOL; default: return DT_INVALID; } diff --git a/tensorflow/lite/python/BUILD b/tensorflow/lite/python/BUILD index a181001d351..02ccf5cc751 100644 --- a/tensorflow/lite/python/BUILD +++ b/tensorflow/lite/python/BUILD @@ -149,6 +149,7 @@ py_library( ":lite_constants", ":op_hint", "//tensorflow/python:tf_optimizer", + "//tensorflow/python/eager:wrap_function", ], ) diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py index 87646a572c8..2eb3d67fdd5 100644 --- a/tensorflow/lite/python/lite.py +++ b/tensorflow/lite/python/lite.py @@ -402,7 +402,7 @@ class TFLiteConverterV2(TFLiteConverterBase): "under development.") frozen_func = _convert_to_constants.convert_variables_to_constants_v2( - self._funcs[0]) + self._funcs[0], lower_control_flow=False) input_tensors = [ tensor for tensor in frozen_func.inputs if tensor.dtype != _dtypes.resource @@ -809,7 +809,7 @@ class TFLiteConverter(TFLiteConverterBase): concrete_func = function.get_concrete_function() frozen_func = _convert_to_constants.convert_variables_to_constants_v2( - concrete_func) + concrete_func, lower_control_flow=False) _set_tensor_shapes(frozen_func.inputs, input_shapes) return cls( frozen_func.graph.as_graph_def(), diff --git a/tensorflow/lite/python/util.py b/tensorflow/lite/python/util.py index 19ca0896f7c..775881ad9f7 100644 --- a/tensorflow/lite/python/util.py +++ b/tensorflow/lite/python/util.py @@ -18,16 +18,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import copy import sys -from tensorflow.core.framework import graph_pb2 as _graph_pb2 from tensorflow.core.protobuf import config_pb2 as _config_pb2 from tensorflow.core.protobuf import meta_graph_pb2 as _meta_graph_pb2 from tensorflow.lite.python.op_hint import convert_op_hints_to_stubs from tensorflow.lite.python.op_hint import find_all_hinted_output_nodes from tensorflow.lite.toco import types_pb2 as _types_pb2 from tensorflow.python.eager import function +from tensorflow.python.framework import convert_to_constants as _convert_to_constants from tensorflow.python.framework import dtypes from tensorflow.python.framework import error_interpolation as _error_interpolation from tensorflow.python.framework import graph_util as tf_graph_util @@ -43,11 +42,10 @@ _MAP_TF_TO_TFLITE_TYPES = { dtypes.string: _types_pb2.STRING, dtypes.uint8: _types_pb2.QUANTIZED_UINT8, dtypes.int8: _types_pb2.INT8, - dtypes.complex64: _types_pb2.COMPLEX64 + dtypes.complex64: _types_pb2.COMPLEX64, + dtypes.bool: _types_pb2.BOOL, } -_LOWER_USING_SWITCH_MERGE = "_lower_using_switch_merge" - def convert_dtype_to_tflite_type(tf_dtype): """Converts tf.dtype to TFLite proto type. @@ -201,26 +199,6 @@ def run_graph_optimizations(graph_def, return tf_optimizer.OptimizeGraph(config, meta_graph) -def _remove_lower_using_switch_merge(graph_def): - """Remove '_lower_using_switch_merge' attributes from the given graph. - - Args: - graph_def: GraphDef to be optimized. - - Returns: - A new GraphDef that with no '_lower_using_switch_merge' attribute. - """ - out = _graph_pb2.GraphDef() - out.library.CopyFrom(graph_def.library) - out.versions.CopyFrom(graph_def.versions) - for node in graph_def.node: - new_node = copy.deepcopy(node) - if new_node.op == "While": - new_node.attr[_LOWER_USING_SWITCH_MERGE].b = False - out.node.extend([new_node]) - return out - - def _convert_op_hints_if_present(sess, graph_def, output_tensors, hinted_outputs_nodes): if is_frozen_graph(sess): @@ -253,7 +231,8 @@ def freeze_graph(sess, input_tensors, output_tensors): # Asides from inlining any simple function, Grappler will also try to lower # while loop into switch merge representation which is undesired for Ophints, # so we simply remove those attributes to prevent Grappler from doing so. - graph_def = _remove_lower_using_switch_merge(sess.graph_def) + graph_def = _convert_to_constants.disable_lower_using_switch_merge( + sess.graph_def) config = get_grappler_config(["function"]) graph_def = run_graph_optimizations( graph_def, input_tensors, output_tensors, config, graph=sess.graph) diff --git a/tensorflow/lite/python/util_test.py b/tensorflow/lite/python/util_test.py index fe861f09a67..f13fad5e821 100644 --- a/tensorflow/lite/python/util_test.py +++ b/tensorflow/lite/python/util_test.py @@ -22,6 +22,7 @@ from tensorflow.lite.python import lite_constants from tensorflow.lite.python import util from tensorflow.lite.toco import types_pb2 as _types_pb2 from tensorflow.python.client import session +from tensorflow.python.framework import convert_to_constants from tensorflow.python.framework import dtypes from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops @@ -54,8 +55,8 @@ class UtilTest(test_util.TensorFlowTestCase): _types_pb2.COMPLEX64) self.assertEqual( util.convert_dtype_to_tflite_type(dtypes.half), _types_pb2.FLOAT16) - with self.assertRaises(ValueError): - util.convert_dtype_to_tflite_type(dtypes.bool) + self.assertEqual( + util.convert_dtype_to_tflite_type(dtypes.bool), _types_pb2.BOOL) def testTensorName(self): in_tensor = array_ops.placeholder(shape=[4], dtype=dtypes.float32) @@ -75,7 +76,8 @@ class UtilTest(test_util.TensorFlowTestCase): b = lambda i: math_ops.add(i, 1) control_flow_ops.while_loop(c, b, [i]) sess = session.Session() - new_graph_def = util._remove_lower_using_switch_merge(sess.graph_def) + new_graph_def = convert_to_constants.disable_lower_using_switch_merge( + sess.graph_def) lower_using_switch_merge_is_removed = False for node in new_graph_def.node: if node.op == "While": diff --git a/tensorflow/python/framework/convert_to_constants.py b/tensorflow/python/framework/convert_to_constants.py index 94576fdcfc1..5eb4dec34d0 100644 --- a/tensorflow/python/framework/convert_to_constants.py +++ b/tensorflow/python/framework/convert_to_constants.py @@ -31,9 +31,16 @@ from tensorflow.python.ops import array_ops from tensorflow.python.training.saver import export_meta_graph -def _disable_lower_using_switch_merge(graph_def): +# TODO(nupurgarg): Handle StatelessIf op. +_CONTROL_FLOW_OPS = set(["If", "While"]) + + +def disable_lower_using_switch_merge(graph_def): """Set '_lower_using_switch_merge' attributes to False in If and While ops. + Sets the attribute to False in the NodeDefs in the main graph and the NodeDefs + in each function's graph. + Args: graph_def: GraphDef proto. @@ -41,14 +48,19 @@ def _disable_lower_using_switch_merge(graph_def): GraphDef """ output_graph_def = graph_pb2.GraphDef() - output_graph_def.library.CopyFrom(graph_def.library) - output_graph_def.versions.CopyFrom(graph_def.versions) + output_graph_def.CopyFrom(graph_def) - for input_node in graph_def.node: - output_node = output_graph_def.node.add() - output_node.CopyFrom(input_node) - if output_node.op in ("If", "While"): - output_node.attr["_lower_using_switch_merge"].b = False + def disable_control_flow_lowering(node): + if node.op in _CONTROL_FLOW_OPS: + node.attr["_lower_using_switch_merge"].b = False + + for node in output_graph_def.node: + disable_control_flow_lowering(node) + + if output_graph_def.library: + for func in output_graph_def.library.function: + for node in func.node_def: + disable_control_flow_lowering(node) return output_graph_def @@ -68,7 +80,7 @@ def _run_inline_graph_optimization(func, lower_control_flow): """ graph_def = func.graph.as_graph_def() if not lower_control_flow: - graph_def = _disable_lower_using_switch_merge(graph_def) + graph_def = disable_lower_using_switch_merge(graph_def) meta_graph = export_meta_graph(graph_def=graph_def, graph=func.graph) # Clear the initializer_name for the variables collections, since they are not @@ -110,6 +122,44 @@ def _get_tensor_name(name): return name.split(":")[0] +def _get_new_function_name(name): + """Returns the function name with '_frozen' appended. + + Args: + name: str + + Returns: + str + """ + return name + "_frozen" + + +def _get_node_defs_list(graph_def): + """Returns a list of NodeDefs in the GraphDef. + + This list consists of all NodeDefs in the main graph as well as all control + flow NodeDefs in the functions. + + The remaining NodeDefs in the functions are not included because the op names + are not unique and the variables are handled differently than the main graph. + The control flow ops need to be extracted because they are need their + attributes to be updated similar to the control flow ops in the main graph. + + Args: + graph_def: GraphDef proto. + + Returns: + [NodeDef] + """ + node_defs = list(graph_def.node) + + if graph_def.library: + for func in graph_def.library.function: + node_defs.extend( + [node for node in func.node_def if node.op in _CONTROL_FLOW_OPS]) + return node_defs + + def _get_tensor_data(func): """Gets the tensor data for all Placeholders in the model. @@ -146,7 +196,7 @@ def _get_tensor_data(func): return tensor_data -def _get_control_flow_function_types(graph_def, tensor_data): +def _get_control_flow_function_types(node_defs, tensor_data): """Gets the types for the parameters to the function. Creates a map from function name to a list of types that correspond with the @@ -155,7 +205,7 @@ def _get_control_flow_function_types(graph_def, tensor_data): from the type of the data contained within the Tensor. Args: - graph_def: GraphDef proto. + node_defs: List of NodeDefs. tensor_data: {str name : Tensor}. Returns: @@ -163,7 +213,7 @@ def _get_control_flow_function_types(graph_def, tensor_data): """ # TODO(b/133793620): Support the "While" op. func_types = {} - for node in graph_def.node: + for node in node_defs: if node.op == "If": arg_types = [dtype for dtype in node.attr["Tin"].list.type] @@ -212,6 +262,24 @@ def _populate_identity_op(output_node, input_node): output_node.attr["_class"].CopyFrom(input_node.attr["_class"]) +def _populate_if_op(output_node, input_node, function_types): + """Updates the type attributes and the function names of the If op. + + Args: + output_node: TensorFlow NodeDef. + input_node: TensorFlow NodeDef. + function_types: Map of function names to the list of DataTypes that + correspond with the function arguments. + """ + output_node.CopyFrom(input_node) + then_func = input_node.attr["then_branch"].func.name + output_node.attr["then_branch"].func.name = _get_new_function_name(then_func) + output_node.attr["else_branch"].func.name = _get_new_function_name( + input_node.attr["else_branch"].func.name) + output_node.attr["Tin"].list.CopyFrom( + attr_value_pb2.AttrValue.ListValue(type=function_types[then_func])) + + def _construct_concrete_function(func, output_graph_def, converted_input_indices): """Constructs a concrete function from the `output_graph_def`. @@ -271,17 +339,19 @@ def convert_variables_to_constants_v2(func, lower_control_flow=True): # Inline the graph in order to remove functions when possible. graph_def = _run_inline_graph_optimization(func, lower_control_flow) + # Gets list of all node defs include those in the library. + node_defs = _get_node_defs_list(graph_def) + # Get mapping from node name to node. - name_to_node = {_get_tensor_name(node.name): node for node in graph_def.node} + name_to_node = {_get_tensor_name(node.name): node for node in node_defs} # Get mapping from node name to variable value. tensor_data = _get_tensor_data(func) # Get mapping from function name to argument types. - get_new_func_name = lambda func_name: func_name + "_frozen" - function_types = _get_control_flow_function_types(graph_def, tensor_data) + function_types = _get_control_flow_function_types(node_defs, tensor_data) - # Get variable data. + # Get variable data for all nodes in `node_defs`. reference_variables = {} resource_identities = {} placeholders = {} @@ -294,7 +364,7 @@ def convert_variables_to_constants_v2(func, lower_control_flow=True): } converted_input_indices.add(tensor_data[node_name]["index"]) - for node in graph_def.node: + for node in node_defs: if node.op == "If": # Get dtype and data for resource Placeholders. then_func = node.attr["then_branch"].func.name @@ -330,7 +400,6 @@ def convert_variables_to_constants_v2(func, lower_control_flow=True): # Reconstruct the graph with constants in place of variables. output_graph_def = graph_pb2.GraphDef() - how_many_converted = 0 for input_node in graph_def.node: output_node = output_graph_def.node.add() @@ -340,13 +409,11 @@ def convert_variables_to_constants_v2(func, lower_control_flow=True): dtype = attr_value_pb2.AttrValue(type=data.dtype.as_datatype_enum) _populate_const_op(output_node, input_node.name, dtype, data.numpy(), data.shape) - how_many_converted += 1 # Convert Placeholder ops to Const ops. elif input_node.name in placeholders: data = placeholders[input_node.name]["data"] dtype = placeholders[input_node.name]["dtype"] _populate_const_op(output_node, input_node.name, dtype, data, data.shape) - how_many_converted += 1 # Update the dtype for Identity ops that are inputs to ReadVariableOps. elif input_node.name in resource_identities: output_node.CopyFrom(input_node) @@ -356,13 +423,7 @@ def convert_variables_to_constants_v2(func, lower_control_flow=True): _populate_identity_op(output_node, input_node) # Update the function names and function's arguments types for the If ops. elif input_node.op == "If": - output_node.CopyFrom(input_node) - then_func = input_node.attr["then_branch"].func.name - output_node.attr["then_branch"].func.name = get_new_func_name(then_func) - output_node.attr["else_branch"].func.name = get_new_func_name( - input_node.attr["else_branch"].func.name) - output_node.attr["Tin"].list.CopyFrom( - attr_value_pb2.AttrValue.ListValue(type=function_types[then_func])) + _populate_if_op(output_node, input_node, function_types) else: output_node.CopyFrom(input_node) @@ -372,7 +433,7 @@ def convert_variables_to_constants_v2(func, lower_control_flow=True): for input_library_func in graph_def.library.function: orig_func_name = input_library_func.signature.name - new_func_name = get_new_func_name(orig_func_name) + new_func_name = _get_new_function_name(orig_func_name) # Do not copy any functions that aren't being used in the graph. Any # functions that are not used by control flow should have been inlined. @@ -404,6 +465,8 @@ def convert_variables_to_constants_v2(func, lower_control_flow=True): # Convert ReadVariableOps to Identity ops. if input_node.op == "ReadVariableOp": _populate_identity_op(output_node, input_node) + elif input_node.op == "If": + _populate_if_op(output_node, input_node, function_types) else: output_node.CopyFrom(input_node) # Convert :value to :output for ops that use the ReadVariableOp. From 25d2acbd3494b3e21466782e93114c14f7dcc98a Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Thu, 11 Jul 2019 13:25:06 -0700 Subject: [PATCH 291/332] [Grappler] Check layout agnostic ops Const fanin dimensions in GenericLayoutOptimizer if they can be permuted. PiperOrigin-RevId: 257669361 --- .../generic_layout_optimizer_test.cc | 1 - .../generic_layout_optimizer_transposer.cc | 131 ++++++++++++------ .../generic_layout_optimizer_transposer.h | 15 +- ...eneric_layout_optimizer_transposer_test.cc | 96 ++++++++++++- 4 files changed, 191 insertions(+), 52 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_test.cc b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_test.cc index 9636d417ec9..a48fde74c09 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_test.cc @@ -472,7 +472,6 @@ TEST_F(GenericLayoutOptimizerTest, DoNotPruneNonAddedCancellableTransposes) { utils::GraphView graph_view(&output, &status); TF_ASSERT_OK(status); - LOG(INFO) << graph_view.graph()->DebugString(); auto* input_node = graph_view.GetNode("input"); ASSERT_NE(input_node, nullptr); diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc index 3d2b18d1326..f2f784a4abb 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc @@ -473,7 +473,7 @@ Status Transposer::UpdateEdge( return Status::OK(); } -bool Transposer::IsFanoutPortDimsN(const utils::MutableNodeView& node, int port, +bool Transposer::IsFanoutPortRankN(const utils::MutableNodeView& node, int port, int n) const { const auto* output_shape_attr = node.GetAttr(kAttrOutputShape); if (output_shape_attr == nullptr || @@ -484,26 +484,68 @@ bool Transposer::IsFanoutPortDimsN(const utils::MutableNodeView& node, int port, return !shape.unknown_rank() && shape.dim_size() == n; } -bool Transposer::IsFanoutPortsDimsN(const utils::MutableNodeView& node, +bool Transposer::IsFanoutPortsRankN(const utils::MutableNodeView& node, absl::Span ports, int n) const { - for (auto port : ports) { - if (!IsFanoutPortDimsN(node, port, n)) { + for (const auto& port : ports) { + if (!IsFanoutPortRankN(node, port, n)) { return false; } } return true; } -bool Transposer::IsFaninPortDimsN(const utils::MutableNodeView& node, int port, +bool Transposer::IsFaninPortRankN(const utils::MutableNodeView& node, int port, int n) const { if (port < node.NumRegularFanins() && port >= 0) { const auto& regular_fanin = node.GetRegularFanin(port); - return IsFanoutPortDimsN(*regular_fanin.node_view(), regular_fanin.index(), + return IsFanoutPortRankN(*regular_fanin.node_view(), regular_fanin.index(), n); } return false; } +bool Transposer::IsFaninPortDimsNIfConst(const utils::MutableNodeView& node, + int port, + absl::Span dims) const { + if (port < node.NumRegularFanins() && port >= 0) { + const auto& regular_fanin = node.GetRegularFanin(port); + const auto* fanin_node_view = regular_fanin.node_view(); + if (!IsConstant(*fanin_node_view->node())) { + return true; + } + // If fanin is a Const, check tensor to see if dimensions match. + const auto* value_attr = fanin_node_view->GetAttr(kAttrValue); + if (value_attr == nullptr) { + return false; + } + Tensor tensor; + if (!tensor.FromProto(value_attr->tensor())) { + return false; + } + if (tensor.dims() != dims.size()) { + return false; + } + for (int i = 0; i < dims.size(); ++i) { + if (tensor.dim_size(i) != dims[i]) { + return false; + } + } + return true; + } + return false; +} + +bool Transposer::IsFaninPortsDimsNIfConst(const utils::MutableNodeView& node, + absl::Span ports, + absl::Span dims) const { + for (const auto& port : ports) { + if (!IsFaninPortDimsNIfConst(node, port, dims)) { + return false; + } + } + return true; +} + bool Transposer::CanProcessNode(const TransposeContext& context, const utils::MutableNodeView& node) const { return !context.nodes_to_preserve.contains(node.GetName()) && @@ -637,7 +679,7 @@ bool LayoutSensitiveOpTransposer::ShouldNotProcess( Status DefaultLayoutSensitiveOpTransposer::TransposeNode( TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsDefaultLayoutSensitiveOp(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4)) { + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4)) { return Status::OK(); } const NodeDef* node_def = node->node(); @@ -654,7 +696,7 @@ Status DefaultLayoutSensitiveOpTransposer::TransposeNode( Status BiasAddGradTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsBiasAddGrad(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFaninPortDimsN(*node, 0, 4)) { + if (!ShouldProcess(*context, *node) || !IsFaninPortRankN(*node, 0, 4)) { return Status::OK(); } TF_RETURN_IF_ERROR(UpdateNode(context, node)); @@ -668,7 +710,7 @@ Status BiasAddGradTransposer::TransposeNode(TransposeContext* context, Status Conv2DBackpropFilterTransposer::TransposeNode( TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsConv2DBackpropFilter(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || ShouldNotProcess(*context, *node)) { return Status::OK(); } @@ -684,7 +726,7 @@ Status Conv2DBackpropFilterTransposer::TransposeNode( Status Conv2DBackpropInputTransposer::TransposeNode( TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsConv2DBackpropInput(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || ShouldNotProcess(*context, *node)) { return Status::OK(); } @@ -708,7 +750,7 @@ bool FusedBatchNormGradTransposer::IsTraining( Status FusedBatchNormGradTransposer::TransposeNode( TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsFusedBatchNormGrad(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsTraining(*node) || ShouldNotProcess(*context, *node)) { return Status::OK(); } @@ -728,7 +770,7 @@ Status MaxPoolV2Transposer::TransposeNode(TransposeContext* context, const auto& data_fanin = node->GetRegularFanin(0); auto* data_fanin_node = data_fanin.node_view(); if (!ShouldProcess(*context, *node) || - !IsFanoutPortDimsN(*data_fanin_node, data_fanin.index(), 4)) { + !IsFanoutPortRankN(*data_fanin_node, data_fanin.index(), 4)) { return Status::OK(); } TF_RETURN_IF_ERROR(UpdateNode(context, node)); @@ -742,7 +784,7 @@ Status MaxPoolV2Transposer::TransposeNode(TransposeContext* context, Status MaxPoolGradTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsMaxPoolGrad(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4)) { + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4)) { return Status::OK(); } TF_RETURN_IF_ERROR(UpdateNode(context, node)); @@ -755,7 +797,7 @@ Status MaxPoolGradTransposer::TransposeNode(TransposeContext* context, Status MaxPoolGradV2Transposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsMaxPoolGradV2(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4)) { + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4)) { return Status::OK(); } TF_RETURN_IF_ERROR(UpdateNode(context, node)); @@ -862,7 +904,7 @@ std::vector LayoutAgnosticOpTransposer::GetVariadic4DFaninPorts( const auto& regular_fanin = node.GetRegularFanin(i); auto* regular_fanin_node = regular_fanin.node_view(); int regular_fanin_port = regular_fanin.index(); - if (IsFanoutPortDimsN(*regular_fanin_node, regular_fanin_port, 4) && + if (IsFanoutPortRankN(*regular_fanin_node, regular_fanin_port, 4) && ((IsAfterDstToSrcTransform(context, *regular_fanin_node) && IsLayoutAgnosticOp(*regular_fanin_node->node())) || IsLayoutOptimizerAddedDstToSrcTranspose(context, @@ -876,7 +918,7 @@ std::vector LayoutAgnosticOpTransposer::GetVariadic4DFaninPorts( Status DefaultLayoutAgnosticOpTransposer::TransposeNode( TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsDefaultLayoutAgnosticOp(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -888,7 +930,7 @@ Status DefaultLayoutAgnosticOpTransposer::TransposeNode( Status AddNTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsAddN(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -900,7 +942,7 @@ Status AddNTransposer::TransposeNode(TransposeContext* context, bool BinaryOpTransposer::IsNDOperateWithMD(const utils::MutableNodeView& node, int n, int m) { - return IsFaninPortDimsN(node, 0, n) && IsFaninPortDimsN(node, 1, m); + return IsFaninPortRankN(node, 0, n) && IsFaninPortRankN(node, 1, m); } bool BinaryOpTransposer::IsFaninShapeSupported( @@ -913,10 +955,10 @@ bool BinaryOpTransposer::IsFaninShapeSupported( std::vector BinaryOpTransposer::Get4DDataFaninPorts( const utils::MutableNodeView& node) { std::vector values; - if (IsFaninPortDimsN(node, 0, 4)) { + if (IsFaninPortRankN(node, 0, 4)) { values.push_back(0); } - if (IsFaninPortDimsN(node, 1, 4)) { + if (IsFaninPortRankN(node, 1, 4)) { values.push_back(1); } return values; @@ -1039,7 +1081,7 @@ Status BinaryOpTransposer::TransposeNode(TransposeContext* context, Status ConcatOpTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsConcat(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1061,7 +1103,8 @@ Status ConcatOpTransposer::TransposeNode(TransposeContext* context, Status FillOpTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsFill(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || + !IsFaninPortDimsNIfConst(*node, 0, {4}) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1089,7 +1132,7 @@ bool MergeTransposer::IsEveryFaninAfterDstToSrcTransform( const TransposeContext& context, const utils::MutableNodeView& node) const { for (const auto& regular_fanin : node.GetRegularFanins()) { auto* regular_fanin_node = regular_fanin.node_view(); - if (IsFanoutPortDimsN(*regular_fanin_node, regular_fanin.index(), 4) && + if (IsFanoutPortRankN(*regular_fanin_node, regular_fanin.index(), 4) && ((IsAfterDstToSrcTransform(context, *regular_fanin_node) && IsLayoutAgnosticOp(*regular_fanin_node->node())) || IsLayoutOptimizerAddedDstToSrcTranspose(context, @@ -1104,7 +1147,7 @@ bool MergeTransposer::IsEveryFaninAfterDstToSrcTransform( Status MergeTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsMerge(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsEveryFaninAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1118,7 +1161,8 @@ Status PadTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsMirrorPad(*node->node()) || IsMirrorPadGrad(*node->node()) || IsPad(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || + !IsFaninPortDimsNIfConst(*node, 1, {4, 2}) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1175,7 +1219,7 @@ bool ReduceTransposer::IsReduceAxisSupported( Status ReduceTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsReduceOp(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFaninPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFaninPortRankN(*node, 0, 4) || !IsReduceAxisSupported(*context, *node) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); @@ -1193,7 +1237,7 @@ Status ReduceTransposer::TransposeNode(TransposeContext* context, Status ReverseV2Transposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsReverseV2(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1206,15 +1250,15 @@ Status ReverseV2Transposer::TransposeNode(TransposeContext* context, bool SelectTransposer::IsFaninScalarVector4D( const utils::MutableNodeView& fanin, int port) { - return IsFanoutPortDimsN(fanin, port, 0) || - IsFanoutPortDimsN(fanin, port, 1) || IsFanoutPortDimsN(fanin, port, 4); + return IsFanoutPortRankN(fanin, port, 0) || + IsFanoutPortRankN(fanin, port, 1) || IsFanoutPortRankN(fanin, port, 4); } std::vector SelectTransposer::GetFaninPorts( const utils::MutableNodeView& fanin, int port) { // Input 0 could be a scalar, a vector with size matching the first dimension // of input 1 and 2, or must have the same shape as input 1 and 2. - if (IsFanoutPortDimsN(fanin, port, 4)) { + if (IsFanoutPortRankN(fanin, port, 4)) { return {0, 1, 2}; } return {1, 2}; @@ -1225,7 +1269,7 @@ Status SelectTransposer::TransposeNode(TransposeContext* context, DCHECK(IsSelect(*node->node())); const auto& regular_fanin_0 = node->GetRegularFanin(0); auto* regular_fanin_0_node = regular_fanin_0.node_view(); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsFaninScalarVector4D(*regular_fanin_0_node, regular_fanin_0.index()) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); @@ -1240,7 +1284,7 @@ Status SelectTransposer::TransposeNode(TransposeContext* context, Status ShapeTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsShape(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFaninPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFaninPortRankN(*node, 0, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1267,7 +1311,8 @@ Status ShapeNTransposer::TransposeNode(TransposeContext* context, Status SliceTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsSlice(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || + !IsFaninPortsDimsNIfConst(*node, {1, 2}, {4}) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1282,7 +1327,7 @@ Status SplitTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsSplit(*node->node())); const auto ports = GetDataFanoutPorts(*node); - if (!ShouldProcess(*context, *node) || !IsFanoutPortsDimsN(*node, ports, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortsRankN(*node, ports, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1298,7 +1343,7 @@ Status SplitVTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsSplitV(*node->node())); const auto ports = GetDataFanoutPorts(*node); - if (!ShouldProcess(*context, *node) || !IsFanoutPortsDimsN(*node, ports, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortsRankN(*node, ports, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1356,8 +1401,8 @@ bool SqueezeTransposer::IsAlongNHW(const utils::MutableNodeView& node) const { bool SqueezeTransposer::IsDimsSupported( const utils::MutableNodeView& node) const { - return (IsFanoutPortDimsN(node, 0, 2) && IsAlongHW(node)) || - (IsFanoutPortDimsN(node, 0, 1) && IsAlongNHW(node)); + return (IsFanoutPortRankN(node, 0, 2) && IsAlongHW(node)) || + (IsFanoutPortRankN(node, 0, 1) && IsAlongNHW(node)); } Status SqueezeTransposer::UpdateSqueezeDims(TransposeContext* context, @@ -1444,7 +1489,8 @@ Status StridedSliceTransposer::PermuteMask(TransposeContext* context, Status StridedSliceTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsStridedSlice(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || + !IsFaninPortsDimsNIfConst(*node, {1, 2, 3}, {4}) || !HasOnlyBeginEndMask(*node) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); @@ -1461,7 +1507,7 @@ Status StridedSliceTransposer::TransposeNode(TransposeContext* context, Status SwitchTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsSwitch(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFaninPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFaninPortRankN(*node, 0, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1474,7 +1520,7 @@ Status SwitchTransposer::TransposeNode(TransposeContext* context, Status TernaryOpTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsTernaryOp(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1487,7 +1533,8 @@ Status TernaryOpTransposer::TransposeNode(TransposeContext* context, Status TileTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsTile(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || + !IsFaninPortDimsNIfConst(*node, 1, {4}) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } @@ -1501,7 +1548,7 @@ Status TileTransposer::TransposeNode(TransposeContext* context, Status UnaryGradTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsUnaryGrad(*node->node())); - if (!ShouldProcess(*context, *node) || !IsFanoutPortDimsN(*node, 0, 4) || + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 4) || !IsAfterDstToSrcTransform(*context, *node)) { return Status::OK(); } diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.h b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.h index ac5c7dc100f..2fb98102683 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.h +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.h @@ -139,12 +139,21 @@ class Transposer { utils::MutationNewNode* added_node); protected: - bool IsFanoutPortDimsN(const utils::MutableNodeView& node, int port, + bool IsFanoutPortRankN(const utils::MutableNodeView& node, int port, int n) const; - bool IsFanoutPortsDimsN(const utils::MutableNodeView& node, + bool IsFanoutPortsRankN(const utils::MutableNodeView& node, absl::Span ports, int n) const; - bool IsFaninPortDimsN(const utils::MutableNodeView& node, int port, + bool IsFaninPortRankN(const utils::MutableNodeView& node, int port, int n) const; + + // Checks if fanin at specified port(s) has dimensions `dims` iff fanin is a + // Const. If fanin is not a Const, no dimensions will be checked and this will + // return true. + bool IsFaninPortDimsNIfConst(const utils::MutableNodeView& node, int port, + absl::Span dims) const; + bool IsFaninPortsDimsNIfConst(const utils::MutableNodeView& node, + absl::Span ports, + absl::Span dims) const; bool CanProcessNode(const TransposeContext& context, const utils::MutableNodeView& node) const; string GetDeviceName(const VirtualPlacer* virtual_placer, diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer_test.cc b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer_test.cc index 6c811a1552f..083a92614dd 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer_test.cc +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer_test.cc @@ -3219,9 +3219,9 @@ TEST_F(TransposerTest, StridedSliceTransposer) { auto attrs = ops::StridedSlice::Attrs().BeginMask(0xB).EndMask(0x7); - auto begin = ops::Const(scope.WithOpName("begin"), {2, 0, 2}, {3}); - auto end = ops::Const(scope.WithOpName("end"), {34, 4, 3}, {3}); - auto strides = ops::Const(scope.WithOpName("strides"), {7, 2, 1}, {3}); + auto begin = ops::Const(scope.WithOpName("begin"), {2, 0, 2, 1}, {4}); + auto end = ops::Const(scope.WithOpName("end"), {34, 4, 3, 1}, {4}); + auto strides = ops::Const(scope.WithOpName("strides"), {7, 2, 1, 1}, {4}); auto strided_slice_op = ops::StridedSlice( scope.WithOpName("stridedslice").WithDevice("/device:GPU:0"), conv2d, @@ -3309,9 +3309,9 @@ TEST_F(TransposerTest, StridedSliceTransposerEllipsisMaskPresent) { auto attrs = ops::StridedSlice::Attrs().BeginMask(0xB).EndMask(0x7).EllipsisMask(0x2); - auto begin = ops::Const(scope.WithOpName("begin"), {2, 0, 2}, {3}); - auto end = ops::Const(scope.WithOpName("end"), {34, 4, 3}, {3}); - auto strides = ops::Const(scope.WithOpName("strides"), {7, 2, 1}, {3}); + auto begin = ops::Const(scope.WithOpName("begin"), {2, 0, 2, 1}, {4}); + auto end = ops::Const(scope.WithOpName("end"), {34, 4, 3, 1}, {4}); + auto strides = ops::Const(scope.WithOpName("strides"), {7, 2, 1, 1}, {4}); auto strided_slice_op = ops::StridedSlice( scope.WithOpName("stridedslice").WithDevice("/device:GPU:0"), conv2d, @@ -3350,6 +3350,90 @@ TEST_F(TransposerTest, StridedSliceTransposerEllipsisMaskPresent) { updated_stridedslice_node->GetName(), 0); } +TEST_F(TransposerTest, StridedSliceTransposerConstFaninBadRank) { +#if !GOOGLE_CUDA + GTEST_SKIP() << "CUDA is not enabled"; +#endif // !GOOGLE_CUDA + GrapplerItem item; + Scope scope = Scope::NewRootScope(); + + auto input = + ops::RandomUniform(scope.WithOpName("input"), + {kBatchSize, kHeight, kWidth, kDepthIn}, DT_FLOAT); + auto filter = + ops::RandomUniform(scope.WithOpName("filter"), + {kHeight, kWidth, kDepthIn, kDepthOut}, DT_FLOAT); + Output conv2d = ops::Conv2D( + scope.WithOpName("conv2d").WithDevice("/device:GPU:0"), input, filter, + {1, 2, 4, 1}, "SAME", ops::Conv2D::DataFormat(kSrcFormat)); + + auto attrs = ops::StridedSlice::Attrs().BeginMask(0xB).EndMask(0x7); + + auto begin = ops::Const(scope.WithOpName("begin"), {2, 0, 2}, {3}); + auto end = ops::Const(scope.WithOpName("end"), {34, 4, 3}, {3}); + auto strides = ops::Const(scope.WithOpName("strides"), {7, 2, 1}, {3}); + + auto strided_slice_op = ops::StridedSlice( + scope.WithOpName("stridedslice").WithDevice("/device:GPU:0"), conv2d, + begin, end, strides, attrs); + auto z = ops::Identity(scope.WithOpName("z"), strided_slice_op); + TF_ASSERT_OK(scope.ToGraphDef(&item.graph)); + + TransposeContext context; + TF_ASSERT_OK(TransposeContext::InitializeTransposeContext( + item, virtual_cluster_.get(), kSrcFormat, kDstFormat, kGPU, &context)); + + DefaultLayoutSensitiveOpTransposer conv2d_transposer; + auto* c2d = context.graph_view->GetNode("conv2d"); + ASSERT_NE(c2d, nullptr); + TF_ASSERT_OK(conv2d_transposer.TransposeNode(&context, c2d)); + + StridedSliceTransposer stridedslice_transposer; + auto* stridedslice = context.graph_view->GetNode("stridedslice"); + ASSERT_NE(stridedslice, nullptr); + TF_ASSERT_OK(stridedslice_transposer.TransposeNode(&context, stridedslice)); + + auto* input_transpose_node = context.graph_view->GetNode( + "stridedslice-0-TransposeNHWCToNCHW-LayoutOptimizer"); + ASSERT_EQ(input_transpose_node, nullptr); + + auto* begin_node = context.graph_view->GetNode( + "stridedslice-1-DataFormatVecPermuteNHWCToNCHW-LayoutOptimizer"); + ASSERT_EQ(begin_node, nullptr); + auto* end_node = context.graph_view->GetNode( + "stridedslice-2-DataFormatVecPermuteNHWCToNCHW-LayoutOptimizer"); + ASSERT_EQ(end_node, nullptr); + auto* strides_node = context.graph_view->GetNode( + "stridedslice-3-DataFormatVecPermuteNHWCToNCHW-LayoutOptimizer"); + ASSERT_EQ(strides_node, nullptr); + + auto* updated_stridedslice_node = context.graph_view->GetNode("stridedslice"); + ASSERT_NE(updated_stridedslice_node, nullptr); + ASSERT_EQ(updated_stridedslice_node->NumRegularFanins(), 4); + VerifyRegularFaninMatch(updated_stridedslice_node, 0, + "conv2d-0-0-TransposeNCHWToNHWC-LayoutOptimizer", 0); + VerifyRegularFaninMatch(updated_stridedslice_node, 1, "begin", 0); + VerifyRegularFaninMatch(updated_stridedslice_node, 2, "end", 0); + VerifyRegularFaninMatch(updated_stridedslice_node, 3, "strides", 0); + const auto* begin_mask_attr = + updated_stridedslice_node->GetAttr("begin_mask"); + ASSERT_NE(begin_mask_attr, nullptr); + EXPECT_EQ(begin_mask_attr->i(), 0xB); + const auto* end_mask_attr = updated_stridedslice_node->GetAttr("end_mask"); + ASSERT_NE(end_mask_attr, nullptr); + EXPECT_EQ(end_mask_attr->i(), 0x7); + + auto* output_transpose_node = context.graph_view->GetNode( + "stridedslice-0-0-TransposeNCHWToNHWC-LayoutOptimizer"); + ASSERT_EQ(output_transpose_node, nullptr); + + auto* z_output_node = context.graph_view->GetNode("z"); + ASSERT_NE(z_output_node, nullptr); + ASSERT_EQ(z_output_node->NumRegularFanins(), 1); + VerifyRegularFaninMatch(z_output_node, 0, + updated_stridedslice_node->GetName(), 0); +} + TEST_F(TransposerTest, ReduceTransposerKeepDims) { #if !GOOGLE_CUDA GTEST_SKIP() << "CUDA is not enabled"; From f58acbab464590dfad82045fe47799a7257eafd8 Mon Sep 17 00:00:00 2001 From: Dan Ringwalt Date: Thu, 11 Jul 2019 13:42:01 -0700 Subject: [PATCH 292/332] Only rotate by multiples of 90 degrees in test_rotate_even. When rotating by 45 degrees with nearest interpolation, we have to round to break the tie between input pixels, and rounding seems to be platform-specific. PiperOrigin-RevId: 257672941 --- .../python/kernel_tests/image_ops_test.py | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py index eedc9633e2f..a386e811d3f 100644 --- a/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py +++ b/tensorflow/contrib/image/python/kernel_tests/image_ops_test.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import unittest import numpy as np from tensorflow.contrib.image.ops import gen_image_ops @@ -32,9 +31,10 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.platform import googletest -_DTYPES = set( - [dtypes.uint8, dtypes.int32, dtypes.int64, - dtypes.float16, dtypes.float32, dtypes.float64]) +_DTYPES = set([ + dtypes.uint8, dtypes.int32, dtypes.int64, dtypes.float16, dtypes.float32, + dtypes.float64 +]) class ImageOpsTest(test_util.TensorFlowTestCase): @@ -49,29 +49,35 @@ class ImageOpsTest(test_util.TensorFlowTestCase): image_ops.rotate(image, angle).eval(), np.zeros(shape, dtype.as_numpy_dtype())) - # TODO(b/133773834) Re-enable these tests. - @unittest.skip("Skipping because of b/133773834.") def test_rotate_even(self): for dtype in _DTYPES: with self.cached_session(): image = array_ops.reshape( math_ops.cast(math_ops.range(36), dtype), (6, 6)) image_rep = array_ops.tile(image[None, :, :, None], [3, 1, 1, 1]) - angles = constant_op.constant([0.0, np.pi / 4.0, np.pi / 2.0], + angles = constant_op.constant([0.0, np.pi / 2.0, np.pi * 3. / 2.], dtypes.float32) image_rotated = image_ops.rotate(image_rep, angles) + # pyformat: disable self.assertAllEqual(image_rotated[:, :, :, 0].eval(), - [[[0, 1, 2, 3, 4, 5], [6, 7, 8, 9, 10, 11], + [[[0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], [12, 13, 14, 15, 16, 17], [18, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 29], [30, 31, 32, 33, 34, 35]], - [[0, 3, 4, 11, 17, 0], [2, 3, 9, 16, 23, 23], - [1, 8, 15, 21, 22, 29], [6, 13, 20, 21, 27, 34], - [12, 18, 19, 26, 33, 33], [0, 18, 24, 31, 32, 0]], - [[5, 11, 17, 23, 29, 35], [4, 10, 16, 22, 28, 34], - [3, 9, 15, 21, 27, 33], [2, 8, 14, 20, 26, 32], - [1, 7, 13, 19, 25, 31], [0, 6, 12, 18, 24, 30]]]) + [[5, 11, 17, 23, 29, 35], + [4, 10, 16, 22, 28, 34], + [3, 9, 15, 21, 27, 33], + [2, 8, 14, 20, 26, 32], + [1, 7, 13, 19, 25, 31], + [0, 6, 12, 18, 24, 30]], + [[30, 24, 18, 12, 6, 0], + [31, 25, 19, 13, 7, 1], + [32, 26, 20, 14, 8, 2], + [33, 27, 21, 15, 9, 3], + [34, 28, 22, 16, 10, 4], + [35, 29, 23, 17, 11, 5]]]) def test_rotate_odd(self): for dtype in _DTYPES: From 258fdb296efe14e60e495114780051bce179daef Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Thu, 11 Jul 2019 13:45:17 -0700 Subject: [PATCH 293/332] Make tf_should_use warnings less verbose. Limit the stack trace to 5 frames. PiperOrigin-RevId: 257673667 --- tensorflow/python/util/tf_should_use.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/util/tf_should_use.py b/tensorflow/python/util/tf_should_use.py index 2d0a447a0cd..b71c96ccf6b 100644 --- a/tensorflow/python/util/tf_should_use.py +++ b/tensorflow/python/util/tf_should_use.py @@ -65,7 +65,8 @@ class _TFShouldUseHelper(object): else: logger = tf_logging.error creation_stack = ''.join( - [line.rstrip() for line in traceback.format_stack(self._stack_frame)]) + [line.rstrip() + for line in traceback.format_stack(self._stack_frame, limit=5)]) logger( '==================================\n' 'Object was never used (type %s):\n%s\nIf you want to mark it as ' From db06a786111a284485c0714d9dbcf8315907ef6b Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Thu, 11 Jul 2019 13:45:21 -0700 Subject: [PATCH 294/332] Fix miscount of output name. Previously the name used for an output's occurrence was not incremented resulting in it being reusable. Also forward a fix from the other UniqueName call (these should be extracted into a common class). PiperOrigin-RevId: 257673679 --- .../mlir/lite/flatbuffer_translate.cc | 22 ++++++++++++++----- .../lite/tests/mlir2flatbuffer/simple.mlir | 11 +++++----- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc index 0862c5fb881..186149437d5 100644 --- a/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc +++ b/tensorflow/compiler/mlir/lite/flatbuffer_translate.cc @@ -451,10 +451,16 @@ std::string Translator::GetName(Operation* inst) { } std::string Translator::UniqueName(llvm::StringRef prefix) { + // Keep incrementing the counter until we find a unique name. std::string name = prefix; - auto& val = name_to_count_[name]; - if (val) name = (prefix + llvm::Twine(val)).str(); - ++val; + int64_t& prefix_count = name_to_count_[name]; + int64_t val = prefix_count; + while (val != 0) { + name = (prefix + llvm::Twine(prefix_count)).str(); + ++prefix_count; + val = name_to_count_[name]; + } + name_to_count_[name] = 1; return name; } @@ -796,8 +802,10 @@ void Translator::InitializeNamesFromAttribute(FuncOp fn) { fn.emitWarning() << "invalid entry function specification"; return; } - for (auto it : llvm::enumerate(fn.getArguments())) + for (auto it : llvm::enumerate(fn.getArguments())) { op_to_name_[*it.value()->user_begin()] = input_names[it.index()]; + ++name_to_count_[input_names[it.index()].str()]; + } } if (auto str = dict_attr.get("outputs").dyn_cast()) { @@ -815,12 +823,14 @@ void Translator::InitializeNamesFromAttribute(FuncOp fn) { // ensure the name that will be assigned to the buffer is the same, or // insert an op so that we can have a buffer named such. This cannot // currently happen due to pseudo_input nodes. - if (auto op = it.value()->getDefiningOp()) + if (auto op = it.value()->getDefiningOp()) { op_to_name_[op] = output_names[it.index()]; - else + name_to_count_[output_names[it.index()].str()] = 1; + } else { fn.emitWarning() << "output is not due to an op and '" << output_names[it.index()] << "' may not be a named output"; + } } } } diff --git a/tensorflow/compiler/mlir/lite/tests/mlir2flatbuffer/simple.mlir b/tensorflow/compiler/mlir/lite/tests/mlir2flatbuffer/simple.mlir index d823216304e..eec837bc62e 100644 --- a/tensorflow/compiler/mlir/lite/tests/mlir2flatbuffer/simple.mlir +++ b/tensorflow/compiler/mlir/lite/tests/mlir2flatbuffer/simple.mlir @@ -1,6 +1,7 @@ // RUN: flatbuffer_translate -mlir-to-tflite-flatbuffer %s -o - | flatbuffer_to_string - | FileCheck --dump-input-on-failure %s -func @main(tensor<3x2xi32>) -> tensor<3x2xi32> { +func @main(tensor<3x2xi32>) -> tensor<3x2xi32> + attributes {tf.entry_function = {inputs = "input", outputs = "SameNameAsOutput"}} { ^bb0(%arg0: tensor<3x2xi32>): // CHECK: { // CHECK-NEXT: version: 3, @@ -14,7 +15,7 @@ func @main(tensor<3x2xi32>) -> tensor<3x2xi32> { // CHECK-NEXT: shape: [ 3, 2 ], // CHECK-NEXT: type: INT32, // CHECK-NEXT: buffer: 1, -// CHECK-NEXT: name: "Input", +// CHECK-NEXT: name: "input", // CHECK-NEXT: quantization: { // CHECK-EMPTY: // CHECK-NEXT: } @@ -38,7 +39,7 @@ func @main(tensor<3x2xi32>) -> tensor<3x2xi32> { // CHECK-NEXT: shape: [ ], // CHECK-NEXT: type: INT32, // CHECK-NEXT: buffer: 4, -// CHECK-NEXT: name: "Const2", +// CHECK-NEXT: name: "SameNameAsOutput1", // CHECK-NEXT: quantization: { // CHECK-EMPTY: // CHECK-NEXT: } @@ -46,7 +47,7 @@ func @main(tensor<3x2xi32>) -> tensor<3x2xi32> { // CHECK-NEXT: shape: [ ], // CHECK-NEXT: type: INT32, // CHECK-NEXT: buffer: 5, -// CHECK-NEXT: name: "add", +// CHECK-NEXT: name: "SameNameAsOutput", // CHECK-NEXT: quantization: { // CHECK-EMPTY: // CHECK-NEXT: } @@ -90,7 +91,7 @@ func @main(tensor<3x2xi32>) -> tensor<3x2xi32> { %0 = "tfl.pseudo_input" (%arg0) : (tensor<3x2xi32>) -> tensor<3x2xi32> loc("Input") %1 = "tfl.pseudo_const" () {value = dense<[[1, 2], [3, 4], [5, 6]]> : tensor<3x2xi32>} : () -> tensor<3x2xi32> loc("Const") %2 = "tfl.sub" (%0, %1) {fused_activation_function = "RELU6"} : (tensor<3x2xi32>, tensor<3x2xi32>) -> tensor<3x2xi32> loc("sub") - %3 = "std.constant" () {value = dense<10> : tensor} : () -> tensor loc("Const2") + %3 = "std.constant" () {value = dense<10> : tensor} : () -> tensor loc("SameNameAsOutput") %4 = "tfl.add" (%3, %2) {fused_activation_function = "NONE"} : (tensor, tensor<3x2xi32>) -> tensor<3x2xi32> loc("add") return %4 : tensor<3x2xi32> } From 36c1a29204496aca6fd5c4393cca057f418893ef Mon Sep 17 00:00:00 2001 From: Juhyun Lee Date: Thu, 11 Jul 2019 14:02:21 -0700 Subject: [PATCH 295/332] TFLite GPU OpenGL: Rename UniformParameter to Variable. This is prep work to introduce shared variables to GeneratedCode. PiperOrigin-RevId: 257677195 --- tensorflow/lite/delegates/gpu/gl/BUILD | 17 +++++----- tensorflow/lite/delegates/gpu/gl/api.cc | 5 +-- .../lite/delegates/gpu/gl/compiler/BUILD | 8 +++-- .../gpu/gl/compiler/object_accessor_test.cc | 13 ++++---- .../gpu/gl/compiler/parameter_accessor.cc | 20 +++++------ .../gpu/gl/compiler/parameter_accessor.h | 8 ++--- .../gl/compiler/parameter_accessor_test.cc | 15 ++++----- .../lite/delegates/gpu/gl/compiler/rename.cc | 10 +++--- .../delegates/gpu/gl/compiler/shader_code.h | 6 ++-- .../gpu/gl/compiler/shader_codegen.cc | 5 +-- .../lite/delegates/gpu/gl/converters/BUILD | 4 +-- .../gpu/gl/converters/bhwc_to_phwc4.cc | 10 +++--- .../gpu/gl/converters/phwc4_to_bhwc.cc | 10 +++--- .../lite/delegates/gpu/gl/gl_program.cc | 3 +- tensorflow/lite/delegates/gpu/gl/gl_program.h | 4 +-- .../lite/delegates/gpu/gl/kernels/BUILD | 13 +++++++- .../lite/delegates/gpu/gl/kernels/concat.cc | 5 +-- .../lite/delegates/gpu/gl/kernels/conv.cc | 5 +-- .../gpu/gl/kernels/depthwise_conv.cc | 3 +- .../gpu/gl/kernels/fully_connected.cc | 3 +- .../delegates/gpu/gl/kernels/max_unpooling.cc | 3 +- .../lite/delegates/gpu/gl/kernels/pad.cc | 3 +- .../lite/delegates/gpu/gl/kernels/pooling.cc | 5 +-- .../lite/delegates/gpu/gl/kernels/relu.cc | 3 +- .../lite/delegates/gpu/gl/kernels/slice.cc | 3 +- .../lite/delegates/gpu/gl/kernels/softmax.cc | 3 +- .../gpu/gl/kernels/transpose_conv.cc | 3 +- .../gpu/gl/kernels/upsampling_bilinear.cc | 3 +- .../lite/delegates/gpu/gl/node_shader.h | 4 +-- tensorflow/lite/delegates/gpu/gl/runtime.cc | 3 +- tensorflow/lite/delegates/gpu/gl/runtime.h | 4 +-- .../lite/delegates/gpu/gl/serialization.cc | 15 +++++---- .../lite/delegates/gpu/gl/serialization.h | 6 ++-- .../delegates/gpu/gl/serialization_test.cc | 33 ++++++++++--------- .../gl/{uniform_parameter.h => variable.h} | 8 ++--- 35 files changed, 149 insertions(+), 117 deletions(-) rename tensorflow/lite/delegates/gpu/gl/{uniform_parameter.h => variable.h} (83%) diff --git a/tensorflow/lite/delegates/gpu/gl/BUILD b/tensorflow/lite/delegates/gpu/gl/BUILD index 87e1f014523..b3385ea1fa6 100644 --- a/tensorflow/lite/delegates/gpu/gl/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/BUILD @@ -12,6 +12,7 @@ cc_library( hdrs = ["api.h"], deps = [ ":command_queue", + ":common_cc_fbs", ":compiler", ":compiler_options", ":gl_call", @@ -23,6 +24,7 @@ cc_library( ":runtime", ":runtime_options", ":stats", + ":variable", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "//tensorflow/lite/delegates/gpu/common:model", @@ -221,7 +223,7 @@ cc_library( ":gl_errors", ":gl_shader", ":portable", - ":uniform_parameter", + ":variable", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "@com_google_absl//absl/types:variant", @@ -298,7 +300,7 @@ cc_library( ":compiler_options", ":gpu_info", ":object", - ":uniform_parameter", + ":variable", "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", @@ -360,13 +362,12 @@ cc_library( ":portable", ":runtime_options", ":stats", - ":uniform_parameter", + ":variable", "//tensorflow/lite/delegates/gpu/common:data_type", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl/runtime:shared_buffer", "@com_google_absl//absl/strings", - "@com_google_absl//absl/types:variant", ], ) @@ -383,7 +384,7 @@ cc_library( ":common_cc_fbs", ":compiled_model_cc_fbs", ":object", - ":uniform_parameter", + ":variable", "//tensorflow/lite/delegates/gpu/common:data_type", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", @@ -405,7 +406,7 @@ cc_test( deps = [ ":object", ":serialization", - ":uniform_parameter", + ":variable", "//tensorflow/lite/delegates/gpu/common:shape", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", @@ -423,8 +424,8 @@ cc_library( ) cc_library( - name = "uniform_parameter", - hdrs = ["uniform_parameter.h"], + name = "variable", + hdrs = ["variable.h"], deps = [ "//tensorflow/lite/delegates/gpu/common:types", "@com_google_absl//absl/types:variant", diff --git a/tensorflow/lite/delegates/gpu/gl/api.cc b/tensorflow/lite/delegates/gpu/gl/api.cc index 60c29abb2e5..2767bc399c6 100644 --- a/tensorflow/lite/delegates/gpu/gl/api.cc +++ b/tensorflow/lite/delegates/gpu/gl/api.cc @@ -35,6 +35,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/object.h" #include "tensorflow/lite/delegates/gpu/gl/portable_gl31.h" #include "tensorflow/lite/delegates/gpu/gl/runtime.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" #ifndef TFLITE_GPU_BINARY_RELEASE #include "tensorflow/lite/delegates/gpu/gl/serialization.h" @@ -168,7 +169,7 @@ class InferenceContextWithBatchImpl : public InferenceContext { struct ProgramParameters { // A list of uniform parameters to be set. - std::vector parameters; + std::vector parameters; // A list of objects to bind to opengl program. std::vector objects; @@ -277,7 +278,7 @@ class CompiledModelImpl #ifndef TFLITE_GPU_BINARY_RELEASE // Called on deserialization - Status OnProgram(const std::vector& parameters, + Status OnProgram(const std::vector& parameters, const std::vector& objects, const uint3& workgroup_size, const uint3& num_workgroups, size_t partial_shader_index) final { diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/BUILD b/tensorflow/lite/delegates/gpu/gl/compiler/BUILD index 551f128556f..6ff34577844 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/compiler/BUILD @@ -35,7 +35,7 @@ cc_library( deps = [ ":preprocessor", "//tensorflow/lite/delegates/gpu/common:types", - "//tensorflow/lite/delegates/gpu/gl:uniform_parameter", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:variant", @@ -82,6 +82,7 @@ cc_test( ":object_accessor", ":parameter_accessor", "//tensorflow/lite/delegates/gpu/common:types", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/types:variant", "@com_google_googletest//:gtest_main", ], @@ -94,7 +95,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:object", - "//tensorflow/lite/delegates/gpu/gl:uniform_parameter", + "//tensorflow/lite/delegates/gpu/gl:variable", ], ) @@ -113,6 +114,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl:compiler_options", "//tensorflow/lite/delegates/gpu/gl:gpu_info", "//tensorflow/lite/delegates/gpu/gl:object", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/strings", ], ) @@ -175,7 +177,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/gl:node_shader", "//tensorflow/lite/delegates/gpu/gl:object", - "//tensorflow/lite/delegates/gpu/gl:uniform_parameter", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/strings", ], ) diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor_test.cc b/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor_test.cc index 2ee6d9de461..0b04210a00d 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor_test.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor_test.cc @@ -23,6 +23,7 @@ limitations under the License. #include "absl/types/variant.h" #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -34,11 +35,11 @@ struct ParameterComparator { const T* v = absl::get_if(&p.value); return v && t == *v; } - const UniformParameter& p; + const Variable& p; }; // partially equal -bool operator==(const UniformParameter& l, const UniformParameter& r) { +bool operator==(const Variable& l, const Variable& r) { return l.name == r.name && absl::visit(ParameterComparator{l}, r.value); } @@ -83,8 +84,8 @@ TEST(Preprocessor, ReadFromBufferByIndex) { EXPECT_EQ(accessor.Rewrite("obj[x,y + 5,z]", &result), RewriteStatus::SUCCESS); EXPECT_THAT(parameters.GetUniformParameters(), - testing::UnorderedElementsAre(UniformParameter{"obj_w", 1}, - UniformParameter{"obj_h", 2})); + testing::UnorderedElementsAre(Variable{"obj_w", 1}, + Variable{"obj_h", 2})); ASSERT_EQ(result, "obj.data[x + $obj_w$ * (y + 5 + $obj_h$ * (z))]"); } @@ -132,8 +133,8 @@ TEST(Preprocessor, WriteToBufferByIndex) { EXPECT_EQ(accessor.Rewrite(" obj[i,j,k] =value", &result), RewriteStatus::SUCCESS); EXPECT_THAT(parameters.GetUniformParameters(), - testing::UnorderedElementsAre(UniformParameter{"obj_w", 1}, - UniformParameter{"obj_h", 2})); + testing::UnorderedElementsAre(Variable{"obj_w", 1}, + Variable{"obj_h", 2})); ASSERT_EQ(result, "obj.data[i + $obj_w$ * (j + $obj_h$ * (k))] = value"); } diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.cc b/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.cc index f3f442bc218..55d7152c0e4 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.cc @@ -68,7 +68,7 @@ struct UniformTypeGetter { }; // Returns GLSL uniform type of the given parameter. -std::string GetUniformType(const UniformParameter::ValueType& value) { +std::string GetUniformType(const Variable::ValueType& value) { return absl::visit(UniformTypeGetter(), value); } @@ -138,7 +138,7 @@ struct ConstGenerator { }; // Appends string representation of a parameter value. -void GetValue(const UniformParameter::ValueType& value, std::string* result) { +void GetValue(const Variable::ValueType& value, std::string* result) { absl::visit(ConstGenerator{result}, value); } @@ -155,11 +155,11 @@ struct UniformDeclarationGenerator { param.name, "[", v.size(), "];\n"); } - const UniformParameter& param; + const Variable& param; std::string* result; }; -void GenerateUniformDeclaration(const UniformParameter& parameter, +void GenerateUniformDeclaration(const Variable& parameter, std::string* result) { absl::visit(UniformDeclarationGenerator{parameter, result}, parameter.value); } @@ -176,7 +176,7 @@ struct VariableLengthGetter { }; // Returns true if value is a vector -bool IsVariableLength(const UniformParameter::ValueType& value) { +bool IsVariableLength(const Variable::ValueType& value) { return absl::visit(VariableLengthGetter(), value); } @@ -222,7 +222,7 @@ struct FieldAccessor { }; // Appends formatted value of the given field. -void GetValue(const UniformParameter::ValueType& value, Field field, +void GetValue(const Variable::ValueType& value, Field field, std::string* result) { absl::visit(FieldAccessor{field, result}, value); } @@ -262,7 +262,7 @@ struct FieldChecker { }; // Returns true if field has field access and field is not out of bounds. -bool HasField(const UniformParameter::ValueType& value, Field field) { +bool HasField(const Variable::ValueType& value, Field field) { return absl::visit(FieldChecker{field}, value); } @@ -322,7 +322,7 @@ RewriteStatus ParameterAccessor::Rewrite(absl::string_view input, return RewriteStatus::SUCCESS; } -bool ParameterAccessor::AddParameter(UniformParameter param) { +bool ParameterAccessor::AddParameter(Variable param) { std::string name = param.name; return name_to_param_.insert({name, std::move(param)}).second; } @@ -353,8 +353,8 @@ std::string ParameterAccessor::GetUniformDeclarations() const { return declarations; } -std::vector ParameterAccessor::GetUniformParameters() const { - std::vector params; +std::vector ParameterAccessor::GetUniformParameters() const { + std::vector params; if (!inline_values_) { for (auto& param : name_to_param_) { params.push_back(param.second); diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.h b/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.h index e6efed0124f..3dacc34d21f 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.h +++ b/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.h @@ -21,7 +21,7 @@ limitations under the License. #include #include "tensorflow/lite/delegates/gpu/gl/compiler/preprocessor.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -51,7 +51,7 @@ class ParameterAccessor : public InlineRewrite { RewriteStatus Rewrite(absl::string_view input, std::string* output) final; // Return true if parameter was successfully added. - bool AddParameter(UniformParameter param); + bool AddParameter(Variable param); // Returns const parameters that need to be inlined in the a shader's code. std::string GetConstDeclarations() const; @@ -60,14 +60,14 @@ class ParameterAccessor : public InlineRewrite { std::string GetUniformDeclarations() const; // Returns a collection of uniform parameters. - std::vector GetUniformParameters() const; + std::vector GetUniformParameters() const; private: const bool inline_values_; // Unique parameter index used for obfuscation. uint32_t unique_param_index_ = 0; - std::unordered_map name_to_param_; + std::unordered_map name_to_param_; }; // Implementation details below. diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor_test.cc b/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor_test.cc index 96182751b9b..d8c634e8c85 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor_test.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor_test.cc @@ -36,7 +36,7 @@ TEST(Preprocessor, CornerCases) { TEST(Preprocessor, Value) { ParameterAccessor accessor(true); - ASSERT_TRUE(accessor.AddParameter(UniformParameter{"var", int32_t(1)})); + ASSERT_TRUE(accessor.AddParameter({"var", int32_t(1)})); std::string result; EXPECT_EQ(accessor.Rewrite("var", &result), RewriteStatus::SUCCESS); ASSERT_EQ(result, "1"); @@ -44,7 +44,7 @@ TEST(Preprocessor, Value) { TEST(Preprocessor, ValueVec) { ParameterAccessor accessor(true); - ASSERT_TRUE(accessor.AddParameter(UniformParameter{"var", int2(1, 2)})); + ASSERT_TRUE(accessor.AddParameter({"var", int2(1, 2)})); std::string result; EXPECT_EQ(accessor.Rewrite("var", &result), RewriteStatus::SUCCESS); ASSERT_EQ(result, "ivec2(1,2)"); @@ -52,8 +52,7 @@ TEST(Preprocessor, ValueVec) { TEST(Preprocessor, Field) { ParameterAccessor accessor(true); - ASSERT_TRUE( - accessor.AddParameter(UniformParameter{"var", float2(1.0, 2.1234567)})); + ASSERT_TRUE(accessor.AddParameter({"var", float2(1.0, 2.1234567)})); std::string result; EXPECT_EQ(accessor.Rewrite("var.y", &result), RewriteStatus::SUCCESS); ASSERT_EQ(result, "2.123456717f"); @@ -61,8 +60,8 @@ TEST(Preprocessor, Field) { TEST(Preprocessor, FieldFail) { ParameterAccessor accessor(true); - ASSERT_TRUE(accessor.AddParameter(UniformParameter{"var", 1.0f})); - ASSERT_TRUE(accessor.AddParameter(UniformParameter{"vec", float2(1.0, 1.0)})); + ASSERT_TRUE(accessor.AddParameter({"var", 1.0f})); + ASSERT_TRUE(accessor.AddParameter({"vec", float2(1.0, 1.0)})); std::string result; EXPECT_EQ(accessor.Rewrite("var.y", &result), RewriteStatus::ERROR); ASSERT_EQ(result, "INVALID_ACCESS_BY_FIELD"); @@ -76,7 +75,7 @@ TEST(Preprocessor, Variable) { ParameterAccessor accessor(true); std::vector v; v.push_back(int2(1, 2)); - ASSERT_TRUE(accessor.AddParameter(UniformParameter{"var", v})); + ASSERT_TRUE(accessor.AddParameter({"var", v})); std::string result; EXPECT_EQ(accessor.Rewrite("var[i].y", &result), RewriteStatus::SUCCESS); ASSERT_EQ(result, "var[i].y"); @@ -86,7 +85,7 @@ TEST(Preprocessor, Variable) { TEST(Preprocessor, InlineVariableFail) { ParameterAccessor accessor(true); - ASSERT_TRUE(accessor.AddParameter(UniformParameter{"var", 1})); + ASSERT_TRUE(accessor.AddParameter({"var", 1})); std::string result; EXPECT_EQ(accessor.Rewrite("var[i]", &result), RewriteStatus::ERROR); ASSERT_EQ(result, "INVALID_ACCESS_BY_INDEX"); diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/rename.cc b/tensorflow/lite/delegates/gpu/gl/compiler/rename.cc index 1c81ebff6b2..e8d1d786b0e 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/rename.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/rename.cc @@ -28,7 +28,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/compiler/parameter_accessor.h" #include "tensorflow/lite/delegates/gpu/gl/compiler/preprocessor.h" #include "tensorflow/lite/delegates/gpu/gl/object.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -66,15 +66,15 @@ class ParameterRewriter : public InlineRewrite { } // Return true if parameter was successfully added. - bool AddParameter(UniformParameter param) { + bool AddParameter(Variable param) { std::string old_name = param.name; param.name = name_func_(old_name); return name_to_param_.insert({old_name, std::move(param)}).second; } // Returns a collection of uniform parameters with updated names. - std::vector GetUniformParameters() const { - std::vector params; + std::vector GetUniformParameters() const { + std::vector params; params.reserve(name_to_param_.size()); for (auto& param : name_to_param_) { params.push_back(param.second); @@ -86,7 +86,7 @@ class ParameterRewriter : public InlineRewrite { const std::string inline_delimiter_; const NameFunctor name_func_; - std::unordered_map name_to_param_; + std::unordered_map name_to_param_; }; // Rewrites names of all objects according to returned values from the diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/shader_code.h b/tensorflow/lite/delegates/gpu/gl/compiler/shader_code.h index 8d6d52b002a..3f3db3796d1 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/shader_code.h +++ b/tensorflow/lite/delegates/gpu/gl/compiler/shader_code.h @@ -22,7 +22,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/model.h" #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/gl/object.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -30,7 +30,7 @@ namespace gl { struct ShaderCode { ShaderCode() = default; - ShaderCode(const std::vector& in_parameters, + ShaderCode(const std::vector& in_parameters, const std::vector& in_objects, const uint3& in_workload, const uint3& in_recommended_workgroup, const std::string& in_source_code, @@ -43,7 +43,7 @@ struct ShaderCode { node_indices(in_node_indices) {} // A list of uniform parameters to be set. - std::vector parameters; + std::vector parameters; // A list of objects to bind to opengl program. std::vector objects; diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/shader_codegen.cc b/tensorflow/lite/delegates/gpu/gl/compiler/shader_codegen.cc index 0b84b8413a8..30da5472565 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/shader_codegen.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/shader_codegen.cc @@ -20,6 +20,7 @@ limitations under the License. #include "absl/strings/str_cat.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/gl/compiler/preprocessor.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -42,8 +43,8 @@ Status ShaderCodegen::Build(CompiledNodeAttributes attr, return OkStatus(); }; - auto add_parameter = [&](UniformParameter&& param) { - if (!parameters.AddParameter(std::forward(param))) { + auto add_parameter = [&](Variable&& param) { + if (!parameters.AddParameter(std::forward(param))) { return InternalError("There is a parameter with the same name"); } return OkStatus(); diff --git a/tensorflow/lite/delegates/gpu/gl/converters/BUILD b/tensorflow/lite/delegates/gpu/gl/converters/BUILD index 1d40b0e0f86..06c78dcab0b 100644 --- a/tensorflow/lite/delegates/gpu/gl/converters/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/converters/BUILD @@ -30,7 +30,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl:gl_buffer", "//tensorflow/lite/delegates/gpu/gl:gl_program", "//tensorflow/lite/delegates/gpu/gl:gl_shader", - "//tensorflow/lite/delegates/gpu/gl:uniform_parameter", + "//tensorflow/lite/delegates/gpu/gl:variable", ], ) @@ -75,7 +75,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl:gl_buffer", "//tensorflow/lite/delegates/gpu/gl:gl_program", "//tensorflow/lite/delegates/gpu/gl:gl_shader", - "//tensorflow/lite/delegates/gpu/gl:uniform_parameter", + "//tensorflow/lite/delegates/gpu/gl:variable", ], ) diff --git a/tensorflow/lite/delegates/gpu/gl/converters/bhwc_to_phwc4.cc b/tensorflow/lite/delegates/gpu/gl/converters/bhwc_to_phwc4.cc index d48d9544025..8b42dedc332 100644 --- a/tensorflow/lite/delegates/gpu/gl/converters/bhwc_to_phwc4.cc +++ b/tensorflow/lite/delegates/gpu/gl/converters/bhwc_to_phwc4.cc @@ -25,7 +25,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/converters/util.h" #include "tensorflow/lite/delegates/gpu/gl/gl_program.h" #include "tensorflow/lite/delegates/gpu/gl/gl_shader.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -89,10 +89,10 @@ Status ConverterBhwcToPhwc4::Convert(const BHWC& shape, const GlBuffer& source, uint3 workload = uint3(shape.w, shape.h, shape.c); uint3 num_workgroups = IntegralDivideRoundUp(workload, workgroup_size_); - RETURN_IF_ERROR(program_.SetParameter(UniformParameter{ - "sizes_", - int4(static_cast(workload.x), static_cast(workload.y), - static_cast(workload.z), static_cast(shape.c))})); + RETURN_IF_ERROR(program_.SetParameter( + {"sizes_", + int4(static_cast(workload.x), static_cast(workload.y), + static_cast(workload.z), static_cast(shape.c))})); RETURN_IF_ERROR(source.BindToIndex(0)); RETURN_IF_ERROR(destination->BindToIndex(1)); if (command_queue) { diff --git a/tensorflow/lite/delegates/gpu/gl/converters/phwc4_to_bhwc.cc b/tensorflow/lite/delegates/gpu/gl/converters/phwc4_to_bhwc.cc index 65f19d4513d..c63fee9f8bd 100644 --- a/tensorflow/lite/delegates/gpu/gl/converters/phwc4_to_bhwc.cc +++ b/tensorflow/lite/delegates/gpu/gl/converters/phwc4_to_bhwc.cc @@ -25,7 +25,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/converters/util.h" #include "tensorflow/lite/delegates/gpu/gl/gl_program.h" #include "tensorflow/lite/delegates/gpu/gl/gl_shader.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -85,10 +85,10 @@ Status ConverterPhwc4ToBhwc::Convert(const BHWC& shape, const GlBuffer& source, // TODO(akulik): simply pass workload as soon as UniformParameter // supports uint3 - RETURN_IF_ERROR(program_.SetParameter(UniformParameter{ - "sizes_", - int4(static_cast(workload.x), static_cast(workload.y), - static_cast(workload.z), 0)})); + RETURN_IF_ERROR(program_.SetParameter( + {"sizes_", + int4(static_cast(workload.x), static_cast(workload.y), + static_cast(workload.z), 0)})); RETURN_IF_ERROR(source.BindToIndex(0)); RETURN_IF_ERROR(destination->BindToIndex(1)); if (command_queue) { diff --git a/tensorflow/lite/delegates/gpu/gl/gl_program.cc b/tensorflow/lite/delegates/gpu/gl/gl_program.cc index 9b0cf3c07db..8e631288181 100644 --- a/tensorflow/lite/delegates/gpu/gl/gl_program.cc +++ b/tensorflow/lite/delegates/gpu/gl/gl_program.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/gl/gl_call.h" #include "tensorflow/lite/delegates/gpu/gl/gl_errors.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -180,7 +181,7 @@ GlProgram& GlProgram::operator=(GlProgram&& program) { GlProgram::~GlProgram() { Invalidate(); } -Status GlProgram::SetParameter(const UniformParameter& param) { +Status GlProgram::SetParameter(const Variable& param) { GLint uniform_location; RETURN_IF_ERROR(TFLITE_GPU_CALL_GL(glGetUniformLocation, &uniform_location, id_, param.name.c_str())); diff --git a/tensorflow/lite/delegates/gpu/gl/gl_program.h b/tensorflow/lite/delegates/gpu/gl/gl_program.h index ff176344d19..dfd6bde4c59 100644 --- a/tensorflow/lite/delegates/gpu/gl/gl_program.h +++ b/tensorflow/lite/delegates/gpu/gl/gl_program.h @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/gl/gl_shader.h" #include "tensorflow/lite/delegates/gpu/gl/portable_gl31.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -61,7 +61,7 @@ class GlProgram { // into this program. Status GetBinary(BinaryShader* binary_shader); - Status SetParameter(const UniformParameter& param); + Status SetParameter(const Variable& param); // Executes program Status Dispatch(const uint3& workgroups) const; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/BUILD b/tensorflow/lite/delegates/gpu/gl/kernels/BUILD index 97b96129a15..50d204c5348 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/kernels/BUILD @@ -49,6 +49,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -85,6 +86,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "//tensorflow/lite/delegates/gpu/gl/workgroups:ideal_workgroup_picker", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", @@ -123,6 +125,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "//tensorflow/lite/delegates/gpu/gl/workgroups:ideal_workgroup_picker", "@com_google_absl//absl/memory", ], @@ -191,6 +194,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -257,6 +261,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -324,6 +329,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -354,10 +360,10 @@ cc_library( hdrs = ["pooling.h"], deps = [ "//tensorflow/lite/delegates/gpu/common:operations", - "//tensorflow/lite/delegates/gpu/common:shape", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -427,6 +433,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -493,6 +500,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -528,6 +536,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -588,6 +597,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) @@ -621,6 +631,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", + "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/concat.cc b/tensorflow/lite/delegates/gpu/gl/kernels/concat.cc index d3aeabcdfc3..c6cdb078a6d 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/concat.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/concat.cc @@ -24,6 +24,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -349,7 +350,7 @@ class FlatConcatByHeight : public NodeShader { GeneratedCode* generated_code) const final { auto inputs = ctx.graph->FindInputs(ctx.node->id); std::string code; - std::vector params; + std::vector params; for (int i = 0, shift = 0; i < inputs.size(); shift += inputs[i]->tensor.shape.h, i++) { code += "if ("; @@ -415,7 +416,7 @@ class FlatConcatByWidth : public NodeShader { GeneratedCode* generated_code) const final { auto inputs = ctx.graph->FindInputs(ctx.node->id); std::string code; - std::vector params; + std::vector params; for (int i = 0, shift = 0; i < inputs.size(); shift += inputs[i]->tensor.shape.w, i++) { code += "if ("; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/conv.cc b/tensorflow/lite/delegates/gpu/gl/kernels/conv.cc index 2c19fcc24e4..0314b959e64 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/conv.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/conv.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/common/util.h" #include "tensorflow/lite/delegates/gpu/gl/node_shader.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" #include "tensorflow/lite/delegates/gpu/gl/workgroups/ideal_workgroup_picker.h" namespace tflite { @@ -50,7 +51,7 @@ class Convolution : public NodeShader { h * attr.dilations.h - attr.padding.prepended.h); } } - std::vector parameters = { + std::vector parameters = { {"input_data_0_h", input->tensor.shape.h}, {"input_data_0_w", input->tensor.shape.w}, {"offsets_count", offsets_count}, @@ -158,7 +159,7 @@ class Convolution1x1 : public NodeShader { int multiplier = SelectMultiplier(input->tensor.shape.w, ctx); - std::vector parameters = { + std::vector parameters = { {"src_depth", IntegralDivideRoundUp(input->tensor.shape.c, 4)}, }; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/depthwise_conv.cc b/tensorflow/lite/delegates/gpu/gl/kernels/depthwise_conv.cc index ac381e1ca08..c82723954b9 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/depthwise_conv.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/depthwise_conv.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/common/util.h" #include "tensorflow/lite/delegates/gpu/gl/node_shader.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" #include "tensorflow/lite/delegates/gpu/gl/workgroups/ideal_workgroup_picker.h" namespace tflite { @@ -49,7 +50,7 @@ class DepthwiseConvolution : public NodeShader { h * attr.dilations.h - attr.padding.prepended.h); } } - std::vector parameters = { + std::vector parameters = { {"input_data_0_h", input->tensor.shape.h}, {"input_data_0_w", input->tensor.shape.w}, {"offsets_count", offsets_count}, diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/fully_connected.cc b/tensorflow/lite/delegates/gpu/gl/kernels/fully_connected.cc index 487db2b5d86..f6c7526b5eb 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/fully_connected.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/fully_connected.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/convert.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -39,7 +40,7 @@ class FullyConnectedBuffers : public NodeShader { ctx.node->operation.attributes); // TODO(akulik): check that input has h,w == 1,1 - std::vector parameters = { + std::vector parameters = { {"src_depth", IntegralDivideRoundUp(attr.weights.shape.i, 4)}, }; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/max_unpooling.cc b/tensorflow/lite/delegates/gpu/gl/kernels/max_unpooling.cc index 610679df2ca..fd9302cb00c 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/max_unpooling.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/max_unpooling.cc @@ -24,6 +24,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -36,7 +37,7 @@ class MaxUnpooling : public NodeShader { GeneratedCode* generated_code) const final { auto attr = absl::any_cast( ctx.node->operation.attributes); - std::vector parameters = { + std::vector parameters = { {"stride", int2(attr.strides.w, attr.strides.h)}, {"offset", int2(attr.padding.prepended.w, attr.padding.prepended.h)}, {"window_h", attr.kernel.h}, diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/pad.cc b/tensorflow/lite/delegates/gpu/gl/kernels/pad.cc index 6d6662c9a54..a27835bbf36 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/pad.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/pad.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/operations.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -46,7 +47,7 @@ class Pad : public NodeShader { attr.prepended.h < 0 || attr.prepended.w < 0 || attr.prepended.c < 0) { return UnimplementedError("Negative padding is not supported."); } - std::vector parameters = { + std::vector parameters = { {"input_data_0_h", input->tensor.shape.h}, {"input_data_0_w", input->tensor.shape.w}, {"prepended", diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/pooling.cc b/tensorflow/lite/delegates/gpu/gl/kernels/pooling.cc index 291c423fe0b..ace3e801c54 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/pooling.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/pooling.cc @@ -24,6 +24,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -40,7 +41,7 @@ Status GenerateMaxPoolingCode(const Pooling2DAttributes& attr, return InvalidArgumentError("Padding is bigger than kernel."); } - std::vector parameters = { + std::vector parameters = { {"input_data_0_h", input->tensor.shape.h}, {"input_data_0_w", input->tensor.shape.w}, {"stride", int2(attr.strides.w, attr.strides.h)}, @@ -100,7 +101,7 @@ Status GenerateAveragePoolingCode(const Pooling2DAttributes& attr, GeneratedCode* generated_code) { auto input = ctx.graph->FindInputs(ctx.node->id)[0]; - std::vector parameters = { + std::vector parameters = { {"input_data_0_h", input->tensor.shape.h}, {"input_data_0_w", input->tensor.shape.w}, {"stride", int2(attr.strides.w, attr.strides.h)}, diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/relu.cc b/tensorflow/lite/delegates/gpu/gl/kernels/relu.cc index c00b9f616f5..aa5c6e855bc 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/relu.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/relu.cc @@ -24,6 +24,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -36,7 +37,7 @@ class ReLU : public NodeShader { GeneratedCode* generated_code) const final { auto attr = absl::any_cast(ctx.node->operation.attributes); // clamp(value, min(0, alpha * value), clip) - std::vector params; + std::vector params; std::string min; if (attr.alpha == 0) { min = "vec4(0.0)"; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/slice.cc b/tensorflow/lite/delegates/gpu/gl/kernels/slice.cc index 66f9abb6b90..678aa7a00ee 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/slice.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/slice.cc @@ -24,6 +24,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -43,7 +44,7 @@ class Slice : public NodeShader { const int4 heights(attr.starts.h, attr.strides.h, attr.ends.h, 0); const int4 widths(attr.starts.w, attr.strides.w, attr.ends.w, 0); - std::vector parameters = { + std::vector parameters = { {"channels", channels}, {"heights", heights}, {"widths", widths}, diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/softmax.cc b/tensorflow/lite/delegates/gpu/gl/kernels/softmax.cc index 000f2b00c5a..04c80937676 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/softmax.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/softmax.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/common/util.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -53,7 +54,7 @@ class SoftMax : public NodeShader { for (int i = 0; i < reminder; ++i) { mask[i] = 1.0f; } - std::vector parameters = { + std::vector parameters = { {"src_depth", IntegralDivideRoundUp(output->tensor.shape.c, 4)}, {"mask", mask}, }; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/transpose_conv.cc b/tensorflow/lite/delegates/gpu/gl/kernels/transpose_conv.cc index f3719c5751c..4682765421a 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/transpose_conv.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/transpose_conv.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/common/util.h" #include "tensorflow/lite/delegates/gpu/gl/node_shader.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -43,7 +44,7 @@ class ConvolutionTransposedBuffers : public NodeShader { const int32_t inner_size_w = (weights.w - 1) / attr.stride.w + 1; const int32_t inner_size_h = (weights.h - 1) / attr.stride.h + 1; - std::vector parameters = { + std::vector parameters = { {"input_data_0_h", input->tensor.shape.h}, {"input_data_0_w", input->tensor.shape.w}, {"src_depth", IntegralDivideRoundUp(weights.i, 4)}, diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/upsampling_bilinear.cc b/tensorflow/lite/delegates/gpu/gl/kernels/upsampling_bilinear.cc index baca806b79a..a30e5ad8e17 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/upsampling_bilinear.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/upsampling_bilinear.cc @@ -24,6 +24,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -69,7 +70,7 @@ class UpsamplingBilinear : public NodeShader { }; return OkStatus(); } - std::vector parameters = { + std::vector parameters = { {"input_data_0_h", input->tensor.shape.h}, {"input_data_0_w", input->tensor.shape.w}, {"scale_factor", diff --git a/tensorflow/lite/delegates/gpu/gl/node_shader.h b/tensorflow/lite/delegates/gpu/gl/node_shader.h index 20491272e35..710d4b6d5e8 100644 --- a/tensorflow/lite/delegates/gpu/gl/node_shader.h +++ b/tensorflow/lite/delegates/gpu/gl/node_shader.h @@ -27,7 +27,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/compiler_options.h" #include "tensorflow/lite/delegates/gpu/gl/gpu_info.h" #include "tensorflow/lite/delegates/gpu/gl/object.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -58,7 +58,7 @@ enum class IOStructure { struct GeneratedCode { // A list of parameters to be set as uniform or hardcoded in a shader. - std::vector parameters; + std::vector parameters; // A list of objects to bind before shader could be executed. std::vector> objects; diff --git a/tensorflow/lite/delegates/gpu/gl/runtime.cc b/tensorflow/lite/delegates/gpu/gl/runtime.cc index c09512c55e7..ca908d5e4c1 100644 --- a/tensorflow/lite/delegates/gpu/gl/runtime.cc +++ b/tensorflow/lite/delegates/gpu/gl/runtime.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/gl_program.h" #include "tensorflow/lite/delegates/gpu/gl/gl_texture.h" #include "tensorflow/lite/delegates/gpu/gl/portable_gl31.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -187,7 +188,7 @@ Runtime::Runtime(const RuntimeOptions& options, const GpuInfo& gpu_info, } Status Runtime::AddProgram(const GlShader& shader, - const std::vector& parameters, + const std::vector& parameters, const std::vector& objects, const uint3& num_workgroups) { GlProgram program; diff --git a/tensorflow/lite/delegates/gpu/gl/runtime.h b/tensorflow/lite/delegates/gpu/gl/runtime.h index 6761d730628..23fff931c2a 100644 --- a/tensorflow/lite/delegates/gpu/gl/runtime.h +++ b/tensorflow/lite/delegates/gpu/gl/runtime.h @@ -30,7 +30,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/runtime/shared_buffer.h" #include "tensorflow/lite/delegates/gpu/gl/runtime_options.h" #include "tensorflow/lite/delegates/gpu/gl/stats.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -45,7 +45,7 @@ class Runtime { // Takes parameters and objects and prepares GL program. Status AddProgram(const GlShader& shader, - const std::vector& parameters, + const std::vector& parameters, const std::vector& objects, const uint3& num_workgroups); diff --git a/tensorflow/lite/delegates/gpu/gl/serialization.cc b/tensorflow/lite/delegates/gpu/gl/serialization.cc index 0b950884239..200ca1fbb01 100644 --- a/tensorflow/lite/delegates/gpu/gl/serialization.cc +++ b/tensorflow/lite/delegates/gpu/gl/serialization.cc @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/data_type.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -276,16 +277,16 @@ void SerializedCompiledModelBuilder::AddShader(const std::string& shader_src) { } void SerializedCompiledModelBuilder::AddProgram( - const std::vector& parameters, - const std::vector& objects, const uint3& workgroup_size, - const uint3& num_workgroups, size_t shader_index) { + const std::vector& parameters, const std::vector& objects, + const uint3& workgroup_size, const uint3& num_workgroups, + size_t shader_index) { Offset fb_workgroups = Encode(num_workgroups, &builder_); Offset fb_workgroup_size = Encode(workgroup_size, &builder_); Offset>> fb_params; { std::vector> offsets; - for (const UniformParameter& param : parameters) { + for (const Variable& param : parameters) { auto name = builder_.CreateString(param.name); auto data = absl::visit(ParameterValueGetter{&builder_}, param.value); data::UniformParameterBuilder builder(builder_); @@ -344,7 +345,7 @@ absl::Span SerializedCompiledModelBuilder::Finalize( namespace { Status ParseParameter(const data::UniformParameter& fb_parameter, - UniformParameter* parameter) { + Variable* parameter) { parameter->name = fb_parameter.name()->str(); switch (fb_parameter.type()) { case data::ParameterType::INT32: { @@ -539,13 +540,13 @@ Status DeserializeCompiledModel(absl::Span serialized, RETURN_IF_ERROR( handler->OnShader(absl::MakeSpan(shader->c_str(), shader->size()))); } - std::vector parameters; + std::vector parameters; std::vector objects; for (auto program : *model->programs()) { parameters.clear(); objects.clear(); for (auto fb_parameter : *program->parameters()) { - UniformParameter parameter; + Variable parameter; RETURN_IF_ERROR(ParseParameter(*fb_parameter, ¶meter)); parameters.push_back(std::move(parameter)); } diff --git a/tensorflow/lite/delegates/gpu/gl/serialization.h b/tensorflow/lite/delegates/gpu/gl/serialization.h index 5c981731ae2..96c0a0b1073 100644 --- a/tensorflow/lite/delegates/gpu/gl/serialization.h +++ b/tensorflow/lite/delegates/gpu/gl/serialization.h @@ -27,7 +27,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/gl/compiled_model_generated.h" #include "tensorflow/lite/delegates/gpu/gl/object.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -46,7 +46,7 @@ class SerializedCompiledModelBuilder { void AddShader(const std::string& shader_src); - void AddProgram(const std::vector& parameters, + void AddProgram(const std::vector& parameters, const std::vector& objects, const uint3& workgroup_size, const uint3& num_workgroups, size_t shader_index); @@ -69,7 +69,7 @@ class DeserializationHandler { virtual Status OnShader(absl::Span shader_src) = 0; - virtual Status OnProgram(const std::vector& parameters, + virtual Status OnProgram(const std::vector& parameters, const std::vector& objects, const uint3& workgroup_size, const uint3& num_workgroups, diff --git a/tensorflow/lite/delegates/gpu/gl/serialization_test.cc b/tensorflow/lite/delegates/gpu/gl/serialization_test.cc index 6256d970f29..38db44122b4 100644 --- a/tensorflow/lite/delegates/gpu/gl/serialization_test.cc +++ b/tensorflow/lite/delegates/gpu/gl/serialization_test.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include + #include #include #include @@ -28,7 +29,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" #include "tensorflow/lite/delegates/gpu/gl/object.h" -#include "tensorflow/lite/delegates/gpu/gl/uniform_parameter.h" +#include "tensorflow/lite/delegates/gpu/gl/variable.h" namespace tflite { namespace gpu { @@ -36,7 +37,7 @@ namespace gl { namespace { struct ProgramDesc { - std::vector parameters; + std::vector parameters; std::vector objects; uint3 workgroup_size; uint3 num_workgroups; @@ -49,7 +50,7 @@ struct Handler : public DeserializationHandler { return OkStatus(); } - Status OnProgram(const std::vector& parameters, + Status OnProgram(const std::vector& parameters, const std::vector& objects, const uint3& workgroup_size, const uint3& num_workgroups, size_t shader_index) final { @@ -107,10 +108,10 @@ struct ParameterComparator { auto v = absl::get(a.value); return value.x == v.x && value.y == v.y && value.z == v.z && value.w == v.w; } - UniformParameter a; + Variable a; }; -bool Eq(const UniformParameter& a, const UniformParameter& b) { +bool Eq(const Variable& a, const Variable& b) { return a.name == b.name && absl::visit(ParameterComparator{a}, b.value); } @@ -138,17 +139,17 @@ TEST(Smoke, Read) { builder.AddShader(shader1); builder.AddShader(shader2); - std::vector parameters; - parameters.push_back(UniformParameter{"1", int32_t(1)}); - parameters.push_back(UniformParameter{"2", int2(1, 2)}); - parameters.push_back(UniformParameter{"3", int4(1, 2, 3, 4)}); - parameters.push_back(UniformParameter{"4", uint32_t(10)}); - parameters.push_back(UniformParameter{"5", uint4(10, 20, 30, 40)}); - parameters.push_back(UniformParameter{"6", -2.0f}); - parameters.push_back(UniformParameter{"7", float2(1, -1)}); - parameters.push_back(UniformParameter{"8", float4(1, -1, 2, -2)}); - parameters.push_back(UniformParameter{ - "9", std::vector{int2(1, 2), int2(3, 4), int2(5, 6)}}); + std::vector parameters; + parameters.push_back({"1", int32_t(1)}); + parameters.push_back({"2", int2(1, 2)}); + parameters.push_back({"3", int4(1, 2, 3, 4)}); + parameters.push_back({"4", uint32_t(10)}); + parameters.push_back({"5", uint4(10, 20, 30, 40)}); + parameters.push_back({"6", -2.0f}); + parameters.push_back({"7", float2(1, -1)}); + parameters.push_back({"8", float4(1, -1, 2, -2)}); + parameters.push_back( + {"9", std::vector{int2(1, 2), int2(3, 4), int2(5, 6)}}); std::vector objects; objects.push_back(MakeReadonlyBuffer(std::vector{1, 2, 3, 4})); diff --git a/tensorflow/lite/delegates/gpu/gl/uniform_parameter.h b/tensorflow/lite/delegates/gpu/gl/variable.h similarity index 83% rename from tensorflow/lite/delegates/gpu/gl/uniform_parameter.h rename to tensorflow/lite/delegates/gpu/gl/variable.h index 90e2c237f90..f2f3979b631 100644 --- a/tensorflow/lite/delegates/gpu/gl/uniform_parameter.h +++ b/tensorflow/lite/delegates/gpu/gl/variable.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_LITE_DELEGATES_GPU_GL_UNIFORM_PARAMETER_H_ -#define TENSORFLOW_LITE_DELEGATES_GPU_GL_UNIFORM_PARAMETER_H_ +#ifndef TENSORFLOW_LITE_DELEGATES_GPU_GL_VARIABLE_H_ +#define TENSORFLOW_LITE_DELEGATES_GPU_GL_VARIABLE_H_ #include #include @@ -27,7 +27,7 @@ namespace tflite { namespace gpu { namespace gl { -struct UniformParameter { +struct Variable { using ValueType = absl::variant>; @@ -39,4 +39,4 @@ struct UniformParameter { } // namespace gpu } // namespace tflite -#endif // TENSORFLOW_LITE_DELEGATES_GPU_GL_UNIFORM_PARAMETER_H_ +#endif // TENSORFLOW_LITE_DELEGATES_GPU_GL_VARIABLE_H_ From 75ba86134502362895f25473555b114065f8da93 Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Thu, 11 Jul 2019 14:15:36 -0700 Subject: [PATCH 296/332] Adding cord support for ZlibOutputBuffer. This improves snapshot benchmark performance by ~2%. PiperOrigin-RevId: 257679909 --- .../kernels/data/experimental/snapshot_dataset_op.cc | 7 +------ tensorflow/core/lib/io/zlib_outputbuffer.cc | 11 +++++++++++ tensorflow/core/lib/io/zlib_outputbuffer.h | 4 ++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc b/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc index d35786c6b32..2e89af6abb3 100644 --- a/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc @@ -91,12 +91,7 @@ class SnapshotWriter { TF_RETURN_IF_ERROR(dest_->Append(StringPiece(header, sizeof(header)))); - // TODO(frankchn): Remove after ZlibOutputBuffer Cord support is added. - if (compression_type_.empty()) { - return dest_->Append(data); - } else { - return dest_->Append(data.ToString()); - } + return dest_->Append(data); } #endif // PLATFORM_GOOGLE diff --git a/tensorflow/core/lib/io/zlib_outputbuffer.cc b/tensorflow/core/lib/io/zlib_outputbuffer.cc index 6355e54aaa2..3b3b4745508 100644 --- a/tensorflow/core/lib/io/zlib_outputbuffer.cc +++ b/tensorflow/core/lib/io/zlib_outputbuffer.cc @@ -190,6 +190,17 @@ Status ZlibOutputBuffer::Append(StringPiece data) { return Status::OK(); } +#if defined(PLATFORM_GOOGLE) +Status ZlibOutputBuffer::Append(const absl::Cord& cord) { + absl::CordReader reader(cord); + absl::string_view fragment; + while (reader.ReadFragment(&fragment)) { + TF_RETURN_IF_ERROR(Append(fragment)); + } + return Status::OK(); +} +#endif + Status ZlibOutputBuffer::Flush() { TF_RETURN_IF_ERROR(DeflateBuffered(Z_PARTIAL_FLUSH)); TF_RETURN_IF_ERROR(FlushOutputBufferToFile()); diff --git a/tensorflow/core/lib/io/zlib_outputbuffer.h b/tensorflow/core/lib/io/zlib_outputbuffer.h index 68f05963226..1eabb2c7b7b 100644 --- a/tensorflow/core/lib/io/zlib_outputbuffer.h +++ b/tensorflow/core/lib/io/zlib_outputbuffer.h @@ -65,6 +65,10 @@ class ZlibOutputBuffer : public WritableFile { // To immediately write contents to file call `Flush()`. Status Append(StringPiece data) override; +#if defined(PLATFORM_GOOGLE) + Status Append(const absl::Cord& cord) override; +#endif + // Deflates any cached input and writes all output to file. Status Flush() override; From 7c4167188208ef9041b4938ac883d90970f61644 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 14:32:14 -0700 Subject: [PATCH 297/332] Fix build with libc++, which follows recent changes to the specification of std::variant, disallowing certain implicit conversions during initialization. PiperOrigin-RevId: 257683060 --- tensorflow/lite/delegates/gpu/gl/object.h | 24 +++++++++++---------- tensorflow/lite/delegates/gpu/gl/runtime.cc | 7 +++--- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/gl/object.h b/tensorflow/lite/delegates/gpu/gl/object.h index 75e9443764d..3340caca8f3 100644 --- a/tensorflow/lite/delegates/gpu/gl/object.h +++ b/tensorflow/lite/delegates/gpu/gl/object.h @@ -85,8 +85,7 @@ inline const ObjectData* GetData(const Object& object) { inline size_t ByteSizeOf(const Object& object); // @return object that references an object created externally. -template -inline Object MakeObjectRef(ObjectRef unique_id, const SizeT& size, +inline Object MakeObjectRef(ObjectRef unique_id, const ObjectSize& size, AccessType access_type) { return Object{access_type, DataType::FLOAT32, ObjectType::UNKNOWN, 0, size, unique_id}; @@ -122,8 +121,8 @@ inline size_t ByteSizeOf(const Object& object) { return SizeOf(object.data_type) * /* vec4 */ 4 * NumElements(object.size); } -template -Object MakeReadonlyObject(const SizeT& size, const std::vector& data) { +inline Object MakeReadonlyObject(const ObjectSize& size, + const std::vector& data) { return Object{AccessType::READ, DataType::FLOAT32, ObjectType::UNKNOWN, @@ -132,8 +131,8 @@ Object MakeReadonlyObject(const SizeT& size, const std::vector& data) { internal_object::ToBytesVector(data, 16)}; } -template -Object MakeReadonlyTexture(const SizeT& size, const std::vector& data) { +inline Object MakeReadonlyTexture(const ObjectSize& size, + const std::vector& data) { return Object{AccessType::READ, DataType::FLOAT32, ObjectType::TEXTURE, @@ -142,8 +141,8 @@ Object MakeReadonlyTexture(const SizeT& size, const std::vector& data) { internal_object::ToBytesVector(data, 16)}; } -template -Object MakeReadonlyBuffer(const SizeT& size, const std::vector& data) { +inline Object MakeReadonlyBuffer(const ObjectSize& size, + const std::vector& data) { return Object{AccessType::READ, DataType::FLOAT32, ObjectType::BUFFER, @@ -153,15 +152,18 @@ Object MakeReadonlyBuffer(const SizeT& size, const std::vector& data) { } inline Object MakeReadonlyObject(const std::vector& data) { - return MakeReadonlyObject(IntegralDivideRoundUp(data.size(), 4U), data); + return MakeReadonlyObject( + IntegralDivideRoundUp(static_cast(data.size()), 4U), data); } inline Object MakeReadonlyTexture(const std::vector& data) { - return MakeReadonlyTexture(IntegralDivideRoundUp(data.size(), 4U), data); + return MakeReadonlyTexture( + IntegralDivideRoundUp(static_cast(data.size()), 4U), data); } inline Object MakeReadonlyBuffer(const std::vector& data) { - return MakeReadonlyBuffer(IntegralDivideRoundUp(data.size(), 4U), data); + return MakeReadonlyBuffer( + IntegralDivideRoundUp(static_cast(data.size()), 4U), data); } // TODO(akulik): find better place for functions below. diff --git a/tensorflow/lite/delegates/gpu/gl/runtime.cc b/tensorflow/lite/delegates/gpu/gl/runtime.cc index ca908d5e4c1..7249ac40ce2 100644 --- a/tensorflow/lite/delegates/gpu/gl/runtime.cc +++ b/tensorflow/lite/delegates/gpu/gl/runtime.cc @@ -543,7 +543,8 @@ Status Runtime::AssignInternalObjects(std::vector* shared_objects) { shared_object.object = shared_ref; if (shared_object.object_type == ObjectType::BUFFER) { // Make a buffer linear. - shared_object.size = NumElements(object.size); + shared_object.size = + static_cast(NumElements(object.size)); } shared_objects->push_back(std::move(shared_object)); is_used_shared_object.push_back(false); @@ -552,8 +553,8 @@ Status Runtime::AssignInternalObjects(std::vector* shared_objects) { Object& shared_object = (*shared_objects)[shared_ref]; switch (object.object_type) { case ObjectType::BUFFER: - shared_object.size = std::max(NumElements(object.size), - NumElements(shared_object.size)); + shared_object.size = std::max( + NumElements(object.size), NumElements(shared_object.size)); break; case ObjectType::TEXTURE: { if (!FitSize(object.size, shared_object.size, From 53f95a95fa0260db57daba790f6e0d2adcde4df0 Mon Sep 17 00:00:00 2001 From: Reed Wanderman-Milne Date: Thu, 11 Jul 2019 14:37:40 -0700 Subject: [PATCH 298/332] Fix issue where AutoCastVariables wouldn't cast in tf.functions. I tried to fix this in 1ec90f6a7b8b4aa6a8ed9a596c0803406b4b7460, but it was rolled back in e737182ec257b55d575fde3a2c87ea5f5f2b0f7a because it broke cases where one layer tried accessed another layer's variables when AutoCastVariables were used, such as in RNNs. This is a much simpler fix. I added RNN mixed precision tests, because RNNs use tf.functions, which did not work with mixed precision before this change. PiperOrigin-RevId: 257684192 --- tensorflow/python/framework/func_graph.py | 5 ++ .../experimental/keras_test.py | 70 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py index ceaf198affb..c11863ccf0a 100644 --- a/tensorflow/python/framework/func_graph.py +++ b/tensorflow/python/framework/func_graph.py @@ -373,6 +373,10 @@ class FuncGraph(ops.Graph): # optimizers. old_graph_key = self._graph_key self._graph_key = graph._graph_key + # Inherit the auto_cast_variable_read_dtype, since this should not change + # inside a function. + old_auto_cast_var_read_dtype = self._auto_cast_variable_read_dtype + self._auto_cast_variable_read_dtype = graph._auto_cast_variable_read_dtype # pylint: enable=protected-access with outer_cm as g: @@ -383,6 +387,7 @@ class FuncGraph(ops.Graph): self._device_function_stack = old_device_stack self._variable_creator_stack = old_creator_stack self._graph_key = old_graph_key + self._auto_cast_variable_read_dtype = old_auto_cast_var_read_dtype return inner_cm() @property diff --git a/tensorflow/python/keras/mixed_precision/experimental/keras_test.py b/tensorflow/python/keras/mixed_precision/experimental/keras_test.py index cb40ad43f9a..820daf405f7 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/keras_test.py +++ b/tensorflow/python/keras/mixed_precision/experimental/keras_test.py @@ -27,6 +27,7 @@ from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import mirrored_strategy from tensorflow.python.eager import backprop from tensorflow.python.eager import context +from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import test_util @@ -38,6 +39,7 @@ from tensorflow.python.keras import regularizers from tensorflow.python.keras import testing_utils from tensorflow.python.keras.engine import base_layer from tensorflow.python.keras.layers import core +from tensorflow.python.keras.layers import recurrent from tensorflow.python.keras.mixed_precision.experimental import loss_scale_optimizer from tensorflow.python.keras.mixed_precision.experimental import policy from tensorflow.python.keras.mixed_precision.experimental import test_util as mp_test_util @@ -124,6 +126,14 @@ class AddLayerWithoutAutoCast(AddLayer): return self._add(inputs, math_ops.cast(self.v, inputs.dtype)) +class AddLayerWithFunction(AddLayer): + """Same as AddLayer, but _add is decorated with a tf.function.""" + + @def_function.function + def _add(self, x, y): + return super(AddLayerWithFunction, self)._add(x, y) + + class IdentityRegularizer(regularizers.Regularizer): def __call__(self, x): @@ -181,6 +191,19 @@ class KerasLayerTest(keras_parameterized.TestCase): self.evaluate(variables.global_variables_initializer()) self.assertEqual(self.evaluate(y), 2.) + @parameterized.named_parameters(*TESTCASES) + @test_util.run_in_graph_and_eager_modes + def test_layer_calling_tf_function(self, strategy_fn): + x = constant_op.constant([1.], dtype=dtypes.float16) + with strategy_fn().scope(): + with policy.policy_scope('infer_float32_vars'): + layer = AddLayerWithFunction(assert_type=dtypes.float16) + y = layer(x) + self.assertEqual(layer.v.dtype, dtypes.float32) + self.assertEqual(y.dtype, dtypes.float16) + self.evaluate(variables.global_variables_initializer()) + self.assertEqual(self.evaluate(y), 2.) + @parameterized.named_parameters(*TESTCASES) @test_util.run_in_graph_and_eager_modes def test_layer_regularizer_runs_in_float32(self, strategy_fn): @@ -694,5 +717,52 @@ class KerasModelTest(keras_parameterized.TestCase): self.assertEqual(backend.get_value(loss_scale._num_good_steps), 1) +class RnnTest(keras_parameterized.TestCase): + """Test mixed precision with RNNs.""" + + # TODO(b/136512020): Support and test recurrent_v2.GRU. + @parameterized.named_parameters({ + 'testcase_name': 'base_simple', + 'strategy_fn': default_strategy_fn, + 'rnn_class': recurrent.SimpleRNN, + }, { + 'testcase_name': 'distribute_simple', + 'strategy_fn': create_mirrored_strategy, + 'rnn_class': recurrent.SimpleRNN, + }, { + 'testcase_name': 'base_gru', + 'strategy_fn': default_strategy_fn, + 'rnn_class': recurrent.GRU, + }, { + 'testcase_name': 'distribute_gru', + 'strategy_fn': create_mirrored_strategy, + 'rnn_class': recurrent.GRU, + }) + @test_util.run_in_graph_and_eager_modes + # RNNs do not work properly with GradientTape in graph mode when V1 control + # flow is used. + @test_util.enable_control_flow_v2 + def test_rnn(self, strategy_fn, rnn_class): + x = array_ops.ones((2, 3, 4), dtype=dtypes.float16) + strategy = strategy_fn() + with strategy.scope(), policy.policy_scope('infer_float32_vars'): + layer = rnn_class(units=4) + def run_fn(): + with backprop.GradientTape() as tape: + y = layer(x) + self.assertEqual(y.dtype, dtypes.float16) + opt = gradient_descent.SGD(1.) + grads = tape.gradient(y, layer.trainable_weights) + return opt.apply_gradients(zip(grads, layer.trainable_weights)) + + op = strategy.experimental_run(run_fn) + if not context.executing_eagerly(): + self.evaluate(variables.global_variables_initializer()) + self.evaluate(op) + + for v in layer.weights: + self.assertEqual(v.dtype, dtypes.float32) + + if __name__ == '__main__': test.main() From dbd23da825b00aedf1f3f70305926d1d8ed6fad0 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Thu, 11 Jul 2019 14:40:42 -0700 Subject: [PATCH 299/332] Automated rollback of commit b9a6fea1f0a501b226394431d0377eef0b40c4b0 PiperOrigin-RevId: 257684824 --- tensorflow/core/grappler/optimizers/BUILD | 2 +- .../grappler/optimizers/meta_optimizer.cc | 8 +-- tensorflow/python/BUILD | 64 +++++++++---------- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index 6dc6cee5bdd..707d242c2b7 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -594,9 +594,9 @@ cc_library( ":debug_stripper", ":dependency_optimizer", ":function_optimizer", - ":generic_layout_optimizer", ":graph_optimizer", ":implementation_selector", + ":layout_optimizer", ":loop_optimizer", ":memory_optimizer", ":model_pruner", diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 7f1302d6b09..057e5a6f5c0 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -32,8 +32,8 @@ limitations under the License. #include "tensorflow/core/grappler/optimizers/debug_stripper.h" #include "tensorflow/core/grappler/optimizers/dependency_optimizer.h" #include "tensorflow/core/grappler/optimizers/function_optimizer.h" -#include "tensorflow/core/grappler/optimizers/generic_layout_optimizer.h" #include "tensorflow/core/grappler/optimizers/implementation_selector.h" +#include "tensorflow/core/grappler/optimizers/layout_optimizer.h" #include "tensorflow/core/grappler/optimizers/loop_optimizer.h" #include "tensorflow/core/grappler/optimizers/memory_optimizer.h" #include "tensorflow/core/grappler/optimizers/model_pruner.h" @@ -121,7 +121,7 @@ std::unique_ptr MetaOptimizer::MakeNewOptimizer( MK_OPT("constfold", new ConstantFolding(cpu_device_)); MK_OPT("shape", new ShapeOptimizer()); MK_OPT("remap", new Remapper(cfg_.remapping())); - MK_OPT("layout", new GenericLayoutOptimizer()); + MK_OPT("layout", new LayoutOptimizer()); MK_OPT("auto_mixed_precision", new AutoMixedPrecision(cfg_.auto_mixed_precision())); MK_OPT("memory", new MemoryOptimizer(RewriterConfig::MANUAL)); @@ -193,7 +193,7 @@ Status MetaOptimizer::InitializeOptimizers( MakeUnique(cfg_.dependency_optimization())); } if (cfg_.layout_optimizer() != RewriterConfig::OFF) { - optimizers->push_back(MakeUnique()); + optimizers->push_back(MakeUnique()); } if (AutoMixedPrecisionEnabled(cfg_.auto_mixed_precision())) { optimizers->push_back( @@ -267,7 +267,7 @@ Status MetaOptimizer::InitializeCustomGraphOptimizers( TF_RETURN_IF_ERROR(custom_optimizer->Init(&optimizer_config)); optimizers->push_back(std::move(custom_optimizer)); } else { - // If there are no custom optimizers with given name, try to initialize a + // If there are no custom optimizers with given name, try to initalize a // default optimizer. This way, custom configurable optimizers can be // mixed with default optimizers in any order. auto optimizer = MakeNewOptimizer(optimizer_config.name()); diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 93f43ab338a..c5fc23c6fcc 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -6455,39 +6455,37 @@ cuda_py_test( xla_enable_strict_auto_jit = True, ) -# TODO(b/131764887) Remove once LayoutOptimizer is swapped out with GenericLayoutOptimizer. -# -# cuda_py_test( -# name = "layout_optimizer_test", -# size = "medium", -# srcs = [ -# "grappler/layout_optimizer_test.py", -# ], -# additional_deps = [ -# ":client_testlib", -# ":framework_for_generated_wrappers", -# ":array_ops", -# ":constant_op", -# ":dtypes", -# ":functional_ops", -# ":math_ops", -# ":nn", -# ":ops", -# ":random_ops", -# ":state_ops", -# ":tf_cluster", -# ":tf_optimizer", -# ":training", -# "//third_party/py/numpy", -# "//tensorflow/core:protos_all_py", -# ], -# shard_count = 10, -# tags = [ -# "grappler", -# ], -# # This test will not run on XLA because it primarily tests the TF Classic flow. -# xla_enable_strict_auto_jit = False, -# ) +cuda_py_test( + name = "layout_optimizer_test", + size = "medium", + srcs = [ + "grappler/layout_optimizer_test.py", + ], + additional_deps = [ + ":client_testlib", + ":framework_for_generated_wrappers", + ":array_ops", + ":constant_op", + ":dtypes", + ":functional_ops", + ":math_ops", + ":nn", + ":ops", + ":random_ops", + ":state_ops", + ":tf_cluster", + ":tf_optimizer", + ":training", + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + ], + shard_count = 10, + tags = [ + "grappler", + ], + # This test will not run on XLA because it primarily tests the TF Classic flow. + xla_enable_strict_auto_jit = False, +) py_library( name = "cost_analyzer", From 14bfae8a0444fc9c95b3b8ed65f9fd59d0f9fbf4 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Thu, 11 Jul 2019 14:44:44 -0700 Subject: [PATCH 300/332] Use declarative rewrite patterns for TopKV2 legalization PiperOrigin-RevId: 257685698 --- .../mlir/lite/transforms/legalize_patterns.td | 3 +++ .../compiler/mlir/lite/transforms/legalize_tf.cc | 11 +---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td b/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td index d4866a4740e..ab4c8cb5ee7 100644 --- a/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td +++ b/tensorflow/compiler/mlir/lite/transforms/legalize_patterns.td @@ -229,6 +229,9 @@ def : Pat<(TF_MeanOp $arg0, $arg1, BoolAttr:$arg2), (TFL_MeanOp $arg0, $arg1, $a def : Pat<(TF_SumOp $arg, $axes, BoolAttr:$arg2), (TFL_SumOp $arg, $axes, $arg2)>; +// TopK in TFL is always sorted so we ignore that attribute here. +def : Pat<(TF_TopKV2Op $input, $k, $ignored_sorted), (TFL_TopKV2Op $input, $k)>; + def : Pat<(TF_MinOp $arg0, $arg1, BoolAttr:$arg2), (TFL_ReduceMinOp $arg0, $arg1, $arg2)>; def : Pat<(TF_MaxOp $arg0, $arg1, BoolAttr:$arg2), (TFL_ReduceMaxOp $arg0, $arg1, $arg2)>; diff --git a/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc index 5be5cfec497..05a604dc461 100644 --- a/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc @@ -72,7 +72,6 @@ DECL_CONVERT_OP(MatMul); DECL_CONVERT_OP(Pack); DECL_CONVERT_OP(Split); DECL_CONVERT_OP(SplitV); -DECL_CONVERT_OP(TopKV2); DECL_CONVERT_OP(Unpack); #undef DECL_CONVERT_OP @@ -207,14 +206,6 @@ PatternMatchResult ConvertTFSplitVOp::matchAndRewrite( return matchSuccess(); } -PatternMatchResult ConvertTFTopKV2Op::matchAndRewrite( - Operation* op, PatternRewriter& rewriter) const { - // TopK in TFL is always sorted so we ignore that attribute here. - rewriter.replaceOpWithNewOp(op, op->getOperand(0), - op->getOperand(1)); - return matchSuccess(); -} - PatternMatchResult ConvertTFUnpackOp::matchAndRewrite( Operation* op, PatternRewriter& rewriter) const { auto tf_unpack_op = cast(op); @@ -239,7 +230,7 @@ void LegalizeTF::runOnFunction() { populateWithGenerated(ctx, &patterns); RewriteListBuilder::build(patterns, ctx); applyPatternsGreedily(func, std::move(patterns)); } From 2cb25a86b2daa1f3e6e582314e33803e9eabe403 Mon Sep 17 00:00:00 2001 From: Ruoxin Sang Date: Thu, 11 Jul 2019 14:47:16 -0700 Subject: [PATCH 301/332] Only enable last partial support in MirroredStrategy in eager mode. PiperOrigin-RevId: 257686233 --- tensorflow/python/distribute/mirrored_strategy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/distribute/mirrored_strategy.py b/tensorflow/python/distribute/mirrored_strategy.py index dcdc70ce94c..811bd2541e8 100644 --- a/tensorflow/python/distribute/mirrored_strategy.py +++ b/tensorflow/python/distribute/mirrored_strategy.py @@ -381,7 +381,9 @@ class MirroredExtended(distribute_lib.StrategyExtendedV1): self._cross_device_ops = cross_device_ops self._initialize_strategy(devices) - self.experimental_enable_get_next_as_optional = True + # TODO(b/128995245): Enable last partial batch support in graph mode. + if ops.executing_eagerly_outside_functions(): + self.experimental_enable_get_next_as_optional = True def _initialize_strategy(self, devices): # The _initialize_strategy method is intended to be used by distribute From f22a30088dff2f047d47d6df0d9d2be40cc2dce8 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 11 Jul 2019 14:48:53 -0700 Subject: [PATCH 302/332] Fix typo in error log message. PiperOrigin-RevId: 257686584 --- .../core/distributed_runtime/rpc/eager/grpc_eager_client.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc index 33c2b17e113..b3164f0956e 100644 --- a/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc +++ b/tensorflow/core/distributed_runtime/rpc/eager/grpc_eager_client.cc @@ -64,7 +64,7 @@ class GrpcEagerClient : public EagerClient { enqueue_dispatchers_.erase(request->context_id()); } else { LOG(ERROR) << "Remote EagerContext with id " << request->context_id() - << " does not seems to exist."; + << " does not seem to exist."; } } From 7b1bbadeda8b4574ce03ed73c88233656da8bc4a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 15:02:43 -0700 Subject: [PATCH 303/332] Create "hello_world" example for TensorFlow Lite for Microcontrollers. PiperOrigin-RevId: 257689385 --- .../experimental/micro/arduino/debug_log.cc | 5 +- .../micro/examples/hello_world/BUILD | 77 + .../micro/examples/hello_world/Makefile.inc | 39 + .../micro/examples/hello_world/README.md | 363 +++++ .../examples/hello_world/arduino/constants.cc | 19 + .../hello_world/arduino/output_handler.cc | 47 + .../micro/examples/hello_world/constants.cc | 19 + .../micro/examples/hello_world/constants.h | 32 + .../hello_world/create_sine_model.ipynb | 1335 +++++++++++++++++ .../hello_world/disco_f746ng/Makefile.inc | 6 + .../hello_world/disco_f746ng/constants.cc | 19 + .../disco_f746ng/output_handler.cc | 80 + .../examples/hello_world/hello_world_test.cc | 112 ++ .../examples/hello_world/images/STM32F746.gif | Bin 0 -> 299034 bytes .../hello_world/images/arduino_mkrzero.gif | Bin 0 -> 541423 bytes .../hello_world/images/sparkfun_edge.gif | Bin 0 -> 639801 bytes .../micro/examples/hello_world/main.cc | 93 ++ .../examples/hello_world/output_handler.cc | 22 + .../examples/hello_world/output_handler.h | 26 + .../hello_world/output_handler_test.cc | 33 + .../examples/hello_world/sine_model_data.cc | 255 ++++ .../examples/hello_world/sine_model_data.h | 27 + .../hello_world/sparkfun_edge/constants.cc | 19 + .../sparkfun_edge/output_handler.cc | 84 ++ .../make/templates/LCD_DISCO_F746NG.lib.tpl | 1 + 25 files changed, 2709 insertions(+), 4 deletions(-) create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/BUILD create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/Makefile.inc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/README.md create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/arduino/constants.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/arduino/output_handler.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/constants.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/constants.h create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/Makefile.inc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/constants.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/output_handler.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/hello_world_test.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/images/STM32F746.gif create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/images/arduino_mkrzero.gif create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/images/sparkfun_edge.gif create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/main.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/output_handler.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/output_handler_test.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/sparkfun_edge/constants.cc create mode 100644 tensorflow/lite/experimental/micro/examples/hello_world/sparkfun_edge/output_handler.cc create mode 100644 tensorflow/lite/experimental/micro/tools/make/templates/LCD_DISCO_F746NG.lib.tpl diff --git a/tensorflow/lite/experimental/micro/arduino/debug_log.cc b/tensorflow/lite/experimental/micro/arduino/debug_log.cc index 94d8d832dd6..4d18f6f97e9 100644 --- a/tensorflow/lite/experimental/micro/arduino/debug_log.cc +++ b/tensorflow/lite/experimental/micro/arduino/debug_log.cc @@ -1,4 +1,4 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -32,9 +32,6 @@ extern "C" void DebugLog(const char* s) { static bool is_initialized = false; if (!is_initialized) { DEBUG_SERIAL_OBJECT.begin(9600); - // Wait for serial port to connect. Only needed for some models apparently? - while (!DEBUG_SERIAL_OBJECT) { - } is_initialized = true; } DEBUG_SERIAL_OBJECT.println(s); diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/BUILD b/tensorflow/lite/experimental/micro/examples/hello_world/BUILD new file mode 100644 index 00000000000..f3340492333 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/BUILD @@ -0,0 +1,77 @@ +# Description: +# TensorFlow Lite for Microcontrollers "hello world" example. + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +load( + "//tensorflow/lite/experimental/micro/testing:micro_test.bzl", + "tflite_micro_cc_test", +) + +cc_library( + name = "sine_model_data", + srcs = [ + "sine_model_data.cc", + ], + hdrs = [ + "sine_model_data.h", + ], +) + +tflite_micro_cc_test( + name = "hello_world_test", + srcs = [ + "hello_world_test.cc", + ], + deps = [ + "//tensorflow/lite:schema_fbs_version", + "//tensorflow/lite/experimental/micro:micro_framework", + "//tensorflow/lite/experimental/micro/examples/hello_world:sine_model_data", + "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver", + "//tensorflow/lite/experimental/micro/kernels:micro_ops", + "//tensorflow/lite/experimental/micro/testing:micro_test", + "//tensorflow/lite/schema:schema_fbs", + ], +) + +cc_library( + name = "output_handler", + srcs = [ + "output_handler.cc", + ], + hdrs = [ + "output_handler.h", + ], + deps = [ + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/experimental/micro:micro_framework", + ], +) + +cc_library( + name = "constants", + srcs = [ + "constants.cc", + ], + hdrs = [ + "constants.h", + ], +) + +cc_binary( + name = "hello_world", + srcs = [ + "main.cc", + ], + deps = [ + ":constants", + ":output_handler", + "//tensorflow/lite:schema_fbs_version", + "//tensorflow/lite/experimental/micro:micro_framework", + "//tensorflow/lite/experimental/micro/examples/hello_world:sine_model_data", + "//tensorflow/lite/experimental/micro/kernels:all_ops_resolver", + "//tensorflow/lite/schema:schema_fbs", + ], +) diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/Makefile.inc b/tensorflow/lite/experimental/micro/examples/hello_world/Makefile.inc new file mode 100644 index 00000000000..59a233c15ec --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/Makefile.inc @@ -0,0 +1,39 @@ +HELLO_WORLD_TEST_SRCS := \ +tensorflow/lite/experimental/micro/examples/hello_world/hello_world_test.cc \ +tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.cc + +HELLO_WORLD_TEST_HDRS := \ +tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h + +OUTPUT_HANDLER_TEST_SRCS := \ +tensorflow/lite/experimental/micro/examples/hello_world/output_handler_test.cc \ +tensorflow/lite/experimental/micro/examples/hello_world/output_handler.cc + +OUTPUT_HANDLER_TEST_HDRS := \ +tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h + +HELLO_WORLD_SRCS := \ +tensorflow/lite/experimental/micro/examples/hello_world/main.cc \ +tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.cc \ +tensorflow/lite/experimental/micro/examples/hello_world/output_handler.cc \ +tensorflow/lite/experimental/micro/examples/hello_world/constants.cc + +HELLO_WORLD_HDRS := \ +tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h \ +tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h \ +tensorflow/lite/experimental/micro/examples/hello_world/constants.h + +#Find any platform-specific rules for this example. +include $(wildcard tensorflow/lite/experimental/micro/examples/hello_world/*/Makefile.inc) + +# Tests loading and running the sine model. +$(eval $(call microlite_test,hello_world_test,\ +$(HELLO_WORLD_TEST_SRCS),$(HELLO_WORLD_TEST_HDRS))) + +# Tests producing an output. +$(eval $(call microlite_test,output_handler_test,\ +$(OUTPUT_HANDLER_TEST_SRCS),$(OUTPUT_HANDLER_TEST_HDRS))) + +# Builds a standalone binary. +$(eval $(call microlite_test,hello_world,\ +$(HELLO_WORLD_SRCS),$(HELLO_WORLD_HDRS))) diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/README.md b/tensorflow/lite/experimental/micro/examples/hello_world/README.md new file mode 100644 index 00000000000..5731234d683 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/README.md @@ -0,0 +1,363 @@ +# Hello World example + +This example is designed to demonstrate the absolute basics of using TensorFlow +Lite for Microcontrollers. It includes the full end-to-end workflow of training +a model, converting it for use with TensorFlow Lite, and running inference on a +microcontroller. + +The sample is built around a model trained to replicate a `sine` function. It +contains implementations for several platforms. In each case, the model is used +to generate a pattern of data that is used to either blink LEDs or control an +animation. + +![Animation of example running on STM32F746](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/hello_world/images/STM32F746.gif) + +## Table of contents + +- [Getting started](#getting-started) +- [Deploy to Arduino](#deploy-to-arduino) +- [Deploy to SparkFun Edge](#deploy-to-sparkfun-edge) +- [Deploy to STM32F746](#deploy-to-STM32F746) + +## Getting started + +### Understand the model + +The sample comes with a pre-trained model. The code used to train and convert +the model is available as a tutorial in [create_sine_model.ipynb](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb). + +Walk through this tutorial to understand what the model does, +how it works, and how it was converted for use with TensorFlow Lite for +Microcontrollers. + +### Build the code + +To compile and test this example on a desktop Linux or MacOS machine, download +[the TensorFlow source code](https://github.com/tensorflow/tensorflow), `cd` +into the source directory from a terminal, and then run the following command: + +``` +make -f tensorflow/lite/experimental/micro/tools/make/Makefile test_hello_world_test +``` + +This will take a few minutes, and downloads frameworks the code uses like +[CMSIS](https://developer.arm.com/embedded/cmsis) and +[flatbuffers](https://google.github.io/flatbuffers/). Once that process has +finished, you should see a series of files get compiled, followed by some +logging output from a test, which should conclude with `~~~ALL TESTS PASSED~~~`. + +If you see this, it means that a small program has been built and run that loads +the trained TensorFlow model, runs some example inputs through it, and got the +expected outputs. + +To understand how TensorFlow Lite does this, you can look at the source in +[hello_world_test.cc](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/examples/hello_world/hello_world_test.cc). +It's a fairly small amount of code that creates an interpreter, gets a handle to +a model that's been compiled into the program, and then invokes the interpreter +with the model and sample inputs. + +## Deploy to Arduino + +The following instructions will help you build and deploy this sample +to [Arduino](https://www.arduino.cc/) devices. + +![Animation of example running on Arduino MKRZERO](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/hello_world/images/arduino_mkrzero.gif) + +The sample has been tested with the following devices: + +- [Arduino MKRZERO](https://store.arduino.cc/usa/arduino-mkrzero) + +The sample will use PWM to fade an LED on and off according to the model's +output. In the code, the `LED_BUILTIN` constant is used to specify the board's +built-in LED as the one being controlled. However, on some boards, this built-in +LED is not attached to a pin with PWM capabilities. In this case, the LED will +blink instead of fading. + +### Obtain and import the library + +To use this sample application with Arduino, we've created an Arduino library +that includes it as an example that you can open in the Arduino IDE. + +Download the current nightly build of the library: [hello_world.zip](https://storage.googleapis.com/tensorflow-nightly/github/tensorflow/tensorflow/lite/experimental/micro/tools/make/gen/arduino_x86_64/prj/hello_world/hello_world.zip) + +Next, import this zip file into the Arduino IDE by going to `Sketch -> Include Library -> Add .ZIP Library...`. + +#### Building the library + +If you need to build the library from source (for example, if you're making +modifications to the code), run this command to generate a zip file containing +the required source files: + +``` +make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=arduino TAGS="" generate_hello_world_arduino_library_zip +``` + +A zip file will be created at the following location: + +``` +tensorflow/lite/experimental/micro/tools/make/gen/arduino_x86_64/prj/hello_world/hello_world.zip +``` + +You can then import this zip file into the Arduino IDE by going to `Sketch -> Include Library -> Add .ZIP Library...`. + +### Load and run the example + +Once the library has been added, go to `File -> Examples`. You should see an +example near the bottom of the list named `TensorFlowLite:hello_world`. Select +it and click `hello_world` to load the example. + +Use the Arduino IDE to build and upload the example. Once it is running, you +should see the built-in LED on your device flashing. + +The Arduino IDE includes a plotter that we can use to display the sine wave +graphically. To view it, go to `Tools -> Serial Plotter`. You will see one +datapoint being logged for each inference cycle, expressed as a number between 0 +and 255. + +## Deploy to SparkFun Edge + +The following instructions will help you build and deploy this sample on the +[SparkFun Edge development board](https://sparkfun.com/products/15170). + +![Animation of example running on SparkFun Edge](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/hello_world/images/sparkfun_edge.gif) + +If you're new to using this board, we recommend walking through the +[AI on a microcontroller with TensorFlow Lite and SparkFun Edge](https://codelabs.developers.google.com/codelabs/sparkfun-tensorflow) +codelab to get an understanding of the workflow. + +### Compile the binary + +The following command will download the required dependencies and then compile a +binary for the SparkFun Edge: + +``` +make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=sparkfun_edge hello_world_bin +``` + +The binary will be created in the following location: + +``` +tensorflow/lite/experimental/micro/tools/make/gen/sparkfun_edge_cortex-m4/bin/hello_world.bin +``` + +### Sign the binary + +The binary must be signed with cryptographic keys to be deployed to the device. +We'll now run some commands that will sign our binary so it can be flashed to +the SparkFun Edge. The scripts we are using come from the Ambiq SDK, which is +downloaded when the `Makefile` is run. + +Enter the following command to set up some dummy cryptographic keys we can use +for development: + +``` +cp tensorflow/lite/experimental/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/keys_info0.py \ +tensorflow/lite/experimental/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/keys_info.py +``` + +Next, run the following command to create a signed binary: + +``` +python3 tensorflow/lite/experimental/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/create_cust_image_blob.py \ +--bin tensorflow/lite/experimental/micro/tools/make/gen/sparkfun_edge_cortex-m4/bin/hello_world.bin \ +--load-address 0xC000 \ +--magic-num 0xCB \ +-o main_nonsecure_ota \ +--version 0x0 +``` + +This will create the file `main_nonsecure_ota.bin`. We'll now run another +command to create a final version of the file that can be used to flash our +device with the bootloader script we will use in the next step: + +``` +python3 tensorflow/lite/experimental/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/create_cust_wireupdate_blob.py \ +--load-address 0x20000 \ +--bin main_nonsecure_ota.bin \ +-i 6 \ +-o main_nonsecure_wire \ +--options 0x1 +``` + +You should now have a file called `main_nonsecure_wire.bin` in the directory +where you ran the commands. This is the file we'll be flashing to the device. + +### Flash the binary + +Next, attach the board to your computer via a USB-to-serial adapter. + +**Note:** If you're using the [SparkFun Serial Basic Breakout](https://www.sparkfun.com/products/15096), +you should [install the latest drivers](https://learn.sparkfun.com/tutorials/sparkfun-serial-basic-ch340c-hookup-guide#drivers-if-you-need-them) +before you continue. + +Once connected, assign the USB device name to an environment variable: + +``` +export DEVICENAME=put your device name here +``` + +Set another variable with the baud rate: + +``` +export BAUD_RATE=921600 +``` + +Now, hold the button marked `14` on the device. While still holding the button, +hit the button marked `RST`. Continue holding the button marked `14` while +running the following command: + +``` +python3 tensorflow/lite/experimental/micro/tools/make/downloads/AmbiqSuite-Rel2.0.0/tools/apollo3_scripts/uart_wired_update.py \ +-b ${BAUD_RATE} ${DEVICENAME} \ +-r 1 \ +-f main_nonsecure_wire.bin \ +-i 6 +``` + +You should see a long stream of output as the binary is flashed to the device. +Once you see the following lines, flashing is complete: + +``` +Sending Reset Command. +Done. +``` + +If you don't see these lines, flashing may have failed. Try running through the +steps in [Flash the binary](#flash-the-binary) again (you can skip over setting +the environment variables). If you continue to run into problems, follow the +[AI on a microcontroller with TensorFlow Lite and SparkFun Edge](https://codelabs.developers.google.com/codelabs/sparkfun-tensorflow) +codelab, which includes more comprehensive instructions for the flashing +process. + +The binary should now be deployed to the device. Hit the button marked `RST` to +reboot the board. You should see the device's four LEDs flashing in sequence. + +Debug information is logged by the board while the program is running. To view +it, establish a serial connection to the board using a baud rate of `115200`. +On OSX and Linux, the following command should work: + +``` +screen ${DEVICENAME} 115200 +``` + +You will see a lot of output flying past! To stop the scrolling, hit `Ctrl+A`, +immediately followed by `Esc`. You can then use the arrow keys to explore the +output, which will contain the results of running inference on various `x` +values: + +``` +x_value: 1.1843798*2^2, y_value: -1.9542645*2^-1 +``` + +To stop viewing the debug output with `screen`, hit `Ctrl+A`, immediately +followed by the `K` key, then hit the `Y` key. + + +## Deploy to STM32F746 + +The following instructions will help you build and deploy the sample to the +[STM32F7 discovery kit](https://os.mbed.com/platforms/ST-Discovery-F746NG/) +using [ARM Mbed](https://github.com/ARMmbed/mbed-cli). + +![Animation of example running on STM32F746](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/hello_world/images/STM32F746.gif) + +Before we begin, you'll need the following: + +- STM32F7 discovery kit board +- Mini-USB cable +- ARM Mbed CLI ([installation instructions](https://os.mbed.com/docs/mbed-os/v5.12/tools/installation-and-setup.html)) +- Python 2.7 and pip + +Since Mbed requires a special folder structure for projects, we'll first run a +command to generate a subfolder containing the required source files in this +structure: + +``` +make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=mbed TAGS="CMSIS disco_f746ng" generate_hello_world_mbed_project +``` + +This will result in the creation of a new folder: + +``` +tensorflow/lite/experimental/micro/tools/make/gen/mbed_cortex-m4/prj/hello_world/mbed +``` + +This folder contains all of the example's dependencies structured in the correct +way for Mbed to be able to build it. + +Change into the directory and run the following commands, making sure you are +using Python 2.7.15. + +First, tell Mbed that the current directory is the root of an Mbed project: + +``` +mbed config root . +``` + +Next, tell Mbed to download the dependencies and prepare to build: + +``` +mbed deploy +``` + +By default, Mbed will build the project using C++98. However, TensorFlow Lite +requires C++11. Run the following Python snippet to modify the Mbed +configuration files so that it uses C++11: + +``` +python -c 'import fileinput, glob; +for filename in glob.glob("mbed-os/tools/profiles/*.json"): + for line in fileinput.input(filename, inplace=True): + print line.replace("\"-std=gnu++98\"","\"-std=c++11\", \"-fpermissive\"")' + +``` + +Finally, run the following command to compile: + +``` +mbed compile -m DISCO_F746NG -t GCC_ARM +``` + +This should result in a binary at the following path: + +``` +./BUILD/DISCO_F746NG/GCC_ARM/mbed.bin +``` + +To deploy, plug in your STM board and copy the file to it. On MacOS, you can do +this with the following command: + +``` +cp ./BUILD/DISCO_F746NG/GCC_ARM/mbed.bin /Volumes/DIS_F746NG/ +``` + +Copying the file will initiate the flashing process. Once this is complete, you +should see an animation on the device's screen. + + +``` +screen /dev/tty.usbmodem14403 9600 +``` + +In addition to this animation, debug information is logged by the board while +the program is running. To view it, establish a serial connection to the board +using a baud rate of `9600`. On OSX and Linux, the following command should +work, replacing `/dev/tty.devicename` with the name of your device as it appears +in `/dev`: + +``` +screen /dev/tty.devicename 9600 +``` + +You will see a lot of output flying past! To stop the scrolling, hit `Ctrl+A`, +immediately followed by `Esc`. You can then use the arrow keys to explore the +output, which will contain the results of running inference on various `x` +values: + +``` +x_value: 1.1843798*2^2, y_value: -1.9542645*2^-1 +``` + +To stop viewing the debug output with `screen`, hit `Ctrl+A`, immediately +followed by the `K` key, then hit the `Y` key. + diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/arduino/constants.cc b/tensorflow/lite/experimental/micro/examples/hello_world/arduino/constants.cc new file mode 100644 index 00000000000..a4aaf74bd75 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/arduino/constants.cc @@ -0,0 +1,19 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/constants.h" + +// This is tuned so that a full cycle takes ~4 seconds on an Arduino MKRZERO. +const int kInferencesPerCycle = 1000; diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/arduino/output_handler.cc b/tensorflow/lite/experimental/micro/examples/hello_world/arduino/output_handler.cc new file mode 100644 index 00000000000..3dbbf247348 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/arduino/output_handler.cc @@ -0,0 +1,47 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h" + +#include "Arduino.h" +#include "tensorflow/lite/experimental/micro/examples/hello_world/constants.h" + +// The pin of the Arduino's built-in LED +int led = LED_BUILTIN; + +// Track whether the function has run at least once +bool initialized = false; + +// Animates a dot across the screen to represent the current x and y values +void HandleOutput(tflite::ErrorReporter* error_reporter, float x_value, + float y_value) { + // Do this only once + if (!initialized) { + // Set the LED pin to output + pinMode(led, OUTPUT); + initialized = true; + } + + // Calculate the brightness of the LED such that y=-1 is fully off + // and y=1 is fully on. The LED's brightness can range from 0-255. + int brightness = (int)(127.5f * (y_value + 1)); + + // Set the brightness of the LED. If the specified pin does not support PWM, + // this will result in the LED being on when y > 127, off otherwise. + analogWrite(led, brightness); + + // Log the current brightness value for display in the Arduino plotter + error_reporter->Report("%d\n", brightness); +} diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/constants.cc b/tensorflow/lite/experimental/micro/examples/hello_world/constants.cc new file mode 100644 index 00000000000..7ac490c9c94 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/constants.cc @@ -0,0 +1,19 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/constants.h" + +// This is a small number so that it's easy to read the logs +const int kInferencesPerCycle = 20; diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/constants.h b/tensorflow/lite/experimental/micro/examples/hello_world/constants.h new file mode 100644 index 00000000000..61ca7df307d --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/constants.h @@ -0,0 +1,32 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_CONSTANTS_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_CONSTANTS_H_ + +// This constant represents the range of x values our model was trained on, +// which is from 0 to (2 * Pi). We approximate Pi to avoid requiring additional +// libraries. +const float kXrange = 2.f * 3.14159265359f; + +// This constant determines the number of inferences to perform across the range +// of x values defined above. Since each inference takes time, the higher this +// number, the more time it will take to run through the entire range. The value +// of this constant can be tuned so that one full cycle takes a desired amount +// of time. Since different devices take different amounts of time to perform +// inference, this value should be defined per-device. +extern const int kInferencesPerCycle; + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_CONSTANTS_H_ diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb b/tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb new file mode 100644 index 00000000000..372021dc08c --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb @@ -0,0 +1,1335 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "create_sine_model.ipynb", + "version": "0.3.2", + "provenance": [], + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python2", + "display_name": "Python 2" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "sblS7n3zWCWV", + "colab_type": "text" + }, + "source": [ + "**Copyright 2019 The TensorFlow Authors.**" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "0rvUzWmoWMH5", + "colab_type": "code", + "colab": {} + }, + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aCZBFzjClURz", + "colab_type": "text" + }, + "source": [ + "# Create and convert a TensorFlow model\n", + "This notebook is designed to demonstrate the process of creating a TensorFlow model and converting it to use with TensorFlow Lite. The model created in this notebook is used in the [hello_world](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/examples/hello_world) sample for [TensorFlow Lite for Microcontrollers](https://www.tensorflow.org/lite/microcontrollers/overview).\n", + "\n", + "\n", + " \n", + " \n", + "
\n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dh4AXGuHWeu1", + "colab_type": "text" + }, + "source": [ + "## Import dependencies\n", + "Our first task is to import the dependencies we need. Run the following cell to do so:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "53PBJBv1jEtJ", + "colab_type": "code", + "outputId": "9b035753-60e5-43db-a78d-284ea9de9513", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 479 + } + }, + "source": [ + "# TensorFlow is an open source machine learning library\n", + "# Note: The following line is temporary to use v2\n", + "!pip install tensorflow==2.0.0-beta0\n", + "import tensorflow as tf\n", + "# Numpy is a math library\n", + "import numpy as np\n", + "# Matplotlib is a graphing library\n", + "import matplotlib.pyplot as plt\n", + "# math is Python's math library\n", + "import math" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p-PuBEb6CMeo", + "colab_type": "text" + }, + "source": [ + "## Generate data\n", + "Deep learning networks learn to model patterns in underlying data. In this notebook, we're going to train a network to model data generated by a [sine](https://en.wikipedia.org/wiki/Sine) function. This will result in a model that can take a value, `x`, and predict its sine, `y`.\n", + "\n", + "In a real world application, if you needed the sine of `x`, you could just calculate it directly. However, by training a model to do this, we can demonstrate the basic principles of machine learning.\n", + "\n", + "In the [hello_world](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/examples/hello_world) sample for [TensorFlow Lite for Microcontrollers](https://www.tensorflow.org/lite/microcontrollers/overview), we'll use this model to control LEDs that light up in a sequence.\n", + "\n", + "The code in the following cell will generate a set of random `x` values, calculate their sine values, and display them on a graph:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "uKjg7QeMDsDx", + "colab_type": "code", + "outputId": "b17a43c6-eba1-4cc7-8807-14fcf5918d01", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 269 + } + }, + "source": [ + "# We'll generate this many sample datapoints\n", + "SAMPLES = 1000\n", + "\n", + "# Set a \"seed\" value, so we get the same random numbers each time we run this\n", + "# notebook\n", + "np.random.seed(1337)\n", + "\n", + "# Generate a uniformly distributed set of random numbers in the range from\n", + "# 0 to 2π, which covers a complete sine wave oscillation\n", + "x_values = np.random.uniform(low=0, high=2*math.pi, size=SAMPLES)\n", + "\n", + "# Shuffle the values to guarantee they're not in order\n", + "np.random.shuffle(x_values)\n", + "\n", + "# Calculate the corresponding sine values\n", + "y_values = np.sin(x_values)\n", + "\n", + "# Plot our data. The 'b.' argument tells the library to print blue dots.\n", + "plt.plot(x_values, y_values, 'b.')\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzt3X2UVPWd5/H3F1pU1ASRjhLhgDNy\nJpJJgrOVZioa4yQGNJsjzE7iqvRKcpwpH+Im2TkrrZNzNg8ziTSZGcnOEUNHozCgxjUjYtZZMEYH\nZyyBZgYThSgswRFWpBWZaFSQ5rt/3NtD3apb/VQPt27V53VOna77rVvd3/ahvv17NndHRERkwJik\nExARkcaiwiAiIhEqDCIiEqHCICIiESoMIiISocIgIiIRKgwiIhKhwiAiIhEqDCIiEtGWdAKjMWnS\nJJ8+fXrSaYiIpMqWLVtedff2oe5LZWGYPn06vb29SachIpIqZvbicO5TV5KIiESoMIiISIQKg4iI\nRKgwiIhIhAqDiIhEVKUwmNkPzWy/mT1b5nUzs/9pZjvN7Odm9nsFry00sx3hY2E18hERkdGrVovh\nbuDiQV6/BJgRPnLA7QBmNhH4OjAb6AC+bmanViknGYXZs6GtDU45BcaPB7PgMXYsnHsu5PNJZygi\ntVaVwuDuG4ADg9wyD1jpgaeBCWY2GZgLPOruB9z9deBRBi8wUkX5PHziE0EROP74oABs2gT9/fDm\nm/D228fuPXoUtm6Fj33sWKE45RTo6koufxGpjXqNMZwJvFRwvSeMlYuXMLOcmfWaWW9fX1/NEm0V\nM2cGH/IbNgRF4PDhkb3/6NHgfUuWwJgxMGkS9PTUJlcRqa/UDD67e4+7Z9w9094+5IpuidHVBe97\nHxx3HGzfXr3v6w6vvQbXXBO0PDo7q/e9RaT+6lUY9gJTC66nhLFycamifB6mTg3+uu/rgyNHhn6P\nGZx44sh/1uHDsHp1ME6hbiaRdKpXYVgLXBXOTvp94N/c/WVgHTDHzE4NB53nhDGpgp4e+OAHgy6j\nPXsGv7etLegSMoOOjqCr6K23gtbAwGPOnGBsAYL7BtPfHxQijUOIpE+1pqveC+SB3zGzPWZ2tZld\na2bXhrc8AuwCdgI/AK4HcPcDwJ8Dm8PHt8KYVKirK+ja2bZt8PsmTYKnnoJ33w0+zI8ehY0b4+9d\nty5obbgH9y1fDhMnDl4kBsYh1L0kkh7m7knnMGKZTMa1u2p5c+fC+vXlXx8/Pvjrf9EiyGar8zN7\neuDLX4ZDh8rf094ODz1UvZ8pIiNjZlvcPTPUfakZfJah5fPBh365ovC+9wXF4De/gQcfrO4HdC4H\n77wDCxbAuHHx9/T1Bd1amr0k0thUGJpET0/woVu49qDQokXwyivQ3V3bPFatCloNy5eXv+faa1Uc\nRBpZKg/qkajp0+HFMsdvTJwIt9wS/EVfTwM/75prSl9zh+uui94nIo1DLYaUO+20wYvCa68l9+Gb\nywUD2xMmlL529GhQNDQoLdJ4VBhSbPZsOFBmDte0aUFRSFo2C6+/HnRlxVm9WsVBpNGoMKTU3LnB\nvkZxFi2C3bvrms6QuruD1sNJJ5W+ds892pxPpJGoMKTQ7NnxM49OPDH48K31APNoZbPwpS+Vxt3h\nggvUchBpFBp8TplyA80dHeUXpjWSgaK1bFmw+G3AkSNBt9KOHen4PUSamVoMKVKuKMyZk64P0+5u\neOONoJgV27QpaBGJSHJUGFKis7N8S2FdSneXuvrq+PimTcG24CKSDBWGFOjsDLpZik2blq6WQrFc\nLlgIN7AxX6Ht2zXmIJIUFYYGN3dufFGYMKHxZh6NRi4XjDfEWb1aK6RFkqDC0MDy+fjZR2PHwiOP\n1D+fWsnlyq9zuOYaTWUVqTcVhgZ22WWlsZNPhiefbL4dSru7y++v9JnP1DcXkVanwtCgpk+PP1zn\nr/6q+YrCgFwumGFV7OBBmDy5/vmItKpqHdRzsZk9b2Y7zeymmNdvNbOt4eMFMztY8Fp/wWtrq5FP\n2s2dGz8DacGC5t90bt26+Gms+/ZpGqtIvVRcGMxsLHAbcAkwE7jCzCKTDd39v7n7LHefBfwN8HcF\nL7898Jq7X1ppPmlXblyhoyPY0roVbNwYzLgqtmmTjgkVqYdqtBg6gJ3uvsvdDwP3AfMGuf8K4N4q\n/NymtGRJaSzt01JHY/fu+H2V7rqr7qmItJxqFIYzgZcKrveEsRJmNg04C/hZQfgEM+s1s6fNbH4V\n8kmtmTNhzZpobMaM5piWOhqPPloae+01zVISqbV6Dz5fDjzg7v0FsWnhGaRXAkvN7Lfj3mhmubCA\n9Pb19dUj17qaOTNY1FVo7FhYsSKZfBpBNls6U+no0fhWlYhUTzUKw15gasH1lDAW53KKupHcfW/4\ndRfwBHBu3BvdvcfdM+6eaW9vrzTnhtLVVVoUIFj41awzkIZrYHX0mIL/Utes0ViDSC1VozBsBmaY\n2VlmNo7gw79kdpGZfQA4FcgXxE41s+PD55OA84BtVcgpNXp64v8CPuec5p+BNFy5HGQy0diSJSoO\nIrVScWFw9yPADcA6YDtwv7s/Z2bfMrPCWUaXA/e5uxfEzgF6zewZ4HFgsbu3VGG4+ebS2EknwbaW\n+qcwtLgN91QcRGrDop/T6ZDJZLy3tzfpNCrW0xNs+VBs+XK1FuJ0dcW3rp56Sl1uIsNhZlvCMd1B\naeVzgm65pTQ2Z46KQjnd3fF7Ki1cWP9cRJqZCkNCOjtLp6GefXZ6z1aol+7uYLuQQjt2qEtJpJpU\nGBLQ01O6lbYZrFyZTD5pEzcu873vaYtukWpRYUjAl79cGrvxRvWTD1fcZnuHDgXjNSoOIpVTYaiz\nuXODD7FCY8YEXSQyfOvWwQUXlMbjxm1EZGRUGOqoqyt+g7yLLqp/Ls1g8eKgC67Q7t1qNYhUSoWh\nTvJ5+O53S+MTJmjAebSyWZgXs12jWg0ilVFhqJOVK6F4yYhZcx3RmYRFi+JbDZqlJDJ6KgwJmTYN\n/umfNOBcqWwWvv/90viSJdqFVWS0VBjqoKsLfvKTYJDZDMaNg3vvVVGollwu6JIrdv319c9FpBmo\nMNTYwDYOe/YEW0Z//OPwxBMqCtUWt1p869ZgFpiIjIwKQ43dfXf0etcuFYVa6O4OzsQutn69ZimJ\njJQKQw11dcH+/dHYb/1WMrm0glWrYPLk0rhmKYmMjApDjcSds2AWzL2X2vnGN0pjL71UGhOR8lQY\namTp0tLY97+vbqRay+WCzQgL9fdrrEFkJFQYauTFF6PX06drO+16iduM8Gc/q38eImlVlcJgZheb\n2fNmttPMbop5/Qtm1mdmW8PHHxe8ttDMdoSPpthZf+ZMeOutaCxuR1CpjWwWOjqisSNHYPbsZPIR\nSZuKC4OZjQVuAy4BZgJXmNnMmFt/5O6zwscd4XsnAl8HZgMdwNfN7NRKc0pSTw9s3x6NHXecWgv1\ntnEjjB8fjW3aFJyDISKDq0aLoQPY6e673P0wcB8Qs4NNrLnAo+5+wN1fBx4FLq5CTomJaxn8wR/U\nPw+BG24oja1erRXRIkOpRmE4Eyic97EnjBX7IzP7uZk9YGZTR/jeVOjshAMHorHx47VJXlK6u+E9\n7ymN60AkkcHVa/D5YWC6u3+YoFWwYqTfwMxyZtZrZr19fX1VT7BS+XzpqWwAt95a/1zkmLgdbfft\nq38eImlSjcKwF5hacD0ljP07d3/N3QeOp7kD+A/DfW/B9+hx94y7Z9rb26uQdnXdVDLkDjNmaGwh\nablc6Q6sDz+s1dAig6lGYdgMzDCzs8xsHHA5sLbwBjMrXI96KTAwPLsOmGNmp4aDznPCWKr09MCG\nDaXxFSNuF0ktdHcHx34O6O+Ha6/VWINIORUXBnc/AtxA8IG+Hbjf3Z8zs2+Z2aXhbV82s+fM7Bng\ny8AXwvceAP6coLhsBr4VxlLlO98pjS1frsVsjeSqq6KtBne47rrk8hFpZObFp8ekQCaT8d7e3qTT\nAI7tnlpo0SKd4dyITjopur7khBPg7beTy0ek3sxsi7tnhrpPK58rVNxXPWGCikKj+sM/jF6/845O\nehOJo8JQga4uOHgwGvvwh5PJRYa2ahWccUY09pd/qbEGkWIqDKOUz5d2IYF2T2103/xm9Pro0fgZ\nZSKtTIVhlOIWSV1wgQacG93A9NVCGzaoS0mkkArDKP30p9FrnbWQHt3dpWdEa12DyDEqDKPQ1QU7\nd0Zj8+aptZAmxYXh4EEVB5EBKgyjEPcBUtw9IY0tbrPD667TQLQIqDCMWNxMpFmz1FpIm1wuODyp\n0NGjcP31iaQj0lBUGEYgn4/flG3ZsvrnIpWLazVs3aozG0RUGEbgiSeCrRQKTZ+u1kJa5XLB1iVj\niv4vePjhZPIRaRQqDCNw4YXBaWyFdGRnuuVypYsSTzklmVxEGoUKwzB1dsJnPxucxjZ/fnCm8PLl\n2la7GSxbFt1gb+9edSdJa1NhGIbOzuAQngMHYP36YDO2jRtVFJpFNgsf/Wg0tnq1pq9K61JhGIZ7\n741er1mTTB5SO1dfXRq788765yHSCFQYhjB3bjCNsZD6oJtPLgdz5kRjmzZpXYO0JhWGIRRvfQGl\nG7FJc7jwwtLYwoV1T0MkcVUpDGZ2sZk9b2Y7zaxkr0oz+1Mz22ZmPzezx8xsWsFr/Wa2NXysLX5v\nkrq6SlsL48drbKFZXXhhdBAaYMcOjTVI66m4MJjZWOA24BJgJnCFmc0suu1fgIy7fxh4ACjcsPpt\nd58VPi6lgdx1V2ns1lvrn4fURzYLV15ZGteUZGk11WgxdAA73X2Xux8G7gPmFd7g7o+7+8Chik8D\nU6rwc2sqn4e+vmjs7LPVWmh2q1bBaadFYwcOaFtuaS3VKAxnAi8VXO8JY+VcDfx9wfUJZtZrZk+b\n2fxybzKzXHhfb1/xJ3YNXHZZaSzuDAZpPt/5Tmnsnnvqn4dIUuo6+GxmnUAGKNxxaFp4OPWVwFIz\n++2497p7j7tn3D3T3t5e0zy7umDPnmisvV1bX7SKuBlKL7+sGUrSOqpRGPYCUwuup4SxCDO7CPga\ncKm7HxqIu/ve8Osu4Ang3CrkVJG4vw6/+MX65yHJWbcuOJFvQH9//FGuIs2oGoVhMzDDzM4ys3HA\n5UBkdpGZnQssJygK+wvip5rZ8eHzScB5wLYq5DRqPT2lrYWOjuDUL2ktixdDW9ux6zVrNENJWkPF\nhcHdjwA3AOuA7cD97v6cmX3LzAZmGX0XOBn4X0XTUs8Bes3sGeBxYLG7J1YY8nm49tpo7Mwzg+0v\npPVkszBpUjQWN/4g0mzahr5laO7+CPBIUex/FDy/qMz7ngI+VI0cqmHlytJttaW1Fa9j2bcvmTxE\n6kkrnwv8+MelsQUL6p+HNI4vfCF6fehQsE2KSDNTYQjNnl26bmHBAo0ttLrubjj++Ghs/XrNUJLm\npsIQ2rw5em0WLHYS+dznSmM3lWz8ItI8VBgI/vorHlvQDqoyYNUqmDgxGnvySbUapHmpMBA/P/27\n3y2NSeu65ZbotbtWwkvzavnCkM/D2qI9XS+4QHsiSVQuB4sWRXdf/cEP1GqQ5tTyhWHlyuiUxDFj\ngoVNIsW6u+HjHz923d8P11+fXD4itdLyheHpp6PXl16qPZGkvHfeiV5v3arV0NJ8WrowTJ8e/I89\nYMyYoLtApJy4s6G//vX65yFSSy1bGObOhRdfjMbe/361FmRwuRzMmhWN7dun8xqkubRsYXj88dJY\n3OldIsWWLSuNqTtJmklLFoZ8Ht59NxqbMEGrnGV4stnSVsPBgyoO0jxasjDErVp95JHSmEg5ca2G\npUvrn4dILbRcYejqgg0bjl2bwfLlGluQkclmSycqbN+uVoM0B/MU7jOdyWS8t7d3VO89/XTYv//Y\n9fveB6+8UqXEpOVMnhzdinvmTHjuueTyERmMmW0Jj1IeVFVaDGZ2sZk9b2Y7zayko8bMjjezH4Wv\nbzSz6QWv3RzGnzezmm5onM9HiwLABz5Qy58oze7UU6PXOq9BmkHFhcHMxgK3AZcAM4ErzGxm0W1X\nA6+7+9nArUB3+N6ZBEeBfhC4GFgWfr+aiBtb0CpnqcRXvxq9PnAAOjuTyUWkWqrRYugAdrr7Lnc/\nDNwHzCu6Zx6wInz+APApM7Mwfp+7H3L3XwE7w+9Xdfl8sCNmoXPO0diCVCaXC7ojC61erT2UpPry\n+WAzx3r8t1WNwnAm8FLB9Z4wFntPeEb0vwGnDfO9VRF3bGfxX3sio1F8yhvAddfVPQ1pYvk8XHgh\nfO1rwddaF4fUzEoys5yZ9ZpZb1/xUWujoB1UpVq6u6Gt6PT0Z55Rq0GqZ8kSOHw4+OP28OHab/le\njcKwF5hacD0ljMXeY2ZtwHuB14b5XgDcvcfdM+6eaW9vH3GSV10F48YF01PHjdPYglTXySeXxuLO\n+RAZqXwe1qwpjdVSNQrDZmCGmZ1lZuMIBpOLTjhgLbAwfP454GcezJNdC1wezlo6C5gBbKpCTiWy\nWXjiCfj2t4OvGluQaoprfa5dq1aDVC7uD4xXX63tz2wb+pbBufsRM7sBWAeMBX7o7s+Z2beAXndf\nC9wJ/K2Z7QQOEBQPwvvuB7YBR4AvuXt/pTmVk82qIEhtdHfD+vXR3XqPHg2a/PpvTipR+N/UgAUL\navszW26Bm0it5PNw/vnRg5/mz4cHH0wuJ0m3rq7SFsOMGfDCC6P7fnVd4CYiQcvg9tuDcz0GrFmj\nLblldHp6SouCGaxYEX9/NakwiFRRLhe0GgotWaI9lGTkvve90ti8efXpmlRhEKmy4uM/Ae68s/55\nSHrl87BtW2m8XidMqjCIVFnc8Z8nnFD/PCS94mYizZ9fv4kMKgwiVZbLBQsoC736qqauyvDErVsw\nq+959CoMIjWweHF0NfS2bcHYg4qDDCWutVCvsYUBKgwiNZDNwh//cTR29Kj2UJKhPf109LrerQVQ\nYRCpmauuKo398pf1z0PSo6ur9EyPG2+s/yJJFQaRGslmg8VIhQ4d0tRVKe/226PXJ58crKqvNxUG\nkRqKW4x03XUaa5BSXV3wxhvR2KRJyeSiwiBSQ9lsMM2w0NGj2nlVovL5+P8mbr65/rmACoNIzS1a\nFN0mA7TzqkTFHTs8a1ZyZ8aoMIjU2MAeSmbHYmo1SKHNm0tjy5bVP48BKgwidZDLBXPRC61Zo1aD\nBGMLb78djc2alex27SoMInUSNxf9+uvrn4c0lrhZakm2FkCFQaRuslk47rho7LnnkslFGkM+DwcP\nRmNnnJH84U4VFQYzm2hmj5rZjvDrqTH3zDKzvJk9Z2Y/N7P/XPDa3Wb2KzPbGj5mVZKPSKM77bTo\n9bvv6ryGVrZyZWnsm9+sfx7FKm0x3AQ85u4zgMfC62JvAVe5+weBi4GlZjah4PUb3X1W+Ig5xE6k\necT9T6/zGlpTPg8/+MGx64GtL5KaiVSo0sIwDxhYwrMCmF98g7u/4O47wuf/D9gPtFf4c0VSKW7n\nVYAf/7j+uUiybroJ+gtOuP/4x5NZ5Ryn0sJwuru/HD7fB5w+2M1m1gGMA/5vQfjbYRfTrWZ2fIX5\niDS8xYth7NhorF1/KrWUfB6efDIaizvgKSlDFgYz+6mZPRvziEy+c3cHfJDvMxn4W+CL7j5wXPrN\nwAeAjwITgbK9rWaWM7NeM+vt6+sb+jcTaVDZLPzJn0Rj99+vqaut5KabwIs+LeMOeErKkIXB3S9y\n99+NeTwEvBJ+4A988O+P+x5m9h7gfwNfc/enC773yx44BNwFdAySR4+7Z9w9064/ryTlrroqel7D\nkSPxA5HSfOJaC9OmNcbYwoBKu5LWAgvD5wuBh4pvMLNxwIPASnd/oOi1gaJiBOMTz1aYj0gqZLNw\n223HupTcg4FIDUI3vyeeKI392Z/VPY1BVVoYFgOfNrMdwEXhNWaWMbM7wnsuAy4AvhAzLXW1mf0C\n+AUwCfiLCvMRSY1cLuhSGtgqo78frr1WXUrN7sILgzPAzYI9tBplJlIh8+KOrhTIZDLe29ubdBoi\nFcvn4bzzov3NF1wA//APyeUktdPTE8xAmzULJkwIikQ9F7OZ2RZ3zwx1X9tQN4hI7WSz8N73Rle/\n6pS35tTVdWzjxPXrYfny5Fc4l6MtMUQS9uEPR69PPFHdSc0m7ryF730vmVyGQ4VBJGHF6xpefDFY\n7KTi0Dzizlto5F58FQaRhGWzwfTFadOOxfr7tfNqM9m1qzT21a/WP4/hUmEQaQDZbHR7BICtW9Vq\naAb5fOnZzXPmNN5MpEIqDCIN4sorS2Of/3z985Dqyefh/PODIg/BFNUFC2DdumTzGooKg0iD6O4u\n3UNp714tekuzhQuDY1wHuMMHP5hcPsOlwiDSQD71qdLY9derSymNenpgx45ozCxYu9DoVBhEGsi6\nddBRtGNYf7/2UUqjO+8sjV15ZeOuXSikwiDSYDZuDFbGFtq2LZlcZHTyedi0KRo75xxYtSqZfEZK\nhUGkAY0bF71upL36ZWjFi9kApk6tfx6jpcIg0oCK9+Z/4w0NQqfJCy+Uxv7oj+qfx2ipMIg0oFwu\n2Etn5szgevt2uOYa6OxMNi8ZWmdnadffggWNvW6hmAqDSIPK5eDkk6Ox1avVcmhknZ3Bv6NC8+en\nZ2xhgAqDSAN7//tLY428+Vory+dLiwIE5y2kjQqDSAOL+1DZtk3rGhpR3JTiWbPSMT21WEWFwcwm\nmtmjZrYj/Hpqmfv6C05vW1sQP8vMNprZTjP7UXgMqIiEstlgrKHYZZfVPxcZ3NNPl8aWLat/HtVQ\naYvhJuAxd58BPBZex3nb3WeFj0sL4t3Are5+NvA6cHX820VaVy4H7e3R2J49wcEv0hi6uo7thzRg\n/vx0thag8sIwD1gRPl8BzB/uG83MgE8CD4zm/SKt5ItfLI0tXVr/PKRU3CE8ZukcWxhQaWE43d1f\nDp/vA04vc98JZtZrZk+b2cCH/2nAQXc/El7vAc4s94PMLBd+j96+vr4K0xZJl+7u0kVvhw+r1dAI\nFi4sjd14Y3pbCzCMwmBmPzWzZ2Me8wrvc3cHyp1JNC08gPpKYKmZ/fZIE3X3HnfPuHumvbhdLdIC\n4g52ufvuuqchBbq6SjfKmzAhKORpNmRhcPeL3P13Yx4PAa+Y2WSA8Ov+Mt9jb/h1F/AEcC7wGjDB\nzNrC26YAeyv+jUSaVHd36QZ7+/drXUNS4rqQIF0L2cqptCtpLTDQkFoIPFR8g5mdambHh88nAecB\n28IWxuPA5wZ7v4gcs3EjnHFGNHbLLcnk0uriZoadfXb6WwtQeWFYDHzazHYAF4XXmFnGzO4I7zkH\n6DWzZwgKwWJ3H1gw3gX8qZntJBhziNmoVkQK/f7vR69371arod7y+WBmWLFm2R7dgj/c0yWTyXhv\nb2/SaYgkIp+H884LTgMb0NERtCakPqZMCU7XK4699FIy+QyXmW0Jx3sHpZXPIimTzQazXgpt3qwZ\nSvWSz5cWBYD7769/LrWiwiCSQt3dwQKqAe7BQKi6lGrvpphlvB0d6Z6eWkyFQSSlFi2CMUX/B8cd\nJynV09UFGzZEY83YjafCIJJS2Sycf3409qtfqdVQS8X/bCdMaL6iACoMIqm2eDG0tR277usLDvRR\ncai+nh44eDAamzAhmVxqTYVBJMWy2aBrY8qUaPwb30gknaaVz8P115fGb765/rnUgwqDSMpls5Ap\nmoD48sswfXoi6TSlJUugvz8aW7SoOVY5x1FhEGkCcTt5vvgizJ1b/1yaTU8PrFkTjc2f3xwrnMtR\nYRBpAtlscOB8scceq38uzSSfh2uvjcbSvqX2cKgwiDSJVatKB0P7+4MD6mV0Vq6MrjAHOOec5lqz\nEEeFQaSJPPJIaWz1ap0RXU1f+UrSGdSeCoNIE8lmgwPoizXL5m711NkJ99xzbBHhmDHNPeBcSIVB\npMnEHUD/k59obcNITJ4ctLR+/Ws4ejQotv/4j8094FxIhUGkyWSzsHw5jB17LLZnjxa+Ddfs2bBv\nXzT2r//a/OMKhVQYRJpQLgdPPqmFbyPV1QWbNpXGL7mk/rkkSYVBpEmVW/imtQ3xenrij+o8+eRg\nxlcrqagwmNlEM3vUzHaEX0+NuecPzGxrweMdM5sfvna3mf2q4LWYYTMRGa24+fbr1+vshjjFZ1wM\nWL++vnk0gkpbDDcBj7n7DOCx8DrC3R9391nuPgv4JPAWUPiP+saB1919a4X5iEiBcrOUlizRFNZC\nnZ3BQHOx5ctba2xhQKWFYR6wIny+Apg/yL0AnwP+3t3fqvDnisgwxc1SgvjD7FtRPh/MQCq2YEFr\nTE2NU2lhON3dXw6f7wNOH+L+y4F7i2LfNrOfm9mtZnZ8uTeaWc7Mes2st6+vr4KURVpLNhvfpbRn\nj8YbIH5coaOj9cYVCg1ZGMzsp2b2bMxjXuF97u6Al/k2mNlk4EPAuoLwzcAHgI8CE4GyPZ/u3uPu\nGXfPtLe3D5W2iBTo7oY5c0rj69e39hTWnh546KFo7CMfac7Dd0aibagb3P2icq+Z2StmNtndXw4/\n+PcP8q0uAx5093cLvvdAa+OQmd0F/Pdh5i0iI7RuXTBHv3g65pe+BB/6UOv1pefzcN110b2QxoyB\n229PLqdGUWlX0lpgYfh8IfDQIPdeQVE3UlhMMDMjGJ94tsJ8RGQQGzfC+PHR2JEj8LGPtdZgdE9P\nsHX20aPR+KWXtl6BjFNpYVgMfNrMdgAXhdeYWcbM7hi4ycymA1OBfyh6/2oz+wXwC2AS8BcV5iMi\nQ7jhhvh43AllzainJ1gFvr+of2NgLyQB8+I9ZVMgk8l4b29v0mmIpNbMmbB9ezR20knw5pvJ5FNP\nkyeXbnlhBt//fvPPQjKzLe6eGeo+rXwWaUHbtsG0adHYb34TfGg282B0XFGA1igKI6HCINKidu8u\nXfy2b1/zbrZ32mnxRaGV1yuUo8Ig0sKWLQu6UYpde21zbZvR1QUHDpTGP/KR1l6vUI4Kg0gLy2bh\nyitL4+7Bwq9mKA7lNscDTU0tR4VBpMWtWhW/+A3grrvqm0u1DcxAKnbiifDUU5qaWo4Kg4iwbl2w\nDUSxvr7gTIc0rnEoVxQmTIDPqs3qAAAHPElEQVS33lJRGIwKg4gAweK3BQuOnXE8YO/e9C2AK1cU\nQAPNw6HCICL/btWq8v3uF16YjtlKnZ3li0JHR+uc21wJFQYRicjl4ruVDh8OPnBnz65/TsM1d278\nFtoQ/E6tvjnecKkwiEiJjRtLF8AN2LSp8YpDPg/nnlv+tLUFC1QURkKFQURi7d4d33KAoDi0tzfG\nuENPTzAGsrXM+Y/Ll2utwkipMIhIWRs3Bh+sx8ccofXqq8EHcpJrHQYbZD7ttGBKqgabR06FQUQG\nlcvB44+Xf33JkqBw1LtAzJ1bviiMHQsPP6wpqaOlwiAiQ8pmg5ZDOYcPBwWis7O2eeTzQReWWfnx\nhEmT4MknVRQqocIgIsOSywVdMyefXP6e1avhlFOq33oYGFz+2MeCLqxyOjqCRXkqCpWpqDCY2efN\n7DkzO2pmZff4NrOLzex5M9tpZjcVxM8ys41h/EdmNq6SfESktrJZeOONYJbP2LHx97z5ZtB6GDsW\nPvGJygaoe3rgve8dfHAZ4Oyzg6KlmUfVUWmL4VngPwEbyt1gZmOB24BLgJnAFWY2M3y5G7jV3c8G\nXgeurjAfEamDVauCI0HLzVqC4NjMDRuCD/XjjgsekyYNvkiuszMYNJ4+HdragjGEX/+6/P1jxgRF\nascOtRKqqaLC4O7b3f35IW7rAHa6+y53PwzcB8wLz3n+JPBAeN8KgnOfRSQlBmYtnXHG4PcdORI8\nXnst+LA3O/Y48USYOjV4vnp1sD32iy9Cf//g33PixOAeTUWtvnqMMZwJvFRwvSeMnQYcdPcjRXER\nSZFcDl5+OTgvua1t5O9/5x3Ys2f497e1BT/rtddG/rNkeIYsDGb2UzN7NuYxrx4JFuSRM7NeM+vt\n6+ur548WkWHo7oZ33w228I47/KdSbW1Bt9G772q/o1obsr67+0UV/oy9wNSC6ylh7DVggpm1ha2G\ngXi5PHqAHoBMJuMV5iQiNbJuXfC1qwuWLg0+yMeMGbprqNjYscHj859Xd1G91aMraTMwI5yBNA64\nHFjr7g48DnwuvG8h8FAd8hGROujuhkOHgkHoI0eCsYiJE0tbEyecEJz50NYG48fDzJnBvUeOBO9X\nUag/Cz6fR/lmsz8E/gZoBw4CW919rpm9H7jD3T8T3vcZYCkwFvihu387jP8WwWD0ROBfgE53PzTU\nz81kMt7b2zvqvEVEWpGZbXH3sksL/v2+SgpDUlQYRERGbriFQSufRUQkQoVBREQiVBhERCRChUFE\nRCJUGEREJCKVs5LMrA94cZRvnwQMsnFvw0t7/pD+3yHt+UP6f4e05w/J/A7T3L19qJtSWRgqYWa9\nw5mu1ajSnj+k/3dIe/6Q/t8h7flDY/8O6koSEZEIFQYREYloxcIwyDEhqZD2/CH9v0Pa84f0/w5p\nzx8a+HdouTEGEREZXCu2GEREZBAtUxjM7GIze97MdprZTUnnM1Jm9kMz229mzyady2iY2VQze9zM\ntpnZc2b2laRzGikzO8HMNpnZM+Hv8M2kcxoNMxtrZv9iZj9JOpfRMLPdZvYLM9tqZqnbTdPMJpjZ\nA2b2SzPbbmYNd1p1S3QlmdlY4AXg0wRHiG4GrnD3bYkmNgJmdgHwJrDS3X836XxGyswmA5Pd/Z/N\n7BRgCzA/Zf8ODDjJ3d80s+OAfwS+4u5PJ5zaiJjZnwIZ4D3u/tmk8xkpM9sNZNw9lesYzGwF8KS7\n3xGeUTPe3Q8mnVehVmkxdAA73X2Xux8mOAOirkeTVsrdNwAHks5jtNz9ZXf/5/D5G8B2UnbGtwfe\nDC+PCx+p+svKzKYA/xG4I+lcWpGZvRe4ALgTwN0PN1pRgNYpDGcCLxVc7yFlH0rNxMymA+cCG5PN\nZOTCbpitwH7gUXdP2++wFFgEHE06kQo4sN7MtphZLulkRugsoA+4K+zOu8PMTko6qWKtUhikQZjZ\nycCPga+6+6+Tzmek3L3f3WcRnFHeYWap6dYzs88C+919S9K5VOh8d/894BLgS2E3a1q0Ab8H3O7u\n5wK/ARpuzLNVCsNeYGrB9ZQwJnUU9sv/GFjt7n+XdD6VCJv/jwMXJ53LCJwHXBr20d8HfNLMUnei\nsrvvDb/uBx4k6CpOiz3AnoKW5gMEhaKhtEph2AzMMLOzwsGey4G1CefUUsKB2zuB7e7+10nnMxpm\n1m5mE8LnJxJMZvhlslkNn7vf7O5T3H06wf8DP3P3zoTTGhEzOymcvEDYBTMHSM1MPXffB7xkZr8T\nhj4FNNwEjLakE6gHdz9iZjcA64CxwA/d/bmE0xoRM7sXuBCYZGZ7gK+7+53JZjUi5wH/BfhF2EcP\n8Gfu/kiCOY3UZGBFOMttDHC/u6dyymeKnQ48GPydQRtwj7v/n2RTGrH/CqwO/0jdBXwx4XxKtMR0\nVRERGb5W6UoSEZFhUmEQEZEIFQYREYlQYRARkQgVBhERiVBhEBGRCBUGERGJUGEQEZGI/w/w1xWP\nb+vxVQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iWOlC7W_FYvA", + "colab_type": "text" + }, + "source": [ + "## Add some noise\n", + "Since it was generated directly by the sine function, our data fits a nice, smooth curve.\n", + "\n", + "However, machine learning models are good at extracting underlying meaning from messy, real world data. To demonstrate this, we can add some noise to our data to approximate something more life-like.\n", + "\n", + "In the following cell, we'll add some random noise to each value, then draw a new graph:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "i0FJe3Y-Gkac", + "colab_type": "code", + "outputId": "60b19cdd-c69c-469e-9446-b738a79c1f51", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 269 + } + }, + "source": [ + "# Add a small random number to each y value\n", + "y_values += 0.1 * np.random.randn(*y_values.shape)\n", + "\n", + "# Plot our data\n", + "plt.plot(x_values, y_values, 'b.')\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJztnX+YVOV597/3mdkdeNNa0tGWKFIS\njUlsuMIKEqc2uqlEg41K3bfVxPddC8j6A4jEq1Jtk5S3MZIQo5ugIqvAyzaaNC0JQoJFMW6kYRoE\nwdKgxh9NEH9Usr7UpGGX3Znn/ePeu89zzpyzO7MzuzNz5v5c116zM/OcmTP74/vc5/5JxhgoiqIo\njYVX7RNQFEVRxh8Vf0VRlAZExV9RFKUBUfFXFEVpQFT8FUVRGhAVf0VRlAZExV9RFKUBUfFXFEVp\nQFT8FUVRGpBktU8gihNPPNFMmzat2qehKIpSV+zdu/cXxpiTRlpXs+I/bdo07Nmzp9qnoSiKUlcQ\n0c+LWaduH0VRlAZExV9RFKUBUfFXFEVpQFT8FUVRGhAVf0VRlAZExV9RFKUBUfFvQLJZYOVKvlUU\npTGp2Tx/ZWzo6gKWLAFyOSCVAh5/HMhkijs2mwV6eoDW1uKPURSlNlHxbyCyWWDxYmBwkO/397OY\nFyPk2SxwwQXA8eNAc3Npm4aiKLWHun0aiJ4eIJ+394nYig8S5hbq6WHhz+X4tqdnbM9VUZSxRS3/\nmCIumnQa6O1lkW9tBZJJFm8AMAY4cMBvwUe5hVpb2eIXyz9s01AUpX5Q8Y8h4qLp72dL3/OskC9Y\nAKxdy8KfzwM33MDH9PbyRhHlFspk+Hj1+StKPFDxrzOKCbqKi0ZcPPk83+/u5vtELP4AW/iyAXge\n3xcSCb+FL5uAoij1j4p/HVFM0DWbBQ4dYveOWPeex0K+YQNb9SL8ggi+MbxOjrn7bhV7RYkrKv51\nRFjQ1RVnd3NIJICODqClhV06hw4B99/vt+xdxDXU2WljBCr8ihJfVPxriJFcOlFBVznu0CG7OQDA\n1KnA9On8XEuLPTaR4CsAcQsRAbNmASefDDz4IHDkCLB7NzB3rm4EihJXyAR9ADXCrFmzTCMNcyk2\njz6YxZNOA8uW8XHi6snl+DU6O+1ziQRw8cX8GpMnAyecANx5J28ATU18OzAQfm6pFPDEE7oBKEo9\nQER7jTGzRlqnln+NMJJLR5DHZKOQIK1Y8YsWscWfTgObNgF9fXZD2LyZ1xDxRnHTTcCkSWzly3Nh\n9PcDq1YBs2fbqwCt9lWU+kbFv0YoJo8+zL1jDG8ARHxcezuvveACK/xBjGEr/847gXvuAb7//ZHP\nb+tW/mpuBpYutVcNbi2AbgiKUj+o+NcIYXn0rpgC/mBucug3J+4d1ze/ciWvG8mjl88D69ZFu3sA\n3lSIbByhvx+44w57pSG1AO75FeO20g1CUapLRcSfiNYD+ASAN40xHwx5ngB8DcDFAH4N4M+NMU9X\n4r3jhJtHH4wBXH21P5jrundc4ZdUz0TCpnqKgBvj3xCSSWDv3ujz8TwOBO/b528L4b4GEb9fd7ff\nbdXdXSjy2h9IUWqHSln+/xfA3QC6I56fC+C9Q18fBrBm6FYJIZsFVqywFbrSjsF1C7nuHXnMDfAm\nk8Cll7JLZ3CQNwOx4JNJYP584I03wn39slmkUsBZZwFu3F0CxLkcr/E8TiH1PP8GITUFrshHxTX0\nakBRxp+KiL8x5kkimjbMkssAdBtOLfoXIppERO8yxrxeifePE2GtGUTs29v9IrlypV3X388BXvfq\n4Ne/5ufkCuCSS/ixtjauAbj++vBzkDjC0qXA22+zyLvk83w8YGsH3PoBCUAbw+e1YgV/tbbaIjKp\nHtarAUWpDuPV1fMUAK849w8PPeaDiDqIaA8R7Tly5Mg4nVpt4bZm8Dxgzhy26MWvfuutVhyPHvW3\ncJgxgwU0keDbtjZ7P5kEtm1jcV22jEW3vZ2t+zDyeQ7qilXvbgC5HPD001w7kBzBfMjngUcfZYE/\ncMC+Ti7H97VbqKJUh5oK+BpjugB0AZznX+XTqQrBrJ+2Nr8rZ/58Fu0DBzjwKngep20+/rjt4TN9\nug0iuxW+fX02dfPGG60A/+u/2s3E82yDN2OAyy7jbB+x8HfvBp55hgvBhksTFfr6+MpkYMCmni5e\nzNlGwSwndQMpytgzXuL/KoBTnftThh5TAgSzfnp6rGsnl+OOnOvX+3P7AX8Tto0bWUw3buTXuvVW\nFtQNG2x66ObNhaItcYLJk4Ff/pKrfQF+nzPO4CDzfffZ9ceP89qJE/kc3YByWKbRjBnAjh32fj7P\nwWr38wLqBlKU8WC83D5bALQTcw6A/1R/fzSZjHXvpNOFmTYDA4U9ev7sz1hA3awb8bdns/xa8+cX\n+u9dcjm+Glizhls8uPT0cIDYxRv66+nsBG67jTeGiRN5I0ok/GtPPx342tf8nyWVstb9rbfyYxLo\nVjeQoowtlUr1/CaAVgAnEtFhAH8DoAkAjDH3AdgGTvN8EZzqOb8S79sI9Pb6WzAT2Wwb2QCIgL//\ne9uV0+3ouWMH8MMfsvC3tAATJgDHjoW/lzG82QBspT/6qH1u717gqaf86+VKZMIEDg5v3gycfTZw\n5pnsmtq8GXjoIeD114EXXvAf63m8aUi2T3c3X9FIzyEJdOvQGEUZGyqV7fPJEZ43ABZX4r0aBfF7\nHz3qd6F85CPAVVcBjzzCPngRfDfDZtZQV489e2wmkIh0Zyfn7a9bx0JL5Pfz9/by92+/7T+fqG6g\nxtgYgrBrF4v/l7/McYjPfjb82N5em+3jViNLoHvFCk0FVZSxoqYCvoq1giVP3gs45n70I14zOMgW\n/sKFbNEvW2ZjA3v28NVBMmkDrCLS+/bZGgHAf6znseWfzbIVHobnFRaLBRkc5M8gbquwtYkEP9fd\n7Rd+qS9whV9jAIoyBhhjavJr5syZptHYtcuYiRONIRJ5Ncbz+MtKrv9r9mw+btcuYy680K5NJIy5\n7jr+SiTs+mTSmFSKH5s4kY9bu9aYpiY+trmZXzPqPRMJY5Yv59ch4turripcd9119vMEnyPi15f3\nlMebmuxxwu232/NPJPi+oijRANhjitDY8Qr4KkUgKZeuFZxIsAskit27gY9+lC3otja2miXPv72d\nLXvX8pZAqqR8dnez+yWft9XEu3fbthBBjGFXzpNPAl/8It9+4xvA8uV2TTJpC9KkOlmQKwdpIe0G\ngFta+DjXspfUV/lMGgNQlMqg4l9DuELX1GRz7d30yDDEp79sGfv0v/AFdo8AnEvvCqwbPDaGff/p\nNL+vK/aeB3zsY/y68+bxOcm0r3S60Ac/bx4/JxtW8PM0N/Oa4bKN9uxhF082ax+T1Ff5TOryUZTK\noD7/KuMOZ9m3D7joIs6dB4CurpH964IxbGVLk7fubq7CdQO1nsd5/A8/bF9zcNDm2ruxhuZm63fv\n6PCfpxSdBfv2yHzgwUG+f+uthTULDz/sP2+36ZxceQRnGejgeEWpPCr+Y8xwmSrBPj5CUxMPT5c+\nOGGIBS2tF4xhMU6n+b1cd4s0YLvkEq7I3bbN3yxOzi2TKewfJMjz0i5a3EerVnG/IGktEZxHEBTu\n5mb+vAB/vkWLbNB5uFkGiqJUFhX/MSSYqRLsu+/28XEZGOCrgLvvtm6bRKKw734yyWtkTq9Y1+46\nIs6937+fU0O3bwdWr+bXBwp97CNZ2W77Cc+zVcKPPsp+/0mThk/JnDsX2LKFN6tk0g6Y18HxijK+\nqPiPIW7Tsv5+4IYbbEtkEWuxhMMs/OnTgWuu4e9bWoBPf9pazeJe2beP3TIimAcOFPbsP+ssLtIS\na723l6t4R4PbfuIb3wAOHrTPPfQQ8O1v8/crV/pnDQCF+fwDA8CSJXxensd9fnQAjKKMDyr+Y4hr\nJQPW/378ODdKO/dcroz96lf9xzU1sah+5CMslKlUYdYOwPfXr/db7729trc+EdcBtLfbfj+VcKvI\ne/3d3/kfP3wYOP98mzkkraFTKTuMxs1k8jx7lZLP2xbT0i4a4LiHXP24IyMVRSkPFf8xxLWSg0PS\njxyJ7oaZy3EKpSC9+sOqbHM5f4C0tZVFUoS+pYWfl8reKEqxrkWQw84n6JoKDqORK5f3vx+YMsXf\nQiK4AWSzfGUg3UWPHbMZTboBKEqZFFMMUI2vuBV57drFxVVRxVrDfTU1cSFWMllYLCWFWsH3uv12\nPmbiRC6OSqW4gMst7nLXy7qw13NZu9ZfmEXERWGplL84zS1Sc4vJ3IKzqC+3QC2s2Ky5efhzVJRG\nBlrkVXvMnQtMm1baMZ7HQd2ODvbdu5x2WrgbRLpk9vb6M3OkG2iwW2axA1Wy2cK6gUSCLfEnngCu\nvdbWC3ge9yC67TZ7jlJMNhLSMG7DBo5ZBGsDBga026eilIu6fcYI140CcBWuuDw8j90ezc2chSNM\nngyccw5nw4hIXnopB35XruTX2r3brr/5Zr/wB103bsxBOn3mcoV+/+AAmWBMQF730CG/eAeDtJkM\nu5kkiPud7/Bm4bqkPC+6SVywAG1wkFNBg7OG3dkFiqKMDhX/EinGNx5M8bzoIiv8gBXQadP84v+L\nXxTm4W/ZwkPY83l+reXL+RiZwztS8VVwUEpUDr+7LrihyGdJJvlLGs7dc48/OAsUtopw4xGZDHDv\nvZz1FLYBGGMrieXzSt2BuyFec436/BWlXFT8SyCswyRQKJrBFM+tWwtf6+BBf5okwOt7e4EFC9jt\nIVWvInrHj3Me/fbthecjFnVQdIN5+1GiGZXf734WgC3xqVNtGqcMihFGuoro6OArmYULgWefLXw/\nY/i5qVPtsYcOcQaUVB67XUkVRRkdKv4lEPSNd3fbFEp3vq4rgG6//JFw3Rkyb1cgKpxxe+iQPR9J\nq3TXVYKgmIvwRrVZHu4qArAtq198Mfz9PM+mrgY3t5kzeWNQq19RykfFvwSCQghY8ZUgpczNFQE8\nehS4666Re/QkEtZ/ns36g5xNTTZfH7CCKFO7gPAK4koQJubBFg/F9uIJG9wiwWFx+dx9d/gVVC7H\nk8QOHOArB90AFKU8VPxLIMyHvnGjFTNpriZNzQAWO6lglcKnILNns3ADLKyHDvnFceFCW5HrCi9g\n3TBjWf0aFPORXDthZLN2Pq/72WS6WNimlU77f27uz1fFX1HKQ8W/RIJCKN0w168vzKRxp1SJdRtE\nLHbAH1iVtshBH3eYG2a8hXAk106QYAM7z+PPuGBB9PlnsxzAlo3TDQIfPcpB9GDQW9s/KErxqPiX\nSVBsXH/1+vVW8IN+f2mvPHeu9d+LOBrDohZm0ZcqvGNFKW2W3QZ2RDxj2K3S7eriCmYR8+AxiYQN\nAh89aucFP/oo8OCDwI9/bIPB2v5BUYpDxb9MghlAYqVLf/swiIC/+AsebiLHuoHhfJ7z5YNplEK9\n9bdvbWVLXwLT+/ez715iIq6YA/y502kbD3CvcC66yP/abhsMdQkpSvGo+I8C183gunbc6tjdu4cP\n8K5eDbz9tvXfuwFeIraE4xLYzGQ4E0rSVwcHufgrLAi+bh1/7mXLrNXvXiW0tfn7AQmyUaTTY/95\nFCUOqPiXSLDoyQ3iJhLhw1SEU04BXn+9sNmZZO4Q2bm2O3YAO3fGx43hdhZ1axKCrRuefpo3VNdN\n1Ntrn5eroXXruFGdmw6by3GX1H37qhMLUZR6Qnv7lEgw11+6WBJxALO3t7CzJcAi//nPFw5Yl/m0\nPT3cH2fOHBvcHK7PTr3hzuK9+277c5gwATjvPLtONtLhhrZ3dHAM4OST/YVw8jNbu7ZwFrCiKH7U\n8i+RqAEsTU3W39/UZC17CewuX84C6E7dksCwkMlwOqRM44pbDxs3VuH+HIDCuEnUOEmAA8TXXhv9\nPpoSqigjo+JfImLBLlvmb7J2+ukcxOzt5efuuMNao9u2sfjL8SJIUe0ixBUSdInEiaiU2SASGHZ7\nE4XNQSACPvQhO8lMZwEryvCo+I+CTIbbK7vif/AgW6MSeHSvCqQFcdAKjWqlPDhoA6ONZL1KTGDD\nBvv53boAIttULohkEREVBokVRSlEff6jpL3dtnhwkbbJrkAlk5zHH/RBiwvJ9W2HPdYIhMVS3NTX\ngQF2tUmrh6i5CJJB5AaJFUUpRC3/UZLJsGBJde/AgD9t8f3vB844g7/fto0btUnfn5GaoNVCEdd4\nEzZ7wLX83SupfJ67m4YhdQGNsmkqymhR8S8D8Vu3t/MmsG6dddk89xzw7//Og8vFWi22CVq9FXFV\ngqjZA+k01zw89ph/cw276rrwQr5ta2u8n5+ilIqK/xCl9IcJrnU3gRUrOEc/n2c3xdNP+/v0qEUa\nTdTsAckMkgyqpiZO8wxeETz6KD+2c6e/QE7aSAOa/68o/00xg36r8TWeA9xLHWDe1MSDxVMpY+bN\n40Hjcoy8lgwzJ+J17hqldHbt4p/hvHl2EH3UMPhEggfYy3GplA5/VxoHjOcAdyL6OBE9T0QvEtEt\nIc//OREdIaL9Q1/XVOJ9K0WpA8wlGNnfz2mH993HM3plqpV06QRYcgYGuCmZWpyjJ5PhttazZ1s3\nWtgoSPH5p9Pc/lqqhQUd/q4oTNluHyJKALgHwMcAHAbwFBFtMcYEhhTi740xS8p9v7FguP70rssA\niJ7K1d/P63p6Cvv6EKm7p1IEe/wDfH/OHPb19/bymk9/2j93WKqum5r0d6EoQGV8/rMBvGiMeRkA\niOhbAC4DEBT/miUq6yab9ffpkfRNovCmbevWhW8Ol1yiVn8lyGa5d8/AgM3nB3jDXrHC/oyvv543\nY4DXzpvH37/2mo6BVBShEuJ/CoBXnPuHAXw4ZF0bEZ0H4KcAPmOMeSW4gIg6AHQAwNSpUytwasUT\nlmEjbRYEEfaowSyy1vNYmGT4iFT3KuXhunCkp89ll9mZCFJhfTBgdrz1Fo+AlAD8Sy9xqmgjpdIq\nSpDxyvbZCuCbxph+IroWwEYAfxRcZIzpAtAFALNmzRqmIfL40Nrq79MjDDeQnYiblslownTa+phV\naEpjpAwsY4CtW4FHHuHfkTG2wtqlr8/2YsrneX6A5/HvKS5dUxWlVCoR8H0VwKnO/SlDj/03xphe\nY8zQhTgeADCzAu875kgh1wc+UPwxnsfC39HBorVsGfC5z2mXyVKRvkfuz6693bp6BOnkKVdickUg\nLbKbm9nVE9wQ4tY1VVFKpRLi/xSA9xLRu4moGcCVALa4C4joXc7dSwE8W4H3HRcyGfblp1J8P5Gw\nxUTCeef5m7BJa4Fis4iUQsJ+dpkMD6x38bzCBnhEwL33Al/8Ih/X0QHcdJN/rVYCK41O2W4fY8wg\nES0BsB1AAsB6Y8xPiOhvwfmmWwB8moguBTAI4C0Af17u+1aaKBeDPP71r/OQkDfeAL73Pft8IgFc\ndRX7lIPZQsNlESnDE/Wzk6Ew/f0s5Jdcwj59d5yjMXagC8AB4PXr+ftEArjiCuDIEWDGDL9LTgfB\nK40EmeFmDVaRWbNmmT179ozLe4W1VhYxcKd2Sc5+MI3zi19kwQj26Zf2BL29KiijYbgNWXoq5XI2\nldONxYjLJ/g7k2C8TBIT339nJ7vogn8DilJvENFeY8yskdZpewewwEhA8Ngx4JprgAce8LseRFiC\ne6VYpSP16VchKZ2oHkcSi3ELvf7wDwutfzcW4D4ezODq62PXXpibSVHiirZ0BlvnrtV48CBw/vn8\nuNteuamJv0+lOHf8uut49GKxffqVyhFsff3bv124JuyiNuqxffv4tRqtlbbSuKjlj/De7wMD/Hhn\nJ3eVbGsrHMEYhfr6x55gYV5wCtiUKcDhw9HHn3468MEPAg8/bDOEFi3iNhzqolMaARV/sIUfrNpt\nauLHly1jl9Djj3NwUWbxDkdUxbBSWVy30IEDtrCuqYlTRJcu9Vdnu1d3N9/Mm/n27fz79TygpYUz\ngxSlEWg48Q8GEbu6gBtu8Au/5wF3382Wf1+ffW7zZi4oCnP1BGnEnvzVIpvlTdoYDv6uXs0i/tJL\nwFe+Yh9ftoxHPba1WZHv7ORmfbkcPz99Oj+uG7cSdxpK/Lu6gCVL+B9dMjzkvov4gN94o9BH3N/P\nIqEzYmsHibHk83wF19vLG8Kdd9rf3+Agt3TYvt1/bG+vdfscP87uI5klrMF6Jc40TMA3rB3zpk0s\nCkGSSc7+2Lw5/LV277YtnJXqEzb3uKfH7+ZJJMJjL8FjAQ3WK41Bw1j+3d1+oU8kuMjn0UftY1Om\n8O2JJ7J7wCUYE9B0wNohKsaSSll//t1382MrV/rHRLa2Fo6PdC1/DdYrcaUhxD+btRWeAAu/+PQl\nEEhks0PCskSi8vuV2iBsBGRQ1KX2wp0H0NQEzJ/vH++owXqlEWgIt48UBAEs8osW2cZrqVRh068o\nPI8nSUXl9yu1QzCw393Nwftcjl1/UrjX3w+sXetvvJfJALfeyt+vXKnuPSWeNITlH8y7P+EE4KKL\n2O1z9tk88HukLhduGwAV/dpGKqzF5XPFFcC3vhX9O5ZqYJnE1trKqaNucoAGfpW40RDi77oAjh7l\nfu6A398fBREPDJk9W90A9YLbriOfBx58MHotkZ0KtmEDx4WSSb6Vq8X+fo3vKPGjIcQfsD7hiy4q\n/hgiYMKE4gq7lNqhtbWwqMtF2jobYwfAnHYa8PzzfEww9TcqU0hR6pmG8Pm7zJhR3LpkErj2Wr3c\nr0cyGeCeeziYG4zneB7/bl3yeeC558I3C0kO0L8BJW40jOUvTJpk0zaJeErXb/wG+3zd9M5PfAJY\ns6Z656mUR0eH7cUkbbXdW7f1AxAdD5DkAO31r8SNWPfzD/uHddstJxL8Tz84GJ7KqX7e+JLNcuxn\nyxZr8Tc12b8Huf/DH/L32qJbqReK7ecfW7dP2AxY4aKLgDPOAE4+uXA4i5DLaXVnXBGj4K23rPAT\n8axfGfcI2Ftt0a3Ekdi6faL+YVtb/Zf7Yeh81/giRoHbsA/gOMAJJ9hGcAAbBnLl6KYKp9O2Uliv\nAJR6JZbin80Chw7ZwJ7ncZ+e3bv9U5xckkme4NXSomMX44wYBcGrvdNPB7761cIRnek0H9PZaeMF\nOu5RiQOxE/+gT//cc3m83+7d0cfMnq3FW42CWPFSByCcdBJn/Lice26h0IddUerfjVKPxM7n7/5z\nDg4C//ZvIx+zcGHhP3A2q6X9cUQK/m67DbjqKr4qJAJ+/GN/CmhTE3DmmYVCH9ZBVFHqkdhZ/kHL\n7q23Rj4mOMZRB7DHGyn4W7mShT+fZ0Nh0SK7pr2dbzdutG0i0mmd0qbEh9hZ/vLPOWdOeIHPvHn+\nx8OsN/fqoa+vcD6sEg+CVnx7O9d2rFljRf3ss23657JlPBBIhV+JA7HN889meeBKf799LJXibpyA\nFXS3la97rJsVJMfpP3v8iCreCvv7AdgdlM/rFaEydpRbUFhsnn/s3D5CJsN92teutdW88+fbH+Zw\nP9RMBliwwB47OKiBvbgSNWtZrv6CSEGgBnuVsWA8Xc6xc/u4tLTY1D1jOI+7WNrbuambBvYaE2kO\nFySZ1L8JZexw506MdUFhrMW/t9d2cASAu+4qPntHYgdf+IJe3seV4TK6Mhng3nsL40Of+QxbZkuX\n8j+mZoMplUImDorBmkyOrYERW7cPwD+4RML2asnlgBUr+KsYMY9yCSj1T/DyWoq4XD+rNIeT+FBL\nCwd9+/p4FgQRxwAWLAiPHSlKKQQnDrpu6rEg1pa/tPZNJPh+Pg/s2FHY60dpPNyMrv5+ntr1uc9x\nkPf66/0jHSUDqLeX17quxOPHC8dAKspoEGNVjApJNx4rYi3+AFtu7qV7Pq/NuRR/mqfn8SYgG0GY\nmEvLENeNKLgBYEUpB/n7Cvs7qzSxdPu4qVLd3f5+PkTanEvxF2tJvx5p9hbM5nFdRMF/ymSS12sA\nWCmXnh6bTTYeGYaxE/+gL/fss/3Pv//9/I++dClvCk1NmrLXqLgxHfHtr1/PVwAi5tksx4iCvYAA\nvmL4xCeAX/8aaGvTvyGlPILdY8famKiI+BPRxwF8DUACwAPGmC8Fnk8B6AYwE0AvgCuMMT+rxHsH\ncYd39/UBL7/sf/7884F9+2wO9/Hj/E+v/7iNjWwE7e32qvHAAWDxYt4MomohH3mErbSdO3kDOXAA\n2LSJx4VOmqRXlsrIdHXx30xbG1+NjldHgbLFn4gSAO4B8DEAhwE8RURbjDEHnWULAfw/Y8zpRHQl\ngC8DuKLc9w4jnbYWmjHA4cP+51taWPwVxcV1Fd56K99fssRmioWRz/PVo8SRVq3i1uGAZgMpxdHV\nxbPCAf6bWb6c+0kdP863tV7kNRvAi8aYl40xxwF8C8BlgTWXAdg49P0/AriAaGxCGsHcfhfP4+fb\n27llAxHfjnVUXaltwqa+dXcPL/xCImGLvl57zf+cZgMpI7Fpk//+hg3AsWPjU+RVCbfPKQBece4f\nBvDhqDXGmEEi+k8AaQC/cBcRUQeADgCYOnXqqE4mmNtvX5uFXi7Dn3hCG3QpTLBHv/j+g64ezwNO\nPZWzfozhYO9nPmPdOwcOhM+N0HYQShjZLLumXY4csd97XgMVeRljugB0AdzYbTSvIbn911/vn8/6\nsY/5i7u0gEsRgoE2wBbbAPz3I8bDN7/Jrp077uA1q1fbS3P5exKf/9tvsyU3OKjZQIqfYPPIMFpa\naj/b51UApzr3pww9FrbmMBElAfwWOPA7JnR08O2SJfwPmkoVX9WrNB7BHv2A7eMvBkQiwVXAAHDn\nnfbx/n5/Smhvr/9vzQ0g699f4xHVobOnJ3qkrLBw4RieGCoj/k8BeC8RvRss8lcC+FRgzRYAVwPI\nAvifAH5gxriXtJTm6z+eUgzBK8HHH2cR37GDhT6fZ2Hv6fGnfBKxG6irK3y2r15hNi7DdehsbeVk\nANfyb27mv6H9+znzR4zYsaJs8R/y4S8BsB2c6rneGPMTIvpbAHuMMVsArAPwd0T0IoC3wBvEmKP/\neMpoyWRY/HfuLMy7TqVsn39jWPg9z24S6t9XgOHnPWcy7DJct467B5955vhnhFXE52+M2QZgW+Cx\nzzvf9wH400q8l6KMF1EjGyUV6Q70AAAfCUlEQVQX+/77bWwgn+cNwPPUv68wwVhSOs2xSID9+W6h\n6Ze+ZF2H4+WtqKmAr6LUGu7Vo/uPOXVqYTaQZABJbCCsfch4/nMr1SXYQmTpUuvmkStFwNaITJ7s\nrzAf61byKv6KUgRB/+3SpfwP7Hb4NIb/cfftC/f/j+eUJqU2EONh5Up/gDfYKmTLFv9j4+E6VPFX\nlCIItoC+6y7r6hHhB/ixgwft2r4+tupmz+bAcJQPWIk3YQFegahwMxgP12HsWzorSiVwW0ATce6+\nBHiD/7g/+pFtI24M1wV89rN8Sa9jIOPLSJPhenqAefMKOxCceGLh+qVLx94wUPFXlCIQ/+2iRXzf\ndfcEMQY4/XT/Y/k8W/zz5+to0DgS1iIkSCbDV4BBLryw8LH9+yt/jkFi6fbRoJoyFmQynOXjVv8G\nIWKr//nn/Y9LFpA2eIsPrs4Ml9bprk2n/e1nPA/4zd/0B4ABzvMfa2In/hpUU6rJyScDr7/uby1y\n2WVs8aXTvHl0d+smUO+EzYCO6sUfXHvFFcBDD/EVoucBTz/tf+0zz+QC1bEmduI/0g6sKOXQ3s6+\n+4EBO/7R5bXX/K6gZJLb9AL+Xi4bNnBzQf3brE+COtPbG14TElzb38/9oeRvZHAQeOopvk/Et889\nx5uFpnqWyHhPw1EaCwncueMf3WpfV/gTCeCP/5i/D/ZyUcOkvgnTmaiOAu5aYwoTBOQK4D3v4eFT\n41UlHruArwTmNKimjBWZDA986ejgv7HbbgPWrOEyfcnkmTePrf6tW9mKS6c51U9Qw6S+KUVnZO0l\nlxQKP8DCn0oBN9/Mt+OVDRY7yx/Qnj7K+OH+rb30EvCd7wCXX849/rdutbn++/axJScj+tTnX/+U\nqjPf+17hYzNmAL/zO7aR23g2o4yl+CvKeNPVxcVcAN8uX86Wv8z/Xb+eBX/NGnuMZqU1DsFusABb\n+M8+y0OAZAb0eBqusXP7KEo1CI7j27+fc/qloGdw0D+Sr5i8cKV+CRZ8tbayMSB4HruBBgZsIHgs\nRzaGoZa/olSAtjYewO3eB/xtH9Jp+7xmpcWXsDTQ3l7g4ouBhx+2mT2AvRoI/n2MByr+ilIBZPDG\nunWc6y++Wyne8TwWALfYR7PS4kkwtVPaOCeT/LuWsZ6TJxf+fYwnKv6KUiGmT2f/7d69wPbtbPGl\nUv5+7mIRJpPA3LksABr8jRfptG345/r5BwfZSJg6ldc88oitCE+lxt8AUPFXlAoRVfgjGT779tnn\nczl2AUyYwOKv1D/ZrH/IT7CBWz4PnHACd3f9m7+xdR8yH3q8DQAVf0WpEBLUy+f5Np3mzJ+tW+2g\nF3leCsLcAfBKfSKiv369v2WzW7Ur3HUXXwG4j8l86PFGxV9RKogb4F282DbwAvj7WbPY2n/ySbtu\nvAN9SuWQ4G5fX3iH16lTueVHLmfbgQTXNTVVJ+aj4q8oFaKnx/5zu60chHwe2LOHRUAsQiJ2Byn1\nibj6RNCDlv5f/ZUN/h89ypY/wIJf7ZiPir+iVAjp4dLfX1jQ46b2BSeArVunQd9ao9gCvKCrb+FC\n9uvv32+rdoULLrBXAF//uv+5aqDirygVQnq4rFgB7NhhN4AzzwRuvNE/wNu1DgcGbFBY2z9Un1Lb\nwrtWf9TvTa4Q8nleVw0ffxCt8FWUCpLJsPi7TdxefJEv/RcssFcAQb/vk08C558P3Hcff7W2atVv\ntQgrwBturbj6crnwtdksZ/jU2ghPFX9FqTCZjL+1g4hCezsHe72Q/7pnn/XHCQYGxr/cX2Hcec0i\n1FHzecPWushVxP338waxaFHtdBtWt4+ijAHt7cDGjYX93sUt9Nhjfuu/VjJAFPt7Ep8/EO0GCq4N\nirp7FQFw9k8tCD+g4q8oY0KUKIhb6Ac/8KeBErGwVDsDRGHc7prXXw8cO8bfHzvGcZlMxt+qA+Dq\nbrkvGVwtLbXbxkPFX1HGiKj2vJkMcNNNwB138P1kkuMBKvi1RzYLPPCA/7F161jUZYqbm9kVTPVs\nbuZ1kv1TS79f9fkryjiTzXI5v2R+rF5t+/yH+ZWj/M3K2CMBXZfBQd4A+vrCRzK6HD/Ouf2PP86b\nQC39DtXyV5RxprvbpnzmciwkgK0ITiY5+0dcCxdcwBam5wH33FP9/PBGorXVVuYKngc8/XR4RW8Q\nIj52vObyloKKv6JUmd27ufJXrMjBQeCWW4CPf5xTBMW1kM8DS5bYiU9KdZg8GXj9dXvfdfUE3T6f\n+hSP9lSfv6IoaG9na99N7Qy6D3bu5C8i/3OSNqriPz709BRa+K++6r/v1m64az0P+P3f5yu6WhzX\nqeKvKONMJsNtAO67zz4mQz0EEZGgmEjfd53/Oz60tvLPvL+f7wc3aSHMNSS/q/Gcy1sKZQV8iei3\niegxInph6PadEetyRLR/6GtLOe+pKHGgvR2YOJFFIpnkgO/atcCUKeHriYA5czhwCOj83/FCUnZv\nu41/R2EFevk8u+IEz7O/q1oUfaHcbJ9bADxujHkvgMeH7odxzBgzY+jr0jLfU1HqHhGVSy8FzjqL\nH5s+HXjzzfD1nmdTBbu7OdOkmPYDSvlkMmzB9/YCn/xk+JpnnrHfex7XctSy8APlu30uA9A69P1G\nAD0A/rLM11SUhuDAAWDzZv5+927gvPMK0woFYzhV8KWXbKsAgK8aaimIGEe6ujjQnstx5XXQRQcM\nX61dq5Rr+f+uMUbi3m8A+N2IdROIaA8R/QsRzYt6MSLqGFq358iRI2WemqLUNps2+e9LgDeMfJ79\nznfc4d8g5s+vfQuzXgirp+jq4grfgQGbrulm9pxySuHvzJj6uBob0fInoh0AJoc89dfuHWOMIaKo\nPe/3jDGvEtF7APyAiA4YY14KLjLGdAHoAoBZs2bVyf6pKKOjrQ149FF73xjgne8EwuwezysMKiYS\nXGm6cqUGfsslrI0zwBZ/sII3meTfQ3Mz8PnP8xWZTPKq1jD20TCi+Btj5kQ9R0T/QUTvMsa8TkTv\nAhDqsTTGvDp0+zIR9QBoAVAg/orSSHR0sBvnjjuswIQJvwR729qAT3/aZp7kcsANN/D3UX3n454V\nVKnPF2zj3N0NvPyyv/8SwOK+ejX7/9Npvu3s9N+vm5+1MWbUXwC+AuCWoe9vAbAqZM07AaSGvj8R\nwAsAzhzptWfOnGkUpRHYtcuYCy80xvMkU9z/lUrxGmOMue668DWJhDG33174us3NxhDx7a5d/HX7\n7fb16pldu4yZOJE/+8SJ5X0m97VSKWOSyfCfM2DM7NnGrF1bufeuNAD2mCL0u9yA75cAfJuIFgL4\nOYA/AwAimgXgOmPMNQA+AGAtEeXBMYYvGWMOlvm+ihIbpNPnzp1sdSYSwDnn8FXASSfxJDDpGNnS\nUlhFCoRXj7ptJI4fB1atArZvL35CVa0TNnRltJ8nk2ELft064D/+A/j5z+1z06YBP/uZvb97N7B3\nL/8OarFtQ7GUJf7GmF4AF4Q8vgfANUPf7wIwPbhGURSL2wJaBn0PDvKQl507/f7kD32Iu0QK06YB\nDz00svi89lrlxLIWkEEqo2mdEHQXZbN+l5rLxImFG24+z5u0tOKuBx9/EK3wVZQaQYT4vPP8vmYR\nHbEyzzmHrwQk+Pvaa+Gv194ObNhgxXHhQj6uFvvMjIaRBqlE4QZ3k0nOmALsVVKQF14Ib9X89a/X\nmY8/gIq/otQQPT2FOeSSV+55LDrt7fz42rX+2bEiQK5V+8QTfnGcPj1eAeDRtE5w3UW5HP8cm5p4\nI5B+S55nvfyyEScSwLnnshsuDrMXVPwVpYZwe8l4Hg99mTQpPJMkOCYS4Lz0xYt5s0il2DJubbV5\n57XaZ6bSDJcFJO4iSc+UDXTRIrumpaXQDWQMd1q99dZx+ADjgIq/otQQxboywtZls5yXLpZqfz8H\nfd1Not6DvMUQlrPvfmb52XV3A+vX25z9lhb/Brtvn7/5HhG32M5m4/EzVPFXlBqjVOtcMoEOHSrs\nLAnEK8g7HGLtuzMQ+vs5kyqs187UqXbE4owZ/L27YbS0FL7H/ffzZhqHTVTFX1HqEHfCl8QDkkn2\nS0tm0D33sI/ftfzT6XhWBLvWPmDjJvk88NhjnDElgh382REBO3ZYF5BsGO95j7+Pj2yscdlEVfwV\npQ5wfdgAi5M7PDyf52Cl9JlJJOzEr85O7iMUZt3Wu4AJbhA3iDF+wZauqO7MBLdfTz7Pm0FYPYUE\n3es9UwpQ8VeUmieYmigZKGK1homYZAABLPj9/X7rNi7WqxAM4rp4Hm+Ghw5xQHz9+vDOm0TAaadx\nW4ewoS3Sp78e2jUXQ7ldPRVFGUOyWWvli99eOkwSRXcBleCkWLkyA1hcQnGxXgUJ4l57LWc5eR6n\nby5fzj2UiNhf7wbEAf/Pzxjg8sv5+DBSqfgIP6CWv6LULGG+6WB3z6je8fk8568HXRdE7Mu++eb4\niJgggfL2dn8W1MqVLPi5nN38ZOMMuonefhu4+mrg4EHgySft4/Pm8UYSp5+Zir+i1Cjix5aALmDd\nNuKbFjFLJOx9sfJlvYsx7NZYtszGBOKGfCZxe4lLKKx2ws3lTya5InpwkNcvX86ZQG1ttjjOff16\nR8VfUWoUt3eNWPsi8IDdBN73PuD88zk1cdMm/4yAMEbTjKyeWkOH5fl3dtppXKtX22D39OnsGhPu\nv9+61yZN4kZ4I9UN1Csq/opSo7iFXOm0zdRJJGxrAmO4+dvzz7NPeunSaPFvarKujjCff5TAjyR+\ntbYxuJk/btqmXBH19bHgi5tIzrmry7rW3J9PJbuH1hIq/opSw7jiJK6HdNoOcRFca95l9mxu6CaV\nq0DpAj+c+NWSVSybUDpt3TyS5+85qS3GsHvH7c+TzfLmKt06Ozvtc+V0D61lVPwVpU6QjWDlyuj8\n85NP9j/+6qt86/ajkUInt9hrOIEP+szTafta5VrFlbpqCG5CS5dym+vDh23vHgnySqrrqlW8OUrv\nI4mvEPFm6f68RtM9tNZR8VeUOsNt/pZIAJ/5DGepvPEGPy8zZo1h8b/2Wn68o4Nvw6z14axbKRRb\nvJhf1w0Wl9tTvxR30nAbhbsJ9fX5R2O6SOzEGGDzZmDLFv5ZdnYO/zmCQeQ4bAAq/opSZwQtUQD4\n6Edt1ornAe94B/CrX9ljNm2y4t/T4+99I69z9dX8fFi74t7e8MlVpVrFroAX605KJICLLwa2bbPx\niuBG0dpqYyFusZtABEyYAJx9tj+FUz5Pb+/wn6OW3FuVQsVfUeoQNxawcqV/EEk+7xd+gNMVARax\n3bv9bSGOHvULW3t7oZU90pVBMUIYFNDhrO1gz/3Nm+1zYe6lTAZYsMDOOHBJJLhds7RpdhE30NGj\nw3+OOAZ9VfwVpc5xffJhELGbRsS3r88+53mcy+4KW1gbaGD4K4NiCArocNa2a8kHP0tUptIbb9hG\nbO4GsGgRsGaNLfaS15FxmMaw//+00+zVUZA4Bn1V/BWlzslkeGLXNddwZWoY4qs+ftwvjIkEXxXI\n8PjmZrsuajOQSWJRuFk3vb3+26CAhlnb2Sy/p9QxSCFbUxOPXJT3l4A1wLdy9RNseXHCCXaN+/7y\nWQXXNRYkjkFfFX9FiQGZDPDAAyxMMopQRH7CBCuSEgwWZG0whuCKPVC8yyOsJYU7fL6zc/i5t+7V\niZx/sKFa0H109dV+t1fQ7XPXXdyeISjgBw6wC0wQ11gUcZuCpuKvKDFBUja7uzmPfWCALfulS63g\nzZ/vn04FWIvXFbbgZrB+vc2Bb22NzrxxUyaBwuHzvb3Dj0GUYLTbYjnYUC3oPpIsJ0Es/2CH02BR\nl9xu2sTCH2X1xxUVf0WJEbIBSMtnALjzThZCCbI2NVmLHwi3eF2RzGatoBKxxRw2FyCb5U6i0nba\nTbUsppNoNgv80z/5jyPyF1wBNh4gm9HkyYWtrV2Syehq5nSan5s+Pfq84oqKv6LEjKieQGJ5//CH\nwC23cIO3T31qZItXNhOZI7BpU2FMQObhDg6yEF96KXDGGexyGRzk8wiKuEs2y/2J3E0J4Pd0C64E\nEftcjn36iYS/VbPLggXh1cyuaymV4rhJnNw6I6HirygxI9gTaOlSO+Xr0CG23J96ioV79WrOchnO\nDx8MlM6YAfzgB7ab6IYNhYHk73+fb2XTiBJxobu7UPiB8KuFnh67NpfjDeamm/hWNjr3+GCAOuha\nAuywexV/RVFqlmJaIojbRlw2YrVL8zJJh+zv5z5BuRwL+b33Fl4JZDK8gXznO8CHP8wbhrRLOOcc\n4Ec/KnS15HLA1q328TDXy0iceSYHsV33k2xowdm6kybxFY08v28fPxeWlppOR89BaCRU/BWljii1\n0tS1koHwlgeS/ZPL8UYQ7PP/l3/JefAA8OKL9nFjgH/+Z/6eyA6PN8bvhiHiQDPgT890N7D2dmDd\nOnuuTU2Fwi9ZRDKQ5oUX7GfavZtfS4LJslGE/fyWLAmPC4yUwho3VPwVpY4otdL06NFwwQfsLIBn\nn7WP5XLs/li1CnjtNRbUO+6Ifv1gcPaee9i9c/So3TCMYb+8266BiDeHZJI3hpYW7j4qmTuTJxd+\nbndgvQi/sHkz996XgrThOpQG3UunnAL8wz80lssHUPFXlLqilErTbJYzfQQi7m2zfz8LbyIBXHIJ\n8NOf+nP/3RYJbh78SAwO2lTOlSuta4aIXUYi3m4Fbi5n308KuSSQu3GjFW63k2gUbkvrYO8it0Np\nsHL4qqsaT/gBHeCuKHWFBHO/8IXiXD6uZZ5McsbN6tU2C+hrXwsf9RiGuHa8CNXwPA4oZ7O286jn\n8eu99JK/6Cvs/USs3e6cCxcCf/In3JNnJD+9MbxZuVc7+bx/48hkuN2De86TJg3/unFFxV9R6oxM\nhq3rkaxVV4CTSeDuu/mYYIfOKLeQCxG3ht65k/3tUdx/P7tcAN6c5syxG4DncWaRWzMw3GYiU8o2\nby7Mzgkjn+e1rpvK8wqzjNrbgYkT+b1TqXj06RkNKv6KElPkKuG227iNcUeHLcRKJPiruZk3hjAS\nCb4lYneMZM5cfnn4WnHXHDvG/v5MhitzUykrtJdf7i/GuvJK3iCKsb6DPXuikM1MWkqE9eYv9uop\nzpTl8yeiPwWwAsAHAMw2xuyJWPdxAF8DkADwgDHmS+W8r6IoxRGs1JVAqOcBM2eyW2XfvvBWyOIX\nlzTRAwf4tcKE+owzOHYgbN7MaaUdHf5WET09ftfPN7/Jt0Hr303lBHjzmDkT2LOnMI//4ouB733P\nX+RFBMyaBZx1Fp+3DGmXDSxufXpGhTFm1F9g0X8fgB4AsyLWJAC8BOA9AJoBPAPgzJFee+bMmUZR\nlPLZtcuY22835rrrjEkkJBnTGCJjJk40Zu1aviWyz4V9eR6v3bXLmFTK/1zYsSecwOuFtWuNmTw5\n+rXPO8+Y0083ZvlyY+bN8z8/ezYfn0rxezU18efZtct+xnnz+PN5njHNzbzW8/yvk0rZY+IKgD2m\nCP0uy/I3xjwLADT89dhsAC8aY14eWvstAJcBiGg+qyhKpQhOxEombbaNMf6++itWADt2RMcA8nng\n+uu5N/4TT7A1/fTTXC0c5o9/+22OEzz5JPBf/+UfyBLGOefwVUU6zdW6Lnv2AM88468Ydgu4Mhng\nu9+1+f2HDnH8IfhZ4jKIpRKMR6rnKQBece4fBvDhsIVE1AGgAwCmTp069memKDHHrQsAbKbL+vV2\nJKIUWq1YYfv6J5PA3LnAz37Goutm5CxezIK+Zg2L7XnnRffVAYAHHxz5PPN5jhN4HrtsgkNcJBNI\nGBy07RiCFc/y2MaN/toAID6DWCrBiOJPRDsATA556q+NMQ9X8mSMMV0AugBg1qxZWoCtKGUSrAsQ\na7m9vbBFRNhsYMncccnn/S2Sb7oJ+MpX+LlkEpg2rbAIazjcGICkg460ToiqeA72Nxqu3UOjMqL4\nG2PmlPkerwI41bk/ZegxRVHGmKgJVK6FLC0X3EBoNsttm48d4/VEVpTdDJpslmsHXPE+ejT6fCZN\nYqv+l7+0j8lr5/O2WZy4d1zc+5J9NFzFswZ1h2c83D5PAXgvEb0bLPpXAvjUOLyvoiiIFsEoqzms\nvXJzM3DjjVwd3NYW3S4hlwOOHIk+ly9/mXv4uJXDJ54I/MEf2PuPPMK3RMDUqcArrxS2kVi40J5D\n3Gbrjhflpnr+CYDVAE4C8H0i2m+MuYiITgandF5sjBkkoiUAtoMzf9YbY35S9pkrilIWUVbzqlWF\n/W/mzuXK4P5+bucMcBpna2u4OyaK3l4Wblf833yTg8Fh/v7Dh9mVJMNpJHdfmrDFcbbueFFuts93\nAXw35PHXAFzs3N8GYFs576UoSmUJTsSS8Yxbt/rXybQstzfPDTewH729nQe3jJTJA7CrxhXor3yF\n2z64LqMgxvAwlqlT7SD4oMire2d0aGM3RWlgxI+fy3ExlLR+cLnySv9aWb92LWfUdHayq8bNxgm+\nvjH+4zs6uHV02LB391ix8lXcK4+Kv6I0KOKvl7YMixdzS+ZUyp8i+eCDNhALhNcJPPGEP7PmjTds\nW+auLlslHAzIBjNyJAU1kWCLPyj8xQyyUYpDxV9RGhTX7QPwbVTBl1jmw9UJhIlxV1d0h02X6dP5\naiAsBdWd4BU1OF43hNJR8VeUBiWT4U6fixezMEsKp1vwJVcAnjdynUCQbJaHvYs7J9hh053O5Xl8\n1dHRET5sPWwYvfTuD3sNZWRU/BWlgRGh3LTJn8IZdMkEA61RdQKCK+wi/MEOm+50LgkiB0dIuhlJ\n8jpE9ooj+BpLlhS+hhKOir+iNDBSzHX8OFv6rnC6ufxhFv5wdQIrVvivGubM4cfc15A0UUFGSAbX\nuHn8nZ2FG1FwmLv27ikOFX9FaWCGq5AdaVh8dzdP25LAb9AN4+blB4Uf4PuXXDJ8muhIefyZDLt6\nlizhz9DIw1lKRcVfURqY4WYCj7QxbNhgUzOlTkCOCbP4wwKzy5cD27Zx1pG0bAgyUh6/pI1q0Lc0\nVPwVpYEZzrIeaWOQTp5EnJYZ1m6hrY3XHjgQnqmTyfDz5Qq3FnqVjoq/ojQ4UcJZysYQ1m7BTc0M\ny9TRBmzVRcVfUZRIRrMxyDErVw6fqaNUFxV/RVFGRXA+cHAjKCZTR6keKv6KopRFMQNVVPBrDxV/\nRVHKQgeq1CdetU9AUZT6Rtw7iYT68+sJtfwVRSkLde/UJyr+iqKUjbp36g91+yiKojQgKv6KoigN\niIq/oihKA6LiryiK0oCo+CuKojQgKv6KoigNCBlpyF1jENERAD8f5eEnAvhFBU+nGtT7Z6j38wfq\n/zPU+/kD9f8ZqnH+v2eMOWmkRTUr/uVARHuMMbOqfR7lUO+fod7PH6j/z1Dv5w/U/2eo5fNXt4+i\nKEoDouKvKIrSgMRV/LuqfQIVoN4/Q72fP1D/n6Hezx+o/89Qs+cfS5+/oiiKMjxxtfwVRVGUYYid\n+BPRx4noeSJ6kYhuqfb5lAoRrSeiN4no36p9LqOBiE4loieI6CAR/YSIbqz2OZUKEU0got1E9MzQ\nZ/g/1T6n0UBECSLaR0Tfq/a5jAYi+hkRHSCi/US0p9rnUypENImI/pGIniOiZ4mopvqexsrtQ0QJ\nAD8F8DEAhwE8BeCTxpiDVT2xEiCi8wD8CkC3MeaD1T6fUiGidwF4lzHmaSL6TQB7Acyrs98BAXiH\nMeZXRNQE4J8B3GiM+Zcqn1pJENFNAGYBOMEY84lqn0+pENHPAMwyxtRlnj8RbQSw0xjzABE1A/gf\nxpij1T4vIW6W/2wALxpjXjbGHAfwLQCXVfmcSsIY8ySAt6p9HqPFGPO6Mebpoe9/CeBZAKdU96xK\nwzC/GrrbNPRVV1YSEU0B8McAHqj2uTQiRPRbAM4DsA4AjDHHa0n4gfiJ/ykAXnHuH0adCU+cIKJp\nAFoA/Li6Z1I6Qy6T/QDeBPCYMabePkMngOUA8tU+kTIwAB4lor1E1FHtkymRdwM4AmDDkOvtASJ6\nR7VPyiVu4q/UCET0GwA2AVhmjHm72udTKsaYnDFmBoApAGYTUd244IjoEwDeNMbsrfa5lMkfGmPO\nAjAXwOIhl2i9kARwFoA1xpgWAP8FoKZikHET/1cBnOrcnzL0mDKODPnJNwF40BjznWqfTzkMXao/\nAeDj1T6XEjgXwKVDPvNvAfgjIvpGdU+pdIwxrw7dvgngu2C3br1wGMBh54rxH8GbQc0QN/F/CsB7\niejdQwGWKwFsqfI5NRRDwdJ1AJ41xtxZ7fMZDUR0EhFNGvp+IjiB4LnqnlXxGGNuNcZMMcZMA/8P\n/MAY87+qfFolQUTvGEoYwJC75EIAdZMBZ4x5A8ArRPS+oYcuAFBTSQ+xGuBujBkkoiUAtgNIAFhv\njPlJlU+rJIjomwBaAZxIRIcB/I0xZl11z6okzgXwvwEcGPKZA8BfGWO2VfGcSuVdADYOZY95AL5t\njKnLdMk65ncBfJdtCSQBPGSM+afqnlLJLAXw4JAh+jKA+VU+Hx+xSvVUFEVRiiNubh9FURSlCFT8\nFUVRGhAVf0VRlAZExV9RFKUBUfFXFEVpQFT8FUVRGhAVf0VRlAZExV9RFKUB+f8FvkT+M2urzAAA\nAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Up8Xk_pMH4Rt", + "colab_type": "text" + }, + "source": [ + "## Split our data\n", + "We now have a noisy dataset that approximates real world data. We'll be using this to train our model.\n", + "\n", + "To evaluate the accuracy of the model we train, we'll need to compare its predictions to real data and check how well they match up. This evaluation happens during training (where it is referred to as validation) and after training (referred to as testing) It's important in both cases that we use fresh data that was not already used to train the model.\n", + "\n", + "To ensure we have data to use for evaluation, we'll set some aside before we begin training. We'll reserve 20% of our data for validation, and another 20% for testing. The remaining 60% will be used to train the model. This is a typical split used when training models.\n", + "\n", + "The following code will split our data and then plot each set as a different color:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nNYko5L1keqZ", + "colab_type": "code", + "outputId": "b9f9c57b-b6aa-4817-8ab4-4a2201732b9a", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 269 + } + }, + "source": [ + "# We'll use 60% of our data for training and 20% for testing. The remaining 20%\n", + "# will be used for validation. Calculate the indices of each section.\n", + "TRAIN_SPLIT = int(0.6 * SAMPLES)\n", + "TEST_SPLIT = int(0.2 * SAMPLES + TRAIN_SPLIT)\n", + "\n", + "# Use np.split to chop our data into three parts.\n", + "# The second argument to np.split is an array of indices where the data will be\n", + "# split. We provide two indices, so the data will be divided into three chunks.\n", + "x_train, x_test, x_validate = np.split(x_values, [TRAIN_SPLIT, TEST_SPLIT])\n", + "y_train, y_test, y_validate = np.split(y_values, [TRAIN_SPLIT, TEST_SPLIT])\n", + "\n", + "# Double check that our splits add up correctly\n", + "assert (x_train.size + x_validate.size + x_test.size) == SAMPLES\n", + "\n", + "# Plot the data in each partition in different colors:\n", + "plt.plot(x_train, y_train, 'b.', label=\"Train\")\n", + "plt.plot(x_test, y_test, 'r.', label=\"Test\")\n", + "plt.plot(x_validate, y_validate, 'y.', label=\"Validate\")\n", + "plt.legend()\n", + "plt.show()\n" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzsvXt8FNX9//+cmd1JEDUpUctHEbR4\ngWBCEvAyRXQwCl6r/eEV26WgpFoQsaiVfj62fIoV64VGBRWoIPl+VD7thxatN5CVEcShKBJuiwiI\nUFRaTU2ol+zszpzfH2c3uwlBbgmX5Dwfjzxwd2d2zq6zr/M+7/O+aEIIFAqFQtG+0A/2ABQKhUJx\n4FHir1AoFO0QJf4KhULRDlHir1AoFO0QJf4KhULRDlHir1AoFO0QJf4KhULRDlHir1AoFO0QJf4K\nhULRDgkd7AHsimOOOUacdNJJB3sYCoVCcVixfPnyz4UQx+7uuENW/E866STefffdgz0MhUKhOKzQ\nNG3Lnhyn3D4KhULRDlHir1AoFO0QJf4KhULRDjlkff4KhaJ9kUgk2LZtG/X19Qd7KIcFubm5dOnS\nhXA4vE/nK/FXKBSHBNu2beOoo47ipJNOQtO0gz2cQxohBDU1NWzbto2TTz55n95DuX0UCsUhQX19\nPQUFBUr49wBN0ygoKNivVZIS/3ZIXZ3Lli0TqatzD/ZQFIpGKOHfc/b3u1Jun3bGkiUu9fXlGIaH\nrpv07h0lL8/ao3Pr6lxqax3y8+09PkehUByaKMu/HeG6MH26A3iATxB41NY6e3RuXZ3LypXlbN58\nLytXlqtVg6LNUVNTQ0lJCSUlJXTu3JkTTjih4bHneXv0HsOGDWP9+vWtPNKWQVn+7QjHgeXLbW64\nwQACwCA/397pONeVx9o2WCkDv7bWIQgaTxrK+le0JQoKCqiurgZg/PjxHHnkkdx5552NjhFCIIRA\n15u3m2fOnNnq42wplOXfVnFdmDiR1dNcJk6UD20bwmHpK5TuQo01axqfNm0anH8+/Nd/QXm5PA8g\nP99G103AQNfNZicNheJAk7rNG+7T1mDjxo0UFhZy44030qtXLz799FMqKiro27cvvXr14je/+U3D\nseeeey7V1dUkk0ny8/O555576N27N5Zl8c9//rP1BrkPKMu/LeK6UF6OiHt0D0xe1qNMyLGIRuHR\nRx0SiSS6LvB9j7lzqwDo0sVh2zabkSMtkkn5NvG4XAFYFuTlWfTuHVU+f8UhQ+o2x/PANCEazaxU\nW5r333+fqqoq+vbtC8ADDzxAp06dSCaTDBgwgKuvvprCwsJG59TV1XH++efzwAMP8POf/5wZM2Zw\nzz33tM4A9wFl+R9m7JGl4zjgeWiBTxiP/oGD58GGKpfi6q0Q6AgBui4YNGg68fgANm++l/r6cnr0\nyLyxYcjVQpq8PItu3cYp4VccEqRuc3xf/us4rXet7t27Nwg/wPPPP09ZWRllZWWsW7eOWCy20zkd\nOnTgkksuAaBPnz589NFHrTfAfUBZ/ocRe2LpuC5s2GpzY8hEFx6JwGSxbnOu4XLjzHKMpMcx/y2o\n+T5oGhiGj4YPgGF49OnjEItZ6DpMntx6lpRCsb/YtvwdpH8P2YZKS9OxY8eG/96wYQOPPvooy5Yt\nIz8/nx/96EfNxtubptnw34ZhkEwvqQ8RlPgfRjRn6WSLc2ZysJhhRJlV4bCj1OayGoshWydiTJcn\n5/wL0hHCGqT2fnV03WTECJvTT2+82atQHIpYljSAmgYntDY7duzgqKOO4uijj+bTTz9l3rx5XHzx\nxQfm4i2IEv9DiN3F0e/K0klH52zdmpkc3sLiua4WdhHgwI7SzMmdFxpsvySJ0AK0JJw6WSNxUV/q\n/1nG5pq5DOwzhiMTx/PJJ5eQSNQoH7/ikMWyDryRUlZWRmFhIT169KBbt27069fvwA6ghdCEEAd7\nDM3St29f0Z6auaTj6IPg25Ov0kJ/eYFLUY3D6gKbs8dYeB6EQiCEFH/ThMpKGDNGTgjnGi7PX1rF\nf7AdOnemzjqa2jcmkb88IG9DGD8Q1J2aYPUkgcisVhHoaFoOpaV7ngymUOwL69ato2fPngd7GIcV\nzX1nmqYtF0L03cUpDSjL/xBhT+PoLQssMs7/HrpJmR9lSSCPHTECunaFggKYMwfq6+Fs4fKKX07u\n3DgQIDSdI57O4fM7ppA3pAaWLUOb+wL/LhGIMA0+ISFA0wKSSY/58x02brQaltfN5QIoFIrDByX+\nhwjpOPq05f9tyVdDtjp0S/l3QsLjAt1hqWZhmhCJyGPLy6XwCwE2DiYeOgEC0EQACY9Zk2q4ZopN\nr5f/Gw1BXjVoCRpZ/smkTjJpMmGCTSwmVxTP3eby3iSHNwK7IYRUTQgKxeGFEv9DhObi6LPFFDKR\nPvMMm2jIxMBDM02uqbTpUJMR3YkT5XFpj56DTRIDIxXVIwCfEG8ENhcsrWLrNR5mHXh58OXjJdSc\navIJx/PWhkvIz69h5UqbNWukmpfFXS5+uJzLA497MBkYj+I4VqPxfVvMtZogFIpDgxYRf03TZgCX\nA/8UQpzRzOsa8ChwKfA18BMhxHstce22RF6e1eDqaRrWOXRo483cZ0dEiXSVPv+XajLumLo6l3PP\ndSgutlm50qJHD5fupQ5/XHkpP177ApoQ+Gg8wzC+7g3cOJ2PNCFdPQHkJNby4J1vsnatDPfs2xey\nS5Wcj4MpPHR8BB625rB1q0VVVeNIpA1VLlYTlT+QSTkKheLbaSnL/xlgMlC1i9cvAU5N/Z0NPJn6\nV9EMrgvjx8sM2yCQYgmNI31OjVi4WI3E9PXXXXxfbhpPmmSyaVMlJ588Bk3z0ESIHXeGyVvlQ8jk\niGERKm+oIvB9KfwCMMAgwe1Dx/BYVSWbNlmUlUH2vvvCwCYIm2i+h9BkDsGS6aDrcHbgch4OdRRw\n48wxkGys8rsKVVWrAYXiwNMi4i+EWKRp2knfcsiVQJWQoUVLNU3L1zTtP4QQn7bE9dsSaes4Lfy6\nToMvPxJpLJITJ0L37i7FxQ6rVtls3OjQrZvcNAaPvn3n8MUXqcca1I67grw/fI0xeDCRCov166v4\nNPv/QACaDqeWLePR0gFs+2QhrmuRXTbcxWJAEOV/KhzexGbJdAvfhzN9l9cpx8Qj8HX0wAcRUNc9\nTu0H48kvHI9tWxQXu/Tq5bB2rY1tW2o1oFAcJA6Uz/8E4O9Zj7elnmsk/pqmVQAVAF27dj1AQzu0\nSFvHaeG/8EIYPDgj+uPGZY4999xpnHnmKDTNJ5HIoWPHSoTIbBofe+xg6uoWy8eEyJ/4CqzyYfFi\nKCqic2GE7dtnIgIPkYSjNgi+PB0wQA883njD4X/+R7p/giCzh/BFD3ihJ/TpA6GZ0pJPbyqH8PEJ\nCNDZfrnGxtsDAuN1jJWLOeWUSiZNGoMI4uiBQWEwmScWVXxr4ppCcaCoqamhvLwcgO3bt2MYBsce\neywAy5Yta5Sx+23MmDGDSy+9lM6dO7faWFuCQ2rDVwgxDZgGMs7/IA/noNA0kWvw4EysfigEw4bJ\nFUC3btPw/VsJhQIADCNO16415OdHWbXKobraJhy26N27SG4iv7iVvFXTwffx6z2WP+gQPWscJ5yw\nkNpah6//VsCIVbex9iGPQECAybvv2vi+FP0rr4S//hVOP93l4YfLCYc9EgmTO++M8tvfWo02lXUE\nX/SC9aNBM0DTBL4f57PP5gBxND1ABAG100dy+feLmGBajRLXlBtIcTDYk5LOe8KMGTMoKytT4p/i\nY+DErMddUs8pmtA0Zd1xMq6d6mqbqVMt/vY3l0mTRiHrMkg0Tdbmj8UsBg2ystwoFpZlQV8XPzSL\nwPdICJPb59osnQtgpf7glVAR09+qIncQPPdWhLVr5fNBAKedJnMIcnOrMM16dF0ghMegQQ6TJlks\ni1s8I4ZTIaaiI6grDkAXaJqcPILA4NgdJdQlFhBooCchf3lAt9MdolGr2agm5QZS7JYDZCnMmjWL\nKVOm4Hke3//+95k8eTJBEDBs2DCqq6sRQlBRUcF3v/tdqqurue666+jQocNerRgONAdK/F8ERmma\nNhu50Vun/P27JjtlPQhcSkszlvbYsVF69XIQItnIF69pd/DEE1ajEg9lcZf4eAfG22BZPDssyvqp\nDm8Im6Xs/EN5y7f4c0eLcf3BuQ8KC11KSuSk89ln8JOfVBGPP42miQZBr662qayEmhroVxBBHzML\nPI+8NQbJhIYuEgihs/6VO7CffJyOpwTU9ob8lZD3YQ7Ydubzui7OeIeyuM2SwFJuIMW3c4A2jNas\nWcNf/vIX3n77bUKhEBUVFcyePZvu3bvz+eefs3r1agBqa2vJz8/n8ccfZ/LkyZSUlLT4WFqSlgr1\nfB6wgWM0TdsG/BoIAwghngJeQYZ5bkSGeg5rieu2B7p0cdj8YRy0AEScsjK5WappISCROkpjxox8\n/ud/ZBnmUAjOES7zg3I6LPDw3zR5dliU+lKL3+dafPNN89cSQmYGA9x4o8txx8lJx/dDhEKCIEgQ\nCklrPgg0XnppOI8+apGbC7fdBr/fANf8fijf96BT3wjbN8DatQ6vvWZzo1/F1sH1dKqGbs8hNzSe\nrATLkjWNVlWRf/sMzl/pMz8wGahHec+0WrVSo+IwZ3eVDluIBQsW8M477zSUdP7mm2848cQTGTRo\nEOvXr2f06NFcdtllDBw4sMWv3Zq0VLTPDbt5XQAjW+Ja7YbUcjb/hFr04wKCEIREwFXFtdxyC+Tl\nXcbnn/8VEPh+DsuXS/98z54uI0Y4FFdvpcMsWdM/iHusn+rw+1yLykpYsQKefhqSSVJCLi+p69KC\nBzj5ZIdEwkPXfTQt7V6S2zBBoOF5ucyfH0EImUn80ksujzwiJ4t3MelbGCFiWUycaMnVy6Sn2RIS\nbE1CyR2Q9z5QU5OpaeTXo98v6D0Wjn7f474LHXLGWw0rArUJoNiJA1TTWQjB8OHDmTBhwk6vrVq1\nildffZUpU6YwZ84cpk2b1ipjaA0OqQ1fBVLoqqpg5kxIJsnTdU4ZBBtuB6HDMX0eIZGo5LPPfDQt\nxH/8xzA+/zzCpk0WZ5zh8tBD5eTmeujFIXZUGxxZDQlh8oawqa+Xwv/AAy5Dhkh3Tk6OxZgxMrRU\n16Xl77pw++02999vEgp5BEEI8GXtf83g5Zdv5rXXIsRiGSEuKXEIhz0MwyeZ9Fi1yqF/f4vLC1xO\nHDiGUDgBGogwbB8ER2zI4f0Cm9pVDr7voWmCIAS1ZRp5m03s8bbcilCxoIpdcYBqOl944YVcffXV\n3H777RxzzDHU1NTw1Vdf0aFDB3Jzc7nmmms49dRTufnmmwE46qij+Pe//90qY2lJlPgfSqSFLl2U\nB0AIEvk6QgvAgEDISBldFySTgu3bP6SkRP4GPvjAITc3VRwOqH10BLXPdeV30wuwfQeEvMSKFeWA\nR+/esnooWIwaJVfPt90GJSWwcqXF2LFRSkoc6uoKGD16NJrmAwa9e0d47DEr1QwGrrsOVqywSSRM\nhPBIJk3WrbPpH3IpGlOOecs3jWJ639HO4ia/EmMUXH76Vno/FEIPQUgPkX/WMPhZJPNDPkBLe8Vh\nygGo6VxUVMSvf/1rLrzwQoIgIBwO89RTT2EYBjfddBNCCDRN43e/+x0Aw4YN4+abb1Ybvoq9IC10\nKeEXmkbSyGHzsbfhJ34PwscPwgghMIwkhhHg+wtYsWIxq1dHKSuz8f2s4nDFEbaug0minBAeHib3\nnTE09bpPENSzfXsVNTUWQZDJJl62TA5n3TqLWMxiyJCJGIbs+wtJzj/fYdEiq5HB9YtfWEyZUkn/\n/nNw3cHcfbcFjiwy1HmetPaFCUnf5IF5lRwVwPygHHONR81Yg8qSEfwjJ0LOCKvxb/lAtmtSKFKM\nHz++0eMhQ4YwZMiQnY5bsWLFTs9de+21XHvtta01tBZDif+hRJbQ+brB08FwZiUjLH3Uose8qxoi\nbwCGDh1Pnz4LMAxZcnnZModf/GIcr78epVs3pyHs89WRE/lVIJOvBB7fWQmJhIFp+oDg009nct55\nEUzTarTgyE4w27rVRggTkJPKtm02ixY1Xmn/8pcuK1aMQQiPs85aTGFhESA/T956j5JxBuvHDWfE\nfTKE9B4mNiSFdYoBsa7M0i1mz27i2TlY7ZoUijaOEv+DTWozc3WBjROHs2cN5fTt8EIswq3TLAIB\nCIjFrEY+9lmzxlNcvLjBzfLeezaeB4sWWdi2xZsPuBz93kT+4RfgYSLw8HUTcUqEefPg8sunpmL1\nk3TpImPts7YaME1ZX0hqrUVdnaw4um2bzUUXWTu54GtrHcBLuYZS/QiscQ3CnWfbRB2LVFQcDjYe\nJprm4Wsmi4TdsPLYybNzMNo1KRRtHCX+rcy3BqqkfPwi7nFcD4PTH9H4dzjJ8gKT7mURjKczkTjZ\nnIPLgHUOs+6qxCyTJZfXr5f1/AsKYJzt8oon6+wMxOQOrZLj9BqOusIm/xKL2U/BwIGzCIU8QiHZ\nO6BbNzm+pvWD0qQrjj73XGMX/Msvu3zwgcP3vlfQfD+CLOG2kRNGadzFxmGsUck9I2rYUWqzYoyF\noTw7CsUBQ4l/K9I0UOX11126dMnq0Zvy8WuBz9clAaEw6IYgmfQIAofJky1GjpQTgGFAIiGFP0o5\nOXgE603evy3Kl7dk/O+OA99PZOrsaJrHJWfWcF31OPy/gjkPKistNmyQm7nFxY378+7OyM52wRcV\nuZx7rgzv/Oork44dKzn55G/v+fubS1xuf7GckPAgZPLljyqhi8Prr6dXLcrIVygOBEr8W5HsQJXu\n3V3q6202b06gaWFKSqQrBNMkiHscUZ3OiE02RMtEIi49e2ZCMkePhgFxKeyG8CHpcfQKh6KKzCbp\n6tXworAbXD1ayGRHmY2/PGOt19TAuHGZsg57Q7YLPggy4Z2IOF++NIduV43H/cRq1GcY224oP31H\nvYMhPAx86k6Ns7J+FMFmH03TGTp0CscfX7HTNVWYv0LR8ijxb0WyreSLL67CMGRh/iDweOKJKpYt\ne5Kht0X55yNVHBPbzo6xsL6kM6/FIpxzDrzzTjmhkAzJNIwoQlgNvnKBrNEzdIbNxKzIyJoaWKZb\nlAdRBmgOPW6yOTViYc5quYCZ9LUmTbLp08cEESecDCh7eQHJ3y/m7iBKEMDtohyhe2g5JhuGRvE8\nizeEzX+mfP21fTQCQ2YpCxHw/vuj2Ly5iH79Mgo/bRoNq5+cHBXmr1C0FPrBHkBbJm0lT5gAV1zR\n+LV//hPmzoXZL6+m/LrpXFA4l6Gxufz6uRl0XAU7djhoWqah+8aNDr4PdYXwmyFDeaRwBOVEecuX\nVnYa25Yi+Y5hUZk7jtJSsJyJ/K3SZcQI2RGsOVxX9gdw3d1/riVLXKZOncj778PYsVE2zbyQM8bq\nfGdtgEh4nOs7nCfkCkUL5HLjfBxMU05MA/Uoz/WcwCdXXNfofYXwmTrVIZ0k6bowapTcgO7Rw+WH\nP5zI9OnuHo1RodhbBgwYwLx58xo9V1lZya233rrLc4488kgAPvnkE66++upmj7Ftm3ezOyI1Q2Vl\nJV9//fVejnj/UJZ/K5P2odfVRaiunonvy+ic+fMjFBa6jHh4FH8P+3ycgN5j4chYgv6Bw4vVmaQp\nTTM55RSbM85weeCBTJG3Z++KYG5qbMlnu2UuL5BJVngehSGTdUJOFrNmNbag9yaJdskSl6+/LufH\nP/a4/nqTu+6KEts4nrxNi/E1uRpxkAPyMDF02We4W8SmshR+9jNY4lt8ATyWdy+GQGb+ChDCYPly\nm83PuQxa4bABG9+3KCzMlI5IJExGjowyZYqlVgCKFuWGG25g9uzZDBo0qOG52bNn8+CDD+723OOP\nP57/+7//2+drV1ZW8qMf/Ygjjjhin99jb1GW/wFE04axZs1PueOOhcRiFiUlDlo4KTN3Q1BbAgnC\nOMgY/bFjo1RVTSA3N0q/fhbDh2d87KGQx+DBTrNCbVmy6UtRjdMoNKdfwmmUKJumuSTa5nBdmD7d\nQdczY+jTx+G6SgtjYZRtP53ApWYUXYMLdId3bqxEu29Cw2xSU5OJXrqipApDy7SQDAKNysrJHB2D\neX45J069lxtnltM/5FJa6hAOxzEMn3A4Tq9ezi7HqGhf1NW5bNkykbq6/V8OXn311bz88st4qb6p\nH330EZ988gmlpaWUl5dTVlZGUVERL7zwwk7nfvTRR5xxhmxf/s0333D99dfTs2dPfvjDH/JNViXF\nW2+9lb59+9KrVy9+/etfA/DYY4/xySefMGDAAAYMGADA/PnzsSyLsrIyrrnmGr788sv9/nxNUZZ/\na5G1S1lXKEsq+L7HaaeZ6HqEwkIoooBQQhAIWd/+n9vP43+veoBlL1oQyNj+006TNXu2bJlIaWkB\nX32VKaFw0UV2I+HfaWM0e9MhZLJE2Bj+zn7/3SXRpt9361ZYvtzm+uvlGIQwGTHCbsgF6GZZPF7q\n0mNUOSHfQ/uzCSMzs5Nty+Qx34ce1dvRE7IjgSbgn5X9eeWViobkLz21oT1rhMOCvgXoeoAQoOsB\nX35ZoMJBFZmigKnw4t69o7uMMtsTOnXqxFlnncWrr77KlVdeyezZs7n22mvp0KEDf/nLXzj66KP5\n/PPPOeecc/jBD36All1TPYsnn3ySI444gnXr1rFq1SrKysoaXvvtb39Lp06d8H2f8vJyVq1axejR\no5k0aRILFy7kmGOO4fPPP+e+++5jwYIFdOzYkd/97ndMmjSJX/3qV/v82ZpDif9eskeRJ038KNtf\nG0QQ1GMYsgHKZcVV9Kp2+PHXy/h4CnzeH76zSOMPiy6myxSL0CuZpu0bN7osX16OrssbvGPHSj78\nsIZTTrHp189qGE9BQabjV8Z1k/EBGbbNRKxmx/5tSbTZHyUUAiEs7rorSp8+DiNG2I02ZyG12gg8\nCHauxWNZ8MQT0vXzTawzxWOhrgTyqiEWK0QAiw0bdFO+R8pddOHxDps360CAEDo//3mNcvkoqK11\nCILMvlhtrbNf4g8Z109a/J9++mmEEPzyl79k0aJF6LrOxx9/zD/+8Y9ddupatGgRo0ePBqC4uJji\n4uKG1/74xz8ybdo0kskkn376KbFYrNHrAEuXLiUWi9GvXz8APM+Tv+UWRon/XtCcb7yw0JVtErNj\n27P8KHXd42xP/rWhAQo+3FU9nbxYQC2CTY9AEIa6YsGGLQV0qIHhw2HqVOkHLy52ECJzg598cg39\n+4/baTxpi3qnLNmswP30w+bYVXx/tksIZDevrl0tCgosFi2S192bWjwVFVBUBH+4KcKw2EyOinkk\nMKkiAsDbwuK5m6JEujoN5+a/uBW9KExAEsMwKS5u/J6K9kl+vt18YuF+cOWVV3LHHXfw3nvv8fXX\nX9OnTx+eeeYZPvvsM5YvX044HOakk06ivr5+r9978+bNPPzww7zzzjt85zvf4Sc/+Umz7yOE4KKL\nLuL555/f78/zbSjx3wua+sbffdclkShvaJDee/Uw8vpGGglgbR8NoQdoSL/2Ca8FdIoJNKTVG4QB\nA3yhkdOnpkErp8t2u1SnNn6ln13e4NlumPR4pEtE1udvySzZploekRq96w3i3dTiSVes/n8bLWIs\nxMbBIdNZTNfh1EhqJkrNbnmeR1GRwZvDRnBsn8h+W3eKtkFenkXv3tGdja/94Mgjj2TAgAEMHz6c\nG26QbUrq6uo47rjjCIfDLFy4kC1btnzre5x33nk899xzXHDBBaxZs4ZVq1YBsGPHDjp27EheXh7/\n+Mc/ePXVV7FTP9R0GehjjjmGc845h5EjR7Jx40ZOOeUUvvrqKz7++GNOO+20/f582Sjx3wuaCmFJ\niaxFDz5B0qd22VTyfpEKpUkJYP4Jtej+7/E1EJ5B5/mJ9B4nedVIv7cAQcZ/7ro0tGiMxSzGjYvy\n2GMyGzcWsxqEN921C+R40u0UWzIZqjktnzhxN1WWd7GMaFqxeikWf9MsdB10IT/P5MlZp2bNtkdW\nwztjuvL7XEvF+isaSJcdaUluuOEGfvjDHzJ79mwAbrzxRq644gqKioro27cvPXr0+Nbzb731VoYN\nG0bPnj3p2bMnffr0AaB3796UlpbSo0cPTjzxxAa3DkBFRQUXX3wxxx9/PAsXLuSZZ57hhhtuIB6P\nA3Dfffe1uPgjhDgk//r06SMORd5+W4j775f/1ta+Ld58s4NY+IYm3nwVUVuIEIYhD0gf3KGDqD1D\nF5sjITHljBtFAA1/s7hRPFB4i/jLo7eI2tq3G9581i1vC8OQwY+aJsQtt2Suf//9ouE1w5Cvpcdz\nIL+DDh3k9Tt02LNr19a+LZ555n5xxhlvi3Rgp6bJ86dObf4zrJr6tvDCHURSM8RXdBDn8Hajr1fR\ntojFYgd7CIcdzX1nwLtiDzT2oIv8rv4OVfFvSm3t2+KjRbeI2lJzJzX86Jb7ha9JpfZ1Q7zGQJFA\nFwJEAl3cw/3CNFOHZylqMqeDON98u1lx3RfhbQ2yJ8Hd0TBJLjTEq692EGec8bYwTTlx7er89Ofs\np78t/tO4X5wXznwfd98txMCBctJIv/9HH90vJ1DFYYsS/71nf8RfuX32k7w8i1jI4s2zI5x/tkO3\niA2WjMIZN8PmNREiTEAiCPF/DKY/ixvKK3/nBzaPXyK9G8dvdegalxmxuvCYVeHwXNedC50dKuXt\n96bKcnZUhml6jBjhcOaZmSStuiXTqN04h/xTBpPXT9b2SXt8lgQWSw2LETfBxV2hthbSOTfz58OK\nFS7XXSc7k7VEuJ9C0V5Q4r+fZCJuLEzTIhqRUTWOI8sSpJueg2ANRdxOJVdrczjlzsH0vyrjv39V\ns3ktMAnjkQhMdpTajNu5xhlw+JW3l1EYJsmkzE945hmb3Fz5HQ06YRpfHvdTgi6g/3s+vZdAXr8K\nLi9w+UZzeEO3ec+0iKTqF2UlXwKyDIbvy6Szlgr3Uxw8RKolomL3iHTnpX1Eif++kBXsX1WV6YCV\nnR371VcuP7lhPF+vSNIpJgjwGUoVEWaRi4f++GKW7CjC8yx8H97SLC4kyvk4LNJsjpxjMb7o8BL5\nXZGXZ7F6dZRlyxzee8/m/fdTTRpzAAAgAElEQVRlqWoh4Lgb5tD9J6T6E8NHy+fQWy+iaEw5ZwQe\n9xom71dGKUp9EYMHS4s/TToaStNk0tm2bbI3geLwIzc3l5qaGgoKCtQEsBuEENTU1JCbm7vP76HE\nf2/JCq73QyaxQFbbBBmtUlAAI0e63H9/OeEBcdb8KKDXWJ0OMZMTToAOnzYtdmY1RO68p1n8LSH7\n6eoLYPHitlPFsm9fi1/8IvNZ0zkJC6oHc2pifkOW80MzBvPbdQ7dUn0OwponE8dSoaAVqdXQ00/D\nihUyNHTBAlmtbt68CB98YDF8OA0rBcXhQ5cuXdi2bRufffbZwR7KYUFubi5dunTZ5/OV+O8t2cH+\ngUc/4bAIC02TyVk1NdCrV7oGT0BC6Py55EKe3Tiex38F2phMbeVuEZtoVucskK0TFyxoJlnrMCd7\nryI7G/mFTRX0fwHCO+awoHowc9ZXMNByiewmUQxg1app/PCHI9G0AM/L4dVXI3ieTJBrWrxOcegT\nDoc5+eSTD/Yw2g1K/PeWVLC/iHvEA5OFqQqW4XAmAepPf7Lx/RCaFiAIs+Xk8Ux0LIosoKjxbq3l\nulg4gHw8fjzEHZd+CYclho1ttx31yt6rKCrKnvQqKC+vaND6UyMWRHa9qz1tGjz6qMujj45C15No\nGoRCcUpKHGIxq5ELTom/QtE8Svz3lpQJ+8IYh98ty2SmnnIKBIFs0/joowXE43IzxvcFr7wCl12W\ndf631FK2gKhWjoaH0EwMouxLx61Dnaab1tGozPxNI6dEi4LVkGoG1lBKY+lSm5ISB12Twi8E6JoG\n2BiGfKx6ASsU344S/33BsphXZrF0WfaTLl9+Kds0go5hCDRNYBh+qgRxM/Xnd1FL2Uh6kKpq2Z7M\n11kpj9jMmVLAk0np/tJ16N3bZdIkGdL5ox+ZzJ10m6yIiswOPqkSPnwZAk3uKVRWtpuvTaHYJ1Q9\n/30kEpHWZZqBA6swTQ8Z2umjaRrJpEEyabJmjc3Wrc10yUrXizCMjKna3HPtgKbzYCKRqf0fBFBY\n6BD4ccBH1+P8uMSheCycPANKbocTXw6wcVK5w3LvRaFQ7Bpl+e8jliUFq6oKZszI1OJJU1NzBUFw\nFitXyno8q1c3swm5q4ytQyGL6wDTqPVAaGfLP7GigNCPglRUUMB3N+eSF4P8mJxuk+g42Oh6u5oz\nFYp9Ron/fpD2W0ci8O67ETRtJuDheSbjx9/Npk0WQ4dKa3aviqAdbllcLUDTeRAykUFz5kD312s4\nY6zOv0sCjl6ps7ljIcexlDAeAQYjeYKjB1pciMwFaGdfn0Kx12j7myXWWvTt21fsrulxS1JX10xd\n/r08tq7OZe5ch4cftlmzRlar7NsXqqul+O+uP66ieVwXxtkur3jlhPHQwiaTLovy4ovQP9i5JHRO\nzs49itObySr+X9HW0TRtuRCi726PU+K/d+3glixxqa8vxzA8wGTBgig1NZnyA03LFqfr6w8bpoRn\nf3Bd2FDl0mO7w92v2Lzlyy8y3WQmG8OACRNkH2PXhQEDIFUZF9NsV3voinbInop/i2z4app2saZp\n6zVN26hp2j3NvP4TTdM+0zStOvV3c0tct6Vorh1cc6QbmIM81vfjHH30eBYtchkwQL5uWalIE1zu\nYSJnC5dEArp2VYKzP1gWRJ60iJ41jrd8q8GV1pS0z7+gQPYdqKrKtMQEuZGsmr8rFC3g89c0zQCm\nABcB24B3NE17UQgRa3Lo/wohRu3v9VqDb2sH57qyY1dJiUN1td3QwBzi6HpAnz4LKC5ezJ13RtlQ\nBZbj0GVZAa+LMZh4eJgM0qJtKlnrYFJQIAU+COTKCuTjCy+Uvv6aGnnM6NGZzeNQSIo+yGQ8tRms\nULTMhu9ZwEYhxIcAmqbNBq4Emor/Icuu2sG5bqZOj+d59OhhAlHuuivK6MgYupctwzAChPC4oncV\nQ56eBYHHxWgIAgwCBB6PXOFwljL79xvXhdtuk0KupeL5QVr648dnVla33ppx8yQScNVV8r8/+QRu\nukmtwBQKaBnxPwH4e9bjbcDZzRw3WNO084APgDuEEH9veoCmaRVABUDXrl1bYGh7TnPt4Bwnu06P\njxAexcUOW2bbDFq0nM0l0voMkiHyq4GEdAfpuk5gGPiBhm6anHW3fUA/S1sl24UjhLT+r7wS7rzT\n5fjjHZYssVm0yCLWxOz417/gnXegLO6y7T2HuZts1uXv3CtBoWhPHKhQz78Czwsh4pqm/RSYBVzQ\n9CAhxDRgGsgN3wM0tl1i27JOTyJhIoSsRV9dbXNXjwf5aKSP0EETUDP5bObGItzGLDTNw8gx0VMN\ndVcX2LzkWNgoodlbsipnN/vdCQEffuiSSJTz4Yce8bjJc89FWbeu8cH19VL45wflmIGH96DJw3qU\nCTmqH7Ci/dIS4v8xcGLW4y6p5xoQQmTnW/4BeLAFrtvqWBZMmWIxaVIU05Q+/1jM4oghnxCEAQNE\nEoy8epZiUU6UC3WHayttiiqs5kr3KKHZQ5r77iIRmD698UZvUZGT2qvxCYXkymztWgvDkCuDcFi6\nera952AGHiF8BB79A4elnqUifxTtlpYQ/3eAUzVNOxkp+tcDQ7IP0DTtP4QQn6Ye/gBY1wLXPSBY\nFvz85xYDBljE49LP/OV3b6JzYllDDfrPj74JTYOlwuIdLI6ogSKaL92jhGbPaO67GzcORoyAp57K\nHLd6tY3vy836IBmiqHorWzSXyBMWNTWZVcPcTTbJh0004ZEQJot1W2UCK9o1+y3+QoikpmmjgHmA\nAcwQQqzVNO03yEbCLwKjNU37AZAE/gX8ZH+v2+LsysfguliOwzuP2TyxwmL7dnjm8SLmzAvxVUmS\nvDUhvjOyiNzcncvPZ5csUEKzd+zqu4tEZJmMeFxu+n7vexZ//nOUvB1V3FE9gwti07n5jBl81XM4\n+cURYjGLW2+FGTMsziTKBYbDd6+z6fiZxVM3yr2Cujq5yb87N5NC0ZZQSV7QvI8hO2Mr1bWrXERZ\nlLD4hZjIBO4lhI+vGRi/nYBrj2skHGkhKSigkQWq2HO+ZT5uqKnk+zKUc2xiImN7/BefDQzYfgmI\nsAZaLj//eZTqaqtRWKhhwOmnuzz0UDm5uTK81zCiXHSRpVx0isOePU3yUrV9ABwHEZdtA5PfeDx/\ns8Mpf7CwmnbtwuFNYeFg42Ei8NBTZuluyvQrIdkHdlXiKF1ULzvR64SKAtZcGRCYgAZogiDwGNSz\nikErZAmIIlYzWMxhTmIwXxXXEA6nk/U8qqudhn7KykWnaA8o8QdWF9h0D0zCeCQweSJms/x8WD7Z\npqih1KTJEmFj+LAiZDH5kijXdXboFrF3Ugnl6299mrqFjr2ghmRYR9cDaeULjaQXYkz1DArw8dEw\nSYKAgcznP1fe3SiKa+ZMu1HegHLRKdo6SvyBl2osXiSKTVaRsIR8nsooNXMcCgbbTCyystwQFrvq\nsKV8/a1P0yqg775rc9RROYSEh+8brF07nOQMuCA2nRA+OnJBALIE9M3xama8FWXLFof33rNZv95i\nxAhZhkO56BTtASX+SL/83zQLBNg4ACwPWxQUwNljLOJxCy0KV1wBd9+9e2HYVZl+RcuS7RY6cjXM\nu3so/yqG12IRbrnF4rkNLqOYhcBD0zX0IAnISeB7dw3msiKL8vJMFFdpaaY5vELR1ml3G75NyzGv\nnubyx585bPcLeBRZjyepm2x4MspLNRYv/afLeSKzIsjJgYULlaAfUqQ2WUTcI2mYvD85SlGFxS9+\nAW895HK+cHjbtHl6zGq6V8+RRYBSKj9tGowcKXMC0qWgQU3cisMXteHbDHVLprGyfhSB4aPrOfQ2\nKukxagy/9j0CNHQCQgRowqNoRRXHbq/iDjGDED4eJuVEIQ7/GONApa2U4VAhtcmiBT5hzaOoxsF1\nLSZNgqSweBsLPQl/zLcYN6+xaV9TkykV4XkyiijdS1ht1ivaMu2nh6/rUjt9JAEJICAI4tRunMNX\np8f5eIjPl4U+AQYJDAgZ1C19mvgRT1FfKLNCw3hEqCJKOZcvuxd/QHkzTXkVB4Vm+h47TqYHMMiX\nmtt7aXoq7LxZr1C0RdqN5b+lyiHv3QD9emRmrmYQ/l4Jqx6aTxAGPSH4ovJaTln7GUedV89HP16U\neh6Kx0LOOhMEmMjJwFdhPIcOzWyy2Eg3TjwuY/snT5Y9FpjoNOoTadk20WhmIx8aW/5qs17RVmkX\n4u+6MG6GzSteDr3Gxvl3X53vVEymtksNwWYdCAg0nV4F/8tJ2wRboKF2TyBgfslZVMYqEcDQ1Aai\nrpTh0KJJUsBO8wFZCXu6gQg0DJFEC4ewhg3DymqzpjbrFe2BdiH+jgNv+bLw2gXrHE4/zybSz4I6\nF13PSdWF0elU7RMiIL9aZ2vSQIiAZNLkv6sriSF78v6yb5Q7ypqP71ccOjTNDt5yq8OJ9R668MEP\nZB4YAhH30aZOleZ+ysGfnkfq6ly2bNmzvs4KxeFGuxD/tF/3Hc9ipWlx29EwaBDceCMcddRQ3noL\ntr1WyszYGGoK43xRovOXx39OTV5+QyXPdGPw6yotuinRP6RJZ1inXT7XXQdbZtvMEzKRz8dABnx6\n6Ai545ve7U3NGEsCGno1766vs0JxONIuxD/bBVBbCw8+CIWFLscdV0447HHRRSZjX4swrLCSikdG\nQdjn0sTjjB0ra8NfdRWcdZZyAxwuOI4U/iCQf88+C6RKbqcT+TTgp70e5LzSv9KpWpD3gQEzZ0Iy\niR8yefXaoQwY2rivsxJ/RVuiXYg/ZFzCgwbBObgMKxlPOBxvaMNYUuJwIlvRwkl0QyCER2mpw+bN\n1h4ldikOHWw70+c3m6VYLMVC06BXL5fvPjSPzWHBlsDgiGmXcuaf/4oW+Ajf46jlkBgiyz9oWuO+\nzgpFW6D9hHqmuLXEJUo5V1cvIJwI8JM6yaRJsrqAO6pnEEoISIKhhTj7bFvFeR+GyCY8spGL3uQO\n13VZBbR3b9meUzcCklrA3JzOfBOYJDDwMJkbizB2bJSqqgnk5iqXj6Lt0W4s/zRX5TsEmscRsYDi\nu3Q23d6XDzqWMbxkBcc855M7Fr4o0YifPIwB96kf/OFKRQUUFTUuq53971NPpdtzxhFCY1VtKRcS\n4fzs+k4xOO88i3792H1PSYXiMKNNl3doWsoBaFRvua7YYMXDGgFJfC9E8VhBp5hPApNLzSgTHUv9\nztsorgux2DS+971RBIFPIpHDuHFR1qyx6Jt0sXF4O2zzwJtWozBRlfarONRp9+Ud6upcVq4sT/V3\nzURruFh8MaiS4tDT/P3aL/HF+xhGQBCC35eMQIt1xcHmHV/1d22rpI34Tp1qECLAMAJ03eOxxxxq\nXoKLHy4nHHigmxhEVY1uRZukzYp/ba1s7J0drRGLWYyzXf73lNtY/4gnM3h18H3p939pZYS1qXj+\nHJXD1SZJL/zq66FnT5tHHjEJhaSB0HFzAd88NJ6QiGMQIBJextWTVaN7dYHNSxOVB0hxeNMmxd91\nZX33oiIT8CAIsfm/t7LsC5frE1V8U+I1ZPCKJLy34kKefXY8/ftbjB6t2i62ZdJGvBAQi1mMHRul\npMThNK+AcX8Zg54S/iQ6gWbyzXkF1B7vkP96JXmLalhdYHP2GNXuUXH40+bEP+PStygujvLbn1ZR\n9tgMCmLTOYeZ6CT5ulrW7AkEBMkw69aN58knlX+/PZA24tN5AGkGHrMCI/AahH8BF/LhiMGc4Y8h\n2JxyHf4syktPWMoDpGgTtDnxz3bPJhLwzdoP8U5O8HGJ4Ohqn7wY5Meg91ioLYGF1Zdxzu07C78K\n7mibZCf8gUvfvuWEQh4JLcSOpQZHVkMCk4nh8dw3xMH3G7sObdtSXdoUbYI2J/5py657d5eHHion\nx6xnkyYgSFXovFMjb60gLwZHx6CeznxR0/g9VAP2tk064W/LFofNm1PiDux4bAQ7nuvKm9g8ELEo\nLISVK02CwMP3TbZts+nXTxV+U7QN2pz4py27Dz5wyM31ACGbthoQ6Dp1v/4BR1//V0QQ4GEy24ww\n0W78Htmrh/p6WfJF/cjbHvn5NrpuNkSE5RdHyOtvEUm9XlcH//rXUBYvhvnzI2zaZFFZqfaEFG2D\nNhvnX1fnUl09ACHiDc9pWg4lJQvJi8n6/m9ic2qkeZePbcsJAFCtG9swzeaCpJ5fsaIc3/dIJEzG\njo0Si1mEw3KvQK0IFa3F/rqc232cf16eRefOw/j006lI01+jc+dh8gduQTcrY+E1xbJg+HCYOlVG\nhSSTamOvrZKXZzVbuiEdKmwYfkPtp1jMIpnMFAFV94SipTmQLuc2Xdvn888j1Nfn4id1/PoQX/2t\ndI/PjUQgN7dRZ0BFOyI/3yYITJJJg2TSpLraBmRdIHVPKFqLqirpaj4QbUTbrOUPsGiRxarnKplQ\nPJJO1T5HbRgDpxbt0VTaTGdARRvj25bXeXkWHTpEmTrVYfly2dPBNGHMGKiuhpKSzA9T3RuKlsB1\nYcYMubIEaWi0poHRpsXftuGbX9Vw0hpBiIBA93DGO+SM37OY/iadARVtiKbL679VuhTVOI1mgn79\nLHTdoqoKzjsPSkul+NfXw/z5oGmycujw4XKlqO4Vxf7gONLiB3lvDRvWuvdUmxZ/y4Ijp9gEPzNJ\n+h5eYPJfC2zeW6w269o72RFdZXGXHqPKIfDwQybPDos2BAJkGwATJ8rksLRllvb9N+kCqVDsE7YN\n5xou/QKHJWGbSKR1b6Y27fMH+LLI4iI9yr1MoJwoSwKr1X1pikOfdD6IYcAFukPIlzNBEPdYP9Wh\nvFyuDtLU1bmce+5EzjjD3em9sjeAFYp9xcIlqpUzgXuJauWymmwr0ibF33Wllea6cgNlUcLiAcY1\ndHEyTVnTPX2Mov2R3tOZMAGumWKj5Zj4mkECkzeE3UjM0xViff9eJk0qp7Awc9OoDWBFi+E4GEkP\nXfgYyda3Jtqc26epL/fMMxu/3qOH9Ns+9ZRLr14Of/qTzZQpqq5PeyTj0rGgKMq2KoehM2Q57wYx\nd11qPxhP0C0OBOjEebQkwv/G7mKGXsFvL3c562uHgsE2ReomUuwPTarHtrY10SLir2naxcCjgAH8\nQQjxQJPXc4AqoA9QA1wnhPioJa7dlOzm3fX18OGHjV8//3yIx13uv182b08kTN59N4qlfrjtG8ui\nm2UxMZIVAbR6GowcSX4PH/0hgR/SMJIBpdUbKeennMYmfv7q49JKW2xCUZRPuq3ms8/msGPHYN56\nq0JFiil2y+ppLjVzUgZENErdu1XUlkB+IeS14nX3W/w1TTOAKcBFwDbgHU3TXhRCxLIOuwn4Qghx\niqZp1wO/A67b32s3R0FBplqjELBtW+PXS0uhZ08Hz2ucwAPqF9quScV9WraNNc6Sj0eNgmSSvDVw\nxliNzSXfoXv1v8iPybTBq4I/oyc8CGRQ9iexB/kgPhcAIebz9tswb3wRs4Y7dIvYahZQ7MTqaS7d\nf1pOTzy8+Sbvzarky96zCHwPfeWshiZUrUFL+PzPAjYKIT4UQnjAbODKJsdcCcxK/ff/AeWapmkt\ncO2dqKmRYVIAhYUuQ4ZMbPDR6rp8vbjYxjBMhDAIhUyKi+3WGIricCHtK7z3XtI7vVuqHIKkjLsT\nwJExg/nP3UxeSvgB/sz/R9IwG5z+n53ySaO3vaL/07zilXPi1Mz7KhTZ1MxxMPEI4RPGY8eKpwmS\n9WRXkm0tWsLtcwLw96zH24Czd3WMECKpaVodUAB8nn2QpmkVQAVA165d92kwti1/i6ed5vLIIxnX\nzl13Rdm0ycK2ZQJPaWm02ZouinZIkzaNW1K+/1dEDiZxAgxGMpkZegXixO6cuXUOfxKDmRmq4PQ7\nruKqfAdsm2O7reaLD5Y1vG1y8fGYLEcXqvi/YmdcF2LfK+DEIdCpGo6I6fSav4J1gwRBCNBD5Ofb\nrXb9Q2rDVwgxDZgGsrDbvryHZcGUKfDWWw7hsHTtaJrHnXc6nHZaZmN3VzVdFO2QJhttb2Lzlm9R\nTpQBOLyp2SzVLHJyYMDzFcydW8HTD4PwYcjjFtGovK+OT7kO0z7/I7sUoeXMg6Qq/q9ojOvCyJEu\nD9w/mr+HfT5OQNGdPt9ZK3uN/KtEY0vOMPIuaD2Nagnx/xg4Metxl9RzzR2zTdO0EHIfo0kV/Zaj\nogJ69bKpr5dtHEMhk6uusslrzd0TxeFLk1oep2JhzoJlcYulgYUGhAyorJSHT5qU2VeKxzMGvdw2\nqMC2K+jfH/r3ByKqRkh7ZlclRBwHevVyCIU9WW5ewI7ego5rQxwR0wjHTL6cuqvSky1DS4j/O8Cp\nmqadjBT564EhTY55ERgKuMDVwBuilWtJ9+tnUVenXDuKPSQrlddCzgXjx8OCBVLog0DuFzlO4/aP\nmgZbt8K0aTKEeKdqjKpGSLvl2yp02jb86U82yYSJKeLoSchfF+bvdz/O36trZORPReveN/st/ikf\n/ihgHjLUc4YQYq2mab8B3hVCvAg8Dfw/TdM2Av9CThCtjnLtKPYVy5Liv3jxzmHXOTnS4gcZUTZt\nmgwmSE8Syr2vgJ22khrdE5YFt9xiMfuPC7nm7Cq+70HelAh5lkX3AzS+NtvMRaFoCZpbtqczx6dP\nzxTi6tXLpazMYcUKm02bLFXnR7GT5V9ZCStWyNdKS+G222Sf8XA4a2Jogebh7b6Zi0LREmR7bbJ/\nl127Zgq8FRa6PPxwOTk5HkOHmuTmRgGLiRN3/g23wG9bcZiQvZVUUCDFPt0dML1SBPncyy+7HJ+s\nIv/2GeSt8g9Iqzgl/grFHtDUinvuNpf/1B2iwuZ7ZQ5mOI6mBRhGnCBwuOgiaydf74Hs0qQ4NEgb\nDxMnSis/Tfa+UWGhy4DzB7DZi6PfL6N98ta3vu9Qib9CsQc0LQF92e/LuTLw+C/d5DfVtxFKBAQC\n9GTA6hcKGo6tr4cHH4SzzpIbw7vyASvaNrYt3Ttpyz+bq8qqMPR4Q9TPFyUaeZsPk9o+CkVbJzsV\nIEIVoWQ9mhAYmkf/NdWcMVbn3yUBR1brzF9fg65LkRcC5s6FF1+UFUBDqV+cCvtve3ybS8+y5GsP\nPggvvJBxGQKcsg30BCnjAdZXn0n1bZVc1cqWgRJ/hWIPSPtvN1S5DJkue+0JICFC/B+D6R9bzFEx\njwQmjm5z5ZUupulQXS1bQAaBnAxGjJD7Bcrn37bYE5eeZckV4AsvNH7+6xMinD52Bl+XJDiiOsxV\nsUqO7mJxVSuPuU2Kv9pUU7QGlgXHVzng+2iAj8ZMhvEHKlhDETYOizSb436wmp/9bBRB4JNI5DB2\nbJT335dlolW7x7ZDts58W1hn9rEFBbL8TDIpn9d1WHOUxfPvO/SPOTjYLMVi6uDWH3+bE3+1qaZo\nTd7E5mpMBNLKr0JmYS7FYikWAy9yue22kWhaEsMAXY8zfrzDxo0WBQUyRLSqSk0ChzvNhXHuqhR/\n02N/8xuXzz+vQgjYvLmUnj1reKvQ5oE14wAoLISiotb/DG1O/Hc3AysU+8OpEYtLZ0Tpl3BYpNss\n9RvfXMce6yBEJpRD1w0GDrTp0kUKQnrDb+ZMWLhQ3ZuHK011pqamUYWQnUo5pI/t3t3lrLMGoOsy\nS1BWINZ56KEc7rorypo1Fu+/LyeL1jZc21wbx+zerGpTTdHSWBZMdCyO/O04fvyERYcOcumu6/KH\nvGKFTSKRgxA6YNCp02WAFIDsUD/V8/fwpjmdsSwYN25nwc4+9srSKnQtjqZlSs9DQG6ux+DBTkP8\n/4G4P9pkhq/y+SsOFNm+3HRtn+Jil8rKKoSYiRBJdN3EMKJccIHVYPnn5CjL/3Bnb3TGdWHxgy4/\n2WCzbpKHCGf6Qmiajq7nYBjRZvND9pZ2neGramkpDhTZ99qmTfDnP8NFF1l06+aweXMS8PF9D01z\ncByLqip5rPL5H/7src58+ZJDp6RPyR3wyUCNdzgTv6PNBadWk3/KYPL6Wbt0HbUGbVL8FYoDzbRp\nMoYb5L+9etl07WqSTHokkyZjx9pMmQJPPpk5p67OVVVn2wmOA28ENr8kxJGxgJNjJo8bN/F4aIzs\nAW0uhmgRlmUdMKNAib9C0QLMmdP48bPPWlx+eZRP/1bF0SvgiPcbBx/U1bmsXFlOEHjoutmqvVoV\nB56mE7ttw7wQ4Elnj6ELfnXFCvQXZQ9oEffQDnB0ihJ/haIFGDwY5s9v/NgCuj87CxOP0cxiU4Es\n+AZQW+sQBB7ZvVqV+LcNmk7shhFl0SKLBy91MF/w0YXA0HwE8E1gEsYjEZhsKrA5ABGeDSjxVyha\ngIoK+e/TT8Pxx8s47SLHQegeWuBj6B5FNY5sZ+Q45J9XgK6bDQLRmr1aFQeW7Ind9z3+8AeHZ5+1\nmBeyiZpmQ1vP1ztHmKZH6B84LNZtLquxlPgrFIcjRUWwejUsXw7z5sHfKm2KcmTmj2aaMiQole2T\nFwrR+85LqB3UmfziiLL62xDbttn4vomue8TjJsuX2wQBLE5aPFsRJdLVYXWBzV9etViqgavL/tAP\n2Qd2nEr8FYoWomniz0s1FkXRKFuqHN7E5vwVDt3SB/g+efe/QN6kXIhG0t4gxeGM67KlyuGe6Tb/\nOj1Kaals7hOLyf+5QQBYsPhkGD0aqqvlaUaqP/SBjv5S4q9QtBC2Lat2BoH8t6AAfvigxV//aiEE\n9A9BNGRiBPXU9RTUlgjyV8XJU2nohzf/f3tnHx9Vde7779p7ZgfbSoKhFpSCgmgBQ8JLbfdBcWtU\nfK32cNvbak8QPNAqaKNolbanNz21pfU1rdIWVLjMtZyeY6lagQo4soXiVkFICAQU0YKgVJs2AV8y\ne2bvdf9YM5lJSIAYNG/r+/nwSWayZ2bt5MNvrfWs5/k9mdZuCxcyyA9YiUVpbZzf1c7JKeRS3d5O\nPrmUVMpn7lyL2bPjTS41ILsAACAASURBVKZ/dXWf/LC1+Gs0x5BMzWQYwsyZWQMvUNv+2ePjfHfs\nXbx55ROEUTCSIcXHF5LfOcPVdJSMcU9jI0iJCUTxcXB5AZvBg+Gtt9Rmb/x4F9NUZwGRiE9JiUtt\nrU002jlOBFr8NZpjhOtmPfxPP92juDhr6QxqQnhgo039iLO4Nu9PIEJCIagPN2vx765kYn3pWT8U\ngqS0cHEA+P731VmQ68Kkkwt5LzAITUkkYnHqqQ7f+U7nFfxp8ddojhEZD5dhwzzuvruUaNQnmVTb\n++3b1f/us0KPkRv3IK8xESLESEkKZj8Cv9Ylv12Jo7ZucByCiAWhj4hEMK6byqq+ZfStUrbMmSww\nGw9Ky2kYFlA/zqBgeiXOnZ3799bir9EcIzINX1591aVPH7W9N2jkjqtjfPhZmyU3eqzwS7G2+bx3\ni6RhNBRUQX5tEmIxPLT9Q1egPbbwHjZzZJwJuKwXDnPLbK6yObQRS3qHkL81JH+7gDPqYMLHfCNH\nQIu/RnMMsW0YOdKhenOEMBVgpiRfWbqQ/HllTJrm0me+jyEDCrZCwVb1moaR8OagTdxwg0dVlVKZ\nhQu1HXln0R5beNeFvwQ2z0kbM2j9Ws+DnXscrolYmLRi+N9J9DhLZ42ms8nPtymumcqpiwXFsyF/\ni1KFIWUORh9L+T+jRP+Vcqi6H961NzJ3bikjR3qAsn/Wls+dQ2t2zZ4Hc+eqr0e6NpfMLmLaQzal\nMs7u6T/pMh2m9Mpfo/kYyB9fRv7ti5u3dsrEhSoqaHhrNdV3S0ILECBE2CUyQDTZP1Mm5g9th4Fa\nXttS03N3EX/BZslgmzmdr/uAFn+N5uOhLVWwbaiooH7+s4TRVM7eW3SJDBCNIteu+frrYehQj8uL\nY5xQDTtjZdi23ayXA0AYeuze7bJ3r8Ojj6oXjxnTdnvHzkaLv0bzcdGW4bttU7dzHoSzwAgQRoQB\nA6YxYEAZjqMVvyvheeB5Hvfdcx5WNIGRhKI5C6lZ4FJabpNIwBe+4DFpUoz331/E66+nSCQs1q5V\nBVyWpZr8VFWlzf660J9Xi79G8wnjeTBnehGXDr+OA2Phkm+XccYZdtvphbo1XafhulBU5BKJ+mBC\nKOHAmUl2PeLS2GgzYoTHvfeWYlmNCCERgmbhO9+H++9XNR7r1qmc/67yJ9Tir9F8wuyMZVM+66SJ\n9zKsB+Y4cHbK5faIwy/Wppt6ZE4MEwl1UDxvXjZ5XPOx4zjw+987pJIWlkxgpOD4LVHuq3WQEkpK\nXKJRH8OQSAlhKEilLKqqHED16Q2C5n15tfhrNL2Uc3Gx8Hl/ZMAr9wYUWPNp/GARj50uKawN8FMW\n/3lHHPdim6v3uAxJJJR6hCHMmtW1lo+9gG3bbG6evYYrSmL0q4YH6stYH6rff1WVcvCU0icITFat\nmsbTT5c1VXVffbVq7alj/hqNhiFlDqlHLP5R0kgYlWBIDOnzQQl8rlYi8THWuSz5B7w/Zg+3jBD0\n2wYC1DKyKy0feziuq5wbamvtJkG//PIFzL2pgrVrJ7N8+Qxmz45TXNzcwRPURm3UKOXx1BWjdlr8\nNZpPGttmyXVx9q+LUZJchCFTCBnhU1WSJAFJLHaNKGyyiNh0tcGY2XDCDonIywPH0ccAnxCOA3l5\nKuoGcOmlC7jllm8D8MUvrmIou5j351+wfbtNEGRfZxjqdZm/T1f8G3VI/IUQJwD/DZwC/BX4upTy\nn61cFwA16Yd7pJRf6cjnajTdneFlNt9ZbDPstjLGjXOZPt3hne/C0p+4/L+9DkPTsWTTDEhJuHfM\ndC4aNBinwsHDPmr7AU3HyM3YLSyEgwdVs2YhAAnXTryHDcuvorHEbvLnNwy44AKoqOjaf5eOrvzv\nAOJSyp8LIe5IP769les+lFKWdPCzNJoeQ0ZUli+HE09Uz71XZHPzOzY+UNhQQxgKpDRIpSyW15Rx\nykwbx4bY9U0Owl3uELEnYttpYzbX5aVTS3ifVZC27u6/VnIeLj+vtvkyHg4u6w2Higq7y/9NOir+\nV0LauxQWAy6ti79Go2lBGHqcfbYK7Rw8aPHkk3GCwObrIxcwY9YshBEQygjz5lWydatNeTn06ePx\n/vsuI0ao+HIk0rUOEXsiNQs8vjCrlEjgMy5qcd9F11B69n/Rf62k//I+rMHhS9IjTikWPqG0sIjT\n1duzddTb53NSyrfT3+8HPtfGdX2EEBuFEC8IIQ4xvMsghJiRvm7ju+++28GhaTRdm9dey4Z2IhGf\n995z+Rfh8bOSGzCjSQxTYhgphg/fTBgqq+iBA0uZMuU/uPfeUkaN8pg6Va/6jxUNDR67d8+loSFr\n4FOzwGP/9RWIZAIRKqe3+mWjmHn7X/jtip8y5eQ4LwobJ53BFSEgKv1uYcx0xJW/EOIZYEArP/pB\n7gMppRRCyDbeZoiUcp8QYijwrBCiRkq5q+VFUsoFwAKA8ePHt/VeGk2P4LTTHA4eVGmCqZTF5s0O\nP+4Xo7AqYG8A0gAhJJMmLWT1anU2EIn4CBEgpc/YsS5jxtjMnasPfjtKQ4NHdXUpYehjGBbFxXHy\na+ELs0oZESYwCUlhEAiL9RGHDYFNtWVT+SNYXg5rGx18qZq2G3ldLKezDY4o/lLKC9r6mRDib0KI\ngVLKt4UQA4F32niPfemvrwshXGAMcIj4azS9iQkTbGKxOKtXqzTBbdts3iVG/rsw4M/w9hUgDIhG\nA2691eW00xySSYtUyicMI/Tvv4cHH/SabARaO/jt6VlBx+r+6utdwlD1YAhDny1bXII7YWLKx0gL\n/zNcwM+MCr71gM2kOnUAXFenmq/X1dnsKoxTVHcMBvMJ0dGwz5+AKenvpwBPtrxACNFPCJGX/r4/\nqoVBbQc/V6PpEZSV2dxwwxxOPtnGMGAxZSTI48RVYPgQpAwMw+KqqxwmTLCpqYmzYsV0pJRcdtlD\n3HVXKWec4TUd/ObieTDH8XjvB3OZ43h4Xuuhje5Kpvj5P/5DfW1pt9weCgocDMMCTMBi/o2F7Fi1\nB19GSGGSIsobDOULqRpOfGQulxd6lJerzy4vV3pfNMOGOXO6hfBDxw98fw78jxDiOmA38HUAIcR4\n4DtSyn8HRgDzhRAharL5uZRSi79GkyZt9Mm6dbDBt7nYXMPt/V1eeqSQARfU4fsOr75qU1cHhYU2\n777rEokEmKYK/5SUuLzxhn1IpKHJRgIf37f40/JKksny5qGN/O4hVK3RnqYrRyI/38Y041RVuexb\nWciC6nIsfFKY1JxyBSP+uoLpLMAkJHjJIHw5j7EyzvrQ7rYZVx0SfyllHVDayvMbgX9Pf/88UNSR\nz9Foejq5+eT19TZX3m+TSoH8g8oplzJbOPTVr6rwT+asoKHBaTXkc27OIaTEZ8SJS/lnTmijvt7t\n1uKfaaTyUawTWoaL1C7CJpGwuYO5TfYb/ygJ+az/FtHdAaYMkUCEkCD0Od90eUHYXc624WjRFb4a\nTRchI94TJ0IqlX1eplMfMuZgffva3H57nDPPdKmqcnjtNZsf/ODQ9xtS5hAssgh8H8OyOGXcZBqC\ndU0r/4IC52O/p4+TIzVSaYvcHr2RCEydqp73ffXVxaFupMkr9waEUYkQm3hvCJz4Z0G/WkkKA2FZ\nfO1XDsfVdZsQ/yFo8ddouhCuq0Q+F8NQzxmGWuGWlQHYzJ9vI6VqIZgbdsiuam3sNVl1zLdtihuK\nqK93KShwuvWqP8NHsU7IDRcFAcyfD9GomgiSSXgBm1+NncaF1nwwJFKm2H8Z/O3iKB8uvZkRFDCk\nzKHItrt1SEOLv0bThcj1kjEMuOUWKCjIZpbkrjIXLz405LFggTISC0P1PvG4zcgbVDZLQYOKbfcE\n0T8SDQ1em5NcJlyUqZKWUk0C06dnr7n0W2UEyUWEYUI56gmQkYARdxQwZMicT/RePi60+Gs0XYij\nDWW0dp3nKcfnTMgokYCNGz2SydIec8h7NLSas59zz5nfXSwGCxcq4bcsuGGM1yxVs+GBqeyp/y11\n6ZcKTP70J4fx47tnmKclWvw1mi7G0YYybDxsXGpqHOa6Nnv2cIizZEmJSxD0nEPew5KOd9Wfvacp\nZz8IfJ54wuX00w/12hk8ONti8foSj6LyFm55Y8fwz4OAACHhyQdv5lfL2q6p6G5o8ddouiPpU0uZ\n8BkWWiw34myI2JhmNjNo3jwYPdqhutpqWgVHo4Xs3j23x8T8m8g5xe07ShD+AmTEIJmyuOceh127\nsoKd2xwtDOHMMz3qTqygfliCgq0hMuHzXIWL+UMI+xhASBgKxLADxySttKugxV+j6QbkpiYCJCpc\nzk34iDAgis85ocvzSVtZDaMOgYuKsvnrb7zhMnRoIa+91nPy/JuRc4pbsAXGzIa6kgg/qKpka63d\n7FA8FsvG+0eO9NJ9ExJsSYaMvs3A2mrxw2ccDu6He+6JKEsNQzJp0iJWrSpj165Dayq6I1r8NZou\nTsvURCnhiymHVaFFnvBJSgsXp+nwErINvwAuvNBm2DC49toKxo1LoFayPSwE1OIUt18tfKZWMpQ6\nDENNhnv2qAPxhQuzv6eLLophWY0YhiQQBlWTL+CHtRWqTeMWWLFiGldcMR8hJJaV4tZbWw8hdUe0\n+Gs0XRjPU9W/uW18AdZLmwtFHEe4PCsdXmhhHyyEErtYTLmBZla3UoYIYfSIPP9m5JziikWLkMkU\nmBZfutlhxgFYtAgeekiFwzLnIqNGeVx22UKEUM3Xk6ko+4dWsCnPhg/VNatWlTFp0mKiUZ9oVNls\n5Od33m0eS7T4azRdlJaxaSGUeGUE7AVsPGnTmv1tGKr8dSHgG9/IWEeHSGnwz39ewIknVvScVX+G\nzEl5WRnCdYk6DlfZNtvnqgyoIMiehwgB48e7GEaAEBAEgpUrp/LBBzZTpkBtLaxdq3r3zp4dp6LC\n5aKLetY5iRZ/jaaLkgljZwq8IJuXLoR6PiNmppl9nLtDkBKqqprbQfzoRxXs2mX3iIyVVrHTeVCu\n6jSViQgNG+YxbpzL+ec77NtnM3Fi1iU1lbKIx8vYsUNNFJYF3/ueygSaPNnmootUrQTQYyYALf4a\nTRcl17sms9rPCDxkJ4EzzoBzz4UxY2DpUti3z6O4WFk/1NbaTavXkhKX6mplHd2yKviIdCNv6Nwz\nkkxa5urVHo2NpZimOuy+8kp12N3QEGfLFpft2x1s22br1qxRXEEBrFx55LqB7ooWf42mi9KyeXh5\nuRIl08xaE0gJ27fDK6+oit7f/tZj4InnYUZ9UkmLW25dw7ZtagLYuVNlA5lm60ZobVbFtqamORNA\nV5sXcu0bEgl1ZvLDH7qYZqbeoZH9+2NN1c7nnGNzzjnqMDgTWsv9/bT0+u8pB+Va/DWaLkxuwVdR\nUXYiuOGG5tc1mb7Vx4ienAATLJngzhkx3uljN1lDQOtCfdjV7WG8k48wL3yiZCahwkI1lsxZyerV\n8PbbDpWVJoYRAJL9+xcxYEBZ0z16nppcw1BNjpWV2fvIeP33FEO8DFr8NZpuQmYimDs3G/rJkFmt\njngH/nY6hBKMFLy/HJiseozkvg+eB3PdplngsKvbdPxJJnxShsWOQqfJ0KyjnvrHatfQchL67W89\n/v73GG++qTJ2ampsli9XaZsgCcMU1dUxhgxROx3XtZvOV4RQPkoZ8vNtiovjPcoQD7T4azTdjlzz\nN9OEm2+GQYM8iotjwH5Ouz+C/+mA4zZHub22jBdWqdfNmJF+g1aW6wUjD7O6tW1qKuM8NtPl2cBh\nU7lNvEiJdUc99Q+7a2gxMxxuoshMQmec4TFpUoxBgx5h8OAkY8bAxRcv5JZbXFatKuOSSxYDPkFg\nIsQi3ngjhWFYTJwYx7LsNu8jP1+FzpYs6TrhrY6ixV+j6Wa0NHUbOdKjquo8wjDB28C+mVHefPhK\ntpcM4ABArToIbhJ/183GRBIJcF1qmUN1tToUHj360NXtsjqbn0mbIAQzZ4XfXk/9XAE/7K4hd2Yw\nTfZfOo05K8r4S9C6t47jwOjRHj/7Wakq2hJqayQERCJJxoxx+eMf57B5cyWwlA8//BQTJjxFZqcz\nZIhLPG63eR9dKbx1rNDir9F0Q3LPAnbvdpHSz/7QSHHSdcsYZEic5GJmz44zebK6uKHBo/60lyj4\nQkh+LRCG7KovTAubjWWpFFBoLuiHW+EfrRFdSwGtrDzMrqGF6f7nnpjPChZTSpwNvn1IeMm24Ze/\ndEmlfISQICFTAGGKCF/6ksP113skk+UEgU8qZRIEkXSOv8VzzzmUlbV9H8eyZWRXQYu/RtPNKShw\nEMJS3vNAGBoYRpgu6vIZM8alqMjOHur2b8S4F4pnQ/4Ogzer6poJWyzWvFdAZjKYMkV9PZxIHo6W\nAlpXd5hdg+PQMNqkfkRAQRXk10qi+JwvXKqtQ711PA9WrnRwHIuI0YhISU54Aax6GHDqdTg32uze\nPZc33vAxzQDDgD17prNq1eCmlNjGxpzdUQs6Et7qqmjx12i6Ofn5NiUla3jssRjbtsHOnWOYNau8\nqairutrBdeGkk9KHukISRuAfJYJP78yjcLKDtS4rbMBhJwPVSewwpGM7NYUOy+psLi9UPvmXFzr8\npEVcvbVdg+fBxo1QdI/qomL4kuLbDD6z0+KMqQ7x9OfPnZsVYcdRO5fHH49z1dgYN21ayAm1AUks\nVn2vjKtonrVjmhbPPVfGkiXZD28WGmvBR20Z2ZXR4q/R9ADy821GjbKZOVO1Ijx+N5w/einx6sns\nel2tlAsKHAwihKkAIwV9qwyuT1byRexmwgbNxR7aEfJoYTX9V1HJMFmONHyK8ixerIyzrM5uU0Az\noaHJk11GjkxhmpKwj0H9rReQf3oFZemD39zw0ZQp2f67maK2ZynDwcXF4eX7bZ67Cmy7edbOl79s\ns2hR9rMnTz787/ijtIzsymjx12h6CLathHlnzOOaReUYtT7XmutYfnMRrmsDNsU1U/nnC/PpVyX5\nVC30p65pxZsrbC0ng4ULsznwjnOYFM10bCdjNf1VuRQL9Rjfp6jOpWhO2wqaOYv2NxXCNQZSSMxI\nHgVXVUD6ELpl+Gj/fvXakSM9SkpcDhwopCh/M303A7VZh1Pbbt7GMrPKX7pUCX9bq/6eihZ/jaYH\nYdtguy6kfAgDIvhsus/lZ1JlybxYWcaIxxYjkz5JlBX01FZWvLmrXM+jqU+AEFBTk602zs188TzY\nucfhmoiFIX2SocUfmcxE1mEYPsaRguWeR8nTLtPCQn5ZW07j7ICGcQYnfLuyWfaR42S9jEwTBgxQ\nDVkyzqWGESJCML4FU2cv5H+/5uI4zSecTDXzqFEOdXU2Rd25E/tHRIu/RtPTyDmdTBkWzwYOQboC\neFmdTdFzcbxYjE194fpRR47hu64yO5NSfV26tPnKe2fM46SYy5yFDqkUNIopXPEVePH0Mv7v/Tbb\nUkWUGi5fq3Qoaitu4nmkzi3lwqRPKQKDkE/VhuRvF5gj6mBC88uFyDZe79sXxo3LOpciAVMVun1Y\nkmTxRJchOZ/bdPAdJAgSJlWPPsiPfzyDNWt6VljnSGjx12h6GjmnkzsKHTbcaCOSWY//9SEE31zM\nqNDHMBazfn2ctWvbjsO3zHS55hqPgQNdXn7ZofBVuGZRKcL3eVpGAEmEALnc4lXKSKXgeWnzorQ5\nrg7aWmDvjrmcnPSJEJDCIMQkiWh1t+C66lwDlPjffz88/LADWEACRAgpVeHcb3uU/HnNX6+qmdV1\nZiTkpyUz2VNbRCzWM5q0HC1a/DWabsZRWSKk4zbvedlVciqlzMs++MBlyhQfw1AFTqsejtF3U4zl\n2yEMy5gwwT7krW68Ef74R5g2zeOUU0q5dkqCa68xGfLHyzAf8kEGRFE+0iaSMPA58JSLlOq9IpHD\nR3yew+F/YSFR4ajvUsnEkXX828PZm8yEaiZOdDAMu8m2Oghg3z6bK69Uh7nRaCHJXZsp2A758w7N\nS9271yH0TQwjVBNEVYiDSz29SPnR4q/RdCvaW2nqujB8uMfo0VmL540bHb75TUv1ppURrvvHw7x5\nT4owCokPF9HQsKZZjD0W89i718WyHN55J0aQakQYEkSIeP8pkjKCIUCaqseklAGBabEm5QBq8pk6\nFWyyfkINI2nmlTO8zOaSR+L8SzKdoRO1mfYwZPQ4azyXAASx2BXceef3qK1Vk8BLL4Hj2NgZw7nd\nsOQ95eff0jHivPNsvjr8QX5aMpN+VSHH1eaxPuLwiyOlsPYwtPhrNN2I9laaTjp5AWfdPQuiAclk\nHrfeGmfbNuXvP3asy9cb90D+fMIoYIIR+uzfH6O6Osa+fTBo0BgGDixn2jQ/XREbQrqCVgRw/MuS\nh8Op7GEwzwuHB+dBUZ3L8nqH5+9SA5MSJvXNzloNo02q7xOEpACLmpo4Th7ErnP57/0OA7C5bkDz\n+2gK1aR3FwMGPMF9963glltcamttnnhCee9nCtLamiAzIaM9tUWsrv13AJ49uYxfPNa7Qj6gxV+j\n6Va0q9LU8yh8diYH/i2lhF0kmDHD5bbbbHbsUP7+k2/26PvnhRhJn1BCgGDv3ocxjBQDBkAiYWKa\nUmXQCCW8QgAhfO5pQYoIu6+Gp6octm+3WVYHRXNU60TDUBk5Z57pUXBcBQ3DEuRvDdnvhIRSgoBU\nyuftF2MM+91ijjN8ZkctSmWcpwKbxYuzwh2NFja7NSEgGk1y0UUxSkrUruaVV+ympvUtrIuahN1x\n4GzTY2VQioWPj8XAa8p6nfCDFn+NplvRrkpT16Xg5RDjGyrzxRQmU6Y49OkDM2eq3cM3fmkzPuny\njQfvYkT5UwgjQIgwJ7UzIAyjhKFAygimqcI6ST/CMzsv4fR7/8yF0Ydwkou5/fY4e/aoIqyM82im\neTx5Caq/HHLaPMH+SUr4VcvJCH03k60FSDTyv4mRAM5rdFl1XyHJGzcj5SLI6VasuphJLrnkYUxT\nkkxaxG6rpPSlOt463SEM1S8mDJW/f+7v79HpLnm/9TEJMAyfqwpc6GXxftDir9F0O4660tRxyP9J\nHsW3JagfZ1Aw/UHy81Vjl0yvX9+H9dJmSP5ZjBRPql7BaVM0CaRSFg888AD9+tVx1lnK/Oz++10e\ne8yhpMRlRPQpTDNACJ/iYpeHHsqu2ONxePVVlz59fCAk7GPw7rShSOt1VPhGsHr1VLwdZZSzUIkx\nkut4hKks5IMRKbZeF5JKCQyjeQODTA/jSCRQP5MJ/nP0TE5ZIhltWNjE8bAxjObe/ABDyhxYrLZP\nR6w96MFo8ddoeirpbUK+65Kf3iY0NHicfbaybd6yxSYSUTuAqioHkVTLcRHA8dvh4HHH8+Cye1ix\nYgbRqOoelp8PH35oU1urPiLTGB4sNm92+GLgcd6HLuvucvje4zYjRzpUV2f7BBzofxuN75cTiSjf\noXHjxnDmQpeVCy7l8uefxERiksIEDpZIwigYhkyv9AUgmxrZBIHakUQiAaQMTqgKiBAiQ59zcXnR\nsMnLa0Xbe6JRz0egQ+IvhPgaUAGMAM6SUm5s47qLgV8CJvCwlPLnHflcjUZzlORsE3JbNd5zj8Wj\nj8b58pdtNm+G+fNtVs2+lWsvugu/H/zjSxBE3mPmzHKkhBNOqCMMHcCmoEC9dW5jeN93KNgOK0nH\n0p+wqFkQp2hGcz+dX//aZsmSIkaPVjYMN91UTjTqI/8zwt9vinJCbUBABMOQfKYqhZEMSUoDYUTo\n27eEgwc3IkRIEAiefvo61qwpY84clyd/WohdW04ynSq6VjiMHw9jx6qK5J0xj3Nx1ao/8zvppaKf\noaMr/63AvwLz27pACGEC84ALgb3ABiHEn6SUtR38bI1GcxRk8uMbG/c0a9WYTLqUl9tUVkKfPrDM\nuIrSSfcTsZJIAYaQRGSC8vJZCBGQSJi89daDOM6Mpk5iGSO1fxEeP5IVWCSIEAIJ3r+tghoqKJqh\n/HQWLIANv/K4fL+Lu9Vh6NUuhqHGI0zYdNN03r5nMCf8q8Orr0L9Ey67ZhcSKakjL8/huusASpHS\nRwiLU04pY948ld45aBA8eFcRB55ycaXDxoiNqFbuoGeFHvH0pBQssjDX9IBOLMeADom/lHI7ZLZj\nbXIW8JqU8vX0tb8HrgS0+Gs0HzO5q30hTISIEIYqlr9pk9PMV//VV10ifdIZPUAQCKQ0MM1UOvQS\nsmPHTPLyilizxiYWg02bILLBY7UsxSKBSUiAgUnI+APP4H97Hc+treS49+vY8EQhj1LelGUztaqS\nZNICmcCUBp87bgxV02aQKoQf3g9JbKUStSpzaMkSmHx6JU7RUp7bNpmZv8mmZ9o22I/beJ7Np10o\n2gMPPaTOBRxcLFT1cNBTOrEcAz6JmP/JwJs5j/cCX2rtQiHEDGAGwODBgz/+kWk0PZzcxuxSwsCB\n03nnncHMnq1SI3N99XPj80JEiEansmrVGM47bxZSJtNdr0Ieesjl29+2+c1vVNHUnye6WCmfCCEp\nDF5nKP1H7uJgScjxVY2cufkGDpSE/HykgVUrVVwen6G1dSyaXclPS2bSvzrA2lbOTUYRLwibIGh+\nH2EIYxMeD9WUY9X4XMM6/hArwm6lt2/GZG7x4nSqZ+jgp6uHe/MBb0uOKP5CiGeAAa386AdSyieP\n5WCklAuABQDjx4+XR7hco9EcgdwGJoZhMWBAGWecYTNv3qHnnfn5zePztbU2990H1dVw000zESIk\nlcrj5ZedZj18T7ylkIMr4ECxoO+2COuH/CvDvnMXYRREIEEGyAgYyYBRs0361prsvUxw6sQn+Py6\nkxj6XxJDhiTxOSd0eV60vir/6pkx/ja6kROqJJ+qVYe6nme3WtCVe6ZbWGjzh83x5jF/zZHFX0p5\nQQc/Yx/w+ZzHg9LPaTSaj5mWgp6xbchdIWc6YuX63Tc0eGzYMJehQx2WLZvBX/9a1FRMtWtXThtF\nz6P/6hupvjtQ9XoEXwAACitJREFUVcKE9K13SUUFhikJhQohYahag2XOFTAaBs94guG8BF+EvUaE\nzy8zSYYW6wyHc0yPs1Muz0qHF9L596NGeRTfvYjdUcmbSSiaYzKkzGGJ23bFc/MzXZvemMt/OD6J\nsM8GYLgQ4lSU6H8DuPoT+FyNRkPzBia5tOUT1NDg8fLLpYwapbKCZs+Os3OnzaWX2rzzjjJ5axJV\n16V+ZLLJHkLKFP36vdRUBdx0HKisgLjq6kuoSi0lDLOGc3/76gDyi8bzSvEAZlXX8LV7yjFIEAiD\nuwbP4z/enEFxsYsZVZXKoSE48Ktp9LNtHHpeb91PCqMjLxZCfFUIsRc1pS4XQqxMP3+SEGIFgJQy\nBcwCVgLbgf+RUm7r2LA1Gk1Hac0nCGDVKhcpVaPzSMSnpMTlkkvggQfg/Wc89s6cS80CT13sOHym\nOoqRRIk96nC2KQVE1XJBCANWQv7aOoYPn9wk/AI44fm9VJ/9BB98dgEDnVkcHNaIkCERmeKOvbM4\nJ+KxZYujDocxMcw+FIxWLmyZ8M5PfnJkkztNczqa7fM48Hgrz78FXJrzeAWwoiOfpdFoji0Zn5sJ\noct608Fx1OHpnXc6/OIXVlMD+JoahwkT1IHrqrAUK/QJb7CIbY4zvMxm3Wku4ewYZ1y0ln6X1apq\nnszKX4JMe+uf+KwFv3Y46SSl0O++eDefXbiLZF/ZFDJKoRrL59dKNZHIgMXTXJYMnsPxx8cZNKh5\n+Ap0yv5HRVf4ajS9FBuPuCgFfILA4pUa1Vx969Zs8VZ1tUNJiVJWR2RTJpOBzyvzXb6z2Kay0uam\nP9uMrfT43c6J7ClPIQWkklFenXcZqQJo3DyAH+0sYy42NnDSSTM4aUARxEtpGJbASIYEwiCZyuOh\nqhv5MfcTEQFGXh5DyhxU218dtz+WaPHXaHorrouR9BEyIAx8HpvpMmieskTYsUMVbwFs26ZCOWeb\nDiEWQeiTlBbPymydwJo14Lo27xWupWR7jFcGwIv7y6gdpIq7wlC9R7MU+xz7ieLjC9kS1jH7Voct\nr9h41lUsnnZods5RNbLRHBVa/DWa3orjkDItCJUlwrOhw2Xpgq+KCnjmGZq6ZYUhrBc2v5+uUian\nLHTYEDSvE7BzVudnoao7M8KfeY/C5s7MeNi42DgGnDOBnBRUu6nvbkbwCwvbbhyvJ4T2o8Vfo+mt\n2DY7Hozz2EyXZ0OHTXk2dztKQCsqYN26rC++YSjBHV6mRHlu2ZEF1/NUs/fM4W5Lh81MttHYhMeH\nhstn5jnYM5o3VcnNSDIMdTidcSPNHFCXlqpxGoaaPGbM+Fh+Wz0OLf4aTS+maIZS2vOXuhROhiI7\nWweQLZJSop0r9G3VCWTIiHYikRX+lg6brquE/5nwPKKhj7zBgqI1zd4oNyMp8z5CZNM6XTc7QYUh\nzJoFRUV6B3A0aPHXaHoznkdReXppvc6Comy+pLJ88A4pEMt5aevtEj2PRIXL2ITD+lB56l9wgdpN\n5Iqy40ChiJFHAgHIIAGxWLOLWnYuq6w8dCLKdAwDNUlo656jQ4u/RtObOUxT4FxTOMOwKC6ON5sA\nNm70mDzZZdOmbAtFGzUjnJvwWRVaXGTE2ZRnHyL8oB6fcgXwhHrcmj3kkaz3bVuFembNUrfQqn+/\nplW0+Gs0vZnDNAXONYULQ5/6erdJ/BsaPIqKShk50ueaayy+//04jmM3TSYiDDjO8LnzApe8CrvN\ng9mB3yuDFQtVV/VoFMrKDhnikfL4Z8xQoR596Ns+tPhrNL2ZwyytW5rCFRQ4TT+rr3cBVQVsGD6/\n/KWLbduA0zSZCMvCmVwI7lxqahxKy+1DQ0R2esLooHLrQq/2o8Vfo+nttKGcbZnCQfOJwTQtRo92\nsu+Ve1Kczs38gmExNoizPrQPMWDTyt05aPHXaDRt0pYp3OEmhiYxnzu36TwhIn3ON1xeELY2YOsi\naPHXaDQfiWYTQ2sB/ZzzBGFZfK3S4bg6HZfvKmjx12g0HaOtnM8W5wlFtk1RZ49V04QWf41G0zEO\nky6q4/ldlw75+Ws0Gk1TeMc0dUeVboRe+Ws0mo5xpEosTZdEi79Go+k4OrzT7dBhH41Go+mFaPHX\naDSaXogWf41Go+mFaPHXaDSaXogWf41Go+mFaPHXaDSaXoiQUnb2GFpFCPEusPsjvrw/8PdjOJzO\noLvfQ3cfP3T/e+ju44fufw+dMf4hUsrPHumiLiv+HUEIsVFKOb6zx9ERuvs9dPfxQ/e/h+4+fuj+\n99CVx6/DPhqNRtML0eKv0Wg0vZCeKv4LOnsAx4Dufg/dffzQ/e+hu48fuv89dNnx98iYv0aj0WgO\nT09d+Ws0Go3mMPQ48RdCXCyEeEUI8ZoQ4o7OHk97EUIsFEK8I4TY2tlj+SgIIT4vhFgjhKgVQmwT\nQny3s8fUXoQQfYQQLwkhqtP38OPOHtNHQQhhCiE2CyGWdfZYPgpCiL8KIWqEEFVCiI2dPZ72IoQo\nEEL8QQixQwixXQjRpWxPe1TYRwhhAq8CFwJ7gQ3AN6WUtZ06sHYghJgIvAfEpJRndvZ42osQYiAw\nUEq5SQhxPPAycFU3+xsI4NNSyveEEFHgL8B3pZQvdPLQ2oUQ4hZgPNBXSnl5Z4+nvQgh/gqMl1J2\nyzx/IcRiYJ2U8mEhhAV8SkpZ39njytDTVv5nAa9JKV+XUvrA74ErO3lM7UJKuRb4R2eP46MipXxb\nSrkp/f1BYDtwcueOqn1IxXvph9H0v261ShJCDAIuAx7u7LH0RoQQ+cBE4BEAKaXflYQfep74nwy8\nmfN4L91MeHoSQohTgDHAi507kvaTDplUAe8Aq6WU3e0eKoHvAWFnD6QDSGCVEOJlIcSMzh5MOzkV\neBdYlA69PSyE+HRnDyqXnib+mi6CEOIzwFKgXEp5oLPH016klIGUsgQYBJwlhOg2ITghxOXAO1LK\nlzt7LB3kbCnlWOASYGY6JNpdiABjgd9IKccA7wNd6gyyp4n/PuDzOY8HpZ/TfIKk4+RLgd9JKf/Y\n2ePpCOmt+hrg4s4eSzuYAHwlHTP/PXC+EOLRzh1S+5FS7kt/fQd4HBXW7S7sBfbm7Bj/gJoMugw9\nTfw3AMOFEKemD1i+Afypk8fUq0gflj4CbJdS3tfZ4/koCCE+K4QoSH9/HCqBYEfnjurokVLOkVIO\nklKegvo/8KyU8ludPKx2IYT4dDphgHS45CKg22TASSn3A28KIc5IP1UKdKmkhx7VwF1KmRJCzAJW\nAiawUEq5rZOH1S6EEP8FOEB/IcRe4P9IKR/p3FG1iwnAvwE16Zg5wPellCs6cUztZSCwOJ09ZgD/\nI6XslumS3ZjPAY+rtQQRYImU8unOHVK7uRH4XXoh+jowtZPH04weleqp0Wg0mqOjp4V9NBqNRnMU\naPHXaDSaXogWf41Go+mFaPHXaDSaXogWf41Go+mFaPHXaDSaXogWf41Go+mFaPHXaDSaXsj/B5mj\nDFi6aXY5AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "t5McVnHmNiDw", + "colab_type": "text" + }, + "source": [ + "## Design a model\n", + "We're going to build a model that will take an input value (in this case, `x`) and use it to predict a numeric output value (the sine of `x`). This type of problem is called a _regression_.\n", + "\n", + "To achieve this, we're going to create a simple neural network. It will use _layers_ of _neurons_ to attempt to learn any patterns underlying the training data, so it can make predictions.\n", + "\n", + "To begin with, we'll define two layers. The first layer takes a single input (our `x` value) and runs it through 16 neurons. Based on this input, each neuron will become _activated_ to a certain degree based on its internal state (its _weight_ and _bias_ values). A neuron's degree of activation is expressed as a number.\n", + "\n", + "The activation numbers from our first layer will be fed as inputs to our second layer, which is a single neuron. It will apply its own weights and bias to these inputs and calculate its own activation, which will be output as our `y` value.\n", + "\n", + "**Note:** To learn more about how neural networks function, you can explore the [Learn TensorFlow](https://codelabs.developers.google.com/codelabs/tensorflow-lab1-helloworld) codelabs.\n", + "\n", + "The code in the following cell defines our model using [Keras](https://www.tensorflow.org/guide/keras), TensorFlow's high-level API for creating deep learning networks. Once the network is defined, we _compile_ it, specifying parameters that determine how it will be trained:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "gD60bE8cXQId", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# We'll use Keras to create a simple model architecture\n", + "from tensorflow.keras import layers\n", + "model_1 = tf.keras.Sequential()\n", + "\n", + "# First layer takes a scalar input and feeds it through 16 \"neurons\". The\n", + "# neurons decide whether to activate based on the 'relu' activation function.\n", + "model_1.add(layers.Dense(16, activation='relu', input_shape=(1,)))\n", + "\n", + "# Final layer is a single neuron, since we want to output a single value\n", + "model_1.add(layers.Dense(1))\n", + "\n", + "# Compile the model using a standard optimizer and loss function for regression\n", + "model_1.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "O0idLyRLQeGj", + "colab_type": "text" + }, + "source": [ + "## Train the model\n", + "Once we've defined the model, we can use our data to _train_ it. Training involves passing an `x` value into the neural network, checking how far the network's output deviates from the expected `y` value, and adjusting the neurons' weights and biases so that the output is more likely to be correct the next time.\n", + "\n", + "Training runs this process on the full dataset multiple times, and each full run-through is known as an _epoch_. The number of epochs to run during training is a parameter we can set.\n", + "\n", + "During each epoch, data is run through the network in multiple _batches_. Each batch, several pieces of data are passed into the network, producing output values. These outputs' correctness is measured in aggregate and the network's weights and biases are adjusted accordingly, once per batch. The _batch size_ is also a parameter we can set.\n", + "\n", + "The code in the following cell uses the `x` and `y` values from our training data to train the model. It runs for 1000 _epochs_, with 16 pieces of data in each _batch_. We also pass in some data to use for _validation_. As you will see when you run the cell, training can take a while to complete:\n", + "\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "p8hQKr4cVOdE", + "colab_type": "code", + "outputId": "3f1a7904-ffcd-4bb7-8bbb-bcd85a132128", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "# Train the model on our training data while validating on our validation set\n", + "history_1 = model_1.fit(x_train, y_train, epochs=1000, batch_size=16,\n", + " validation_data=(x_validate, y_validate))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Train on 600 samples, validate on 200 samples\n", + "Epoch 1/1000\n", + "600/600 [==============================] - 0s 412us/sample - loss: 0.5016 - mae: 0.6297 - val_loss: 0.4922 - val_mae: 0.6235\n", + "Epoch 2/1000\n", + "600/600 [==============================] - 0s 105us/sample - loss: 0.3905 - mae: 0.5436 - val_loss: 0.4262 - val_mae: 0.5641\n", + "...\n", + "Epoch 998/1000\n", + "600/600 [==============================] - 0s 109us/sample - loss: 0.1535 - mae: 0.3068 - val_loss: 0.1507 - val_mae: 0.3113\n", + "Epoch 999/1000\n", + "600/600 [==============================] - 0s 100us/sample - loss: 0.1545 - mae: 0.3077 - val_loss: 0.1499 - val_mae: 0.3103\n", + "Epoch 1000/1000\n", + "600/600 [==============================] - 0s 132us/sample - loss: 0.1530 - mae: 0.3045 - val_loss: 0.1542 - val_mae: 0.3143\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cRE8KpEqVfaS", + "colab_type": "text" + }, + "source": [ + "## Check the training metrics\n", + "During training, the model's performance is constantly being measured against both our training data and the validation data that we set aside earlier. Training produces a log of data that tells us how the model's performance changed over the course of the training process.\n", + "\n", + "The following cells will display some of that data in a graphical form:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "CmvA-ksoln8r", + "colab_type": "code", + "outputId": "1b834831-81e8-4548-dd8c-f5edf2c3ff43", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 295 + } + }, + "source": [ + "# Draw a graph of the loss, which is the distance between\n", + "# the predicted and actual values during training and validation.\n", + "loss = history_1.history['loss']\n", + "val_loss = history_1.history['val_loss']\n", + "\n", + "epochs = range(1, len(loss) + 1)\n", + "\n", + "plt.plot(epochs, loss, 'g.', label='Training loss')\n", + "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", + "plt.title('Training and validation loss')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('Loss')\n", + "plt.legend()\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzt3Xd8FHX6wPHPk5AQamhRWiBRUHqN\nYA6BIIjYQJTzQFHh9FB/Kp7lFMspopzlPAse56l32FCxIIoKogIRPKIUpRcJECDUEDoB0p7fHzNJ\nNstuNm0JhOf9eu0rM9/5zsx3djb7zLfsjKgqxhhjTFFCKroAxhhjTn0WLIwxxgRkwcIYY0xAFiyM\nMcYEZMHCGGNMQBYsjDHGBGTBwpwUIhIqIodFpFl55q1IItJCRMp97LmI9BORFI/5dSLSszh5S7Gv\n/4jII6Vdv4jtPi0ib5f3dk3FqVLRBTCnJhE57DFbHTgO5Ljzt6nq+yXZnqrmADXLO++ZQFXPL4/t\niMitwHBVTfDY9q3lsW1T+VmwMD6pav6XtXvlequqfu8vv4hUUdXsk1E2Y8zJZ81QplTcZoaPRORD\nETkEDBeReBH5SUT2i8gOEZkgImFu/ioioiIS485PdpfPFJFDIpIkIrElzesuv0xEfhORAyLyqoj8\nT0RG+Cl3ccp4m4gki8g+EZngsW6oiLwkIukishEYUMT786iITPFKmygiL7rTt4rIGvd4NrhX/f62\nlSoiCe50dRF5zy3bKqCrV97HRGSju91VIjLQTW8P/BPo6Tbx7fF4b8d6rH+7e+zpIvK5iDQqznsT\niIgMdsuzX0TmiMj5HsseEZHtInJQRNZ6HOuFIvKLm75LRP5e3P2ZIFBVe9mryBeQAvTzSnsayASu\nwrnoqAZcAHTHqbGeA/wG3OXmrwIoEOPOTwb2AHFAGPARMLkUec8CDgGD3GX3AVnACD/HUpwyfgFE\nAjHA3rxjB+4CVgFNgfrAPOdfyOd+zgEOAzU8tr0biHPnr3LzCHAxcBTo4C7rB6R4bCsVSHCnXwAS\ngbpAc2C1V97rgEbuObneLcPZ7rJbgUSvck4GxrrT/d0ydgIigH8Bc4rz3vg4/qeBt93p1m45LnbP\n0SPAOne6LbAZaOjmjQXOcacXAcPc6VpA94r+XziTX1azMGXxo6p+qaq5qnpUVRep6s+qmq2qG4E3\ngN5FrP+pqi5W1SzgfZwvqZLmvRJYqqpfuMtewgksPhWzjM+o6gFVTcH5Ys7b13XAS6qaqqrpwLNF\n7GcjsBIniAFcAuxT1cXu8i9VdaM65gCzAZ+d2F6uA55W1X2quhmntuC5349VdYd7Tj7ACfRxxdgu\nwA3Af1R1qaoeA8YAvUWkqUcef+9NUYYC01V1jnuOnsUJON2BbJzA1NZtytzkvnfgBP2WIlJfVQ+p\n6s/FPA4TBBYsTFls9ZwRkVYi8rWI7BSRg8A4oEER6+/0mM6g6E5tf3kbe5ZDVRXnStynYpaxWPvC\nuSIuygfAMHf6enc+rxxXisjPIrJXRPbjXNUX9V7laVRUGURkhIgsc5t79gOtirldcI4vf3uqehDY\nBzTxyFOSc+Zvu7k456iJqq4D7sc5D7vdZs2GbtaRQBtgnYgsFJHLi3kcJggsWJiy8B42+jrO1XQL\nVa0NPI7TzBJMO3CahQAQEaHwl5u3spRxBxDtMR9oaO/HQD8RaYJTw/jALWM14FPgGZwmojrAt8Us\nx05/ZRCRc4DXgDuA+u5213psN9Aw3+04TVt526uF09y1rRjlKsl2Q3DO2TYAVZ2sqj1wmqBCcd4X\nVHWdqg7FaWr8BzBVRCLKWBZTShYsTHmqBRwAjohIa+C2k7DPr4AuInKViFQB7gGiglTGj4E/i0gT\nEakPPFRUZlXdCfwIvA2sU9X17qKqQDiQBuSIyJVA3xKU4RERqSPO71Du8lhWEycgpOHEzT/h1Czy\n7AKa5nXo+/AhcIuIdBCRqjhf2vNV1W9NrQRlHigiCe6+/4LTz/SziLQWkT7u/o66r1ycA7hRRBq4\nNZED7rHllrEsppQsWJjydD9wM84Xwes4HdFBpaq7gD8ALwLpwLnArzi/CynvMr6G07ewAqfz9dNi\nrPMBTod1fhOUqu4H7gWm4XQSD8EJesXxBE4NJwWYCbzrsd3lwKvAQjfP+YBnO/93wHpgl4h4Nifl\nrf8NTnPQNHf9Zjj9GGWiqqtw3vPXcALZAGCg239RFXgep59pJ05N5lF31cuBNeKMtnsB+IOqZpa1\nPKZ0xGniNaZyEJFQnGaPIao6v6LLY0xlYTULc9oTkQFus0xV4K84o2gWVnCxjKlULFiYyuAiYCNO\nE8elwGBV9dcMZYwpBWuGMsYYE5DVLIwxxgRUaW4k2KBBA42JianoYhhjzGllyZIle1S1qOHmQCUK\nFjExMSxevLiii2GMMacVEQl0JwLAmqGMMcYUgwULY4wxAVmwMMYYE1Cl6bMwxpxcWVlZpKamcuzY\nsYouiimGiIgImjZtSliYv1uDFc2ChTGmVFJTU6lVqxYxMTE4N/s1pypVJT09ndTUVGJjYwOv4IM1\nQxljSuXYsWPUr1/fAsVpQESoX79+mWqBQQ0W7j171rnP7B3jY/kIEUkTkaXu61aPZTeLyHr3dXMw\ny5m0NYln5j9D0takYO7GmErHAsXpo6znKmjNUO7dPyfiPE4yFVgkItNVdbVX1o9U9S6vdevh3Io5\nDuce9kvcdfeVdzmTtibR992+ZOZkEh4azuybZhMfHV/euzHGmNNaMGsW3YBk9znDmcAUCp5HHMil\nwHequtcNEN/h3AO/3CWmJJKZk0mO5pCZk0liSmIwdmOMKWfp6el06tSJTp060bBhQ5o0aZI/n5lZ\nvMdejBw5knXr1hWZZ+LEibz//vvlUWQuuugili5dWi7bOtmC2cHdhMLPCk7FeUC7t2tFpBfwG3Cv\nqm71s+4Jj8oUkVHAKIBmzQI94dK3hJgEwkPD82sWCTEJpdqOMebkql+/fv4X79ixY6lZsyYPPPBA\noTyqiqoSEuL7uvitt94KuJ8777yz7IWtBCq6g/tLIEZVO+DUHt4pycqq+oaqxqlqXFRUwFub+BQf\nHc/sm2bzVJ+nrAnKmCA7Gf2DycnJtGnThhtuuIG2bduyY8cORo0aRVxcHG3btmXcuHH5efOu9LOz\ns6lTpw5jxoyhY8eOxMfHs3v3bgAee+wxXn755fz8Y8aMoVu3bpx//vksWLAAgCNHjnDttdfSpk0b\nhgwZQlxcXMAaxOTJk2nfvj3t2rXjkUceASA7O5sbb7wxP33ChAkAvPTSS7Rp04YOHTowfPjwcn/P\niiOYNYttFH6wfP4D2vOoarrH7H9wHq+Yt26C17qJ5V5CV3x0vAUJY4LsZPYPrl27lnfffZe4uDgA\nnn32WerVq0d2djZ9+vRhyJAhtGnTptA6Bw4coHfv3jz77LPcd999TJo0iTFjThiXg6qycOFCpk+f\nzrhx4/jmm2949dVXadiwIVOnTmXZsmV06dKlyPKlpqby2GOPsXjxYiIjI+nXrx9fffUVUVFR7Nmz\nhxUrVgCwf/9+AJ5//nk2b95MeHh4ftrJFsyaxSKgpYjEikg4MBSY7plBRBp5zA4E1rjTs4D+IlJX\nROoC/d20cpeZCfPmwbZtgfMaY0rvZPYPnnvuufmBAuDDDz+kS5cudOnShTVr1rB6tfc4G6hWrRqX\nXXYZAF27diUlJcXntq+55poT8vz4448MHToUgI4dO9K2bdsiy/fzzz9z8cUX06BBA8LCwrj++uuZ\nN28eLVq0YN26dYwePZpZs2YRGRkJQNu2bRk+fDjvv/9+qX9UV1ZBCxaqmg3chfMlvwb4WFVXicg4\nERnoZhstIqtEZBkwGhjhrrsXeAon4CwCxrlp5W7/fujdGz7/PBhbN8bkyesfDJXQoPcP1qhRI396\n/fr1vPLKK8yZM4fly5czYMAAn783CA8Pz58ODQ0lOzvb57arVq0aME9p1a9fn+XLl9OzZ08mTpzI\nbbfdBsCsWbO4/fbbWbRoEd26dSMnJ6dc91scQf0Ft6rOAGZ4pT3uMf0w8LCfdScBk4JZPoDQUOdv\nbm6w92TMmS2vfzAxJZGEmIST1vR78OBBatWqRe3atdmxYwezZs1iwIDyHVzZo0cPPv74Y3r27MmK\nFSt81lw8de/enQceeID09HQiIyOZMmUKDzzwAGlpaURERPD73/+eli1bcuutt5KTk0NqaioXX3wx\nF110EdHR0WRkZFCrVq1yPYZAzvjbfeQNkqiAQG3MGaci+ge7dOlCmzZtaNWqFc2bN6dHjx7lvo+7\n776bm266iTZt2uS/8pqQfGnatClPPfUUCQkJqCpXXXUVV1xxBb/88gu33HILqoqI8Nxzz5Gdnc31\n11/PoUOHyM3N5YEHHjjpgQIq0TO44+LitDQPPzp4ECIj4R//gPvuC0LBjKmk1qxZQ+vWrSu6GKeE\n7OxssrOziYiIYP369fTv35/169dTpcqpdT3u65yJyBJVjfOzSr5T60gqgNUsjDFldfjwYfr27Ut2\ndjaqyuuvv37KBYqyqlxHUwrWZ2GMKas6deqwZMmSii5GUFX0j/IqnNUsjDEmsDM+WFjNwhhjAjvj\ng4XVLIwxJjALFu47YDULY4zx74wPFuAEDKtZGHN66dOnD7NmFb4L0Msvv8wdd9xR5Ho1a9YEYPv2\n7QwZMsRnnoSEBAINxX/55ZfJyMjIn7/88svL5b5NY8eO5YUXXijzdsqbBQucfgurWRhzehk2bBhT\npkwplDZlyhSGDRtWrPUbN27Mp59+Wur9eweLGTNmUKdOnVJv71RnwQKrWRhzOhoyZAhff/11/oOO\nUlJS2L59Oz179sz/3UOXLl1o3749X3zxxQnrp6Sk0K5dOwCOHj3K0KFDad26NYMHD+bo0aP5+e64\n447825s/8cQTAEyYMIHt27fTp08f+vTpA0BMTAx79uwB4MUXX6Rdu3a0a9cu//bmKSkptG7dmj/9\n6U+0bduW/v37F9qPL0uXLuXCCy+kQ4cODB48mH379uXvP++W5Xk3MPzhhx/yH/7UuXNnDh06VOr3\n1pcz/ncWYDULY8rqz3+G8n4AXKdO4H7P+lSvXj26devGzJkzGTRoEFOmTOG6665DRIiIiGDatGnU\nrl2bPXv2cOGFFzJw4EC/z6F+7bXXqF69OmvWrGH58uWFbjE+fvx46tWrR05ODn379mX58uWMHj2a\nF198kblz59KgQYNC21qyZAlvvfUWP//8M6pK9+7d6d27N3Xr1mX9+vV8+OGHvPnmm1x33XVMnTq1\nyOdT3HTTTbz66qv07t2bxx9/nCeffJKXX36ZZ599lk2bNlG1atX8pq8XXniBiRMn0qNHDw4fPkxE\nREQJ3u3ArGaB1SyMOV15NkV5NkGpKo888ggdOnSgX79+bNu2jV27dvndzrx58/K/tDt06ECHDh3y\nl3388cd06dKFzp07s2rVqoA3Cfzxxx8ZPHgwNWrUoGbNmlxzzTXMnz8fgNjYWDp16gQUfRt0cJ6v\nsX//fnr37g3AzTffzLx58/LLeMMNNzB58uT8X4r36NGD++67jwkTJrB///5y/wW51SywmoUxZVVU\nDSCYBg0axL333ssvv/xCRkYGXbt2BeD9998nLS2NJUuWEBYWRkxMjM/bkgeyadMmXnjhBRYtWkTd\nunUZMWJEqbaTJ+/25uDc4jxQM5Q/X3/9NfPmzePLL79k/PjxrFixgjFjxnDFFVcwY8YMevTowaxZ\ns2jVqlWpy+rNahZYzcKY01XNmjXp06cPf/zjHwt1bB84cICzzjqLsLAw5s6dy+bNm4vcTq9evfjg\ngw8AWLlyJcuXLwec25vXqFGDyMhIdu3axcyZM/PXqVWrls9+gZ49e/L555+TkZHBkSNHmDZtGj17\n9izxsUVGRlK3bt38Wsl7771H7969yc3NZevWrfTp04fnnnuOAwcOcPjwYTZs2ED79u156KGHuOCC\nC1i7dm2J91kUq1ng1CwsWBhzeho2bBiDBw8uNDLqhhtu4KqrrqJ9+/bExcUFvMK+4447GDlyJK1b\nt6Z169b5NZSOHTvSuXNnWrVqRXR0dKHbm48aNYoBAwbQuHFj5s6dm5/epUsXRowYQbdu3QC49dZb\n6dy5c5FNTv6888473H777WRkZHDOOefw1ltvkZOTw/Dhwzlw4ACqyujRo6lTpw5//etfmTt3LiEh\nIbRt2zb/qX/l5Yy/RTnA2WfD4MHw73+Xc6GMqcTsFuWnn7LcotyaobCahTHGBGLBAuvgNsaYQIIa\nLERkgIisE5FkERlTRL5rRURFJM6djxGRoyKy1H0FtYHIOriNKZ3K0ox9JijruQpaB7eIhAITgUuA\nVGCRiExX1dVe+WoB9wA/e21ig6p2Clb5PFnNwpiSi4iIID09nfr16/v9sZs5Nagq6enpZfqhXjBH\nQ3UDklV1I4CITAEGAd6/aHkKeA74SxDLUiSrWRhTck2bNiU1NZW0tLSKLoophoiICJo2bVrq9YMZ\nLJoAWz3mU4HunhlEpAsQrapfi4h3sIgVkV+Bg8BjqjrfewciMgoYBdCsWbNSF9RqFsaUXFhYGLGx\nsRVdDHOSVFgHt4iEAC8C9/tYvANopqqdgfuAD0SktncmVX1DVeNUNS4qKqrUZbGahTHGFC2YwWIb\nEO0x39RNy1MLaAckikgKcCEwXUTiVPW4qqYDqOoSYANwXrAKajULY4wpWjCDxSKgpYjEikg4MBSY\nnrdQVQ+oagNVjVHVGOAnYKCqLhaRKLeDHBE5B2gJbAxWQa1mYYwxRQtan4WqZovIXcAsIBSYpKqr\nRGQcsFhVpxexei9gnIhkAbnA7aq6N1hltZqFMcYULaj3hlLVGcAMr7TH/eRN8JieCkwNZtk8Wc3C\nGGOKZr/gxmoWxhgTiAULrGZhjDGBWLDAahbGGBOIBQusZmGMMYFYsMBqFsYYE4gFC6xmYYwxgViw\nwGoWxhgTiAULrGZhjDGBWLDAahbGGBOIBQusZmGMMYFYsMBqFsYYE4gFC6xmYYwxgViwwKlZWLAw\nxhj/LFjg1CysGcoYY/yzYIHVLIwxJhALFlgHtzHGBGLBAuvgNsaYQCxYYDULY4wJxIIFVrMwxphA\nghosRGSAiKwTkWQRGVNEvmtFREUkziPtYXe9dSJyaTDLaTULY4wpWpVgbVhEQoGJwCVAKrBIRKar\n6mqvfLWAe4CfPdLaAEOBtkBj4HsROU9Vg3L9bzULY4wpWjBrFt2AZFXdqKqZwBRgkI98TwHPAcc8\n0gYBU1T1uKpuApLd7QWF1SyMMaZowQwWTYCtHvOpblo+EekCRKvq1yVd111/lIgsFpHFaWlppS6o\n1SyMMaZoFdbBLSIhwIvA/aXdhqq+oapxqhoXFRVV6rJYzcIYY4oWtD4LYBsQ7THf1E3LUwtoBySK\nCEBDYLqIDCzGuuXKahbGGFO0YNYsFgEtRSRWRMJxOqyn5y1U1QOq2kBVY1Q1BvgJGKiqi918Q0Wk\nqojEAi2BhcEq6K6M7RzPziRpa1KwdmGMMae1oAULVc0G7gJmAWuAj1V1lYiMc2sPRa27CvgYWA18\nA9wZrJFQSVuT+Hj1h2Rl59L33b4WMIwxxodgNkOhqjOAGV5pj/vJm+A1Px4YH7TCuRJTEslBIDeE\nzJxMElMSiY+OD/ZujTHmtHLG/4I7ISaB0BABDSU8NJyEmISKLpIxxpxyzvhgER8dz02drgcNZfZN\ns61WYYwxPpzxwQKgWV1n4FX3JhYojDHGFwsWOL+zAPuthTHG+GPBgoJgkZ1dseUwxphTlQULoIo7\nJsx+mGeMMb5ZsKAgWFjNwhhjfLNggQULY4wJxIIFFiyMMSYQCxZYsDDGmEAsWGDBwhhjArFggQUL\nY4wJxIIFFiyMMSYQCxZYsDDGmEAsWGDBwhhjArFggQULY4wJxIIFFiyMMSYQCxZYsDDGmEAsWGDB\nwhhjAglqsBCRASKyTkSSRWSMj+W3i8gKEVkqIj+KSBs3PUZEjrrpS0Xk38EspwULY4wpWpVgbVhE\nQoGJwCVAKrBIRKar6mqPbB+o6r/d/AOBF4EB7rINqtopWOXzZMHCGGOKFsyaRTcgWVU3qmomMAUY\n5JlBVQ96zNYANIjl8Wt1+nIAVuxYUxG7N8aYU14wg0UTYKvHfKqbVoiI3CkiG4DngdEei2JF5FcR\n+UFEevragYiMEpHFIrI4LS2tVIVM2prEnTNvA+DR758gaWtSqbZjjDGVWYV3cKvqRFU9F3gIeMxN\n3gE0U9XOwH3AByJS28e6b6hqnKrGRUVFlWr/iSmJZHEUgOxsJTElsVTbMcaYyiyYwWIbEO0x39RN\n82cKcDWAqh5X1XR3egmwATgvGIVMiEkgrIoAEEoECTEJwdiNMcac1oIZLBYBLUUkVkTCgaHAdM8M\nItLSY/YKYL2bHuV2kCMi5wAtgY3BKGR8dDzvXjsJgEd+9zjx0fHB2I0xxpzWgjYaSlWzReQuYBYQ\nCkxS1VUiMg5YrKrTgbtEpB+QBewDbnZX7wWME5EsIBe4XVX3BqusF0R3BiA2smWAnMYYc2YKWrAA\nUNUZwAyvtMc9pu/xs95UYGowy+bJhs4aY0zRitUMJSLnikhVdzpBREaLSJ3gFu3ksWBhjDFFK26f\nxVQgR0RaAG/gdFx/ELRSnWQWLIwxpmjFDRa5qpoNDAZeVdW/AI2CV6yTy4KFMcYUrbjBIktEhuF0\nQH/lpoUFp0gnnwULY4wpWnGDxUggHhivqptEJBZ4L3jFOrksWBhjTNGKNRrKvfnfaAARqQvUUtXn\nglmwkykvWGRlVWw5jDHmVFXc0VCJIlJbROoBvwBvisiLwS3ayRMa6vy1moUxxvhW3GaoSPcOsdcA\n76pqd6Bf8Ip1coWEOC8LFsYY41txg0UVEWkEXEdBB3elUqWKBQtjjPGnuMFiHM5tOzao6iL3fk3r\ng1esk8+ChTHG+FfcDu5PgE885jcC1warUBUhLMyChTHG+FPcDu6mIjJNRHa7r6ki0jTYhTuZrGZh\njDH+FbcZ6i2c24s3dl9fummVhgULY4zxr7jBIkpV31LVbPf1NlC6R9OdoixYGGOMf8UNFukiMlxE\nQt3XcCA9mAU72XLkGL9uW27P4DbGGB+KGyz+iDNsdifO87GHACOCVKaTLmlrErsytvHr9hX0fbev\nBQxjjPFSrGChqptVdaCqRqnqWap6NZVoNFRiSiIakoXmhJKZk0liSmJFF8kYY04pZXkG933lVooK\nlhCTgITkgIYRHhpOQkxCRRfJGGNOKWV5rKqUWykqWHx0PC0aHKFKvTD+e9Ns4qPjK7pIxhhzSilL\nzUIDZRCRASKyTkSSRWSMj+W3i8gKEVkqIj+KSBuPZQ+7660TkUvLUM5iqV2tBudEnmeBwhhjfCiy\nZiEih/AdFASoFmDdUGAicAmQCiwSkenu7c7zfKCq/3bzDwReBAa4QWMo0Bbndx3fi8h5qppTvMMq\nORs6a4wx/hUZLFS1Vhm23Q1Idm8NgohMAQYB+cHCvZNtnhoUBKZBwBRVPQ5sEpFkd3tBG6ZkwcIY\nY/wrS59FIE2ArR7zqUB370wicidOZ3k4cLHHuj95rdvEx7qjgFEAzZo1K1NhLVgYY4x/ZemzKBeq\nOlFVzwUeAh4r4bpvqGqcqsZFRZXtB+UWLIwxxr9gBottQLTHfFM3zZ8pwNWlXLfMLFgYY4x/wQwW\ni4CWIhIrIuE4HdbTPTOISEuP2SsoeEbGdGCoiFQVkVigJbAwiGW1YGGMMUUIWp+FqmaLyF04D00K\nBSap6ioRGQcsVtXpwF0i0g/IAvYBN7vrrhKRj3E6w7OBO4M5EgosWBhjTFGC2cGNqs4AZnilPe4x\nfU8R644HxgevdIVZsDDGGP8qvIP7VGHBwhhj/LNg4bJgYYwx/lmwcFmwMMYY/yxYuPYe38W+jIP2\nLAtjjPHBggXOw49mbPiSg0cz7OFHxhjjgwULnIcf5cpxyKliDz8yxhgfLFjgPPwoJFQh1x5+ZIwx\nvliwwHn40R86XEMY1ZltDz8yxpgTBPVHeaeTmPqN0RwsUBhjjA9Ws3CFhTlDZ3NzK7okxhhz6rFg\n4QoPd/5mZVVsOYwx5lRkwcJlwcIYY/yzYOEKC3P+ZmZWbDmMMeZUZMHCZTULY4zxz4KFy2oWxhjj\nnwULV17NwoKFMcacyIKFy5qhjDHGPwsWLmuGMsYY/yxYuDYeWAvAkq3LK7gkxhhz6glqsBCRASKy\nTkSSRWSMj+X3ichqEVkuIrNFpLnHshwRWeq+pgeznElbk3jsh4cAuH36aLtFuTHGeAlasBCRUGAi\ncBnQBhgmIm28sv0KxKlqB+BT4HmPZUdVtZP7GhiscoJzi/JsyQAgOzPUblFujDFeglmz6AYkq+pG\nVc0EpgCDPDOo6lxVzXBnfwKaBrE8fiXEJFClqvNM1Sq5Ne0W5cYY4yWYwaIJsNVjPtVN8+cWYKbH\nfISILBaRn0Tkal8riMgoN8/itLS0Uhc0Pjqe169+GYAnL3re7jxrjDFeTolblIvIcCAO6O2R3FxV\nt4nIOcAcEVmhqhs811PVN4A3AOLi4rQsZbgwtqOz05rnl2UzxhhTKQWzZrENiPaYb+qmFSIi/YBH\ngYGqejwvXVW3uX83AolA5yCWlWrVnL9HjwZzL8YYc3oKZrBYBLQUkVgRCQeGAoVGNYlIZ+B1nECx\n2yO9rohUdacbAD2A1UEsqwULY4wpQtCaoVQ1W0TuAmYBocAkVV0lIuOAxao6Hfg7UBP4REQAtrgj\nn1oDr4tILk5Ae1ZVLVgYY0wFCWqfharOAGZ4pT3uMd3Pz3oLgPbBLJs3CxbGGOOf/YLbVaWK87Jg\nYYwxJ7Jg4aF6dThypKJLYYwxpx4LFq6krUlI1YMk79gdOLMxxpxhLFjgBIq+7/blgGzhm1UL7N5Q\nxhjjxYIFzr2hMnMyIfwgucdq2b2hjDHGiwULnHtDhYeGQ8RB5Hik3RvKGGO8WLDAuTfU7Jtm0z66\nGdFV29q9oYwxxosFC1d8dDxdm7chN7NaRRfFGGNOORYsPFSrZr+zMMYYXyxYeKhe3YKFMcb4YsHC\nQ3pWKkePKgu22NBZY4zxZMEY0jgoAAAd/klEQVTClbQ1icmr30RV6DvpcvuthTHGeLBg4UpMSSQn\n9DAAmcftOdzGGOPJgoXLeQ53FgDhufZbC2OM8WTBwhUfHc+YhLsBePfKT+y3FsYY48GChYdOzVsC\n0LJWlwouiTHGnFosWHjYkb0KgPlrVlVwSYwx5tRiwcKVtDWJ++ePAOCB6X+z0VDGGOPBgoUrMSWR\nrKo7Acg6HGmjoYwxxkNQg4WIDBCRdSKSLCJjfCy/T0RWi8hyEZktIs09lt0sIuvd183BLCe4d56t\ndQiA0GNn2WgoY4zxELRgISKhwETgMqANMExE2nhl+xWIU9UOwKfA8+669YAngO5AN+AJEakbrLKC\nMxpqzh9nEl4tkyGxo2w0lDHGeAhmzaIbkKyqG1U1E5gCDPLMoKpzVTXDnf0JaOpOXwp8p6p7VXUf\n8B0wIIhlzVet1lEO7qtyMnZljDGnjWAGiybAVo/5VDfNn1uAmSVZV0RGichiEVmclpZWpsLmP1o1\nZCPfrFhsHdzGGOPhlOjgFpHhQBzw95Ksp6pvqGqcqsZFRUWVqQz5j1atlk5uRh3r4DbGGA/BDBbb\ngGiP+aZuWiEi0g94FBioqsdLsm55ynu0qlTfixytbx3cxhjjIZjBYhHQUkRiRSQcGApM98wgIp2B\n13ECxW6PRbOA/iJS1+3Y7u+mBU3eo1VbN2tI1czGwdyVMcacdoIWLFQ1G7gL50t+DfCxqq4SkXEi\nMtDN9negJvCJiCwVkenuunuBp3ACziJgnJsWdL9lJHHsUHUufruf9VsYY4wrqMN+VHUGMMMr7XGP\n6X5FrDsJmBS80p0oMSWRnGppoKFkHq5FYkqiDaE1xhhOkQ7uU0VCTAKhjVYCELKju/VbGGOMy4KF\nl5Cz1jgTe2MrtiDGGHMKsWDhoeD+ULlk743myyWLKrpIxhhzSrBg4aF+9fpoSDZU2wtJ9/PM4NEV\nXSRjjDklWLDwkJ6RToiEQPU9FV0UU0mtXQs33gjZ2RVdEmNKxoKFh4SYBKqGVoWGy/PTcnOdvxkZ\nMGkSqFZQ4UylcOONMHky/PprRZfEmJKxYOEhPjqelwe8TEjbqflpc9YuBGDMGLjlFpgV1J8GmsrO\nLjbM6cqChZdfd/xKbqtP8ufvfGQLANu3O/OHDlVEqYwxpmJZsPCy8/BOCFE4/wsAfvtiSKFfcn/i\nxpHcXEhOrogSGlMgIwOeeebM6gP54gtISSn/7ebklP82i+Nf/4LvvquYfZeEBQsvDWs2dCa6vpGf\n9tbnG1i3zpn+5BNYuRKeegpatoT16wuvn5EB48ZBZmbJ9vvee3DNNWUo+Blq1Sr46KPSr//llzB1\nauB8p6onn4RHHnE+PyXx8cfwpz8Fp0zBdvXV0LVr+W5zwQKoUgXeead8t1scd94J/fuf/P2WlAUL\nLzd1vIkQQqD+uvy0N0cPZ+XKgjy//QbffutM79pVkL5smdOv8cQT8N//+t7+3LkgAnu8BlzddBNM\nm1ZOB3EGadcOhg4tmN/itBqSng779wdef+BAGDIkOGU7Gfbtc/4eP150Pm9/+AP85z/lX55gyxtw\nsrcc7xT34Yfw9tvO9Pffl992KxsLFl7io+Np1aAV1N3oN8+11zpXIgChoc7fnBzo1AmmTHHmDx/2\nve7zzzt/F/n5vd/IkU7NZPdu38v9OXgQGjSAOXNKtl5FUg3cnLB/f/H7iWbPhubNndpfgwZQv/6J\n+1u7tvjlGzMGbr+9+PlLIiMjcB5vubnw7LNw4EBBWl7TSZVK+HDHl15yLqyOHi1IK2mNvTiuvx7e\nfNOZFin/7VcWFix8OK/+eU6/xY1+73OY73//g88+O/Gf1V/7Z96H0d+omLffdmomt93mzE+bBhdd\nVHBF5c+yZc7V9F//GrDIJ0VyMmRlFZ3nX/+C2Niih5HWrQtNinq+ois7G5YscabzmqW837N33oHW\nrZ2/IvDHP/re1ldfOV9Kzz0Hr78eeN9563z9dfHyAiQkFPR/FdeMGfDww3DffQVpeZ+zvIsWEadZ\no7iOHIE+fZzmvLJIT3fOU945KI6UlKKDZt6FlWcNMRjBorzs2FHRJQguCxY+PNjjQQSB5vMC5v3L\nX+Cep1afkO4vWIS473jeF5mqcwXlLe/q8ZprnIDkr6aSx7OGA87VeKB1fMnIgEsvhTVrSr4uwPLl\nThBo2RIeeKDovImJzt9AAwWKU7M4erTgPfXXB5H3hfivfzl/33rrxDw//ABXXVU46L77btH7zsx0\n1rnyysJXwYF8/nnx8wIcO+b89fzy9A4WUHB8xTFnjnMeAp2r4mxn+3ans91Tdrb/C6PYWBg0yAkw\nG92K/OrVTsCb4XGvas+g79nc5mu7WVmBL6yK4mubqvDgg0Vf1CxeDI0bFzRn5dm/P/Dn53RhwcKH\n+Oh4/vfH/1G/Zm24rXPA/KnbTvx0PjnnKeo9V4/oF6Op9bdaVB11MZH3JPDD5kQAJv78GoOnDKbf\nC/cUulLMM3du4fkjR3zvW9XpP8n7kOd9edSu7f+KPCcH7r0XfvnlxGXz5zvbu/tu3+sG0rFjwZVt\noN+k5JXZs+q/ahX89FPJf4+QkRH4S6JOHeevry/08ePhm28g71HueVe1AJ9+WjD9wgtOTdLTgw8W\nTD/0UMH01Klwxx3w5z/DaB93jinuMf7yi/MebXWfSv/ZZ05t4NChwsGiNFfdJe3r8LZoUeHOde9j\nCgvz/VnKq3V+/z3ExcG55zp9gR9+6KR79t95ltFzev78E7cbHu6/xpiW5pyrko4cO3wY/v536N3b\nf568CxHvZuARI+Dmm0+8+Jowwemo93VRGRLi9H2Cs94HH5SsvEGjqpXi1bVrVy1vC7YsUBkryoN1\n1fk38POKSD8x7ZxZyuX/p0TsVUb2KEg/7wvn77V/UMai3NrN73Zr//mi/OlGf75aGw78p5739wv0\nrN/N1AGv3ay9JvXSc255TEG1Ra+F+XlH3rM1f/pv8/6mC7YsKHRcM2YU7MPzWP8272/6z49WKqhe\ndFHp3jPP8rdsWXTea6918j32mO/1o6JOLKdn+VevLlj+/vuqY8ac+B56mjDBSYuN9X8uP/nkxLQb\nbzyxfJ569ChIHzzY97HkrdO1q//yff656qRJqjk5ql9+qZqb66T/6U9O3ksvLbzukiWq113nTL/3\nnurevb6364vn+5a37dLwft88jz8z0395Dhwo4v8J1dtuU23UqOA850lOLsjz1VeFt5mTU7DstddU\np00rvPz6651l3un+znWeXbucZRER/t+Hd95x8gwfXji9fXsnfdky3/s8cqRg+tZbVRcuLPyeFXU+\n8z4fZQUs1mJ8x1bCbrHyk1fDuHnazawfUxsW/R/88ifYd27hjMfqnbjyxv7OC2Cqx6WBuJdeU6dA\n1BpI8lGtcB18ueDSacfLzqXWzi9vBw3lmwUDYKxAahcAkpdF5ed965Wm+dOPzHkEgFAJJTTEaavI\nXT0QcBrMqzxRnarhQka223i88TtgDj/+CBEjryTsvB9oVLMRmTmZiAjNIptx8NhBth3aRpPaTUBh\nT8Yeru9wvbvH5/L3vWnvZmJfSaBORB2OZx/n/Abnc98FDxFaRflhcyJTpz4MwNNPQ9crl3B240yg\n4GFTeVf5AHd8dQcAnRt15t+vxAPtGTGiYPkNN/h+D+/46g46N+pMekY6u9KHArGkbs8CwnzmX7R+\nA1D4/LZo4XvbeTyv6DccXE3S1gPFfmhWv35OLaZ2bedKE5wa0q23OreXGTmy4Grdu6bmWbPIyirc\n7Ji0NYk5GxOpvnUQf76hDatXQ2QkNG3qv1mnLH7/e+fvuj3rSNq6l7iG8YwZ4z9/oOa6Pcd2cjiz\nBlCLDz5wOqFbt/Zd3gULnJp3jx4FaXc4H5f89w4KrtBXbPuNNfOnkhCTQLfGhc+TqlNLaN3aaRpr\n0aKgX8VXzVXVOUd5zcue+4OCWoy/Zum8UZXgjE7zbsbKs32708zlKSwMLrvMGf59Moh6H91pKi4u\nThcvXhyUbT8z/xkem/sYuerxadnfDA42gakfwoHmTlqH92DV7yEnIijlOMETAotvg6//DaHHIafq\niXnGerTxHI6Cb/8B+2NgS08n7S9RUGMPHGwE1fbBlGmwYcCJ6+eGwFevwbnfQbP5UGuXkya5IMDO\n9lB/PYz3+BaoegBG9nLutZUVASiMPwYXvgT9xsDTHv/5t1wI0T/DWD+fx7xyZIfBR5/B+iuL9x49\nWA+yakBkKvz4IHz/XOB1vFTt/RI1Ln+a2lVrk/LnTQA0fKERDWs25Hj2cTY8M5XMba2dzB3fgcEj\n4Pk0yGhQaDu1/labw//8Ad1edNNmzf5/5/C3f6HaBR+itVM5NvsvPvM1aLOCw7npHFubQM0BzxLR\nbiZ7XvgBABkbgv50F3wzgaiRd5D21msAXP3hYNbNb8eafz0FQIsbXyL5vXup2+5nat/yB0QkP7hH\n1Yji4LGDrE9PJmvBnTTsuIwaTTYTVSMKFFbuXsneMemFC9VqGuE3DKXvgfeY+cJ1+ck3T36A7Go7\nSUxJpPbhriA5rHn8K/9vQvdXnP+lw+43pOTQ8O9Nqb2vF7+Nd0YwNL3t/6jSemb+OWn394tZ+ZfC\n7UBXfziYhjUbsubzq/jhv5c7idfcAB2cyBEV1py0R1Py84fV207W3saE1NhL7pF6XPXq/Szdtpqt\nz8503vOnY6hbO5yj+2sjAlE5nfhlbMEY5OiuK7nygc9JTP+Afcf2see5BWSnxdJ//JOsrPomUTWi\niK0Tw+fDAo+Tb/3PNqy5q6A/tNek3qRlpBFVI4p6EfXyt9Hsrlt5dEQ3RnUdFXCbvojIElWNC5TP\nahbFkHeDwcycTMJDw7m7+918tPIj9h5dSZUHu5N5pBqZx4UqtfaSdemjZM+7HxbfDioQmuV8WQXD\nS1vgYLQz7StQAGyJh8ZLYG8LeGseHPUaT/rWDxD3b/hmAkRuLgh83t5YDDs7wy/uB7LuBqeG1ekt\nGHAP/Hs5tPX6ddzxSPj3MieojT8K1d2qwk/3wnavX1UdbApZS4s+3uRLYPK3Refx9rw7IH+sQHbp\ngvjxH+7lePwT7M0uGLqz8/BO59f+AMc9AlyIeynpFSgADmUeAg3c+3o42/nxxNFFw4rMt2d1ewhz\nOrMOfzOGwxva5y/T+Q/BIeeLNi9QAHy+7nPYWvAZSN7u/FBo39F97Duw2UnMioCke1nzu39AlUzY\n0RFmPMuWJUvhjs6s2bgfdnSF83z82EGFzLSmzHyzb6Hkd4a/4EzU3sq2g9FwXoDL4ZDsglo4gIY6\n7/mSs/OTUtPTYc/2/PmVqScOd/f5pawFXbVpa84vtChrv3Peco84rQVfLlkEYQUXQHv+tpA9D56d\nf1Gz1WvTW5e047Vh7WDsX0GBvY0A+HbdPJg+n+2tprFswP3+j9vDmj2FOzrmbZlXkH68Zn76ln/+\nh9tqRjJv8zwmXzO5WNsujaAGCxEZALwChAL/UdVnvZb3Al4GOgBDVfVTj2U5wAp3douqDgxmWYsS\nHx3P7Jtmk5iSSEJMAvHR8TzXz/8VatLWJBJTXmT/8f3MWT+fs6s1p0HKbSzfsZK1yyLJOh5GiISQ\ns+ccqJdMTs/H4Z+/QUgW1NoGB2KKLpDkgIYWBIqiTFpQ9PI9bZxAAb4DxViFS+91AoWnvKa4pSOh\npTtmdMMlvvex0f3iyChoKmNLr8J5DjSHxCeLLmtJA4W3rGqlX3feX2GBxxX+8ZpQ1W33yfbY7q+3\nQJSfcage/+BFCvHogQ3JhNxw/3k9L0TWX1EwnfgEdH3zxPyLbndqg/llql0wvSXe2V5qd5j7NCy8\nG7q8CfV/c5bvbg/J/WGy2x52vcf+8gl8NvnEi5I8eZ/Z367yf0xQuIx5Vg8u+KwC7D23cO10ge8a\n2AnyLhoONCk4ljze7/WxOoWCCxlnwe7WgfehwNxxBa0MR+vC/lj46T4oZrDgO6+hZYpzAbo9Dqoe\nLLzsnTm8HxFHr+a9Sl3DCKg4HRuleeEEiA3AOUA4sAxo45UnBidQvAsM8Vp2uCT7C0YH98myYMsC\nveqd67TVhLba8V9dNOS+5srDNZT7muh5j16nLR64WRnVRbnlQqXTJK11++Uaes5crdr6W61Sf4tS\nO7Wgk67uegVVaf2Zhpwz208HYk6RnYun5Gvw8LKtf8GrZVu/2bzC8+0+UFp9psS/UP7HetH48tmO\nZAXOc863zt9zZxakdXulcJ4ubxRMR24qmK637sTtnf+5cvbSspe92ytKra1l346/V42dziCU4uS9\n/rLy3fclD5RuvbvOU6occabbv1d4Waupyli0/7v9S/z9QzE7uANmKO0Lp6dylsf8w8DDfvK+fSYH\nC295I5M8RzH5SvO3jq9REuvWqb7w9mq9/4PXdMGWBfrrr6qPPKI6ZMR2rV5vn0Zf/p7yu+eVyE0q\n50/XGg23FfowhjVepY2ufkURJ9DUar1AQbVK1MYTP9Qd3lHO/Sbwh7/hkhPTRvRS2k4J3peEvU6P\nV/XdFV+G0+nV5mNlLPr64tdL/H1T3GARtA5uERkCDFDVW935G4HuqnqXj7xvA19p4WaobGApkA08\nq6on/IRJREYBowCaNWvWdfPmzcE4lDOG03xW0NTmT06OM64/K8sZkXHsmDMevE4dqFkTko85I3H2\nrWtHVtWddKh3IXt2hzHv11Q6dj+I1kylfe1exLQ8yjPvLkRzhWZHB5Fw2V5GzO/B8ezjcLQeXWpf\nRvamC1n70U3ExEJ2Rk169XJGCQG06LyDTc2eICfpbkJjFzCg+WDWrs1lw/KGtLjic+rntuXnmS0B\n6Dl4DUt+aETj9r+R/EM3QmunERaZxsjnp/HasEf9Hmv1jjPJWHbZCelh9VMJb76EI78MAkBCs7nx\nrUdZ8G1DkiffC8DwR+cxeXyvE9YFnKaqtLZ+9xtacy+aWZ3cTKcZI7zJajK3tXEWXvBP6Pgu/Gdh\nfv7IVr9yZPP5ZB+t7pT77FQydjU9YbulFV7zIGFNV3BkbY8TF3Z50xklWAJnjW3H7injYXNPOOpj\nNKGner/B3vPgd8/Dggdh4C3Q9iNCX00h57DbNxSaCTleTUiSg5y1Bt3VLnCB6q+F9FYlOoZTSbO7\n/8ijN19Yqiao4nZwn8rBoomqbhORc4A5QF9V3eBvf8EcDWVOnuIGrNLm90UVfkp1ttO7eQLdm8QX\n+kX0/zYnMfXH5Qzq3pHOZ11IrVoFPyQ8ftwZslq7thM4wRlKqwpVq8K2bc49qo4dc34w9nPqz8xc\nuoSB3ToTHx2PiDPscs8eiIhwgu2xY852Q0OdwFy7trO/nBznYVyL987OP95du6BePed2M7m5zisr\nC6pXd349PHu2Mzx39cGC96lb43hSU2HdoYW8My2Vo7ubEBG1nQtbnI+EKPPTphEReYjOMoIhCa35\n6CPnvmf9+jk/otsb/ivL9s9j2VJh5vHHyQ07gCwczTUdL6FDtSupVs35UVndurBpE+wKS+Kdpe8i\nAlc3H0lc427Ur++cu9nJP1B7x0CGD2jDuAmb2J6zgo5xR/j24xh2bq1JnYRJjLiiDXWPxrEp9Bt0\nXwzUSaFPbAKta8WzcKHzPvfq5QwtDo1eyEffr6ddg86MHtaGI0fgnx+uQ1t9xvmhlxIT0YXDh51h\ntFdc4axXp45Tlv/+9BGZGRG0rNWZ9CMH6BzVncu6dOTxf2xhe+RUdv8WQ9927Rn1+xZERzu3eGnf\nHlJTnXOflLyaTz7PIPxILN071GfcOOdcrtyfxISvv+WzBcvIafEF8svtXNLiYr7fMJuc3S0J7fw+\nF2Y/SEyDRpx3cRIcaI7W2srG6UM5q1pTdhzaRvKunVzVvybNa51Pt27O3QciI53PxYMPlu2eVqdC\nsIgHxqrqpe78wwCq+oyPvG/jFSxKshwsWBhzsiVtTaLvu33zRwnOvml2qQP2mcD7wqY8LnTKw6kQ\nLKoAvwF9gW3AIuB6VT1hqIh3MBCRukCGqh4XkQZAEjBIVU+8CZPLgoUxJ9+p8oVnSq/Cf2ehqtki\nchcwC2dk1CRVXSUi43A6VKaLyAXANKAucJWIPKmqbYHWwOsikotz/6pniwoUxpiKER8db0HiDGG/\n4DbGmDNYcWsWdtdZY4wxAVmwMMYYE5AFC2OMMQFZsDDGGBOQBQtjjDEBVZrRUCKSBpT2fh8NgD3l\nWJzTgR3zmcGO+cxQlmNurqpRgTJVmmBRFiKyuDhDxyoTO+Yzgx3zmeFkHLM1QxljjAnIgoUxxpiA\nLFg43qjoAlQAO+Yzgx3zmSHox2x9FsYYYwKymoUxxpiALFgYY4wJ6IwPFiIyQETWiUiyiIyp6PKU\nFxGJFpG5IrJaRFaJyD1uej0R+U5E1rt/67rpIiIT3PdhuYh0qdgjKB0RCRWRX0XkK3c+VkR+do/r\nIxEJd9OruvPJ7vKYiix3aYlIHRH5VETWisgaEYk/A87xve5neqWIfCgiEZXxPIvIJBHZLSIrPdJK\nfG5F5GY3/3oRubm05Tmjg4WIhAITgcuANsAwEWlTsaUqN9nA/araBrgQuNM9tjHAbFVtCcx258F5\nD1q6r1HAaye/yOXiHmCNx/xzwEuq2gLYB9zipt8C7HPTX3LznY5eAb5R1VZAR5xjr7TnWESaAKOB\nOFVth/OsnKFUzvP8NjDAK61E51ZE6gFPAN2BbsATeQGmxFT1jH0B8cAsj/mHgYcrulxBOtYvgEuA\ndUAjN60RsM6dfh0Y5pE/P9/p8gKauv9AFwNfAYLzq9Yq3ucb56Fc8e50FTefVPQxlPB4I4FN3uWu\n5Oe4CbAVqOeet6+ASyvreQZigJWlPbfAMOB1j/RC+UryOqNrFhR88PKkummVilv17gz8DJytqjvc\nRTuBs93pyvBevAw8COS68/WB/aqa7c57HlP+8brLD7j5TyexQBrwltv09h8RqUElPsequg14AdgC\n7MA5b0uo3OfZU0nPbbmd8zM9WFR6IlITmAr8WVUPei5T51KjUoydFpErgd2quqSiy3ISVQG6AK+p\namfgCAXNEkDlOscAbhPKIJxA2RiowYlNNWeEk31uz/RgsQ2I9phv6qZVCiIShhMo3lfVz9zkXSLS\nyF3eCNjtpp/u70UPYKCIpABTcJqiXgHqiEjes+Y9jyn/eN3lkUD6ySxwOUgFUlX1Z3f+U5zgUVnP\nMUA/YJOqpqlqFvAZzrmvzOfZU0nPbbmd8zM9WCwCWrojKcJxOsqmV3CZyoWICPBfYI2qvuixaDqQ\nNyLiZpy+jLz0m9xRFRcCBzyqu6c8VX1YVZuqagzOeZyjqjcAc4Ehbjbv4817H4a4+U+rK3BV3Qls\nFZHz3aS+wGoq6Tl2bQEuFJHq7mc875gr7Xn2UtJzOwvoLyJ13VpZfzet5Cq6A6eiX8DlwG/ABuDR\nii5POR7XRThV1OXAUvd1OU577WxgPfA9UM/NLzgjwzYAK3BGm1T4cZTy2BOAr9zpc4CFQDLwCVDV\nTY9w55Pd5edUdLlLeaydgMXuef4cqFvZzzHwJLAWWAm8B1StjOcZ+BCnXyYLpxZ5S2nOLfBH9/iT\ngZGlLY/d7sMYY0xAZ3ozlDHGmGKwYGGMMSYgCxbGGGMCsmBhjDEmIAsWxhhjArJgYUwAIpIjIks9\nXuV2d2IRifG8q6gxp6oqgbMYc8Y7qqqdKroQxlQkq1kYU0oikiIiz4vIChFZKCIt3PQYEZnjPldg\ntog0c9PPFpFpIrLMff3O3VSoiLzpPqPhWxGp5uYfLc7zSJaLyJQKOkxjAAsWxhRHNa9mqD94LDug\nqu2Bf+Lc9RbgVeAdVe0AvA9McNMnAD+oakecezitctNbAhNVtS2wH7jWTR8DdHa3c3uwDs6Y4rBf\ncBsTgIgcVtWaPtJTgItVdaN708adqlpfRPbgPHMgy03foaoNRCQNaKqqxz22EQN8p87DbBCRh4Aw\nVX1aRL4BDuPcxuNzVT0c5EM1xi+rWRhTNupnuiSOe0znUNCXeAXO/X66AIs87qpqzElnwcKYsvmD\nx98kd3oBzp1vAW4A5rvTs4E7IP9Z4ZH+NioiIUC0qs4FHsK5tfYJtRtjTha7UjEmsGoistRj/htV\nzRs+W1dEluPUDoa5aXfjPL3uLzhPshvppt8DvCEit+DUIO7AuauoL6HAZDegCDBBVfeX2xEZU0LW\nZ2FMKbl9FnGquqeiy2JMsFkzlDHGmICsZmGMMSYgq1kYY4wJyIKFMcaYgCxYGGOMCciChTHGmIAs\nWBhjjAno/wGVkooxFkdVNgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iOFBSbPcYCN4", + "colab_type": "text" + }, + "source": [ + "## Look closer at the data\n", + "The graph shows the _loss_ (or the difference between the model's predictions and the actual data) for each epoch. There are several ways to calculate loss, and the method we have used is _mean squared error_. There is a distinct loss value given for the training and the validation data.\n", + "\n", + "As we can see, the amount of loss rapidly decreases over the first 25 epochs, before flattening out. This means that the model is improving and producing more accurate predictions!\n", + "\n", + "Our goal is to stop training when either the model is no longer improving, or when the _training loss_ is less than the _validation loss_, which would mean that the model has learned to predict the training data so well that it can no longer generalize to new data.\n", + "\n", + "To make the flatter part of the graph more readable, let's skip the first 50 epochs:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Zo0RYroFZYIV", + "colab_type": "code", + "outputId": "e6841332-0541-44bb-a186-ae5b46781e51", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 295 + } + }, + "source": [ + "# Exclude the first few epochs so the graph is easier to read\n", + "SKIP = 50\n", + "\n", + "plt.plot(epochs[SKIP:], loss[SKIP:], 'g.', label='Training loss')\n", + "plt.plot(epochs[SKIP:], val_loss[SKIP:], 'b.', label='Validation loss')\n", + "plt.title('Training and validation loss')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('Loss')\n", + "plt.legend()\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzsnXl4lNXZuO9nZhJQQbGRFpFAcKkC\nRhYjGgEJov1A0Wqx1q3giqJYqbV1aVWq9odrxQUVKiKpWvWTT9xArUDYDPsiRURRIomCQiooAknm\nfZ/fH2dmMjOZJJNkJpkk576uXJl3O+95t+c5z3LOEVXFYrFYLJb64mnqClgsFouleWMVicVisVga\nhFUkFovFYmkQVpFYLBaLpUFYRWKxWCyWBmEVicVisVgahFUkliZHRLwiskdEuiZy36ZERI4WkYTn\n1ovIGSJSFLa8SUQGxbNvPc71rIjcUd/jayj3PhF5PtHlWpoOX1NXwNL8EJE9YYsHAmWAE1i+VlVf\nrEt5quoA7RK9b2tAVY9NRDkicjVwmarmhZV9dSLKtrR8rCKx1BlVDQnyQIv3alX9oLr9RcSnqv7G\nqJvFYml8rGvLknACrotXRORfIvIDcJmI5IrIUhHZJSLbRORxEUkL7O8TERWRrMDyC4Htc0TkBxEp\nFJHudd03sH24iHwqIrtF5AkRWSIil1dT73jqeK2IbBaR70Tk8bBjvSLyqIiUisgXwLAa7s+fReTl\nqHWTReTvgd9Xi8jGwPV8HrAWqiurRETyAr8PFJF/Buq2ATgxat+/iMgXgXI3iMi5gfXZwJPAoIDb\ncGfYvZ0Qdvx1gWsvFZFZInJ4PPemNkTk/EB9donIPBE5NmzbHSLytYh8LyKfhF3rKSKyOrD+GxF5\nKN7zWZKAqto/+1fvP6AIOCNq3X1AOXAOprFyAHAScDLGCj4S+BQYF9jfByiQFVh+AdgJ5ABpwCvA\nC/XY96fAD8AvA9tuBiqAy6u5lnjq+AZwCJAF/Dd47cA4YAPQBcgAFprPK+Z5jgT2AAeFlf0tkBNY\nPiewjwCnA/uAEwLbzgCKwsoqAfICvx8GCoBDgW7Ax1H7XggcHngmlwTq8LPAtquBgqh6vgBMCPz+\nRaCOfYC2wFPAvHjuTYzrvw94PvC7R6Aepwee0R3ApsDvXsCXQKfAvt2BIwO/VwAXB363B05u6m+h\nNf9Zi8SSLBar6luq6qrqPlVdoarLVNWvql8AU4HBNRz/mqquVNUK4EWMAKvrviOAtar6RmDboxil\nE5M46zhRVXerahFGaAfPdSHwqKqWqGopcH8N5/kC+A9GwQGcCXynqisD299S1S/UMA+YC8QMqEdx\nIXCfqn6nql9irIzw876qqtsCz+QlTCMgJ45yAS4FnlXVtaq6H7gNGCwiXcL2qe7e1MRFwJuqOi/w\njO7HKKOTAT9GafUKuEe3BO4dmAbBMSKSoao/qOqyOK/DkgSsIrEki+LwBRE5TkTeEZHtIvI9cA9w\nWA3Hbw/7vZeaA+zV7ds5vB6qqpgWfEzirGNc58K0pGviJeDiwO9LAsvBeowQkWUi8l8R2YWxBmq6\nV0EOr6kOInK5iKwLuJB2AcfFWS6Y6wuVp6rfA98BR4TtU5dnVl25LuYZHaGqm4A/YJ7DtwFXaafA\nrlcAPYFNIrJcRM6K8zosScAqEkuyiE59nYJphR+tqgcDd2FcN8lkG8bVBICICJGCL5qG1HEbkBm2\nXFt68qvAGSJyBMYyeSlQxwOA14CJGLdTB+D9OOuxvbo6iMiRwNPAWCAjUO4nYeXWlqr8NcZdFiyv\nPcaF9lUc9apLuR7MM/sKQFVfUNUBGLeWF3NfUNVNqnoRxn35CDBTRNo2sC6WemIViaWxaA/sBn4U\nkR7AtY1wzreBfiJyjoj4gJuAjkmq46vAeBE5QkQygFtr2llVtwOLgeeBTar6WWBTGyAd2AE4IjIC\nGFqHOtwhIh3E9LMZF7atHUZZ7MDo1GswFkmQb4AuweSCGPwLuEpEThCRNhiBvkhVq7Xw6lDnc0Uk\nL3DuP2LiWstEpIeIDAmcb1/gz8VcwG9F5LCABbM7cG1uA+tiqSdWkVgaiz8AozFCYgomKJ5UVPUb\n4DfA34FS4ChgDabfS6Lr+DQmlrEeEwh+LY5jXsIEz0NuLVXdBfweeB0TsL4AoxDj4W6MZVQEzAHy\nw8r9CHgCWB7Y51ggPK7wb+Az4BsRCXdRBY9/F+Niej1wfFdM3KRBqOoGzD1/GqPkhgHnBuIlbYAH\nMXGt7RgL6M+BQ88CNorJCnwY+I2qlje0Ppb6IcZtbLG0fETEi3GlXKCqi5q6PhZLS8FaJJYWjYgM\nC7h62gB3YrJ9ljdxtSyWFoVVJJaWzkDgC4zb5H+A81W1OteWxWKpB9a1ZbFYLJYGYS0Si8VisTSI\nVjFo42GHHaZZWVlNXQ2LxWJpVqxatWqnqtaUMg+0EkWSlZXFypUrm7oaFovF0qwQkdpGaACsa8ti\nsVgsDcQqEovFYrE0CKtILBaLxdIgWkWMxGKxNC4VFRWUlJSwf//+pq6KJQ7atm1Lly5dSEurbqi1\nmrGKxGKxJJySkhLat29PVlYWZtBlS6qiqpSWllJSUkL37t1rPyAG1rVlsVgSzv79+8nIyLBKpBkg\nImRkZDTIerSKxGJpJAoLYeJE8781YJVI86Ghz8q6tiyWRqCwEIYOhfJySE+HuXMhN7epa2WxJIak\nWiSBkVc3ichmEbktxvbTRGS1iPhF5IKobQ+KyAYR2Sgijwdmt0NEThSR9YEyQ+stllSmoMAoEccx\n/wsKmrpGLZvS0lL69OlDnz596NSpE0cccURoubw8vmlLrrjiCjZt2lTjPpMnT+bFF19MRJUZOHAg\na9euTUhZjU3SLJLA3A+TgTMxczCvEJE3VfXjsN22ApcDt0QdeyowADghsGoxMBgowEyAcw1mUp7Z\nmIlw5iTrOiyWRJCXZyyRoEWSl9fUNWrZZGRkhITyhAkTaNeuHbfcEiFmUFVUFY8ndnt6+vTptZ7n\nhhtuaHhlWwDJtEj6A5tV9YvAzGUvY+amDqGqRYGZ26KnyFSgLWbK0TZAGmbmtsOBg1V1qZphi/OB\n85J4DRZLQsjNNe6se++1bq3qKCwuZOKiiRQWJy+ItHnzZnr27Mmll15Kr1692LZtG2PGjCEnJ4de\nvXpxzz33hPYNWgh+v58OHTpw22230bt3b3Jzc/n2228B+Mtf/sKkSZNC+992223079+fY489lg8/\n/BCAH3/8kZEjR9KzZ08uuOACcnJyarU8XnjhBbKzszn++OO54447APD7/fz2t78NrX/88ccBePTR\nR+nZsycnnHACl112WcLvWTwkM0ZyBFActlwCnBzPgapaKCLzMVN6CvCkqm4UkZxAOeFlHhGrDBEZ\nA4wB6Nq1a91rb7EkmNxcq0Cqo7C4kKH5Qyl3ykn3pjN31FxyM5Nzsz755BPy8/PJyckB4P777+cn\nP/kJfr+fIUOGcMEFF9CzZ8+IY3bv3s3gwYO5//77ufnmm3nuuee47bYq3npUleXLl/Pmm29yzz33\n8O677/LEE0/QqVMnZs6cybp16+jXr1+N9SspKeEvf/kLK1eu5JBDDuGMM87g7bffpmPHjuzcuZP1\n69cDsGvXLgAefPBBvvzyS9LT00PrGpuUzNoSkaOBHkAXjKI4XUQG1aUMVZ2qqjmqmtOxY62DV1os\nliakoKiAcqccRx3KnXIKigqSdq6jjjoqpEQA/vWvf9GvXz/69evHxo0b+fjjj6scc8ABBzB8+HAA\nTjzxRIqKimKW/atf/arKPosXL+aiiy4CoHfv3vTq1avG+i1btozTTz+dww47jLS0NC655BIWLlzI\n0UcfzaZNm/jd737He++9xyGHHAJAr169uOyyy3jxxRfr3aGwoSRTkXwFZIYtdwmsi4fzgaWqukdV\n92BiILmB47vUs0yLxZKi5GXlke5Nxyte0r3p5GXlJe1cBx10UOj3Z599xmOPPca8efP46KOPGDZs\nWMz+FOnp6aHfXq8Xv98fs+w2bdrUuk99ycjI4KOPPmLQoEFMnjyZa6+9FoD33nuP6667jhUrVtC/\nf38cx0noeeMhmYpkBXCMiHQXkXTgIuDNOI/dCgwWEZ+IpGEC7RtVdRvwvYicEsjWGgW8kYzKWyyW\nxiM3M5e5o+Zy75B7k+rWiub777+nffv2HHzwwWzbto333nsv4ecYMGAAr776KgDr16+PafGEc/LJ\nJzN//nxKS0vx+/28/PLLDB48mB07dqCq/PrXv+aee+5h9erVOI5DSUkJp59+Og8++CA7d+5k7969\nCb+G2khajERV/SIyDngP8ALPqeoGEbkHWKmqb4rIScDrwKHAOSLyV1XtBbwGnA6sxwTe31XVtwJF\nXw88DxyAsVRsxpbF0gLIzcxtNAUSpF+/fvTs2ZPjjjuObt26MWDAgISf48Ybb2TUqFH07Nkz9Bd0\nS8WiS5cu3HvvveTl5aGqnHPOOZx99tmsXr2aq666ClVFRHjggQfw+/1ccskl/PDDD7iuyy233EL7\n9u0Tfg210SrmbM/JyVE7sZXF0nhs3LiRHj16NHU1UgK/34/f76dt27Z89tln/OIXv+Czzz7D50ut\n/uCxnpmIrFLVnGoOCZFaV2KxWCwtjD179jB06FD8fj+qypQpU1JOiTSUlnU1FovFkmJ06NCBVatW\nNXU1kkpKpv9aLBaLpflgFYnFYrFYGoRVJBaLxWJpEFaRWCwWi6VBWEVisVhaHEOGDKnSuXDSpEmM\nHTu2xuPatWsHwNdff80FF1wQc5+8vDxq604wadKkiI6BZ511VkLGwZowYQIPP/xwg8tJNFaRWCyW\nFsfFF1/Myy+/HLHu5Zdf5uKLL47r+M6dO/Paa6/V+/zRimT27Nl06NCh3uWlOlaRWCyWlCCRUxFf\ncMEFvPPOO6FJrIqKivj6668ZNGhQqF9Hv379yM7O5o03qo6yVFRUxPHHHw/Avn37uOiii+jRowfn\nn38++/btC+03duzY0BD0d999NwCPP/44X3/9NUOGDGHIkCEAZGVlsXPnTgD+/ve/c/zxx3P88ceH\nhqAvKiqiR48eXHPNNfTq1Ytf/OIXEeeJxdq1aznllFM44YQTOP/88/nuu+9C5w8OKx8cLHLBggWh\nib369u3LDz/8UO97G5Pg5C4t+e/EE09Ui8XSeHz88cd12v/DD1UPOEDV6zX/P/yw4XU4++yzddas\nWaqqOnHiRP3DH/6gqqoVFRW6e/duVVXdsWOHHnXUUeq6rqqqHnTQQaqqumXLFu3Vq5eqqj7yyCN6\nxRVXqKrqunXr1Ov16ooVK1RVtbS0VFVV/X6/Dh48WNetW6eqqt26ddMdO3aE6hJcXrlypR5//PG6\nZ88e/eGHH7Rnz566evVq3bJli3q9Xl2zZo2qqv7617/Wf/7zn1Wu6e6779aHHnpIVVWzs7O1oKBA\nVVXvvPNOvemmm1RV9fDDD9f9+/erqup3332nqqojRozQxYsXq6rqDz/8oBUVFVXKjvXMMMNZ1Spj\nrUVisVianGRMRRzu3gp3a6kqd9xxByeccAJnnHEGX331Fd9880215SxcuDA0YdQJJ5zACSecENr2\n6quv0q9fP/r27cuGDRtqHZBx8eLFnH/++Rx00EG0a9eOX/3qVyxatAiA7t2706dPH6DmoerBzI+y\na9cuBg8eDMDo0aNZuHBhqI6XXnopL7zwQqgH/YABA7j55pt5/PHH2bVrV8J71ltFYrFYmpzgVMRe\nb+KmIv7lL3/J3LlzWb16NXv37uXEE08E4MUXX2THjh2sWrWKtWvX8rOf/Szm0PG1sWXLFh5++GHm\nzp3LRx99xNlnn12vcoIEh6CHhg1D/84773DDDTewevVqTjrpJPx+P7fddhvPPvss+/btY8CAAXzy\nySf1rmcsrCKxWCxNTjKmIm7Xrh1DhgzhyiuvjAiy7969m5/+9KekpaUxf/58vvzyyxrLOe2003jp\npZcA+M9//sNHH30EmCHoDzroIA455BC++eYb5sypHIi8ffv2MeMQgwYNYtasWezdu5cff/yR119/\nnUGD6jRnHwCHHHIIhx56aMia+ec//8ngwYNxXZfi4mKGDBnCAw88wO7du9mzZw+ff/452dnZ3Hrr\nrZx00kkJVyR2rC2LxZISJGMq4osvvpjzzz8/IoPr0ksv5ZxzziE7O5ucnByOO+64GssYO3YsV1xx\nBT169KBHjx4hy6Z379707duX4447jszMzIgh6MeMGcOwYcPo3Lkz8+fPD63v168fl19+Of379wfg\n6quvpm/fvjW6sapjxowZXHfddezdu5cjjzyS6dOn4zgOl112Gbt370ZV+d3vfkeHDh248847mT9/\nPh6Ph169eoVme0wUdhh5i8WScOww8s2Phgwjb11bFovFYmkQVpFYLBaLpUFYRWKxWJJCa3CbtxQa\n+qysIrFYLAmnbdu2lJaWWmXSDFBVSktLadu2bb3LsFlbFosl4XTp0oWSkhJ27NjR1FWxxEHbtm3p\n0qVLvY+3isRisSSctLQ0unfv3tTVsDQS1rVlsVgslgaRVEUiIsNEZJOIbBaR22JsP01EVouIX0Qu\nCFs/RETWhv3tF5HzAtueF5EtYdv6JPMaLBaLxVIzSXNtiYgXmAycCZQAK0TkTVUNH9VsK3A5cEv4\nsao6H+gTKOcnwGbg/bBd/qiq9Z8swGKxWCwJI5kxkv7AZlX9AkBEXgZ+CYQUiaoWBba5NZRzATBH\nVffWsI/FYrFYmohkuraOAIrDlksC6+rKRcC/otb9TUQ+EpFHRaRNrINEZIyIrBSRlTZzxGKxWJJH\nSgfbReRwIBsIn3z5duA44CTgJ8CtsY5V1amqmqOqOR07dkx6XS0Wi6W1kkxF8hWQGbbcJbCuLlwI\nvK6qFcEVqrotMHlXGTAd40KzWCwWSxORTEWyAjhGRLqLSDrGRfVmHcu4mCi3VsBKQUQEOA/4TwLq\narFYLJZ6kjRFoqp+YBzGLbUReFVVN4jIPSJyLoCInCQiJcCvgSkisiF4vIhkYSyaBVFFvygi64H1\nwGHAfcm6BovFYrHUjp2PxGKxWCwxsfORWCwWi6VRsIrEYrFYLA3CKhKLxWKxNAirSCwWiyXFKCyE\niRPN/+aAHUbeYmnGFBZCQQHk5UFublPXxpIICgth6FAoL4f0dJg7N/WfrVUkFkszpTkKnETTEhVp\nQYF5po5j/hcUpP61WUVisTRTmqPASSQtVZHm5ZnrCV5XXl5T16h2rCKxWJopzVHgJJKWqkhzc41S\nbE6WllUkFkszpTkKnETSkhVpbm7zep5WkVgszZjmJnASSWtXpKmEVSQWi6XZkixF2hKD+MnEKhKL\nxWIJIxjELysDjwcmT4YxY5q6VqmN7ZDYSDS3DkYWS2uloMAoEdcFvx/GjbPfbW1Yi6QRaKlpipbm\ni3XdVE9enrFEXNcsO07LyQhLFtYiaQRipSlaLE1FsGFz553mv21tR5Kba9xZaWlGobRp07IywpKB\nVSR1pD4uqmCaotfb8tIUG5tEuAhbu5vRNmxqZ8wYWLAA7rvPehDiwbq26kB9XVQ2TTExJMJFaN2M\nLbv/RSJpzanVdcUqkjrQkJ609qVsOInoydxSe0PXBduwsSQaq0jqgG3JNS2JuP/2GRpsw8aSSKwi\nqQO2Jde0JOL+22dosSQeUdWmrkPSycnJ0ZUrVzZ1NSwWi6VZISKrVDWntv1s1pbFYrFYGkRSFYmI\nDBORTSKyWURui7H9NBFZLSJ+EbkgbP0QEVkb9rdfRM4LbOsuIssCZb4iIunJvIa60tpTSy0WS+sj\naYpERLzAZGA40BO4WER6Ru22FbgceCl8parOV9U+qtoHOB3YC7wf2PwA8KiqHg18B1yVrGuoK7aj\nl8ViaY0k0yLpD2xW1S9UtRx4Gfhl+A6qWqSqHwFuDeVcAMxR1b0iIhjF8lpg2wzgvMRXvX7Yjl4W\ni6U1kkxFcgRQHLZcElhXVy4C/hX4nQHsUlV/bWWKyBgRWSkiK3fs2FGP09Yd24PdYrG0RlI6/VdE\nDgeygffqeqyqTgWmgsnaSnDVYmJTSy2W5o8d0LLuJFORfAVkhi13CayrCxcCr6tqRWC5FOggIr6A\nVVKfMpOK7ehlsTRf7BA69SOZrq0VwDGBLKt0jIvqzTqWcTGVbi3UdHqZj4mbAIwG3khAXWvFZmNZ\nLC0fG+esH0mzSFTVLyLjMG4pL/Ccqm4QkXuAlar6poicBLwOHAqcIyJ/VdVeACKShbFoFkQVfSvw\nsojcB6wBpiXrGoLYVorF0jqwQ+jUj6TGSFR1NjA7at1dYb9XYNxTsY4tIkYgXVW/wGSENRp2oL/U\nJejPzsiA0lLr17Y0DBvnrB8pHWxPFWwrJTUJn1vbdSsnIbIWo6Uh2Dhn3bFDpMRBsJVy771WSKUS\nQUsxOCWq61q/dkvFxihTG2uRxIltpaQeQUsx3CKxFmPLw8YoUx+rSBqRROSn2xz3SsL92YmKkdj7\nm3rYGGXqYxVJI2GniU0OibQU7f1NTWyMMvWxMZJGIhH56TbHPbnY+5ua2Bhl6mMtkkYivFXl9cLW\nraYFXJePwrbMkkuq39/W7HazMcrUxs6Q2IgUFkJ+PkyfDn5//dwnrVmYNAapen+t283SFMQ7Q6K1\nSBqR3FwjpPz++gcObcssuaTq/bUBZ0sqY2MkjYwdat5SH+x7Y0llrEXSyLTmIRjicRsl2rWUqq6q\nutKa3xtL6mNjJJZGIR4ff6LjAKkYV2gpis3SOog3RmJdW3Fgh2doOPGk1iY6/TbV0nmDiu3OO83/\nZLxP9l21NAXWtVULqdiqbY7Ek1qb6PTb6PIyMoyQbSprINkBc/uuWpoKq0hqoTGyZVqDuyMeH3+i\n4wDRQ6iMH9+0QjbZ/VRSIbOrNbzLlqpYRVILyf74W1MrMp7U2kSn3wbLmzix6YVssgPm9X1XEyX8\nW9O7bInEKpJaSPbHnwqtyNZAqvRaT2Y/lfq8q4kU/vZdbr1YRRIHyfz4U0XAtXSS1SBINVdOXd/V\nRAp/+y63XqwiaWJs/4DGI9ENgpbgykmk8LfvcuvFKpIUIFWH5Whp1GQ91MeyaAmunGQkODS3e2Bp\nOHEpEhE5CihR1TIRyQNOAPJVdVcyK2exxEttiqAm66G+lkVLceVY4W9pKPF2SJwJOCJyNDAVyARe\nqu0gERkmIptEZLOI3BZj+2kislpE/CJyQdS2riLyvohsFJGPRSQrsP55EdkiImsDf33ivAZLEkiF\nDnDxdPSrqXNifTsu2nkyEkMqvEOWhhGva8tVVb+InA88oapPiMiamg4QES8wGTgTKAFWiMibqvpx\n2G5bgcuBW2IUkQ/8TVX/LSLtADds2x9V9bU4654QUi2omgqkSowgHhdTTdZDQywL25pvGKnyDiWD\n1iQz4lUkFSJyMTAaOCewLq2WY/oDm1X1CwAReRn4JRBSJKpaFNgWriQQkZ6AT1X/HdhvT5z1TAot\n+WVvCKkSI4hHEdQUC4jeBk3bA741UdM71JwFcWuTGfEqkiuA6zAWwhYR6Q78s5ZjjgCKw5ZLgJPj\nPN/PgV0i8n9Ad+AD4DZVdQLb/yYidwFzA+vLogsQkTHAGICuXbvGedrYpIrATDVSJUYQb8C4Jush\nuK21CYCmprp3qLk/h9YmM+JSJAF31O8ARORQoL2qPpDkeg0C+mLcX69gXGDTgNuB7UA6Jl5zK3BP\njDpPDWwnJyenQUMcp4rATDVSKd0zUS6m1iYAmprq3qHm/hxam8yIN2urADg3sP8q4FsRWaKqN9dw\n2FeYoHyQLoF18VACrA1zi80CTgGmqeq2wD5lIjKd2PGVhJJKAjORJMJ10NJiBK1NAKQCsd6h5v4c\nWqrMqI54XVuHqOr3InI1Ju33bhH5qJZjVgDHBNxgXwEXAZfEeb4VQAcR6aiqO4DTgZUAInK4qm4T\nEQHOA/4TZ5kNoqUJzObuOkgWrU0ApCqp/hziaYS1NJlRE/EqEp+IHA5cCPw5ngMCWV7jgPcAL/Cc\nqm4QkXuAlar6poicBLwOHAqcIyJ/VdVequqIyC3A3IDCWAX8I1D0iyLSERBgLSZ2k9KkYtCwubsO\nkklrEgCpTKo+B9sIq0q8iuQejEJYoqorRORI4LPaDlLV2cDsqHV3hf1egXF5xTr235iOj9HrT4+z\nzilBqr50iXAdpKKCtLQumuIdbGgjrCV+N/EG2/8X+N+w5S+AkcmqVEsiVVv+DXUdpKqCbApaomBo\nCup6H5vqHWxII6ylfjfxBtu7AE8AAwKrFgE3qWpJsiqWitRHYKRy0LAhroNUVZCNSWEh5OfD9Ong\n97cswdDY1EfANtU72JBGWEv9buJ1bU3HDIny68DyZYF1ZyajUqlIfVsSqR40rC+prCAbg+D7sH8/\naCC5vCUJhsamPgK2Kd/B+jbCWup3E68i6aiq08OWnxeR8cmoUKpSUABlZeC65n9dBEaygoZN6VJp\nqQoyXoKCL6hERFqWYGhs6iNgm+M72BzrHA/xKpJSEbkM+Fdg+WKgNDlVSk0yMowSAfM/I6Np65MK\nvtZUzappDMIFn9cLV14Jo0a13vtRE/GmytZHwNb0DqZq7Kq+302qXg/Er0iuxMRIHgUU+BDT07zV\nUFoKHo9RIh6PWW5KWqqvtSmpy4famC3LVBYgtVGXBk8iGyap0NBKJKl+PfFmbX2J6dkeIuDampSM\nSqUieXnQpk3ifZv1FRIt1dfaVNTnQ61N8CVCAaS6AKmNpmrwNNeGVnXvTKpfT0NmSLyZVqRIEtkC\nDb4sGRkwfnz9hERL9bU2FYn8UBOZzRWrXuvXw8yZMHIkjBlTvzo2Fk3V4GmODa2aGg2pfj0NUSSS\nsFo0ExJhek+dCuPGGcEQdJW5bv2EV2uOUSSC8NZfbR9qvNZForO5ouu1axfccYfZ9v775n8qK5Om\navA0x4ZWTY2ZVL+ehiiSBo2o2xwoLC6koKiAjNIRlG7MToglcsMNppUKRtD4fMnL+GnOvvVkE6v1\nV92HWhf3UqKzuaIFyIQJkdtnzmx6RVLbe9ZUDZ7GcD0mktoaM6nccKxRkYjID8RWGAIckJQapQiF\nxYUMzR9KWVE/3Bk34XGVNul2zFsRAAAgAElEQVRSrRCJ56UsKKjM/AKjRJ580gTuE/0yN3fferKJ\n1fq7/fbY96gubq9kZHOFC5CRIystkeByU9Jc37NUrHddrI5UU4I1KhJVbd9YFUk1CooKKHfKcbcM\nAn86rkq1QiTelzIYsC8rM26tJ59MXmsy1YNzQZrqg6iLz7ku+ybbBRF8X1IlRtJc3rNoUrXe8Vgd\nqagEG+LaatHkZeWR7k2nrPsiXF85HtdLerrEFCLxvpT1FTItbWiWIE35QcR6FtXd57o+t2S7IMaM\naXoFEiQZ71ljNC6aw/dRHamoBK0iqYbczFzmjppLQVEBu059n7VLOzByeAa5udlV9q1ri7UuD72h\nQ7Pk58d/rsamqT+I8GdR231OZf90U5JoC6yxGhepHryuiVRUglaR1EBuZi7rv13P3SUX4ndPYt5T\nQwHIPnEPBUUF5GXlkZuZm9SXsqHCdsYMc9yMGalhAoeTl2fiCK5r/jflB9HUSq05k0glW1BQ/6GI\n6kpzbRykohK0iqQGpq6ayth3xuJu7Q8z/o3fSef6BS7ey3+Bc8Ri0r3pzB01N6RMkvFAG9L6aA7C\nUSTyf3Uk292Riq281kiqDUWUqqSaErSKpBoKiwu5YfYNuOpCUR446aA+HL8f9/NT0c4LKCvqx4T7\nyphwefLiHQ1pfaS6cCwoMKnQquZ/dYquMdwdjdnKS7WMm1Qi1YYissSHVSTVUFBUgOM6ZiGrALzl\n4Adw0QN2QHEu7oz3+UAPYNE/4xNuDYl31EfgpKIJHE5Q0ZWVGYukutZnY1lWjdHKS8WMm1QiWUMR\nWZKLp6krkKrkZeXh9XjNQuZSGHYTeFxQD7z7GKz7LTjpuI5JC86f9SUTF02ksLiw2jJjCcT6UFgI\nEyea/7WRm1t9/4imJjcXJk2qjJOMHx/7moIKx+ttHOFSl/tbVxL1DrRUgo2fe++1ShaS+y4mEmuR\nVENuZi6Tz5rM9e9cj6MO7DvMKBH1gV9gz8/AW464gi8Nnts1Gv+8RXg8HiafNZkxJ1bNz4zX1VST\n66OltWhLS2sfJqahllVdXEnJvr/VvQPW3VVJoi3D5npvm9O3bhVJDWT/NBufx4fjOMa95fGD4wU8\n8NlZ+EbczNXH3sr2jq8w68cFALiuy7jZ48j+aTa5mZFPvTaBGM9gf80hgF4X4nVv1Ve41PVjTPb9\nra7/SnMRGM2N5nxvm9O3bl1bNVBQVIDfDQyMlbkU+k4HXEDA9TKiy+WMGvc1s/ffGXGcow4FRQUx\nywy6miDSZA2+8FOmGKFaneujsd08ySZe91aQupr6dXUlNcb9jXY3WndX8mjO97Y5fetJtUhEZBjw\nGOAFnlXV+6O2n4YZiv4E4CJVfS1sW1fgWSATM97XWapaJCLdgZeBDGAV8FtVLU9G/UO92/1luLjQ\nOx/WjgYnDbwVLOCv/HfuHiqcisprQmjjbUNeVl5o0Mdgf5PwQSDHX5Id0UoKvvC1DfaX6gH0+hCP\newvq17qsa+ZaU9zf6DpmZBhl2VKeb1OS6pmLNdGcvnVRTc4gviLiBT4FzgRKgBXAxar6cdg+WcDB\nwC3Am1GKpAD4m6r+W0TaAa6q7hWRV4H/U9WXReQZYJ2qPl1TXXJycnTlypX1uo7C4kLGvzue5V8v\nNyuKTzHpwAfshO39zLre+ZC5FA8ezj3uXIYfPZw129Ywfe10KpwKRIQBXQewrGQZftePLL4Dd95f\ncR3B6zWBxby8SiHZkqdujeWvjldBTJwId95ZOQT/GWeY0XDjiXuk+seYiDlqLLFpDs8/VRGRVaqa\nU+t+SVQkucAEVf2fwPLtAKo6Mca+zwNvBxWJiPQEpqrqwKj9BNgBdFJVf/Q5qqMhigTg/FfOZ9Yn\nsypXrLwa3nnKBN4BvGVw+RDIXIrP40MQKtyK2IUBFOfi/WcBOOkRwqKlv/A1KYx4rj14fLDns8dj\nUkVbkrANV5bBRkbQFWqxNDbxKpJkxkiOAIrDlksC6+Lh58AuEfk/EVkjIg8FLJwMYJeq+msrU0TG\niMhKEVm5Y8eOel4CTJ21nree7WUsETD/Z08OKBExf06asVIAv+uvXokUnwKLbgMUz+gzueYPX0YI\nwVRO1U0ENfmr47n2oKl/xhmVndaam9+7NpqTX9xiCZKqwXYfMAjj8joJOBK4vC4FqOpUVc1R1ZyO\nHTvWqxKFhTDuouNw5k6AGXOhOJeee64H9WKUiJo/b4XJ6qqJ4lNMGfPuhRlzcdSh64iXamx9N4f8\n8bqQCCGZm2vcWW3atExha/tRWBpKU8iOZAbbv8IEyoN0CayLhxJgrap+ASAis4BTgOeADiLiC1gl\ndSmzzhQUgOP3gQo4im/rGdx0fR/Gv+llf5mLqgPHvgUDHjJZXVF4xYtHPMZCCRtmBUehaDAZB8bO\nda3RBRQVwK8rDT2+ISQqeFjfcpqL6zDVxlGKh+Zyb1s6TZXunExFsgI4JpBl9RVwEXBJHY7tICId\nVXUHcDqwUlVVROYDF2Ayt0YDbyS+6oa8PGiTLpSVK14fPHn9rxlzXja8tJ4bnvpf/F3/jWQu48wj\nz+TAtPN4Y9MbaNiEkhkHZHB538v5fv/3TPvqQyoWlBsl4q3A7TaPcbNXAVC6tzRCsFeXPx6ctbHc\nKY8YMDJeGnp8IkiUkKxrOanWn6CugjeVBXWq3dt4SOX72RCaqu9J0hRJIBg+DngPk/77nKpuEJF7\nMErhTRE5CXgdOBQ4R0T+qqq9VNURkVuAuYEA+yrgH4GibwVeFpH7gDXAtGRdQ2XLV8jLSyM3N5vC\n4kJm/jAB/wAz36kC73/xPqd1O61SiQQyu77NKuDBvQ/ypwF/4qpze7L950+ybMkBbMt4CTKXUuHC\n2HfGIkiEYK8uZTE4a6OjDuVOOQVFBXVSBA09vjkTzwfWWMKlroI31QV1c+o4B8m/n02ppJoq3Tmp\n/UhUdTYwO2rdXWG/V2DcU7GO/Temf0n0+i+A/omtaXyE5nH3l1XZtmbbGvMjGAtx0s1Aj6OH8tCS\nh/CIB6/HS0WfCgizWlw1Y2bv29KH8X/5hknXVe+6CfZrCVoUeVl5dap/rOMbU3g2ZQuwtg+sMYV1\nXQVvqgvq5tZXI5n3s6mVflP1PbFDpNRA9Esx+pHPzDzuxf0r+5LsOwyyCvghGCOpEgvJQzOX4qiD\n67gRrq8QxafAjA9Y7qQz5FWHxx/zUloKGT3WU+B/G4or3V6je48GYFTvUVCSy8QX6jAkfdisj3lZ\neVCS2ygvfVN/XFD7B9aYwrqugjfVBXVN97apGxCxSOb9TAWl3xQxNqtIaiD6paBoMN5dA3FmzAZ/\nOuAFcU0/ktFDTcA9q8BYIk7VbK6YSgQilE9ZmcO4ceC4ius5Cs/od2iTdS+Thk1i/LvjQ9ZEX//1\njL+kHkPSZ+aGlNLEFxrnpU/Ux9VQoVTTB9aYwrqurcZktDITLeBj3dtENyDi7WtU2z7JbLU39D1K\nRcUbD1aR1ED0SzHqvG4wawZT3LZoMHNaveCkIUWn4+22En/mUqNUghZLoH9Jz37f88nOT0KuLA8e\nM+wKhA0IKYCL3/GgroCbhrtlEOWZS5n58cyI+MbMOaUNFs6NJTwTcZ5ooXTjjbB2LYwcCWPGNLyO\nje0SCAreYKpmbees6/410VgWYiJb5/HUuS7XlaxWe0Peo1Sw3OuLVSQ1sH49ZGWZca9uuin4ULsx\n44nI3tVen4e+B40kr8tJTPrqN5QH3VxhsZLDut5DG+8Wyp1yRMQolFgGiriopxzwgqcCshbg8/jo\nc3gf5hfNR1HSvemMHJ7Bon9CWbni8fnJ6PEJkF2n62ss4ZmI84QLpf374cEHzfr3Tc5DwpRJY364\nyQ66h7duofJ3Y7lfEtlQiafOqeBWgvq/R6lS//pgFUk1TJ0K115buTx2rPk/ZkylUMzIgDlz4K23\nvKx8ux/r/92PJ15ayRrfUyzceCofh8VKFi308sfbbqRDmw5kHJjBjXNupMKpwCMenKI8cH2AF9SF\nPs/CIVuNpZK5lHJHeLTwURzXwePxMGnYJMacWJmG7HSbx/gNq8k+sWo6b039RhrTjG6okA4XSqqV\ng1sCzJyZGEXS2CQz6B6udLxe0xgKTk0waVLjWKKJbKjEo5SSZWGHj4NWWpq876Wu9U8lN5hVJNUw\nc2bksuvCuHGQnR3pZrjhBvNhg7FSSjdm8/TtT1OYAYNeL8epqABvBZo1j0cLV7HgcjNviaqiqJk0\n64CdJtaC38RVeuebAgNuMc1cGhp2RVQo3Wsmsi7NeBsd+P9w1aHc8Uak8xYWF5K/Lp/pa6fjd/1V\n+o1MnbXe9Nr3+2iTLilvRocLpV27Ki0SMO6thtBUH2Qyg+7hSscNeFBVzbrS0sZz4yWy31BtdU5W\nLKmxxnerS/1TzQ1mFUk1jBxZ6TYJ4jhm4qlwF0HwIwXT8gt+3Lm58NQrmxg7+WXcbvMgcymOeigo\nKmDr7q2V43EVn2Km7nU9ZirfYTeZ9VEpxMGe8x7xhNJ+w4e5B1j+9XKmrpoaGnm43CkPBfjLnXLy\n1+WbYewPzOCGp0rwl98FajpcFhRIwl/EZAZ0jzrKKPuGxkiaIiAcpDbBEV1WXQRNuNKJtkjCy2tO\nxFPnRF9XUCEHv/PapjpoKPHWP9XcYFaRVENQOE2aBJs2md8+X+XshV4vnHVW5YRMXi88+WTkwxxz\nXjZkFnL9Mz7cRbfjOXIxW3dvZfue7WaH4lOg4G6jMPABfpNOHCOFOKhIMg7IYPy74+l8cGf+dOqf\nuPHkG3loyUO46jLrk1mRoxRHMW3NNFx1ERGcbv3Bexs4ptd+Xl5aQu9fsltMY8Ykxp3V2AHhaKoT\nHNWVFb1/dYorWukErzUV3CDNiaBCDrdIUiEFO9VSwq0iqYGgsAp+rFu3mtiJ6xrBMysgs0XM35w5\nsGZN1Dwiq8bA81ejjuJfUMYUPZO0bivxlAzAnfF+WBqxHzzllenC1aQQb/9xO9t/3A5fwxufvIGI\nVJ9WHMAjHlx1jRsN8KgHX9cV+EefiXyZx+8vOYnc3PPqfH9qir+kWoupOho7IJzIsmpTXNFKJxXv\nf6oTrpCTHSOpb71SoT5WkcRB8IMMKpFoVI2VElQs06fD/Pkm62vsWHBdATzgb4NuOQ1/5lJyym9h\nhdsGxQv44cgPIO+vlYM/BlOIsxbEHBASTL+UeOaTCaYcQ2AGR18bftX+IV4q+ho3az6PljzE929f\nxajeoyJmcgy60ILusPAxwWobtyvVWkzV0dgB4USUFd6waQ7KurmTqm7AmurV2HE/q0jqQGlp5TwY\n4YhEZhGVlZlZ7latCu4bHHLeAwfsxCterjr/KNa9CmVlJhgfVCKC4BEPAwam85MzPuGtT5fjKJUz\nMwYyuWokfBbHQM97yVxGmjeNs44+C0py+dcfr0YrvOD9MxWjhzLFncKMdTOYNGwSN059iYrPT0W6\n/xFv1+U4roOLGxoTbP7o+RHjdu337yd/XX6EImmKFlN9RzZOdEA4P79ux8X66GO5piZOjJxB0es1\n7lZIbWWdaqRStlMyaIpAvFUkdSAvz2RshE+H27evcWm98UakMlm+PPxIxSgTP7Lvpzx51pOMOTGb\nNX/PZ8rMTWjWvIhgukc8LNm6BMC4o6LH7xp2U0hBVFEqwX1DLjMHvOX0vvUWso7dxpzNcygv6GmU\niPrAL7BuFJq5lHKnnEn/u4zy52aDk456y3HDAv2KUuaUkb8un1G9R+H1eHEcB0WZvnZ6yKIJEi6g\nEzl8fayyUmFk4yAzZph3ZMaMhvUNCc8ODO4jUjm/PcA110DXro0jFFuCAE61bKdk0BRuZatI6kCs\nVnZhoWkh1o4DvnI83ReyZtvxFBYXMmrEMcz473Xs9+8PRTlcdXE1akyu8OC7n8AMjZ4qGV0R++ID\nNBSwX7v0ENalPWPKzZoHnjvB8QIeWHMF0ucFpOtyPln5s2oD/dEcfejRfLzzY6AyKyyW8K5OyEe4\n0Epy40t7rKasmkY2bsw5WJLRNyR8H4+nMgsrPT0qHpdEGiqAG2vY/NqOay6xu3Dqei+awq1sFUkd\nCT7I/PxKF0awk1w4Ho/5UwXHUfA4MOwmnC6LmbJqCTPWzWDuqLnMHTU3or+H1+M1c747FZFDqASD\n7xDovOiJLeiD+/ohFMQPBOxDyilzKfSdDivHmH1cLz1+uI5NugzNml8l0O/BE+qNn+ZNo+/hfcmb\nkUe5Ux46raL8Y/U/6Ht4X8acGJlOFUvIAyGF4P1qIJI/F3+F13SYe2k9a3xPAVSxcgqKCigr6oe7\nZRBl3ReFFEZeVh5ejxfXcfF6vKH4TmNYKuGKKi8vt/rYRgyFVp+OdpMm1R70TbTyrE0A19bxtTGG\nzY/nuOYSuwtS30zAxnYrW0VSRwoLzcMpD8hQr9f893iMvzro7lqzBlavhpUrATxmlsV9ZspfRUMt\n+K6HdGVU71GM6j0qIsA9oWACH2z5wATKM5fiufwXuGsvhdVXBuaLDyinsIwuwCiJ8LG+qnOB9c6H\ntaPNfPPeCooOnWHcaMHj140K7aooAzMHst+/n7bbh/DYwwdQflC/yDKLT8EpyuPardN5ceCL9Dys\nZ0gJZByYEcocExEyDsyIUC7O6t/AfgHMkC/XT34FZ+AzAExfO535o+eHhFNG6QjcGTeBPx3XV07G\n2Z+HqiCYMlSV/HVGyyd7DpZYimru3KrWVXUKLRkd7ZKhPGsM/tdyvmRYafU9LtWynWqjvveisRME\nrCKpIwUFUFFRuRzs1e7zwRNPVKYL/+53JugORsmkpXsYPqwDc/a3CVke0b3Obx90e6jcCXkTWLR1\nUejjnHTdKGZO/TkfrEnHRQDHWBWx3E6ZS2sPyIcrnKwC9naK2n/taOPiWjsaHT2UhSwMxF/uCsRq\nRla61aJiOAsZysLMZ5i+djqPD3+c8e+Ox+/6TU9+12H8u+OZNGySGR5m60mw5gqCCQnicXC6zQtV\no8wpY0LBBEa2f5jSjdls3ZqNx1VcFTyul9KN2XAe5K/LD3XArPjyRJ5ZcCjPHXU7T4y5JK45WOrb\ngo+lqG4flFvVPVWDQov+6KfOWs/MOaWMHJ5h+iLF2Aeqd3kkQ3nWJIDDz1fmN89rQt6EWq2u6uqf\n0WM9Ht9xKD58aS5bO7xIYfExtV5DvNZGlb44CbLeCgsrvRR9+1ZajVB/xdVcLCirSOpIXh6kpVVa\nJEFUzYsD5mUKKhGAnBy46iovpaV/YvjxwynNeJutu7fyj9X/wNl6EvuLTif/J5+ROzYsUB01d0hu\nZi7ZlxMaqNGVssqhVMIQaulXEp39FUvhVNchMt7160ZBUR5lWQuY+fFMyor6oVsGGfdaIKi/Ztsa\nY20V5QVcdUY5dhzwNtszCyPq+/7b5/H+mmPwoPi8QppP8APp6UJGj/WMffsppq2ZZq47TKmVLyhn\nTZ/XmDRsEjM/nsnIniNZv6od4y6qiBgahi71b8FXN9lYtHCKd1KyqbPWc+2FR4G/B+9PL4dX14eU\nSTjRY2ldeWVlvKShE6BVR3Wt3PARFlxcPtjyAYu2LqrR6qrOZVNYXMj4DUNxftsPKcrD7b6If+xY\nwoz82p9LfayNulpv1cX1INJTAZVeivBRBeoaW2ouFpRVJHUkN9c81Px82L7dZGwFLZTly80HEk3n\nzpUpm+np2dx414Gs3rgedgnMfhh10pm+ROh7+HpKM94mo3QEpRuzycvLJc+XS8ELQB7QpZDRj3wG\nRYPZ3vEVZn0isOg2JGshZw5ux8ieI1mzbQ0Lv1wYCoJHsPLqmgP1YATx7q5mWHs3ECc5YCcsus38\nj9VRMnysMI9jLAzXZ/ZtPxv3+bMiMs6cfR159/si3PZulflbth/1QGRdZswFfxvAg4vgYDKVOPhL\nPj7oacZ+9FBEP5lopfbxyo5M2/lLHHUo+LIAZ8GfcMrvjhgahoGRLfjwoWSi+85UabmW5DL6+42Q\ntYBRI44BYOzbY5m2ZlrI2gy65sIVWnXCauacUvD3CCRWKDPnlJJ9YtXzhrs8HEeZMgVmzAgoRiLr\nVF0CRE2t8Lq00oONnnB3bG1WV7TLJn/WlxT4X2Lr7q1m8rguS5AuH+JiXJXxWlbx9vwP1aMOSRrh\nSic6rjd6dKSnAkxmXXBdcJyz+gT36+qmaszkkiBWkdSD8Ac7dSpcf31lT/c5c8yQ88GhU9LSoFOn\nsCHQy5QH/5wJ2g04OxDvECoqlBue+l/cbvNwZ9yExzWt72BrxpfmoKNuxzliMekHp3NjxkuQ/wH4\n01FfOSNHfE72T/cw/t3xMacCpvgUo0TcNEAQx8txX9/P5m5nRo77FXRRefxw4rPQabUZCywq9bjL\nCZ/T/qjdfLp2IE74WGHHzIZN54YE+b/fOjhmxlnRgnIYvcWct88M8793fjUZaF5AEVHS04WD+8/i\nkeILQj31w6/R+/2RiA8cx8GXJizx/D+cwPWVO+XQbS54bwdHUY+f5WlPMPzAjFALPuhyDCY7CILX\n4+Xm3Jt5YtkToX2u7HNlYHKxbMrLu5GePoq+h69n/IaTA1l4xioMpksDoYnJFm1dRPZPjZURLagO\nPGY5ePsbxeqroM8puxgy46JQizmolPLyzDthXKseVIWycpf8fE8g/djUaVQfKCyJsgZqaYXH20qP\nFljR7thYllDQ/bN9e2UfGF+aw7Pf/RZn3mK8Hi8+jw9cQoknQYVcnbVXXZ2CM4CWlRnrYPJkM+hq\n+L2IOf10NQOehisd9/MBUC6oW2mFRHsqoi0SX5pTxU0Xb0ZWrE7C1V1/U6TBW0XSQEpLIzsolpXB\nww9Xjr81fjx8/715oczw52omwwoIxyDicYwS2TLIBJJVqAgbtdVV4PMBaOcFlDvlrF3aAY97QESs\noCBjomnN4eLBQ07nHDq370zR+sNZW3CuEfaBWIQqbJ47iN//+lXe2nsHG3dujGzNu2qGst93WKTb\nat9hMOh+SgDZKXiK7kCCPfS1Atp9E2FhaI//hS8HmmVRYw2FucA8665AnTTUU9VVJ1kL8aS5uH4H\nnw/OvnAH9M7nkeI7KvvXBN10ADPm4jjpiMfhhLNW06bvK6zwLol8YJmFodiQZhUw68elvPWOlz+c\n+ge+3/89q7etZuW2laGMOUXxu34e/vBhwKRnO47DM6uewbP4J2hZL9T1UF5urInyI8pjuhajW775\n6/KZsW5GZQwsbAZM3xVv8/Pvr+HnOdtYKu9Q5piGQVApBd1FV/z9RZ557seABejFlQo+3rGF8vJe\nlS39/Mp+LUHXSoG/5s6k8cRYqhNY0e7YiGOiElXS0uCci7ez6Yg/s/GARQD4XT8jjh1B/yP6xxSa\nNQnK6G2jv99IWVm3UL+bsWMrOw9XjuAbNf00JpswvDGwf0tfRv1hI78a3qOywXHUEnSxi79C8aXB\nqFFeRo2qjJEc3O1z1m4pZuTwDLJ/lk3+rC95btdo/rFjcchNZ9ystY/AXVhcyJAZQ8y74fHhEU/M\nEb3jfXbJwCqSBhIrZhJULI5jlAqY1okIuCoE4wFgBLsInHPhDt7LWs3+b3qh4uIRxecLt0hAj1qC\nI17Sven06Z7JfI+AQpt0MX7aLpGtq0nDJkFJLoMvr4AywSivyvNWVPh55KVVuAM/MZUMdzP5/MZl\ntXk4oBFpxEEUxc2ajy/tbly/F1+ah+G/+YG3+vwPzpaBlXGYn/2nMovs3ccqXWOA6/cZxappVVKZ\n07NW8fgrm1hTeDDbO77CnP13Uf5jeZVYCN5yY9UEFJ66yrp9b4L3sSrPSxA0KjbkqMNDSx7C5/FF\n9OIPVwiqis/jCw3/D5hRnT1/xiNtSU/3mMnGNlTGCoLWDEBGmNWT7k0HiAhQ37vg3pDw8hyxhM8y\nl7LxRz/6Y6RSWr1tNYXFheRm5jJqxDE8u/M0/L3zQwp1ifjwpRWgePD4/GzfU0p5eacIFxKDtppE\nB43dmTRmKz18kqwuhUwomECZU4arboQyCp/KOZroRJUKv8tb2yfjHj09Yr9O7TpFJJ7EKyijt23v\n+ArIHwi+764bvJdCWVmlmyli+ulFEyNGzab4FHTGv9nsT+fBmS6X3jaHXsM+NHMKcQZ8fir+7otY\nn/5bxpw4JtLiO6KcRRvSmXviXLqOKMCZvziiIfHsUx1rHIE7aIW8+/m7ocZEhVsRejdjNQKSFR+r\nDatIGkgwZjJ+fHRvdkP4XBDmhyAeJSPzv/y35DCjCNrAn244nOHfLGPcvcfhx4fHIzzxRLgp7oUu\nEykoKmDXkgt59K6jQqMQT5oUbMlUbRFOfAEcfzCY7YcjVsL2PuB6wVuB020uIcsokMnl+XIobttv\nYc4T4LQJXgmc/FiVmIpkLuXCB6eyY0OvQJbRnxj79haeWXV/5U4Bwd2nUx/W/uwMKBpslMw3x0fO\nwxKVynxFnyvI7r3HuIt+3E+1nTSD/WuqGehSEHweH66aPiaqWunOCz4WKtcJQuf2nel4UEfWf7M+\ndNzvc3/P0uKlLNy6MOJ+adEQbry0P2POOw8yTRykz+F9+HTnp7y56U2mrJqC1+NlxDEj6NSuE30P\n78uabWuMYnIUF5eSH0oq6ysSynKLZuW2lQzNH8qkYZMo3VvKb3r9hhfdF0PPxUXodP1lbF13JP7u\nBcz2pOFLmwt48aU5PLdrNM7qxaHrVJQKpyIi0yoYz5m2ehqdD+7M+lXt+N3FDuXlgjfNj/72T7hd\nloTqF1RGfQ/vGxFTiqay0RW4Lk85TrcPCLfMveLl4LYHM3HRxJjl1CQow7d5PV7m7L8LPetzeOcJ\n875TaZGLR8nL84SODQrtjCg350//+1tK/OmAsdL/9cAAFo8YTMHeifg7L0I7L8ABxs1eBsCabWtY\nvW11SMkGlV10vQHz7QVG4BavsrXDyyG3V2FxYZW+WqH3I/DcYjUCarMKk4XEM+hfvQsXGQY8hmkK\nP6uq90dtPw2YBJwAXMpip5UAACAASURBVKSqr4Vtc4D1gcWtqnpuYP3zwGBgd2Db5aq6tqZ65OTk\n6ErToSNpFBbCwIGxB3WEynGRKiqMZeLxVPZUnjzZpA1PnAh33mnWe71w771w++1Vz3PaacZKAXP8\nffdV3S98/6FDTaaXeCsY8Oe7KCwpxP/FQLxHLsLbdXkoHfmso8+iU7tOAEyZdCg6917MowPTb6UC\nrsgz7qEoBKGtry1zR80FYOBzAys7VAboeVhPNu7cGGlR+NNNbOWsGyDn2dC+aZ40zj7mbD4t/TR2\n4kDIIknDm+Yy4M93GwEfVFJRCq/HYT0Y3G1wSIhv37Odol1FrPtmXewst5DbbAGezGW4uHjFa6yw\nYHA/zLWW1m0Vv8/9vZnJUh18Hh8VTkWVstO96SG/v9fjpXO7zhTtLorY57xjz+OtT9+KiAGF+sgY\nWwOvx1utsglHEE5yfke/st9D1gL+seNKHHVCZYRbYD6Pj6v6XkXfw/tyw5QX8H8xwIzTVjQEnXeP\nUdpSAaffBYPur3KeoLIOulxijVYwddZ6rv3bEkCrxMQ84glZfB7x4PP4uLLPlRzc9mAKthTQNq0t\nPQ/rWUVhTV01NUJ5byrdxD7/Pr7c9WXlu7ZulHEBOj7wuPQY/RTT/noy679dz7TV01izfU1oBtLf\n9PoNO37cwcieI6E4l2tHHhuKLSJ+rvvjV4wa9zWnPX8aftcfqr9XvBHPzCMe2njbhNxPU1dNZdrq\nabRNawuKeV8D75C3+2LILAx5Eqatnsbyr6u2TD14OO6w40LfkQcPZxx5BiN7jqw9MaQeiMgqVc2p\ndb9kKRIR8QKfAmcCJcAK4GJV/ThsnyzgYOAW4M0oRbJHVdvFKPd54O3wfWujMRQJwK23Rs7cF8Tj\ngaefNr/HjTNKIHjbwxVGdErkpEmmYyNUpnZOnAh/+UulwkpLgwULwvpDxOojUVg12FpT4K6wuJC8\n+26n/Nn3wE0PXIX5iHpfMpPy3L9Suq+Ub3/8NuI6BeGkzifRuX1n3tj0RhUhF+wh76hjssDm3RsS\nTt6hf+Wcqzbw333/ZcfeHXxa+qnpYxKeqhxN4CP0dF9EWreVVDjGojjsoMOq1A2Mcgr6l0UEVQ19\n+IKQ5k2jz8/6sHyZp9qJxSLOHbHPGXi6LovMIItBuEIILmtUi/yps5/i+neujxBKl2Zfyv9t/D/K\nnXJzD12nViUSJCiUzzr6LN757B0q3Aq8YuJCBVsKIgSWIHhKBuA8/15kgsW7j4U6rwbvR5VU8zDl\n279zf9Y99Egoqyno/x/79lieWfVMrfelJtI8aaGZRh9c8iCzNlU/B0/w+l11TdbixpHQYyaek54L\nvQuxCLolJ581mRenH8jCpy8MZTv2v/0OJl3zGyavmMyL61+s9rxHH3o0v+r5Kzq06cCGHRt4af1L\ntU/5EFDw0RYzmHfDI54q24LPIai4bjz5Rh758JGQUg/v0FtX4lUkyXRt9Qc2q+oXgQq9DPwSCCkS\nVS0KbKv562smPPCAmblv2jSjAGJZHK4bOZyKiBnRFSJzxjMy4MYbK2MvwaHp8/KMKyyYiRI+mVa8\nkyFF+7GjX7LczFyeGP4E10/34gSfjDj40l02HDgZ/86NMa9f0ZitqPDt5/78XNPaDovHeNJcBgwy\nH/Syr5YZH3XxybUL84DLzAXKAjLXK15O7XJqSGCGE7Ec9T0rSteDu9LW1xaKTq19vLGiIVX2cWNY\natEEBVSwLtGC5aLjL2La6mlVMtLap7cPDaezfc923vz0zVqnEAgKZ3drf8qL8phVVACZ5ryOOjy2\n9DGGHz28yn1wtgysmmAR1nk1NIhn8SmVFiBEPK/lfWZAmYIaazh/1lYK/C9VsS4FQURC1khwnLma\n+kNVuBVc/871rP92fdXMvRgMzBzI4g8d3GD24Zen4f7sP7jVddotPgUtysOfVcC42eN48oonWVrx\nCyo+PxXNms8K7zJOe35ylfsfXefPv/ucB5fEaFlWgyB4PFWVm1e8XNPvGgCmrJpS7Tldddnn38dD\nSx6qkjWYbBdXMhXJEUBx2HIJcHIdjm8rIisxSaP3q2p4s+NvInIXMBe4TVWr5LuKyBhgDEDXrl3r\nWvd6Ez0ZVngHrK1bK1Meg9kjrmviK9mBPmfBY6IDk8Ec9Ntvr6GHcUHiBqQr3ZiNupWjFnuOms+I\na9fwxo+LI/aTklPRLYOR7gvQLh/WWKaiDD9mOJ3adWKKTkFD2VMLWagfwqawnaNjILGEeZAwF5On\n2yrmbJ4Ts0VXG5u/28zm7zZDVnm18RYwH++g01wWLgjfZ35c5zj32HPp1K5TzFY5UG0Ld/ue7aGU\n1Ihx2KgUxgA+j49TjjiF/f795HXP4++vFuKf8W5MhVzulNOpXaeQ7z5EVN+eCIuwKK9yvxkfxEx2\niI5ZuVLBP/57Ge68JVUsGA1kz9F1OaoaSrX+fv/3VaaLDmftN2vjnlqhsKQQ3XJLRP2kaAiSuTzS\n/RruAgv0g6oYPZSHljzE+F//ioItc1n+9XIUYloyQVdT1w5dK91qNeAVL1p8CrrlNLxHLubqc3sZ\nt+LsG0LlC8I1/a7h6RFPU1hcyNTVUyMUmIhJuIlIDInTUk0kqRxs76aqX4nIkcA8EVmvqp8DtwPb\ngXRgKnArcE/0wao6NbCdnJycRr+zEUOoR/VCPucc+PRT2LjRKJPg/CVr1lT2gL3ppkjLxeer7EFb\nXaerjIy6DadQkx81Lw+8Pj+uI+Bx0R6v0ek4D2lr/3975x5eRXXu/8+ayd4Bj1UwakEJBJEq2lQC\nFomUkIpFsag5pb961NOgUmnwSmvLkd4OWg+0tFbqpTZ4vECrrZ5S8Qbe0ACSILeAUdACEgIKFoOo\nFUiy96zfH2vWzJrZs5NAQETm+zzzJHv2zJp12+ud9X7fS8JbdBLvlMCf5pNqsbBfSeF89xycHkFz\n24BuX1g07mqkqHuRmuzujsJrprkw6MUshTIb7vx+dCNCKqYTb/geDUc9mnHZcUccx/Zd21vvEA2X\nRBf15yB6L8CRArFoEhQsQOQvITcnl8vPP4nqLed7PEKbIWlQKpaRfUdSeHwh/1v7v1nVKmEkrARP\nr3s66/UXn3IxE4dM9HYrz6x7hpSTYuW2lTgb/yurQBYItn2yjQmDJ/Db6t/6ajm3/QU7r6S+y0OR\nYXBaExzYLYr/MKzJ0j1aVw06rn9SqmABHxV9RPkZ5Wx7szfv1n2JLqeuYmXOPby/6/2s95sC0hxr\nzUfJzttBOAjhkJMQOL0X4YTVcobzKwivv9bn/4ppi6dxcteT2xwrB4eGnQ2tLubdjuzG4B6DGZl7\nKzdMPY2mZnAWtnBU8bOMG6UylV439zrSMk2unUv5GSruXXF+MZd++dLAy8a5vc9l/sb5WXdmAkFR\n96I2691RHEhB8g6Qb3zu4Z5rF6SU77h/3xZCVAFFwAYp5Vb3kiYhxIMofuUzDXOnICU89ZQfowvU\nrsS0+NKmiWbCrKIilXExiv8wna5++EPlt9IW2nJcKi6GH0xucJ0nLeSzd1B01QaqxpR7DnbsupkZ\nKRvpCFLNAjaWYOcvofCLhdS9V4dEmcyaTmV5R+Qxe83szAoZC4PIaUGWn6N089oT/9nfKzPisNVY\n/deRxoK2aXUBDA1xM8Li6NyjfUFiCCyR/2okl3PRud3oduQOjtr+I26vuIB0i9KPW1ecx/WXnM3s\nNbNJn/gKnLio7c524UiHa565hqsHXM0Pi38YXLwjIBBcfOrF7Ni9g4WbFma9rtuR3aj7Zx33PfG6\na3a9DfKXKMHTaz7YP/EW+Msv6sEnx5fx5FtP4uAw5805nlopgPwlbO65DCHd5Ta8Q/zXFz2LO5GT\nYuiFm1jsmn2LgoUM/VqSRZsWKVPrKJiqwRTwzB8BCXYz94nzuP/Jm2l5cJ4rKEZgXfEiVo8d/g6i\nlR3rlUVXes6jQgjSmwap+eNYYKcp/t6jLD6+JthmXZ727xISkZNSuyUX6z9Yn3UMTITnk0AwtOdQ\n9qT2sGJpkvc2ljC3z2K6dTmK5mYBjuJwpj38Ks8338LgEwdz9wV3U7u0k1IdbulFDcpJ8rE3HguU\n25oQ0Zjw7AQKjy88oOqtAylIlgF9hRC9UQLkP4DL2nOjEKIrsEtK2SSEOBYYAkxzv+supdwq1F6+\nDHj9gNR+P6K01N8pCBEUItnQqVNmoqylS9X9nTr5/EdVlRIi2unq9tvV7iWVaj2xUnscl7rIPlhC\n4ji+02Nxmc+pzEjV4YjdIBKeeseRDm/88w2klFiWxV0j74LNxcye10j/wTuZ8Oxl0Z73xsIgU9JX\noZgOjPWlWPlLvZD2tmXz5d49WGWYEMuClwkTII50/EUg9CY79Oe3Ui1u9972LWHxo7N/5C9ErxyH\nk7pQ+bqkFX9we/VvMnPGtBNpmaZyRSWdcjpx6Zcv5S+v/0UZxFlWBoEukcxbN4+Tup6UtTyB4OPm\nj7nmj38KEuT6DT0UnHOl/SFf+PgLAZWOI51IYZKWaf+8qe6y0rDuAi+awY9vfZdf3/wrZqyYwXVz\nryPlpFjcYHFM52No3N0YXXGdrkCnO5CqNaQE6VWXkT56c0BQOBuHYvWo8SyjRMFCsNPItPCjYG8e\njKj/Oku2n0bhx4/Qqc8Savgdsr4EnaNHOi0senMt1vHh+gTbV1BaxZCLNvDwB23vNE2EyXKdqE5H\nzk4/9AsvDtySKx4C+0rVRleFuGrbKlZtW0XinRKsP71EqsXmwTtVVIvmExYEniWR0ULEyJAqdx8b\nSLdwoHDABImUMiWEuA54DiXmH5BSviGEuBVYLqV8UgjxVeBxoCtwoRDiFinl6UA/oNIl4S0UR6JZ\nuoeFEMehlPergIoD1Yb9hdZI9CjYNpx2GrzySqY5sVaFaf6jtDSY/tdx/Pwoe/YoT9uMqLGba2j4\nsMELRZHNcam0VDk7KlWZyFCVNeY9DedvhLXfgn6zlSWP6+jm4CCkoHZpJ2beVEhzM8x/MIVzxu+Q\nZ8zEyl/KKbuvQNSX8tYXZpAuWBDNSxjnrN6LuHfUvRQeX6hs/htHccOU05QXvuWoHYyhtoEIfXHo\nTXbPhsHIPr4F1bgB4+iS28UTslavlxA5P0W2OF692kPwtgaJpCnVpN4uXSFyzwX3ADD+mfGBXUpT\nuoldLbtaLevhuodh483ZOSXDAXNtFg1hNuLeka5zpiGQxIcFyJXfA2wEaT76IIepi6bS8GGDJwzT\nMp1diGj0nwlbi+CdM/ESsWEpjmLk9aH5sMAzzX3sjcdoQbqhItQ4896X4dnfI1NJFs7XmUFHwJgF\nGZyPLHgpw1Q3LHDr85dwRM5prZL+UbjolIuYt36eZw5d3KOYhQ0LlRHKonMCY7Rq42YoPyeS52l5\n+2xEM0gH0lLChrMhJEgiEZEhNZxu4UDggHIkUsq5wNzQuV8Y/y9DqbzC91UDmSFP1Xfn7Odqfiow\neY3CwmDQR73wa/+SCy9UqixtnRUWJratBJLO4X322UrogB+KpaVF/X3wwWAWvUDgOUtZg4STR5l1\nbi3yaF7jKHi2j2cJY3Vby48uGeK9zSftJNuqR7Bnj7u7Stuw/GpYVY71zR/x9nN30dJi4YjRMOZc\n7CvOc8n3l8ktWM31Z13PU8dez+71Z9F/8E5Gnnwvjc8XQilMGlrM1KmQ0py6bFHWRS70rsWD8ZZm\nJ9I4KUEyaTH23/tQ94bvKKb10Z5TWq9lyDEjcN4+u91cSFuLj+cIqB0gpaB2ay09j+7JpV++NMNM\ndNOHm9p8JtkEcTuRtb4uIa7VgHav5QxouZalq5pAJpCWItLlS9WK+FX+flnbnEFoWymwU5AWeNyE\nY6vcPd7CvsA1rYZH33iUtJOG1d9V5shY6vrasS6/4aqmTIE69FcZVmfNmyKI+lDEgzXb1wTrHtE3\nZhm5di7djuzm+fc40uHdj9/1r4/i/bJF4C6oUmGDZEL5b2ljDveZomAhX+jzOh81h/TYnorOz5Bq\npls4UPgsk+2fW4SJeL1Tqa1VpsNPPAFz56r8Jo2N8MYb8Je/KIFiWfCd7yhyPixkLAsuuED9r3PI\nt7So8kH9beiyznvbxoGeR/dUDkxZgse1Fnm0cW0hllS5QYQjGHfMw/z63F6UnVLm7Riuv7WboaJT\nYVqEk8uAD/6HFS02ThqlGqsfBiW/4eqLv0zPo79JacFvYEsxXY6G0pvU3WHTZq0ybGqWOJbasVhu\nGBMtRLRfhPzTCzgtbkTiCybw/VNvprysF8XFhRQOnO95NWvjA+0d3PBhA5XpSjjR5yi0lZQZLgVQ\nQSM3ncuQkhYWOdMiHcaqXmni+fnN0Hm7l3RM5i/1ogXr8mxhc8qxp7B2+9p2vRH3/+puXhMjPAsg\nu2etZxK9L7CwkFtUeBCtLjv1pus5pf8OnnjrThjzaiaRLl1foYiFt0/XPpz0yeU8P3NiiNAGceJK\nhn65DzUvd6WlRako7d6LyOlVS0v+UiVbGs6C+mHIgoVqGtVe6ZYhFVeztcj/DJkhffKX+HxYK0R9\nNvQ9pi/rdqzzPostZ3t9YyXSXPQ/v2fiJUOp+2edUgciSdpJzmIC6xdt8QVWO3g/Xd8Mk2uj3tJu\n5qOoepvCChthOeQmrQOexyQWJAcZ5kI9frxv8tvcrBwVhw2Dxx4Lhlp59FGfuDfhOEqA2Lb/nePA\nzp3+IpyTuBy7/AE48RVPpTVjhnKUTKfVLqg9qVxBCT9bx/vKtSkv66Xa5PqpjP+vTbSktHbSzyaZ\nTNqMvbwrdctcIUAa8WEvxJazYQCByK1acIwZk2na7JtCC/L6baAx75s0fFjIfSvvCyziJ+2aQWUq\n4fMcn3SBob+iuPher75AICTF5YWX8+dv/ZkZK2ZkLIoSSY7Ioah7kQrwKB3YPBhr1stIJ5eaRWms\n7y5G5leTa+d6oUdmzKnj+V/0CagdsJtxxgzP8GmQSEp6lrCucV3AlFmgfC4uPOVCvpT3Jao2VlG7\nrZa69+rI6WVz1cWFlJ/xa+r+WefxFZawMoSSdsCMChdjCTVebCwNqGL+seIE3uz8YMDiTl/v7f4E\n9Mvrl+Evsv6D9WxY1BwktJHq/61fZdkOwQ9u2cDt8x8i3Ws+Ob1WcuPgG7mj5g5aNp3pmRpLu5mT\nz6lmvZHDhu618O6Z/udTn0CcuJyTB77DuiP8fv3KF7/C6vdWZ6g3rU3DuejcbsqooWEhYUgk63as\nU17lx53KjWfdSO1j51HpdEJKC5HOYVDLROpW1HHNPQ2kO41B7D6eoYVF/PXWi6BFxT0795Zf8fzu\npgDvJ+q/jui51IvzptVtCStBuufS4LxoxcDAE3SBDKmN9DtyKDde0p/i4kgFz35DLEg+w9iyBR4O\nuRVI6YdHiYLKER88Z1qNgc3VXWbS8+uPeAv2tdf6Ze7Zo4SK47SeiKemRu2KtNPl9Onq/NSpfmC/\nB3ZOQlpzQSbIzbW48b/rvYio48oKXRWf4P4HkqRWXk1qVTkz5Ahmrh7OmI/W0tzcyxMcEG3a7Avi\nQqCQms01gai6k0snw8m9uG96M2mD54D+gfboDIsaD9c9zIlHnUiX3C6Rb9gtTgsfN39MwkqohW7B\nfyNTSaRUwQHFxqHYPZcw/fzpnqC6//ENkO6HqXbI5iNjC5ui7kWIVcL7fNPZN9Elt0vAXHvqoqms\n2LpCBWBMS1ZuXemVYRoElPQsYcOODTSnm7GExcDuAyntXcrvl/iBLTUx7EhHCYaCl8D+qacucwpe\nCvSDQPDjIT+mT9c+XPPMNR5pb765m5CaYHcJ7eP7vMv29b2RjgpauGrjZhg6FWSalrTF39f8XS2s\n9cMCC+iGHevJSZaQakmB1QxF98N7X/HVekN+g8xfwjrw/Dr6d+vPyJNHct3c62gJcSY/uuxMyoZ8\nk6r6KgbnD6ZqYxXNTrMXZ81z+MNhXaPbtoIFJJOXk2pR8zEvD8Zf8iWc5smAjRRpnn/Z8YSGk5Ls\nfLM/FEwLPPvHlw+iy8m3eRylNuHudmQ31ry/JmitV1ClLBpTmerLQJ8bQn4N93HD67kUDtx37/b2\nIBYknyGUlytOw8yuuC8I71RWrPAdIZNJXJWOCs419c9B9Zi2KtOkfTanRi2cHEfdU1trJu+CMbev\nU+axrj/GyPOP5q6myV5E1MKB8ykuVrGYnLSFdACZwNk4lOb8Je4Ptdwrr7wcis7zU9CG37B81VxE\n0Lp8+MOjb3HNPY+R7jWf3IJays/4XZv9+Pc1f2fWv8+iU04n9mwsQtYPU7pq90f65vtvugmOXiTd\nkqOIX+Fbj0kpadzlE84nFP4jaKkUEVEZlND45pe+Se1WFf9Jo0tuFyYNnUTN5hovqGE4O+HSd5ey\n9N2lJKxEwJjiqE5HKdXZ5rNI15eyrPdCVm77nbeTsLA4t/e59E+N53d/WYnT88VAyH0KqsjpuYK0\nYwUiG3+0R+noc6wc5TwoZUaMNQ8hdc37wka+/bwad6uF405/A+tDV5BhWNpp/sflFmS3FTj9/wwb\nS/zx0BGmQzyWg0P9znoaPmzgufXPcfcFdzN7zWxeEN9AbizB6r2Ij44rZPismRmm8DovyX0r7/N2\nCiknxXVzr1NWg+UPcOEnf6HbF7ozb9E2nJZjCbwkyJQyAqEF7BZOKPwHnZtWsWfMNxCbSvnRZV/l\n11eWYZIXeiep47WZRgGJXiso/tktLF6UUAEfo8LURODTCCcfC5LPEIqLVRiUadNUkqz9BceBK6+M\n/q60NBhy5Yc/VNyMXsA1qR9Wc5kmzcmkOmeqnqgfRvKoJM09l5HsvZpup46heWWmuXEUz5G0k5SP\n6kt5/2Do8gkvDQ8KIiM5UJA/KWbS0OCPZlxZIYUD/0VV/RGUFtye8aMqP6OcGStmBBbBb532LRUJ\n93Q3KnOLjbT2eDp1iST99lBI5ag3T1Jw0otQegsi/1WSdqeANdzES4by5FsjVM6Zzu8jdh/PN4Yn\neDm1grT042HNWz+Pp956KpDkyVNDuia22lltfvl8LzvhC2+/4C0qKSfF9wd+n55H9yTviDyunXut\nil1m6NhTY4Zj91yG7aYmGP2F3zLhskKc5oux7Z9B+XDS+a94RPKNg3/AI689wpaPtyCRpDadyR8X\ndMXu/SdkfotHMOuQJ5awcBwn0KdWz6VIl6twIECoP/bh8oDg9JBfgzXyhzhPu1F8592Fc8U5avfi\nXZOFtHahI/E27mp0E3ANpzn/VTcSb2GkKbw+iroXeX2uw+870kE6KZ7527GkUxJJV2U44O761UuC\nnwgucVI1Ey/5FRPx+bjGXWup2fzFQIw706s95aQYN2AcoCIbPPPSDha9bWOftICKi86gqPsV1G6t\n9RJw6cCrQCBE0KcRTj4WJJ8xFBfD44+rzIuzZ0P//srB8L772ud/EoVkEo46Cu64Q5XxwAPBHN+m\nZRb4Do1FRcFdRpg7Cd9nJlAqL+tFeY9gwiBT5ZTXOMoTUGGeo7TgN95OQguvqYui/V5qNtcw+aEm\nmpqH4aRF5C7K3K1MGloceKM3w2+/ctUr3Pzizbz9wdtc9pXLKDulTJm11lyGk0ogHbBEJ07Y8V3e\n67VCvZX2WYxYLGlpkThWM9bXbyOn10qu6v/9SGu4nF7LaPa8/wWLnE7cfcHdXuTWqvoqP/KvA1cP\nuJqeR/f0+tBcaJrSTVTVVzFp6CQml06malOVp57TPIcu03GcaB17z2We5d6su49iT5ODdCxsklx9\nzJ9hoIryW9S9iOvnXe+r/wziN203Y19xHlaPaiU0pHq+NsG2hc2QnkO8yL0Tnp2gcq9sPssPbdNz\nKWlJ1t2Ms7U/OpsoaQtr9RjsXsvbDIVjBjTUC2o41Hp4boYX3XEDx/km50fk+VlI64eRarGQjgBh\nqYyiRzcoayzXkELvGr556sXU/bOOxl2N5B2Rxw3zbvCed+fIO2nc1UjDhw1qnAwUdS9i3MBxjL93\nFi0PfhvSSVILmqHobxSe0ZfGXY3e/eHAq9MWT+Pdj99l7ICxh3SsrRgdgI7ZpVFU5EcO1h7v2mRY\nCF89ZVnqvGXBkCHKH6WoiAAP0tyMm+M7GNgx/Gavr02n1Y4lijsxF+ywqXBNTTG8Ugw5ruBxf7x5\njaPcFLV+WSoMfiFZrL6zpkQdPms4Tc4AHOt5LDpn+LtkRFR+pI4JbwynqX4A1qbd3HPNkYwrU88s\nzi9mwZXKVj9gJr3zOXIS85FYONYe3s17hBwhuLroasqvLIcrbGXO/a+P6DZgHOWjfhP5w62qrwq8\ncUuk95ZsJnIy22kKo6mLpgYWGlvY5B2R5wnFu0be5UUNTss0M1bOYObqmUw/fzq5ObnsKViItJvB\nwTUpDfrEPLBzjMdp5SQsVwV6r/dsHV0ZyBBKF+bezq6Tfu7lbHek45VtY3N+n/O9NhYeX8i0Rxcx\nZ+Z1nuXUudfNZf7rq5QnfmhnYQlL+ScZ576W921O+9f5bDvuUbqdupFtn2zjiTefiLSUU2//jZ5V\nnl7QzYW3rcyOVVXFlJYWUzxQnbtu7nWkTBNdHRYmYlckkcx5cw5z3pwDm4tVNIaCIshfQlO6iWue\nuUb1k2WTsBNeEisppeeVHuaJtr1xKsN3lAbUcWxR+YfUDh6e2/Aczelm6p6tO6Q922PsR4wb5ye5\nysuL3ink5SlyXjsyLlsGv/qVuifKsTH89m6S8mGCuz3cSbb4Yn4d1Y+xau3eB5eMStijs9k5PRZj\njRnBudZtTL6itNVAlrPnNSrB89DzOOkk1y2UFL6c+XzT858TX+Hq3z3M26t68qLzM5wei0k7tm86\nvUXvxrqRnF1OkQ1VEVZvYT7DfEturZ3m/TnvDqVlw9lYvRfxg0vO9tLzJu0kY84YE2iDqc7xhPio\nDdz/+AaWJn4dWPSq6qsCnNaVo0+huLg88OyEbcRZO6kaXoFUS5pk0mLi5YOgh8rZrtunkWPleIJf\nt2tQy0SedE3HaAHyYAAAIABJREFUSdnMv+vbOPLfwfqJSq6Wv9QTBrVba9l2xE6eWS1JtQgsW7L4\npaN55cVjyE1OZP584Iwanlv/XMDIwuy7GXPquPaX/0e653xkfnVGrpBsmR3DcfKuugq29dmp+KYe\n1Ygx33B9n17yrdhcK8WM3dXmwTDzRRXSx/6ppyL1CH0HLvzShTz5jye9c3rXWV42ift/n6KlJUUi\nIeh2+ps0bzdSNz+9jpk3FQc4yk8z5W4sSA4hhJ0ag2//wXzYoCywJkyAE07w0wGbRLxlBQM7hnmP\n8nJ1RAmvbHbpWo3U0JB9NzN9usuLNAHCYemOZ6nZ3LXNiR7+sQd2KQUrmVyeS3F+8J5wm0aPzOOl\nP5yD477dpVMyUpCFd0Dlo/rCKFg0ayXNaTsgAExhFbVz09eUliq+RYeL6XLy2sg34KhFraYGpt1T\nQOqxFyBtkbMYPhrwcGCxALWbUYYBJYjeC0kW1FJaUErdiiOpmlfM6JEw/bYvUjpzJS1pZQIccMJ0\nOa3yUfMz6lQ1psqLs1Z+VTl1F6wNGT8Ue3yN3pkIBFf2VwSdGdtt+umvkmMX0uxuM9Jp1xRYJhD1\n55Dbe7UnRDwO4Iq5XJyYxlNLV5FefhVIwZ49klmzBPfem10A19TAdf9xqkpra98Mrrl1WwtsWG2a\nTkNlpUTa18GYxyF/CcmCFYz8xvHMeQGVY6egCpn/Kt8f+H1AcRueqnJ1ue806aoW7Z7LvCRoAYdO\n7XjY+xV3ntUgxkxCbBiC6LOYokGXkXzWyLhYPyyao/y0Uu5KKT/3x8CBA+XnHVOmSCmEVnhFHwUF\nwWtsW8rKSnVvdbUqp7JSykGDpCwr889pVFcHrw2julrKzp1VucmklLm56v+cHCkty3/mlCnqOXZO\nWiJSkpxPZHLcMFndkKVg8xkN1XLKwineteHP2eoVaOPjr8lEbrO0bEd27txKeyLKrm6olhV/mCkr\nJtZ795ntDre1oiK6T1p7brjuFRVSJpJpCSmp4sFIadmOrJhYLzvf1lnat9iy822dZXVDtdc2YaVl\nIrdZVj7+mqx8/DVJ4hOJaJEkPpGVj7+WtW2t9aXZj9UN1RnPNssJfzdl4RRp32JLJiPtW2w5ZeEU\nWVEhvfaovymZyG2RFX+YKSuXV8rOt3WWYrKQTMa7b8SsEdL63hCJvdu9x5G5udF9qZ9bMbFeWrb7\nHNEsGX6ztG6xMuodvrfzbZ3VsxKfSCEc/7ckmiVn/kEyfJIs+82vVf/muP2b84lMXF0SKLdyeaW0\nv/e1QJ2xd0vGFsvcX+bKiS9MlIlbE9K6xZLJXyZl4uoSr7yc3CZZWSnliHEvq7oY/WeOlzkH9dxq\nz2+jLaDCWbW5xsY7ks8JSkv9XQcEIwdr1Nf7HAqot5drlHqWZFLFALv9dp/UnzcP7rzTV5tFOSma\nHvFBfxW4+mro2TN6NzNrFqTTQlk7pRO0bBjS5vY7W8TiNncyIZVb49pC7r4TajdsUqalPfoCbW/7\na2pg1qxiHnywWAXFvMvnisxYamZbwe8T06m0NZWeGe3Aj8umQ4hIII2d42QaNGwpZva9kG5RMZqc\nlEXjWkUSk+qn9Ospyex5jYwrCxKzugzNY4RTDISjTI+a8AHNX4hWnWRTz4U5rjovurk7IU99krE3\nfMy948s9taXpQJm0k4w+bTSLGiawu+ghFW4Hm1QqwsAixHElEvNploprufSs8zn960eR1ziKqj8X\nUhcxt7VqU6tNz9x+J6ufHUBLSjnQ6hAvTy1y4Ds7sGRnL8LD2K6zKM7v5dVl3MBx1B5zHpUyiUQo\nT/yihyC/hpRjs2rrKp9XctKc8tH3WOPumFNNKa651kHKYYoHHDOCZMHKjB1GdDijtn8b+wuxIPmc\nQEcCnqU0DxQVKSERtvQKcyX6+z174Le/DX7f1KTKkNL3F0kk/JArs2YpvxedQ0WrrEzVGKjrr78e\nVq2C0aPVuQcewF0/lHNaos9iSgumtpoq2Azvkk0tkS3Ui/7O9/BPI8vHkN7+CjNnZYbRDwut6ae/\nyoTLCv24YQSFQTa1I/jWbLat+lD3V5R60KyjOT7u6IHdjD1gFnf/pNhTJ4UXeh1KRz8j7708nn+w\nGVoAIenf29f/RQlnIONcVVVxIMr0U9PPx77qa4EICSaisnCGhUtVozJ2ko5KnpaTX0v5KJWx0VQt\n2pbNVf2vChgeXFP/J9KryiGdwLItGhpsamoyBYHmuIZ861Ve+evZSMfm73cMo6TXMCZMwBvPcFTt\nsNp0+i+aYIuyLly6ZidPPHIcUtqkW1p46q2nSCSuIoVNMulHeDBRXtaLmXfpuSeRA/5KWptdnzaa\nRQ2LvP7+0plbWTPbdVoU0nvhsuiseMDy3MgxKi4u3udEdh1FLEg+R4iKi1VRkbkziYJWeIVhLmT6\nTXraNHjuOTIW1cZG/61o504YOxbeessv27Jg0SIV7kSVKxBC8tVRa5j+s6kZYVE0v9BaeBcT0QS/\nL1TMHZMjgQ1DkCcsiBRK5kLUVD+A3zzxbzQ1+e0VIrswyGbNltevjtqttVA/zLWIyrzXrKOb9NCF\noKDvLo7v9xalxcNpXNuHmi9mGkpoIXLuuTB5slsXCtlw6wZ++7MCpExw1619KBvm9onRTt0PDR82\nKPNc16qsqr6K0tLiQJRp6VhcZURIaM+bbwbHVQqdcpVXu50jufua/0dxvm9Bl43zqF3aCWdjifLR\n2DYQZ/X3uO++oBViQBC98zUWPzbYq3tTk4ppt3u3X7dwVG3z+X4MNpg0qZiamm48838ttDSrSAlO\nt+WcccJABnQfEAiQGp4T/o7Bhh5Tg21zUy2MHplH4cB/MXfdBSq1b+ftKiZXOkEiYTF6YClVf4aG\nLrMC4zbr6XVU7SxuM6zRAUN79F+H+nE4cCTZUF0t5Wmntc6d7O0xaJDPA+jD1FNXVma/N8wbmFzB\nlCnqnMmlhM9VTKzPqvc1r7UsKROJkM7Y0CPndkrJ5LhhkTp+KSN05JbSkQuh7o/ikHR/m3yM5jjK\n/nNrq88z7zc5lURCPTORUH2s+92ygn0XpSPP1jdCqDqZ7dT1qlxeKZO/THq8RO4vc726VlaqepjP\nrqyUcsQI9be9CHAtbfBuUfcmkinFF9m7pf3VSonwx0aXNWWK4sI8jsQKzsFEInNuJhIRvGAWLqjy\n8ddkzjd+LsWF4ySJT9rk29rqgyh+Y8SsEdK6xZKMHSzF8J/Ish89Ezl/k+OGydxOKWlZiqPbm7Fo\nC7STIznoi/yncRzOgkRKNTGTSf8HY1mZgiB8WJa/gJWUBL+7/HI1YfVnc2GSUi0s2crNzVUTvaxM\nCaTKSuOHXxnxg2pjgQy3MxvpPWWKf41JGLdKLjdUK5LT9hcq3XfhRTyq/pWVZr9rgnWwR5Zmg7k4\n67IrKnxBYC6Iui0VFapPKyqyCzhzDuhxMBfcMCkuJgtZ8VRFRjm6rWVlwfq0ZwFrz3i2Ni6KoPf7\ntG+/jzLmZrY5pBfasrLMvgQ1z70x1UT9UxUZRgLZ5oc5z1prf0VF0OjCHNvAXA0JsYqJ9ZEvVWFB\nGSUQ9xWxIIkFSQB6Ausj6ocUJUiSSXV92NqrpET9jVpUwwuMKXDKyoILmn7T1m/gUYvh3ry16msn\nTvTf5s23tPZYnoV3FHphsm2/HyzLX+yzCa8RI8KWdCkphv+k3TuSqB1HeEdSWRkU6npnGCVczHHU\nOzbLdjzrLimj38Cz9Ul4fE8+uW0LuPBiGF54s+0A9P1l/7k18Mzjj8+sg7krHTEic+ej6x+2chTC\n7dPHX/PqkPxlUub+Mjf7zrUNwRhlWWU+V/8mspXRlmWWroM5ByyrbYHWXsSCJBYkWRFeHKMWfL1g\nCaGERrYdTEmJ/2ZrLt6WpcyNS0r8c1FCST8j48dcuXcqD90u/bacmxssUwspsy5R5s3hHYUur6Ii\nUx2i33DNxd1UpwV3JMqEt+IPM1s1x4xS70W1z9yJhMekoiL4XHMHYgo9ra5DNMucb/w80qQ6avEy\n6xi144xaTJPjhkkx/Ccy5+LxMrdTKuvCW/GHmVIM/0lg52YKl+S4YTKRTHu75XA9Jk6MFrhRY613\nBuGXpBHjXg7sQiqeqmi/WXRox5ttRxE1Nu2Z71FC3fztZWvvvqK9giQm2w9DhM1VzfAptg033aSI\nau3AuHBh9rIWLlQkd0so5JGU8M47KvTJq6+qc0IoazLTTDmRUOSwfpaU2cOxtAaTaJcyaH0mpTpv\nBsJsbs5MQ9yaY+GYMZkWb/qztsaKIvh1NkyA8nIr4C2u621aeDU0BCM1m2R+lDGFLtvEtm2Z42Ea\nQ+jsnE89k1ZpXO0WnF4vUVXf2SfEtxQrUtdwLNV9Bn4dhYCuXWH7dvW5qcnvV922pTs+oPmBuSpO\nlN3MqAnPMeiYCyJNyR/84eXIJgn2T7GvusCLFWZaYF14w7Pseu0CjjgCnnrKv7+kBH79aygrUybY\ny5Zlj8Sg+7K8PGh9aNtwxK5Tsd/xLdK0tVhNDX4IEoLWgVEhhsLe5RQswLbLMywpUyk1NpMm0SZa\nix5x993tyyN0IBALksMUUal/wQ/k+NFHKh6XlG2XFV60NNJpZR2jF3f9g7nrLnX+hBNg4kR17bRp\n8OSTvjAxE3y15W8R5cPSXphlmF7w4ZAwoBYZs3zL8hOB1daqc4WF0QtWtmeb4Te0abBtKx8cPRat\nmTSXlwcDegoB3boFhTUEhZI2R5YI+MJWKHyE3IKVKitlRL200LDt4IKr6zhrFvzxj9nbhjhfmbK6\nicW6Wad7Y6b7CNTnVIuNSnoouKrLTM8fw7TAmjfrPFItqg6W5bf91VfVc0GZmuu5m5OTPRKDKVCm\nTVOC6clHupFIzufq3z1MUfciz9dE+weFzbj1i05GiKGQd3n5qL7wWmZftVa/1hB+XnuF0QFBe7Yt\nh/oRq7b2HmGdfFte81FH2DomkVDqpbB3d5gINg+TOGzN0sVUq4XVT/qZZlvCqh6tdjPVWWGVTlgt\nN2hQZl10ORMntm3NFLak0mWHSfRs3vCtqTV0hIL+/X2jhvAz3b2bBEdOnLo+sl7ayi5M+IcNGJLJ\noMowbEGnoxjYOWlP/ZSNBwqrFk3DiDDHMmhQZr9ls1Bra76HeQbTutBUYUaNVVT9oww6ws+Jql97\nOcE2+Zl2ltMaiDmSWJB0FGGd/MSJQa5E8yfZBInJtegfnbkQa1KwtfAugwb5dTF/NOai5hHHli+8\nwqaQUfxCeFHV/EyU4NKfTcGo+YDWOANo3VRY6+hNowO9iIaJ2dYWrcCiW52dJ9AmvOE6jhgRrJdp\n5WT2YVZSOKKvogR9mFcyBZXJMUQJ+agXiI5a+mmE52AiEZxjpsVea6Fu2rN4m2bUuZ1SAd5sb+ue\n7Xn70gdRaK8giVVbMbIiSi1TVhZUg4FSYZjOXRpSKtWDVgWE88w7juJoCgszVTEaffuqxFqmrn7P\nHqXjj1JD2bZyhAR1TW0tnsdzlIrJVFdJqcrWKpcodZLJk7S0BDkDx4lWBc6ZA3PnBnPAmH0Eqg06\nHI2pqjO/N9VTrak1pk71Pdx1nTW/MXOmnx7ZbEv//sEEZtOn+2kLrr1WXTNuXFQYDgWTJ5g6VY3r\nmDFqDLp1U6pS7RWv+12IoLrMTCkwdarfPhUsUdV9+nRVLvh9GY4kUFWVyVW1hby84Nj94Adqrs+c\nGexL21YqLp2zJzyerakyNXQk71lzNvHAzjHcZ0RXMCMImA6S2ZDteeH50Z4I2x3BARUkQojzgd8D\nNvC/Uspfhb4vAaYDXwH+Q0r5N+O7NFDnfmyQUl7knu8N/BXIA1YA35VSRixBMQ4EoiauSdzPm6e4\nDjM/ytixmTlRQC0kjY3B8C7btsEbb8A6NwX1ww+rMnJyfH24lJlxwMz4VkVFZowqtQhdfLHiY8Jh\nVwYPDhoTSKm88sOhw3UU5DCJf//9fviYqPhmGlE5YGbNUsJIStWu2loVmwyCfI1ZB13/cFRjU8e+\nc2dm+mQhVN9q73fbVgvlrl1KiJhZMefPV/2q+zqVUkJF8z9tcT5mxAM9ByzL/1/zSo2N6gVBczxa\n2Om5lEz6ZckIIwz9IhMmu81sn2EeJhvf1NjoC1ch1DX/+Aecdx68+y4sX+4LwjvuyKxDNqj4bOr/\noqJQpIXUI6RffoX0sivZvfbbTNv5ASP7Bl8A8vJaLz8bWpsfBwTt2bbsy4ESHhuAk4AksBo4LXRN\nAUqIzAK+HfruX1nKfQwldAD+CIxvqy6xauvThcklmOqYysqgCihsKtoaVwJSHndcUP1QUuL7xWgT\nXe3oGHW/6WNhqn7CnMqIEdEqr4kTW1fjhVV+/fpFczZaFWG2VZtim2qoKHVcNlNTsw/DarawSias\nsoriFNryTYh69pQpbTu6hj2vTTVPWGWkx7S1KNJmfTL9doL+Nm3xTVG+JVrN1VYdso1FeD4Hxreh\nWuZcPF56EYFx5BmDdkpEOqPP2zPu+reQTTW7L+BgcySocKrPGZ8nAZOyXPtQewQJKgTq+0BO1DOy\nHbEg+XTRli49POE12hMKv7WjLSdL8PXx2fxnshH3lhUkdls7NJEfJRS1kAkLuyjuSMrsC21OTpBE\n130bFVXAdELs1y9aYIW93qurfU4sfG22BTksfLKNUZjnMT3OTV4nvIhG8SHhcqKeqTm0bHyTRmVl\ndBlauLZm2BDlw5FtPpvPHlSyQ4ZD6WPvlohmmdspFWnM0ZaxivnC9HkQJN9GqbP05+8Cd2e5NkqQ\npIDlwBKgzD13LLDeuCYfeD1LmePc+5f37Nlz33syxj5hXyZxWzuS/XHYtu+0pn/k2vu9rMz32A8b\nCoQXe/Po0SNo0aMFZUfqefnlmREITGsh89ChVKLeqG07aLEWbk9ZmRqnsrLgvf36Ze5eop4Rtjoy\nd51RoXi0FV3YSEHHLzOvbW9OnPCOSkdl0M8zdxH6mmzxyKL611yYtaOrbft9km3n1taORPdX8Hkp\nyZl/kGL4T2TFH2ZGts8sq6IicyemxyQm2xV6SSnfEUKcBLwkhKgDPmzvzVLKGcAMgDPPPFMeoDrG\nyIL2kI5R92iuZM0aWLx47/1CTNg2XHih0nWvWaPOpdNKx/2d78Ajj6hz+if4zDNBnxgZmjXmZ53X\nJZmEn/88yNFo3woTxx3nO+21Bw8/nMm5aK6oqSl47ezZSvcejk6seSFQPNXatZntmTNHcVo5OerQ\n7Tev1VxW2ABAl3H//er/oiLF85hcQ1QagzlzlL/GTTcF9fjdugWNAJ54QkWZNrNMRnEGYT7ATD0d\n5tBsW/Fie/ZAXZ1fbmlpJuEO6vo771Rzc/x4v+81p9XYmMmb5eX5XMxdd/k+RkcdFUylMH684q38\nNksQDqL7Kjqd9ScvS2VeXrBPdR0dR/V9KhWsdzKpytX80qFOtr+D2jFo9HDPtQtSynfcv28LIaqA\nImA20EUIkSOlTO1tmTE++wh77k6eDM8/73+vPe9XrVIk8ZIl2T3vHQcGDVKLaUmJT/SnUkqImD8+\nbR3UHmgB1a1b0HJIGwvMmqUW1dxcN/x9DuzYsdddEahfTg7cc496zs03B9u8Z48i2E0IodquSeh/\n/CP7cxxH9UleXrSwkxKWLoWRI/0FW98npRI+2snOXOha608t0E1vbFDGCCbBblqbmblWcnN9o4Xo\npE5BaMuunTuV4yGoNiUSvvFB//6ZwltKJagLC6PbUVqqxkb3iZTKIMCygpZo4BtwVFX5/RaEAJlD\nzvN3c/3ZE6n6cx/PEVJfGxZ0ZhlCwNChcMwxSlDra/fV4XGv0J5ty74cKCH1NtAbn2w/Pcu1D2Go\ntoCuQK77/7HAOlyiHvg/gmT7NW3VJeZIDl1k82kwoaPlTpwYrSvW15h+DPuqcjLjaYX9G8LBKMvK\n1NGv374/Tz8zHAm4PVyNSaa3RYK359DcjzZqMOOZ7csRCGdv8CAmB5NMRqctEMIPyGjOlbaMElqL\nTG2WHf5sxk4zHS+rq6ONO8LGJu0dM31oHmtvxy3KKbesbN9/fxxsjkTVgQuAf6Cst37qnrsVuMj9\n/6vAFuAToBF4wz1/Nsr0d7X7d6xR5knAUmC9K1Ry26pHLEgObewN39IamW8uVlFcjCbUs5HqQqjv\noiy69nahMJ/ZnuvCud+jnArD5WqCWJPUemEK69P3duHXTpS2LWWXLpnX6MV/bwwTopwK9bOy9ZHJ\ndWhBrvku0yjBNC4IW95FLbyDBmU63+rz2lCiNV4qLBDCLzhRY9W//96NQ7ZnRp1vj2d/NrRXkBxQ\njkRKOReYGzr3C+P/ZSj1VPi+aiByMymlfBsYtH9rGuOzjL3hW1q7Niq+2LZtyifFTBcMShVgOkhq\ndcrYsUq3Hla/QHanymzIyYFRoxQXoN6RopFIqL9mLLGBA5VqxoRWy2gu5cEHlepD+1R06RLOA9/6\nc8OQUqnJTJ7AVKmZKr+PP1a+F2b5OnPjEUf4bU6llOrIdDadNQvuvdeNvZXyOYgePVQgUF2m7vuq\nKtUXuk2pVNBfSaOpSfVBZaV6Zv/+yrFwwYIgJzRggLou3Ddmf69eDVdemekzE0Y6rVJYh2FZkJ8P\nmzer+19/PXsZYQwdqgJSak7MfH5YNZdItO3rsj/wWSbbY8Q4YAhzMWH9+lVX+UErw6lrtRAyPbLL\ny32SXQunlhZfp6/JeSnVIq8dDEERyk1NahEwSdW+feGUU3zCXAdbTCZ9gWY632lBoR39ZswIOtEt\nWKDqX1vb/oCcYaxenf27IUMyDRZAtVdKVe/Jk/0267aMHq36XztAPvig4pgaGoJBGd97T11v9msy\nqdpsRgHOBiH8/hk92ifgw/1QVKTG2LajBRKofl+ypH19qMdTQwv6LVt8IZntOWEkEooDOeUUxROG\nERYqY8d+OpGAhdyX2XSI4cwzz5TLly8/2NWIcQghHKI7KpR9a1F59Xd64YoKPR5VTrZrop7X1vNN\nAwPLgttuU+FHTA/wcGh8DS3U2muAoEnrqOsHDVJe9GY9w3UfP94XbtoTH4Jv3ELA97+vvP/Nfq2q\ngp/9zG+Lea+JkhL1Jq/D6pghW8x2/8//qH6aMcMPE7O/lkkdNRl8Qd8eWBZcdFG0oM6G3Fx4+eWO\nCRIhxAop5ZltXtge/dehfsQcSYx9wf5w6DqYyOYoJ2V2T3CtszfJ/XBgTjMNc1mZykc/6KIVfqKs\n0LE3KXjb4owmTmz93pyczORWOtpzVKDPqLZHEfjaCCCckdG8T/d1WxyH5u9M3qqte3QQyb3htEpK\nOj6H+CyQ7Z+VIxYkMQ5XtCYMw2RxOC2xeZ0m2MPZ/HQWQOt7QySJTyTCCSyuUQt/a3XNJtz0YUYp\nbq2d2cLLhCMHh50g26pvpgNhMKyM+cxsKaejLNXaQ9pnS5GtoyWcdlrmPR19CWqvIIlVWzFiHMaI\nUsG1lUTM/H7qoqn8/OWfk5ZprC1DONe6jdEDS/c5U5+pdoNM1U9lpYqeu6+IaseMGYp8Hz26fWXP\nmKEcAWtrW8/iGQ5iKYRSAUY5B+p67dyp+Kx0WvEo4QRadXVwzTV+BOeLLvKDkdbUKCJeqxdNdea+\nor2qrViQxIgRY59Rs7mG4bOGe1kA55fPV6l6O1KmsdjX1SlLOiHgxhs7JkT2N1rjqMLXtCWos5UL\nmc9o7bma10mng06b+4pYkBiIBUmMGAcONZtrqKqvorSgtMNCJEbH0R4B117EgsRALEhixIgRY+/R\nXkFifRqViREjRowYn1/EgiRGjBgxYnQIsSCJESNGjBgdQixIYsSIESNGhxALkhgxYsSI0SHEgiRG\njBgxYnQIh4X5rxBiO7DpYNfjIOFY4P2DXYmDiLj9cfvj9u87ekkpj2vrosNCkBzOEEIsb48d+OcV\ncfvj9sftP/Dtj1VbMWLEiBGjQ4gFSYwYMWLE6BBiQfL5x4yDXYGDjLj9hzfi9n8KiDmSGDFixIjR\nIcQ7khgxYsSI0SHEgiRGjBgxYnQIsSA5hCGEyBdCvCyEWCOEeEMIcaN7/hghxAtCiHXu367ueSGE\nuFMIsV4I8ZoQYsDBbcH+gRDCFkLUCiGedj/3FkK86rbzUSFE0j2f635e735fcDDrvT8ghOgihPib\nEOJNIcRaIUTxYTj+P3Dn/+tCiL8IITp9nueAEOIBIcQ/hRCvG+f2esyFEGPc69cJIcZ0pE6xIDm0\nkQJuklKeBgwGrhVCnAbcDMyXUvYF5rufAUYCfd1jHHDvp1/lA4IbgbXG518Dd0gpTwY+AMa658cC\nH7jn73CvO9Txe+BZKeWpwBmofjhsxl8IcSJwA3CmlPLLgA38B5/vOfAQcH7o3F6NuRDiGOC/gbOA\nQcB/a+GzT2hPYvf4ODQO4AngG8BbQHf3XHfgLff/SuBS43rvukP1AHq4P5xzgKcBgfLkzXG/Lwae\nc/9/Dih2/89xrxMHuw0daPvRwMZwGw6z8T8R2Awc447p08B5n/c5ABQAr+/rmAOXApXG+cB1e3vE\nO5LPCdwtehHwKvBFKeVW96ttwBfd//WPTmOLe+5QxnRgIuC4n/OAnVLKlPvZbKPXfvf7D93rD1X0\nBrYDD7qqvf8VQvwbh9H4SynfAX4LNABbUWO6gsNnDmjs7Zjv17kQC5LPAYQQRwKzgQlSyo/M76R6\n3fhc2ngLIUYB/5RSrjjYdTlIyAEGAPdKKYuAT/BVGsDne/wBXHXMxSihegLwb2SqfQ4rHIwxjwXJ\nIQ4hRAIlRB6WUv7dPf2eEKK7+3134J/u+XeAfOP2Hu65QxVDgIuEEPXAX1Hqrd8DXYQQOe41Zhu9\n9rvfHw00fpoV3s/YAmyRUr7qfv4bSrAcLuMPcC6wUUq5XUrZAvwdNS8Olzmgsbdjvl/nQixIDmEI\nIQRwP7BWSvk746snAW2FMQbFnejz5a4lx2DgQ2M7fMhBSjlJStlDSlmAIlhfklJeDrwMfNu9LNx+\n3S/fdq+6BakOAAADBklEQVQ/ZN/WpZTbgM1CiFPcU8OBNRwm4++iARgshDjC/T3oPjgs5oCBvR3z\n54ARQoiu7q5uhHtu33CwSaP46BDh9jXUFvY1YJV7XIDS+c4H1gEvAse41wvgHmADUIeydDno7dhP\nfVEKPO3+fxKwFFgP/B+Q657v5H5e735/0sGu935od39guTsH5gBdD7fxB24B3gReB/4E5H6e5wDw\nFxQf1ILalY7dlzEHrnL7YT1wZUfqFIdIiREjRowYHUKs2ooRI0aMGB1CLEhixIgRI0aHEAuSGDFi\nxIjRIcSCJEaMGDFidAixIIkRI0aMGB1CLEhixNhHCCHSQohVxnFz23e1u+wCM7prjBifZeS0fUmM\nGDGyYLeUsv/BrkSMGAcb8Y4kRoz9DCFEvRBimhCiTgixVAhxsnu+QAjxkpsXYr4Qoqd7/otCiMeF\nEKvd42y3KFsIcZ+ba+N5IURn9/obhMpB85oQ4q8HqZkxYniIBUmMGPuOziHV1iXGdx9KKQuBu1ER\nigHuAmZKKb8CPAzc6Z6/E1ggpTwDFSvrDfd8X+AeKeXpwE5gtHv+ZqDILafiQDUuRoz2IvZsjxFj\nHyGE+JeU8siI8/XAOVLKt92gmtuklHlCiPdROSNa3PNbpZTHCiG2Az2klE1GGQXAC1IlKkII8V9A\nQkp5mxDiWeBfqJAoc6SU/zrATY0Ro1XEO5IYMQ4MZJb/9wZNxv9pfE7zm6j4SQOAZUaU2xgxDgpi\nQRIjxoHBJcbfGvf/alSUYoDLgUXu//OB8eDlnz86W6FCCAvIl1K+DPwXKgx6xq4oRoxPE/GbTIwY\n+47OQohVxudnpZTaBLirEOI11K7iUvfc9ahshj9GZTa80j1/IzBDCDEWtfMYj4ruGgUb+LMrbARw\np5Ry535rUYwY+4CYI4kRYz/D5UjOlFK+f7DrEiPGp4FYtRUjRowYMTqEeEcSI0aMGDE6hHhHEiNG\njBgxOoRYkMSIESNGjA4hFiQxYsSIEaNDiAVJjBgxYsToEGJBEiNGjBgxOoT/D+Vislm1Q+UtAAAA\nAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "W4EQD-Bb8hLM", + "colab_type": "text" + }, + "source": [ + "## Further metrics\n", + "From the plot, we can see that loss continues to reduce until around 600 epochs, at which point it is mostly stable. This means that there's no need to train our network beyond 600 epochs.\n", + "\n", + "However, we can also see that the lowest loss value is still around 0.155. This means that our network's predictions are off by an average of ~15%. In addition, the validation loss values jump around a lot, and is sometimes even higher.\n", + "\n", + "To gain more insight into our model's performance we can plot some more data. This time, we'll plot the _mean absolute error_, which is another way of measuring how far the network's predictions are from the actual numbers:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Md9E_azmpkZU", + "colab_type": "code", + "outputId": "39b97561-b01d-49f2-c35c-fbd8db663806", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 295 + } + }, + "source": [ + "plt.clf()\n", + "\n", + "# Draw a graph of mean absolute error, which is another way of\n", + "# measuring the amount of error in the prediction.\n", + "mae = history_1.history['mae']\n", + "val_mae = history_1.history['val_mae']\n", + "\n", + "plt.plot(epochs[SKIP:], mae[SKIP:], 'g.', label='Training MAE')\n", + "plt.plot(epochs[SKIP:], val_mae[SKIP:], 'b.', label='Validation MAE')\n", + "plt.title('Training and validation mean absolute error')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('MAE')\n", + "plt.legend()\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzsnXmYFNW5/z9v98wALoiOUSIMYIiJ\noqOAhNjXJU0gRo3EBe+9GnPHuBFZvEGNXk00GRMTlBglUWPAhTC/GDAJEVfckFHEVgQBUVwQHQEV\no6OIiszSfX5/nD5d1dVVvcx0z8b5Pk8/3VV16tSp01Xve95dlFJYWFhYWFhkQ6izB2BhYWFh0fVh\nmYWFhYWFRU5YZmFhYWFhkROWWVhYWFhY5IRlFhYWFhYWOWGZhYWFhYVFTlhm0cUhImER+UxEBhWz\nbWdCRL4qIkX32RaRcSLS4Np+TUSOzqdtG651u4j8rK3n9zSIyGYRiRa5z7+KSG0x+7RoO8o6ewA9\nDSLymWtzF6AJiCe3f6yUuquQ/pRScWC3YrfdGaCU+nox+hGR84AfKqWirr7PK0bfFsWBiPwVeEMp\nVdvZY+mpsMyiyFBKpYh1cuV6nlLq8aD2IlKmlGrtiLFZWFi0H37vbKHvcXd8760aqoMhIteIyN0i\nMk9EPgV+KCIREXlWRLaKyHsi8kcRKU+2LxMRJSJDktt/TR5fJCKfikhMRPYvtG3y+PEi8rqIfCIi\nN4nIMhH5UcC48xnjj0XkDRH5WET+6Do3LCI3ikijiLwJHJdlfn4uIvM9+24RkRuSv88TkVeS97Mh\nueoP6iulGhGRXUTk/yXH9jJwuKftlSLyZrLfl0Xk+8n91cDNwNFJFd+HrrmtdZ1/QfLeG0VkoYh8\nOZ+58RnzNSIyP/l8fCYia0RkaHJ8H4jIRhEZ52rfT0TmJP+TzSLyKxEJJY8dICJLROQjEfkwef97\neObnYhFZm3wG5olIr4BxZe0riW8m/5uPReQO05eI7CMiDyWfnY9E5ClXvweLyJPJY2tF5HsB1z9P\nROpd26lnXUQmA/8N/Cw5Z/ck2wwUkXuS8/aWiEzJMu+9ReQGEdkkIu+LyJ9EpHfy2DgRaRCRn4nI\nFuA2v33Jtrmeg8ki8gbwatBYuiyUUvZTog/QAIzz7LsGaAbGo5l1H+AbwDfRkt5XgNeBqcn2ZYAC\nhiS3/wp8CIwCyoG7gb+2oe0+wKfAScljFwMtwI8C7iWfMd4L7AEMAT4y9w5MBV4GBgKVwFP60fO9\nzleAz4BdXX3/GxiV3B6fbCPAt4EvgEOTx8YBDa6+NgPR5O/rgXpgT2AwsM7T9r+ALyf/kx8kx7Bv\n8th5QL1nnH8FapO/j02OcTjQG/gT8EQ+c+Nz/9ck72lc8ty/AW8Blye3JwHrXe3vT15vF2BfYCVw\nbvLY14CxQEXy/14GXO+Zn2eB/sn/5XW0JOw3rnz6ejH5H++d7NfMz+/QDLc8ef4xyf0VyXu7LHls\nXHLev+ozx2n/Af7Peq3reAhYDfwseZ2vot/HsQH3dxNwT/L56As8BPza9Vy1Ar9N9tUnYF8+z8HD\nyWv06Wz6VDA96+wB9OQPwcziiRzn/RT4R/K330vxZ1fb7wMvtaHtOcBS1zEB3iOAWeQ5xiNcx/8F\n/DT5+ylcRAg4gQBmkTz+LPCD5O/jgdeytH0AmJL8nY1ZbHT/F8Bkd1uffl8Cvpf8nYtZzAV+6zrW\nF22nGphrbnyuew2wyLV9CvAJEEpu75nsbzdgAJqx9HK1/x/gsYC+TwOe98zP6a7tG4Cb8/z//fpy\n/8ffN/8bmqD+Cxjq6WMM8A4grn3/AK70meNCmcWRwJue610F3OZzLyFgBzDYte9okkw5+VztACpc\nx/325fMcHJPP/HbFj7VZdA42uTdE5EDg92jVyC7oB+u5LOdvcf3eTnajdlDb/dzjUEopEdkc1Eme\nY8zrWsDbWcYLejV9RvL7B8lvM44T0S/9AeiXfBfg+Rz9gZYaAscgWv12EVrqIDn2vfPoF/T9PWM2\nlFLbRORjNDE3c1LIf/a+6/cXwAdKqYRr24xvMNALeF9ETPsQepGCiPQH/ogmnLsnj33guZZ3XHv5\nDSjPvrzzu1/y97XA1cBiEYmjFzC/Sx7fqJKU1XXeAL8xFIjBwCAR2eraF0ZLl170R8/jGtc8iqfN\n+0qp5hz78nkO0t797gRrs+gceN1GZ6FXsl9VSvUFfkHmw1psvIde8QAg+i3J9pK2Z4zvAVWu7Vyu\nvX8HxonIALSa7G/JMfYB/glMR6uI+gGP5jmOLUFjEJGvALeiVTyVyX5fdfWby833XRwmg4jsjpYA\n3sljXO3BJpIEXinVL/npq5Q6NHn8OrQ3XnXyP/sRbX+u8unLO7/vgiaaSqmLlFJDgJOB/xORbyWP\nV4mLQifP85u3z9ELA4P+nuPe/2gTWjLo5/rsrpQa79P3+2jV8NddbfdQSrltMn7PgHdfPs9Bt03z\nbZlF18DuaFXD5yJyEPDjDrjmA8BIERkvImXAT4AvlWiMfwemicgAEakE/i9bY6XUFuBp4C9oVcb6\n5KFeaP3wB0A8KWWMLWAMP0sahAeh7SgGu6Ff4g/QfPN84EDX8feBgZI06PtgHnCuiByaNOpOR6v4\nAiW1YkAptQl4ErheRPqKSEh0DMsxySa7o4nsJyJShVYdthX59DXV9R9fgbaRkXzGhiaZwido1UwC\nvQpvBS4RkXIR+TZaRXm3T99rgENFpDq5aPil5/j7aFuWQQxoFpFLksbrcPLcwz3nobTL+e3ATBH5\nkmgMFJFj85wbg055DjoKlll0DVwCnIU2OM/C/2UpKpRS76M9SG4AGoGhwCr06rHYY7wVWAysRauM\n/pnHOX9D64VTKiil1Fa0qugetJH4NDTTywe/REs4DcAioM7V74toA+fyZJuvk65iewxYj1b3uNU2\n5vyHgV8lx/UeenV8Zp7jai9+COyKNth/jNb5m1X3L4HRaAJ9H7CgHdfJp695wOPABuA1tK0C9Hw+\ngTZeLwP+oJRaqpRqQjssnIR2xPgj2la13tuxUmpdsr/6ZN9PeZrcDhyW9MT6p9JuqSckx9yQ7H8W\n2o7gh0vQKrDlyXt8FK3qzBud/ByUHJKuLrTYWSEiYbQYfZpSamlnj8fCwqJrwUoWOzFE5LikWqYX\n2mjcgl5ZWVhYWKTBMoudG0cBb6J19d8FTkmqBiwsLCzSYNVQFhYWFhY5YSULCwsLC4uc6DFBeXvv\nvbcaMmRIZw/DwsLColth5cqVHyqlsrnNAz2IWQwZMoQVK1Z09jAsLCwsuhVEJFdGBcCqoSwsLCws\n8oBlFhYWFhYWOWGZhYWFhYVFTvQYm4WFhUXHoKWlhc2bN7Njx47OHopFAejduzcDBw6kvDwoxVl2\nWGZhYWFREDZv3szuu+/OkCFDSE8Ya9FVoZSisbGRzZs3s//+++c+wQdWDWVhYVEQduzYQWVlpWUU\n3QgiQmVlZbukQcssfBCLwfTp+tvCwiITllF0P7T3P7NqKA9iMRg7FpqboaICFi+GSKSzR2VhYWHR\nubCShQf19ZpRxOP6u76+s0dkYWHhRmNjI8OHD2f48OH079+fAQMGpLabm72VT/1x9tln89prr2Vt\nc8stt3DXXXcVY8gcddRRGbaCE088kX79+qXtu/7669lll1349NNPU/sef/xx9thjj9Q9Dh8+nCVL\nlhRlXIXAShYeRKNaojCSRTTa2SOysLBwo7KyktWrVwNQW1vLbrvtxk9/ml64TymFUopQyH89PGfO\nnJzXmTJlSvsH68Luu+/Os88+yxFHHMFHH33E+++/n9Fm3rx5HH744SxcuJD/+Z//Se0fM2YMCxcu\nLOp4CoWVLDyIRLTq6de/tiooC4tiIbYpxvSl04ltKp0h8I033mDYsGGceeaZHHzwwbz33ntMnDiR\nUaNGcfDBB/OrX/0q1faoo45i9erVtLa20q9fPy6//HIOO+wwIpEI//73vwG48sormTlzZqr95Zdf\nzujRo/n617/OM888A8Dnn3/OhAkTGDZsGKeddhqjRo1KMTIvTj/9dObPnw/AP//5T0477bS046+/\n/jqtra3U1tYyb968os9Pe2GZhQ8iEbjiCssoLCyKgdimGGPrxnLVkqsYWze2pAzj1Vdf5aKLLmLd\nunUMGDCAa6+9lhUrVrBmzRoee+wx1q1bl3HOJ598wre+9S3WrFlDJBLhzjvv9O1bKcXy5cv53e9+\nl2I8N910E/3792fdunVcddVVrFq1KnBs3/nOd3jiiSdIJBLcfffd/Pd//3fa8Xnz5nH66acTjUZ5\n6aWX+PDDD1PHlixZkqaGamhoaMPstA+WWXhgPaEsLIqL+oZ6muPNxFWc5ngz9Q31JbvW0KFDGTVq\nVGp73rx5jBw5kpEjR/LKK6/4Mos+ffpw/PHHA3D44YcHEuJTTz01o83TTz/N6aefDsBhhx3GwQcf\nHDi28vJyjjjiCObPn088HmfgwIFpx+fPn8/pp59OOBzm5JNP5p//dErVjxkzhtWrV6c+nZFh29os\nXLCeUBYWxUd0SJSKcAXN8WYqwhVEh0RLdq1dd9019Xv9+vX84Q9/YPny5fTr148f/vCHvnEGFRUV\nqd/hcJjW1lbfvnv16pWzTS6cfvrp/Od//ifXXHNN2v5Vq1bx5ptvMmbMGACampr42te+xgUXXNCm\n65QCVrJwwXpCWVgUH5GqCItrFvPrMb9mcc1iIlUdswLbtm0bu+++O3379uW9997jkUceKfo1jjzy\nSP7+978DsHbtWl/JxY1oNMrll1/uq4K65ppraGhooKGhgXfffZe33nqLzZs3F33MbYWVLFywnlAW\nFqVBpCrSYUzCYOTIkQwbNowDDzyQwYMHc+SRRxb9GhdeeCE1NTUMGzYs9dljjz0C24dCIS699FKA\nlHSilOLuu+9m8eLFqXYiwsknn8zdd9/NYYcdlrJZGPzyl7/klFNOKfr9ZEOPqcE9atQoVYziR7GY\nliiiUauCsrDwwyuvvMJBBx3U2cPoEmhtbaW1tZXevXuzfv16jj32WNavX09ZWddch/v9dyKyUik1\nKuCUFLrmHXUiIhHLJCwsLPLDZ599xtixY2ltbUUpxaxZs7oso2gveuZdWVhYWHQA+vXrx8qVKzt7\nGB0Ca+C2sLCwsMgJyyySsPEVFhYWFsGwaihsfIWFhYVFLljJgrbFV1hJxMLCYmeCZRY48RXhcH7x\nFUYSueoq/e1mGJaJWFiUFmPGjMkIsJs5cyaTJk3Ket5uu+0GwLvvvpuRxM8gGo2SywV/5syZbN++\nPbV9wgknsHXr1nyGnhW1tbWICG+88UbatUQkbUyrV69GRHj44YfTzg+Hw2n5o6699tp2j8kNyywo\nPNNskCSSjYlYWFgUB2eccUYqe6vB/PnzOeOMM/I6f7/99kvLu1QovMzioYceyqhL0VZUV1en3ds/\n/vGPjHxT8+bN46ijjsrITNunT5+0/FGXX355UcZkYJlFEoVkmg2SRGy6EAsLfxRT4j7ttNN48MEH\nU4WOTHqMo48+OhX3MHLkSKqrq7n33nszzm9oaOCQQw4B4IsvvuD000/noIMO4pRTTuGLL75ItZs0\naVIqvfkvf/lLAP74xz/y7rvvMmbMmFQepyFDhqQyxN5www0ccsghHHLIIan05g0NDRx00EGcf/75\nHHzwwRx77LFp13Hj5JNPTo15w4YN7LHHHuy9996p40op/vGPf/CXv/yFxx57rF01tQuFZRYu5PtA\nB0kihaqzLCx2BhRb4t5rr70YPXo0ixYtArRU8V//9V+ICL179+aee+7hhRdeYMmSJVxyySVky1Jx\n6623sssuu/DKK69w9dVXp8VM/OY3v2HFihW8+OKLPPnkk7z44ov87//+L/vttx9LlizJqFa3cuVK\n5syZw3PPPcezzz7LbbfdlkpZvn79eqZMmcLLL79Mv379WLBgge94+vbtS1VVFS+99BLz58/PyCH1\nzDPPsP/++zN06FCi0SgPPvhg6tgXX3yRpoa6++67C5vYHLDMIolCH2g/ScQWTrKwyEQpJG63Ksqt\nglJK8bOf/YxDDz2UcePG8c477/hWpDN46qmn+OEPfwjAoYceyqGHHpo69ve//52RI0cyYsQIXn75\n5ZxJAp9++mlOOeUUdt11V3bbbTdOPfVUli5dCsD++++fyu2ULQ06OEWSFi5cmJH/ydS8MO3cqiiv\nGsrLaNoL6zqbhN8D3RZib9OFWFikoxQJOk866SQuuugiXnjhBbZv387hhx8OwF133cUHH3zAypUr\nKS8vZ8iQIW1S1bz11ltcf/31PP/88+y555786Ec/apfKx6Q3B22IDlJDga7NfemllzJq1Cj69u2b\n2h+Px1mwYAH33nsvv/nNb1BK0djYyKeffsruu+/e5rHlCytZJGFVSBYWpUEpJO7ddtuNMWPGcM45\n56QZtj/55BP22WcfysvLWbJkCW+//XbWfo455hj+9re/AfDSSy/x4osvAjq9+a677soee+zB+++/\nn1J5ga6l/emnn2b0dfTRR7Nw4UK2b9/O559/zj333MPRRx9d8L3tsssuXHfddfz85z9P27948WIO\nPfRQNm3aRENDA2+//TYTJkzgnnvuKfgabUFJJQsROQ74AxAGbldKXes5fgEwBYgDnwETlVLrXMcH\nAeuAWqXU9aUcq3mgbcZZC4vioxQS9xlnnMEpp5yS5j105plnMn78eKqrqxk1ahQHHnhg1j4mTZrE\n2WefzUEHHcRBBx2UklAOO+wwRowYwYEHHkhVVVVaevOJEydy3HHHpWwXBiNHjuRHP/oRo0ePBuC8\n885jxIgRbSqBalRNbsybNy9DLTVhwgRuvfVWampqUjYLg+OOO66o7rMlS1EuImHgdeA7wGbgeeAM\nDzPoq5Talvz9fWCyUuo41/F/Agp4LhezKFaKcgsLi+ywKcq7L9qToryUaqjRwBtKqTeVUs3AfOAk\ndwPDKJLYFc0YABCRk4G3gJdLOMYM2KA6CwsLi0yUUg01ANjk2t4MfNPbSESmABcDFcC3k/t2A/4P\nLZX8NOgCIjIRmAgwaNCgdg84FoMxYxxD3JIlVh1lYWFhAV3AwK2UukUpNRTNHK5M7q4FblRKfZbj\n3NlKqVFKqVFf+tKX2j2WujpoagKl9HddXbu7tLDokegpFTZ3JrT3PyulZPEOUOXaHpjcF4T5wK3J\n398EThORGUA/ICEiO5RSN5dkpEWGLc1q0ZPRu3dvGhsbqaysREQ6ezgWecC42fbu3bvNfZSSWTwP\nHCAi+6OZxOnAD9wNROQApdT65Ob3gPUASqmjXW1qgc86glHU1MCdd0JLC5SX6+1CYdOdW/R0DBw4\nkM2bN/PBBx909lAsCkDv3r0ZOHBgm88vGbNQSrWKyFTgEbTr7J1KqZdF5FfACqXUfcBUERkHtAAf\nA2eVajz54pxz9HdNTduIfLGC+ywsuirKy8vZf//9O3sYFh2MksZZKKUeAh7y7PuF6/dP8uijtvgj\ny4RXImiLVAFOcF9TE4hAZWVRh2lhYWHRKeh0A3dXQbHy10QiMHOmjgRPJGDaNOuGa2Fh0f1hmUUS\n0agm8CL6uz3pPhobNaNIJGyqcgsLi54ByyxcMI4d7XXwsHmmLCwsehoss0iivh5aW3WMRUsL1Na2\nXX1kU5VbWFj0NNgU5Um4DdOJBDz+OCxd2nZi702cZmMvLCwsujOsZJGEkQbGjYNQqLj2Blub28LC\norvDMgsXIhGtfurVSzOMYrm+2trcFhYW3R1WDZWEW000cyZMnaqJ+7Rp+nhjY3YVUjY1U6GVwqzK\nysLCoqvBMgsyA/LOOstxfW1q0owjkQhO35ErxUchhZVsuhALC4uuCKuGIlNNBI7rayik98fjmnH4\neUnlo2aKROCKK3ITfquysrCw6IqwzILMuIiaGr2iP/98OPFEnVTQGL0ffzzTSF3MuAobo2FhYdEV\nYdVQOCk6FiyACRP0diwGc+bo1X1ZGYwaBStWpHtJGSmhmPW7bS1wCwuLrgjLLNCMYdo0zQSWLoXq\naqcQEuggvU8/1RJGa6v/ir+YBelLUdzewsLCoj2waijysxO8+qqO7j7/fGt0trCw2PlgmQX+doKa\nGv3bQCnNTAYNsozCwsJi54NlFsDChbDXXnDkkY7UEIloCeOCC3SQXiEG51gMpk+3kdoWFhY9Bzu9\nzeL//g9mzNC/33lHMw634dqNfKrnueMkwmFdec8UUrJG654PG1Bp0VOx0zOLf/0rffuuu+C66/Tv\n2bOdSO5evfKrnue2f8TjMGuWrust4hjHrc2jZ8IGVFr0ZOz0aqhTT03ffv99/dLHYjBlivaEMpHc\n+QTIGfuHqYlhUp7bQLueDxtQadGTsdMzi+uug2OOcbaV0m6ztbX6pTfIt3qeiZP48Y8dW0d5uQ20\n2xlgAyotejJEKdXZYygKRo0apVasWNGmc712BhFHojBlVm+5BSZOLLxfo78Gq8veGWBtFhbdDSKy\nUik1Kmc7yyw0zEu+cSPcdpuWKkIhXd+itrawF98SDAsLi+6CfJnFTm/gNjDusrEYzJ3rGCkNo5g9\n20kHkk3CsEZOCwuLngjLLDwwNoe6Or29dq12rV24UG8/+qj+njjRX4LwM3JaZmFhYdHdYZlFAObO\ndepxe7Fggc4f5SdBeAsdVVbqAL32qKSsWsvCwqKzYZmFD4x04McoQKui3BKEqXNhVFYma2xlpZOg\nsK0qKavWsrCw6AqwzMIHRjowkoWIdqkFna68ulr/drd5/HGdsXbmTKcEa5DffSFSglVrWVhYdAVY\nZuEDr3SwYIFmBomEJto1NXDppbpNba1zzFuCdebMTJVUoVJCofW7LSwsLEoByywC4K4pUV2tpYYd\nO7SE8cYbOuhu1izNLJYu1cRcRDMTUyCpsTG9kFFbpARbDMnCwqIrwDKLABijcmWlJvoXXgi33w4f\nfeS0WbBAe0UF2SgMcXcT+GxSQpAh2xZDsrCw6GxYZgHENsWob6gnOiRKpCqSMir72Szc2LHDSUO+\ncaP+uG0WXgKfTUqwhmwLC4uujJ2eWcQ2xRgzdwzN8WYqwhUsOWsJ9fWRNG+ooCD3pUs10TfJAkHn\ng1qyJJjQB0kJ1pBtYWHRlbHTM4u6NXU0xXWx7aZ4EzOWzeCy6D2+3lDeb8Mk3MykqUkH9AUR+qB8\nUdaQnQ4bW2Jh0bWw0zMLLxa+tpBdK37IWb8/Fhq+xYihg1m1CrZsgf79YcQIWLUK5szR9SnCYad2\nhcFtt/kXSvJLWOiucVEKQ3Z3JLpWJWdh0fWw0zOLEV8ekbHvrrV3IfyN3n17M2Kf55g7tzpFuGpq\ntFG7psYxat9xByxf7pwfj6dLF4ZgL1/ueFQlEo5EYmplXHFFph2jPYS+uxJdq5KzsOh6KCmzEJHj\ngD8AYeB2pdS1nuMXAFOAOPAZMFEptU5ERgOzTTOgVil1TynG2Li9EUFQpBsmFIodrTu4454NNDdX\nZxCuSETnjZo6VUsHQfAayw1CIUcaSSQ00/E7rz2EvrsSXauSs7DoeigZsxCRMHAL8B1gM/C8iNyn\nlFrnavY3pdSfk+2/D9wAHAe8BIxSSrWKyJeBNSJyv1IqC1luG6JDopSHy2mON2ccUyhW9bqRsvLx\nQBgRnVCwslLHXkyZks4oQiEtLRgJBIJTh3z96/Dqq3p/KKQ9qNwohNAHSSDdlegWK7akO6rgLCy6\nKkopWYwG3lBKvQkgIvOBk4AUs1BKbXO13xX08l4ptd21v7fZXwpEqiKccMAJLHx1oe/xxMBlnHvD\nXWx5tIaFC7UqaflyXV3PzSjKy+HmmzPdZg3B/uKL9H6/9jV4661gQp4voc8mgXTngL72xpZ0VxWc\nhUVXRSmZxQBgk2t7M/BNbyMRmQJcDFQA33bt/yZwJzAY+B8/qUJEJgITAQYNGtTmgfbftX/GvrJQ\nGYlEAhFhxOgdLPDwkqVLnd/hsGYUfnUu3CnP77hDM5jycjj+eG0wB39jeL6Evr7eUXEZ20dnBvR1\nldV8d1XBBaGrzKvFzotON3ArpW4BbhGRHwBXAmcl9z8HHCwiBwFzRWSRUmqH59zZJG0bo0aNarP0\n4WfkjifiCEI8EWfaw9O4cMxYHn10qOva+lsExo/XEkUs5v8iG4LtNoq7I72NyirovGyorHRUXH62\nj45EV1rNd1cVnB+60rxa7LwIlbDvd4Aq1/bA5L4gzAdO9u5USr2CNn4fUtTRuWCM3GnXRZEggULx\nResXrB4wmcumb2D0aM0gDEIhuP9+uPJK/UKbiG43YjFd0wK0x1Njo3822jaNvVGPwYzFa/soFsw9\n+N2fQVCW3Y64thdGMvv1r7s/cS3VvFpYFIJSShbPAweIyP5oJnE68AN3AxE5QCm1Prn5PWB9cv/+\nwKakgXswcCDQUKqBRodE6V3Wmx2tO1AoX++ox958jKVl1Xyj30aU2ju13x1f4VUDxWJa/WRiMsyq\n0J0CXSS7NJBL/RCN6qjxUq6g813ZlmI1355VdU/JqdWTpCSL7ouSMYskoZ8KPIJ2nb1TKfWyiPwK\nWKGUug+YKiLjgBbgY5IqKOAo4HIRaQESwGSl1IelGmukKsLimsXUN9RTuUsli9Yv4t7X7k1jGApF\nU2sTb25qCuwnHHYq4xlVk4mrAGdVeMUVOofU1Kma2Uybpr2rsgXxBRHKjjBi56v/z3cshejfe5rt\noS3ozo4KFj0HJbVZKKUeAh7y7PuF6/dPAs77f8D/K+XYvIhURXQSwU0xpj08zbdNggTfOuUN7npl\nQMYxEbjoIscWIZIeeCeSvipsbNTHTTpzPyJYCJEuFQGJxXSCxLLkk5JrZZtrLIVKCnZVrdFTpKS2\noCcb97vTvXW6gburob6hnqbWppQ6auieQ9nw8QYUipCEOPi4Z5g1+FvccQesXOmoocrKYNs2h7gb\nu4aIljjOOy/d6ykfItjZhNKbnuT88/09twpBoZKCd1UN7a9pbtF9UGrjfmcS6+7muFBKA3e3xNam\nrSTQ7kUKxanDTqUiXIEglIfKiQ6JMnEiPPecJp6GKRiPpIoKJzjPfEQyiWw+BtjONtK6CXs8DoMG\ntX8MhgGGw/kzwEhEq+5Av1wkobxhAAAgAElEQVRXXRXsTGDRs1BK434sBmPGwM9/rr87+nnqbo4L\nVrJwIbYpxg2xG1LbIUK8/uHrtCZ0iEdcxalbUwdotVVNDcyd66wMRiQ9cF94AZ5/3lFBtbb6r6C9\nqgW/VU5nqh9KIdm0R/9u7ReFozupOfxQSum6rk47mUDubNGlQGdrDgqFZRYu1DfUk/Dk5XAbulsT\nrcxaOYu5a+ayuGYxkUjEt0peOKzVUqbGRSiU7vFkvKTAkTi6okhaKsNqWxlgd3u5Ohtd8ZkqFD3Z\nuN/d7s0yCxeiQ6L0KutFU2sTItp9VqnMBINN8SbqG+q1UTxJ+KZPd1a9oFVUW7boGIxEQueRAu31\nFI3qtqDdapcsSV81NzVpxjNyZPttBO1FVzKs5qo02F1euo5CT5HESvUM1tTAnXfqRV15eXBwbCnR\nld6vXLDMwgW3C+3yd5cH5osShOiQaNq+aFRLFImE/jbR2vfdp9VRra3aVfbccx2JA5yX2B17kUg4\nOagMM+kuD1Sp4fdyeQ3x55yTm8nuDMzFSmLZEYnoZ6CnPwfFgmUWHkSq9BNT+2RtYJvxXxufaueG\n2wNq7VrtcuqGkTrKyx3JwrzEZtVcWwuPPZYZm2Ef5GAC7zXEz5qlbUlBapeeoJ7JB91NzdEZ6E4r\n+86GZRY+qG+oJ56I+x4LSYjjDzie2KYY9Q31RIdEiVRFqK/X0oMptWoC7twmEBHo2xdOOAFee02n\nKb/sMs1YamthwgT9bYgf2BWhQTYCb1bQJgBSqexMtqeoZ/KBJYYWxYJlFj6IDolSEa5Ipf9wI6ES\nTH5wMmWhMloTrVSEK1hcs5hoNJIS+UUyGQXofTNmONtvvqlTlZt9jz6qV8X19ekGcLCxBdkIvDuz\n75w5mllnS6Ni1TMWO4MastgQrwG3u2LUqFFqxYoVRevPRHIvf3d51nZhCXP+yPMZtMcgKhtPpPGV\n6pRnlLE/iDhqJTdEYMAA2LzZ2XfssfDII87DvHUr3HijJpK9ejkr6p3tYc9XdTR7tiPVuefLr7+d\naf4sHOwsash8ISIrlVKjcrWzkkUAIlURZh43k+jcqG8VPdCGbhFhzuo5SSnj19qltipCdbXjUrtq\nlSZiXkmjrAzeey9934QJwaVYTaJCKMyg2xMIY77693zSqJj+uuJc9IT/qqtjZ1JDFhOWWWRBpCpC\n/Vn11DfUs7VpK79b9ruM5IJKKVpUCwmVoDnenOFSa7Bliy7JajBsmK62d9ttzr5jjtEFlIwbrpe5\nhMOaiBRi0DVRqmYV1Z09q/Ih8NlUTF2dENsVb8fAqiHbBssscsCdYPD+1+7nlQ9fSTseV3HCEiYs\nYSrCFRkutQaXXQaLFjkP6O236/133ul4ST33nCYYXjdak1/q5psd4pGvQbezo1Q7GkESSEcT4rYw\nJrvi7RhYL7G2wTKLPBDbFGNs3Vh2tO7wPa6U4qjBRzFs72GBfUQiTvCd+wE95xwtGZhYjPp6nQfJ\nHRnurevtNeiaWhlddYXU0St6PwkkKA9PPuMqdPxtZUx2xdtx6KpqyK4MyyzyQH1DPc3x5gzPKIME\nCZ56+yme3vh0KhWIOc+41oLzgJrKb9EoqfxSphDS1q3OMZM8zw+mLxP8F0TIOjtK1W1/CYXgllv8\na5WXGl5CXFmZH0FvC+Fvq4RgV7wWXRmWWeQB40rb1NqUykjrh4RKsKN1BzOWzeCRDY/QHG9OudYa\nhuFHPGfOhMmTtYQwY4ben82Tx41sKySzIr7ppkzppKNQX++o0xIJmDRJ7+9ohuElxPkS9LYQ/vZI\nCPmseLu67cWiZyIrsxCRvkqpbQHHBimlNvod62lwpwHZ2rSV3z/ze+LKP2hPobj3tXsRkQyjN2QS\nz8mTdXCeuzyrnyePm0CYfnJVo+ssY6l7rNGoZn7GWJ9IaNdWv8qApYaXEOdD0NtC+EspIfRkI7hl\ngl0buSSLemAkgIgsVkqNdR1baI7tDDCG7ulLp5NQwdIFkCqc5Gf09hLPeBzWrcvsw62Scme0NTEb\nSgVLH7GYjgQ3TKkjjaV+xOyWW7RE4b7nzjbe5kvQ20r4S6UT76lG8J7MBHsKcjELcf3eK8uxnQbR\nIVHCoXCqxkUQlFJMPHwiNYfVpOWRikQ08Zw6NT2hoEE4rL9NtHco5DAXryutibtwSx/uKOZEQp/b\nkcZSP2JmbC9Tp2pVmzdle2chX4Ju2hijeGcSsZ5qBO+pTLAnIVelPBXw2297p0FYwjnbKBQvvPdC\n2r7YphjTl06n+vgYTz4Jo0ennzNwIIwfnzw/ObuJhCawvuMIO8TCrMxmzXIkilAIxo3r2FVaUCW8\niRO1629ZmR7btGmZlcmM4b+rVcAzc9sVKvQZSact1RO76vxC2yooWnQsckkW+4jIxWgpwvwmuf2l\nko6si6K+oT6nVGGw/N3lHHnnkVx65KUAXP/M9Sil6F3Wm8U1i5k5M8KYMU4cxDvvpKf+cMMvQO+i\ni9JdQJubHSYjotVUtbUdm+4im9omW3R1qdQQ3vtsy33X1TkxLV1h1dsWFVcx57cUz471BOv6yMUs\nbgN29/kNcHtJRtTFYTyjmuPNKSN2NhuGQjFj2Yy0fU2tunjSFUdHWLIkMy15Phg/Xns5Ga+qiy92\n1BMmBciIEf6qk1Lqh7MRkmwqlFKoIbz3OXOmY/vJ975jMe16bP6bsrLuueot1vy29dnJh8HY2Ieu\njazMQil1ddAxEflG8YfT9eH2jDKG63y8pNIgpM6NRDSzeOKJYHWTF+Xl0L+/s9pNJHSywZtvdlxk\nIfilLpV+OBchybZ6LIUu3nufCxY4KjqvvSdbH8ZTTQTOPrt7ErRizW9bnh1rvO4ZKCjOQkSGAWck\nP1uBnJkKeyKMZ5R7e/rS6Wlt+u/any2fb/E9/6f/8VNfo/cFF6RLF8OH6/oXy5Y5BGvIEMdg7G4b\nj2tGYY65y7x6X2o34QiHdZGmWKz9L3A+hCRo9ehmJJWV+nvt2vbFh3gJ5PDhOg08aIaRj5Hd20dn\nlN4sBoql5mkL07HG656BnMxCRIbgMIgWYDAwSinVUMqBdTe41VMV4QquHnM1kx+cnCZp7NVnL/rv\n1h+A6Uunp0V3T5yoYw8uv1zXufjWt+Bf/3II+vjxOrfUpk1alXLWWempz93Gbsj+UkciWiVzxx06\nI+5tt2WvLJf3HGS5Zj4w13Zn3M03QNFPzeEXiGc8y0IhzYjyGVNH6NI7IsagGGqetsxHd07umA96\nwj3kg1xBeTGgLzAfmKCUWi8ib1lGkQmveqq+oT7DlvHRFx/x0Rcfse4DHVhREa7gpuNvYtV7qwCo\nOayGJ5/UT9v06TB/viNRbN+u1VTxuFY/bdkCvXs7NoubbyZ1nnlog17qWEwzHKPGAifJYK7Av2wv\nRjEIq1mFuoP4cq1Gs6k5vASyV69gZhZ0b0FEtlhEorupaQplOkHPRSH33VUJcnf779qDXJLF+8AA\nYF+099N6dmKX2VzwqqdCEspqw2iONzPpgUmpFCJzVs9hyVlLiFRFMlZjEyboRITxuCbwDz6Yn40i\nWwoLtxorkdCSRiKhpRQRJ0HhYp3qKiNxod+L0d7Vq7lvt2SRS0rJV83hp+oy+wt96d3t86kpkg09\nRU2TayHh3ZfvfXeVbATZ3qXu/t/lg1wG7pNFZA/gVKBWRA4A+onIaKVU9hJyOzkiVRH+9L0/8eMH\nfpy1nTvXlLcehns1BukpQVpaNHGfOVM/nNlsFF4YguyWLAxzMAZzcFxF6+q0msrdvtDMrfnCS9Dz\nsVkUov5yq7rcxKfQl97dPldNkVzwG//s2dogP2FC5yReLBRtIej5/m+dRZDzuaeermJzI6fNQin1\nCTAHmCMi+wL/BdyYzA1VVeoBdjfENsVSqqjqfaodN1uEb+z3jYwyrSFCKYYhIlTu4lhd3aux6dMz\nXWuXL9cFk265pTCjtSHIRlLwShlKaY+rREL3CZkxHBUV+WduLRTFUnMEwY/4FGpv8TLc9sRgeMe/\ndi38OLnGMAb5rs4wCiHobiKaz//WWVHr+TpsdIX6KR0CpVSbPsDgtp5bis/hhx+uOhvPbHxG9bmm\njwpfHVZ9rumjLrj/AhW+OqyoRYWvDqsL7r9Alf2qTFFLat/wW4entqlF9fp1L/XMxmcy+35GqbIy\nQ5bSP+Xl+vgzzyh18slKhcNKhUJK9emj1KxZSv32t85x89vgsssy+wuFlLrggvTz+vTR/VZU6GOm\nr3BYnxMO6+3uAPf99OnjzIff/OTq54ILlOrVq/19uXHssen/x7HHFt5HRyNoTtvazu88v/lszzzn\nc822jFWp7vVuACtUHjQ2l4H7vhy85vvFYlo9AabuRVzFU3W73R5SgGG0gE5pvvr91Wl9NMebqVtT\n51sL47zz4M9/zrxuPK6lBID773fUVTt26HxMQXaISARWr87sr1evTP170AqwFCu+QjPsBp2bLfjL\n737aItFEIpk1Rdq7qpwwwZEozHZXR77SXVtVSn7/TalX7+1x2OiJObxyqaEiwCZgHvAcO2nywHxh\n3Gd3tO5Aoejbuy+LaxZTt0ZT8hFfHpFWFyOomNLsF2YD0CvcK60WRk2NdnN12y5AM4PZs521qIGI\nbutOQqiUZiKmvKqXMJ18si4Bm8tAaYjyzJnFrZXhJgD5ZNgNOjcX8SiGG2lQX0EEMV8dtlE5dSeb\nBeQ3p8Ukoh1hy2jrc5LLG7E72jJyMYv+wHfQMRY/AB4E5imlXi71wLojIlURLvzmhcxYNgOlnDQf\nc9fMpTneTDgU5oSvnsC7n76bYbsAEARFSs1HU7wprRZGJAJ/+lN6um8Dv9xRRx6p63q3tuptpbRh\nXCltq6ipaRthCiLK2SQCcyyX0dpNANzIJ+K6EO+aUr6sfgSx0FXwxIndh0kUgmK4Vxt09dV7W6Sh\nrsxIcnlDxYGHgYdFpBeaadSLyNVKqZs7YoDdDavfS9fr/Gvdv1KqqXg8zr2v3Us45J+11itpKKVS\nBu+U4fz4KE8/HaGuDl54AZ5/PtPwPWSITkq4bJlmEuefrxlDXZ1T77ulRacZqa0tnDD5EWVIdyV1\nq7xMTqZ8Au38PLUgM+jQD/kQj44wPPoRxEK81Xo6iiXV5cN4uhrxzbag6epG8XwiuHsB30MziiHA\nH4F78ulcRI4D/gCEgduVUtd6jl8ATAHiwGfARKXUOhH5DnAtUAE0A5cqpZ7I8546FROGTeDRNx29\nzqnDTmXmszOJJ5fKCkU8kRl7EZYwCpUWyKdQTHt4Ghs+3sCNsRuJq3hKNXXrrRFiMf0SNDen99XQ\nkL49aJDz0BkX2ERCJy9cujT/hHrmpfMjyu6XwOt6u2BB/oF2Xk+tlhYn6DDXGPMhHh3lhukliN45\nq6xMD6C0aBuyMZ6uSHyzLWg6y0U4X+QycNcBhwAPAVcrpV7Kt2MRCQO3oNVYm4HnReQ+pZS7Ltzf\nlFJ/Trb/PnADcBzwITBeKfWuiBwCPIIODuzymHi4XqIvWLeACcMmMPHwiWzbsY0/r3Qs0yEJEZIQ\nLQld/SgsYS75j0t49I1HMwzeX7R+wfXPXJ9iIiZjrYnFqK/Xhu+ganvuBzIS0av8SZMcW4A3cjvf\noCg/oux23XVLFhMmaKaUb6BdkOE4H+RatXak6sK7qnXHjxSa/daicHRF4pttQdPV1Wq5JIsfAp8D\nPwH+VyRl3xZAKaX6Zjl3NPCGUupNABGZD5wEpMiaSq/vvSvJ6HCl1CrX/peBPiLSSynVlPOOugAm\nHj4xxTRAp/GYu2YuTa1NhEIhbjnhFqr3qU4zfF+46MKUB5UXbmkjQSIjFuP22+HoozP1/CJw4YWa\nGcyYoTPVeiGSOyrb76W74or0dt6XwJxnXojq6sIC7Tqj3kYxEbSqtSqp0sH7zHRV4hu0oPF7NruS\nGi2XzSLUjr4HoD2pDDYD3/Q2EpEpwMVoldO3ffqZALzgxyhEZCIwEWDQoEHtGGpp4c0bFamKENvk\nlCtb9MaiQEYhSKpuhsGCdQsAaNzeqPuLRPjTnzKz1iYS8PvfpzORcFh/jDvt+PGOu63xkoL0B7Sy\nUksDSuUnEbi3g45lQ0e4RJb6xcu2qg0iYl2JMHQEinm/Qc9Mdyuo5H42u5oaraAU5aWAUuoW4BYR\n+QFwJXCWOSYiBwPXAccGnDsbmA0watSoLp2zyp03KrYpRnRuNJBBGIQIURYuY9jew3jx/RdTkd6P\nvvkoj775KCEJpWwYEydG2LBBSxCp80OZXlJum4K7nck5dccd6ZKGMU7H47rdhRem51TyQ3uq08Vi\n2uhuVFbFWnl3NCHOtqoNWkF2JcJQasyerWOA4vH83KJzIYg5d8TCoFToamq0UjKLdwB3OpCByX1B\nmA/cajZEZCDakF6jlNpQkhF2Euob6mmJt2RtEyLEqP1GsWrLqgw7hkFCJdJsGNddB0OHaoLf3KwJ\n7uuvZ6qnjPQRj8O992omYGIaWlqc337G6RtvdNKA+Ln9eZMN5ludzn1uS0v+SQTzQSkIcS7mk2tV\nm29sRk9ELAZTpjjFvvItRJUNXVXl1B50tXsqJbN4HjhARPZHM4nT0bEaKYjIAUqp9cnN76Gz2iIi\n/dAxHZcrpZaVcIydguiQKOXh8qySRXm4nJFfHukbj+FGggRbm7amto0LrMktBHDAAfDGG/5lW03i\nQHdtDKUcQm2M0yZIzkgm3mAzvzxTbmaTjQgaYu52lw2FYNy44BrihaDYhDhf5lPIqnZn8paqr0+X\nbvNxi86F7qhyyoWudk8lYxZKqVYRmYr2ZAoDdyqlXhaRX6FzkdwHTBWRceiiSh/jqKCmAl8FfiEi\nv0juO1Yp9e9SjbcjEamKUH9WfcrA3bd33zSPJ0E4e/jZ9O2dzX/Awe+f+T0nf/3klJprwYL04xs2\nwKGHwpo1/ucrpRmBG6NGORlt3cZpt5TgDjbzxkUYTyw3s/FbHbnVTu5Ehb16FYdRQPFXaKWQAorl\nLVVqdVsx+o9G9f/rrsVSrLgLv4VIVyG2bUFXUqOV1GahlHoI7Xbr3vcL1++fBJx3DXBNKcfW2fDW\nvhi651CmPjQ1FUtRc1gNtfW1efUVV3HOu+88fnLET2jc3gjDDoBHTUIhQSl48cXg80Uy7Rhr1ujs\np+ZFM+Vaq6sdIzg4hNNN6MvL0+s7GGbjl/bAWxWvrCyzNoTfC58PESg0u2m+KJT55Euw2ustVWq7\nR7b+CyHKne2RZtFG5JNtsDt8ukLWWTee2fiM+u1Tv/XNIJvvOZc9dllaRlpqUYNvHKxCtaGM/Rmf\nI3+rkFYlEldlZUqJ+GesdWeudbcR0fv8sqm6M3HOmuWfkTYfuDNzglKjR/tnFe3TR2fCLSvT15s1\nS4/NZNb1u157Mobmg3yznQZl7M3WV1vHXupMp0H9l3qu24rulPm1M0Exss5atA2xTTHG1o1NZZt1\nJwPMBiNtxDbFmPTAJOasnpPRZtMnm9IKJgXiOz+DA+9jwEf/w6ihQ1h083dpbgr72i1CITj3XF2q\ndeFCvU8ppxiSO/GgVwXT2Ni+zJzhsGOA91OT1dc7kkciAZMn6/3mnCDjaFcxGLvH4VckqZgun6U2\niAb131Xm2otSz0dnq7g6+vqWWZQA3lTl7mSAuWAYjclcC06CQSA/RmFQ9SzvD17J/SpBuOYoTvp8\nHov++eWUx5Nxra2ocKKl77033dBtvk3iQT9DbNADm4/H0DnnOPmqWlszCU00mu4CnEjklzOqlISi\nEPWGGUdQkaRsLp+gt9euzR7IWCp1mxdBTKzYc12oijGoTSnVXZ2t4uqM61tmUQKYVOVGsogOieZ9\nbn1DPU2tTWlJBQWhIlxBS7wlkFkYhhIixKH9D6UiVMF+fffj/tfu13XABzzN6DF/4bIpV6ReHnDs\nD2vX6up6bq8oN5qbddtbbw02xLrTlUN+D3NNjeNFVVaWSWgiEV0J0Pjkl5XpMebKGVVsQuEmToWs\npM04vC7F5j6DvKDM3OZKvuhHNIx9qRTwM7gWc67zIYKdlYrejc6Wpjrj+pZZlAB+Edv5IjokSigU\nIuGyOCdI8JMjfsLq91bz+FuPp7ymwhImoXRdDLfksXrLairCFZw78lweeeORNKYVqUqqPzbFqHtg\nPXP+ciYtzeEM91kv3NKFnyH2iy+0isi43Z54Yv4Ps1eS8a4aq6u1mgz09aEwg3F74SVOM2cWtpI2\n4/DLdRXkBWWcDnIlX+xsomVQrLnO535Kcc/5ptA36OwYiM64vmUWJYLX28nAXaPb73ikKsItJ9zC\nBQ9ckCZdrH5vNbXRWpZuXJqqjTFs72GBAXvN8WZWvbcqrfjS2n+vpb6hnspdKpn28DR2LLkI1aTA\nQ6yD0NKSmbbCbXMw34mETiFSVua45VZW+vWo+zPR46bi39y56YTZLb24mVU25KvPzaddsew0QeP2\nY76hkJ5byB6g2NlEq9jI535KofbyeuXliirv7BiIzri+ZRYdiHwN3xMPn8iGjzekiieBTn1uJJYZ\ny2Zw/+v3BzIKgy2fbaFuTR1zVs+hOd6MQiEI4VBSIhnyBIR/DnEBFU5JFkESRiKRSfS/+lX/jLdK\nwfHHO3mn/vd/tYTgfai9Lz6kE+Z8gvq8yFdNkW87P+JUjJV0rsR3Rq2XbbXb2USr2Mjnfop9z2Yx\nkE8Kfe84OnO+O/r6lll0IAoxfF837jqG7jk0LdW5wQPrH9B2iCwISziVoNAtoZh6GiEJQdWzcNZY\nwm+P45KxP6KfGsrWrfC73wX3+6tfwV13wV57wYMPamnDQMQJ7isrS081YlKh+/nle7PVuiULb1Bf\ntshm0+fGjfkxmELUGWclw0W9tcnbikK8oMx9+d1rsRhXUL8dfT7kdz/FlC4Nk843hX4xUGi+tK6w\nGLDMogNRqOHbm+ocNMNx2zNChKjcpZIPtn+Q2jdw94F8Za+vsPTtpb51vhWKIwcdyVNvPwVVz5Ko\neo7VA5ZTG62l7reRAHWU3vnOO/DOO8Gl2MvK4IQTYNGiTInjqaf0gw/ZjbJeghkUQe4NCnNX6itL\nPtnZXvp81Bleom5sJu1FNi+oIAN2OKy9x0aM8J+HYhD69njYdLaHUFvH4rUbFbOmfHvH1pXm1DKL\nDkR7DN8GlbtUEg6FUQlFOBRO1cYYWzeWptYmEiTY/OlmNn+6OXVOiFCaEVwQ9uq9V+q4QvHom4/y\nRMMTHLz5KeAIdMkSjeHf/ITVa3fA9n3S9nth7A7btztJ4txYtw7GjIGzz86+ovcSTD+dvvc8N/EF\nXUp20KD2u1aWyoCcr97dfX0Tp2FSzLvVJdB+otLee+0oY3tb7ExdSaVUyNhyte1IqcMyiw5GkOE7\nH8Q2xZj28DTiiXiKURjJY3HNYqY9PM038aBCcdi+h/Hi+y+i0Ezmox0fZbRrTbSyZt+LIbwE4uWI\nCJdeGqLfiX9i9cQD4dWTUz16mUYopJlFWZlWHRnDtRfNzfq7LQbKbATWeyxfdZEhErGYv3rL3W84\nrFVcsVj7X8x89e5+cRqmFom7EmIxCHV7DccdYWxvj52pWNdvrwRSyNiyte1oqcMyi24EY/NIkECU\n6DxQSUSqIoFZahUqzRjemmjVKig/VD0LPxoDDVFk/6WcPPV3rP13JeGjbiD++gmQKMePUYTDTsR3\ndbVWRZlocDdEtITx3e/qyn2F2ACyEVivKsGstvPpO9tLZ/o1cRK33ZYegd0e5Kub94vTcMe0mD7a\nSxzbazh2j3XLFv2dK6CwUPgxRbM/yCW5WNdui9eUHwoZW7a2HSXJGVhm0Y0QZPMw7rgjvjyCsIRz\nGr9zoupZqHoWhVC3po65a+aiBjYRPmccR348i6X/GpZm1xg0CDZtSnd/feih9C6Nh1UioW0XoF80\nYwMoNNle0DHIjInIRaxyvXSRiN7X2qrbNDXpjLjZsuIWYwXqvn4kkr0mebGIYzHUMXfe6UiQkD2g\nsNDxelfalZX+jL4U6pm2ek35oZB5Dmrb0W7Tlll0IwSVZzXuuOFQmCOrjmTZpmXtZxhAWaiMLZ9v\nSaUeCVfFOK7mXvhsX556ZC+MhDFoEGzanEBCUFaugHBgwSU32qpvdxMCSCcKbsLf1KQjv4OKNRkU\n4ttvVpWPP669tLJFGLd3BepFNpVZV/GYqa9P95ADf6LaVhWKlyl6GX1dXaYUlk/fpiYLZEq7bsbv\nfgaM919lZed4kZVCesoGyyy6Gbw2D7c7bjweZ+nGpdotNgD9evVLK5YUBEGIDIxw/2v3pwzjZaEy\ntjZtZVnVyRB+DOLlhMuE2LMQb02AQOs3bqTv4FNRamhaf6FQZhp0pfTK/+GH8xenvd5BIulEwU34\ng4o1ZcxpHi9dJKLH+rvf6fog2fos5go02/2be4a2667bUwLX7/xoVKeo90oWXibcHhWKd6VtCDjo\nKpFGHQr59R2L6bGZMc+ZA0uWOPPhJ6lu3aqrRsbjutSw9zksRIJqj+2hIw3zlll0cxjVlFn9K1RW\nqSIbowgRQkRIqAQhCfH0pqfTCjJ9c8A3dZGmgQlt11hTw64fRdn25tcBHa4df3oa1z8jaYxBBL7/\nfe1Oa15qg3//W3/KyjTxzyVOu4mMuYY7Od8VV6TrzRctyszH5Idchu5YzMnV5K4kmI8UUky//SCd\nfVsIrx8hzFV0ySvV+RG5+npnlT5ihL8arlgqFMPEp0511IQGbgeAoPuvr9dOC25pyD2H3vlubNTP\n2PTpjkeaOdebJNJcIxcj6GjbQ1thmUU3h1FNmUjt1kQrIkJrwsd31QOvlHFo/0PpW9GXZZuWpXJO\nGQiSxjwAWH0W21or0OqopIeUCpGIp0s2ZWVw2WX6Y1QEXqax7766LnO2FW1sU4yN/dZTVn4mEE5J\nFiaLrju63AT2hcPajTYfQ3q2F9stLYRCmSVfvavHUvntBxHZtnhseYmUO1renZY+aH7OOiu/WBE/\ntFeF4p7vxkb/bMTjx1imPDsAACAASURBVOtnLtdq3sTlGKLvnteg+fZ6ybkli0IlqI62PbQVlln0\nABjVVM1hNSl7xtp/r2XBugUM//Jwtu3Yxh2r7qAlka5M3ta8LS39+eotwelDEiRIi+9riEK8Av0I\ntSISAlE6GE5J6sULh9MzwxpD7eWXO4ZugDPP9M+WmtIXH7SWCxddQfMLpxPa/36+f/gRXDa5P2vX\nOhlpp03T5yxYkO5qOmhQfsSovt6RBrx1MrwvdG2t3u/OEOtlMqVYHQYRWbfH1OzZcPvtOluvqcnu\nB7cEJALDhzsuz97EkaD7N/Oarwu0m6ivXav/m+HDoV8/vS9XhtygKolBiR2NI4VS8Mgjmln49WlK\n+Rrp9PzznePuew6ab+9+8Gd8+TCCYjLOUkokonJlj+smGDVqlFqxYkVnD6PLYtIDk5i1claatGDU\nTm0yhm86AuYuhng5hFs45sf/YtiuRzMiso1V761i3eJR7GjZwblnlzPx5OqM02MxzTBeeQUGDIAj\njsjMKAsOUSDUSjweh0QFAOHyOH+6uYw//MEVKS5xwmEhEQ+lrTJnzcokmn4v2OzZ8OMf+5/nNYC6\nx+bOECui+7j11sKntL0v/fTp8POfOyvs8nJ48snsfc2e7TDbXr20S7OpaeKWoCBdr9+rl9brQ/CY\n3UTdrLwNRKB37+zeasaW0NKi78Uw70mTnBoo4TD8+teOsXv5cmf85pibIZXK+cCLbE4Y2dq2hVG0\nN9ZCRFYqpUblamcli50ENYfVMHfN3FSUd0hC9Ar34rtf/S4LX00PiHBLG4FI5pWiIUpo/6Us2+dZ\nnhFBvai0CqtaJy1c81IF1YcvSTPKew2KH3wAq1dr42QopIlKWXmcw45dQ1PzCBJxQRJhUCGMB1a8\nJcykSWZlaNLmQrw1fdyhkCZGbgS9YI2NjiHefZ5fyg/3KjsUcnJiKaXvAwqLIfES7Xw9eNxEprIy\nXRXT2prbxdeocIwRvn9/TcS9Xl9nneXYA0R0FL57le03ttpaZ468UCq3t1pdnfOMGE+ntWt1rIvp\n09RAMefV1jrHQqFMlVwudWIx4GZIoZCW8IIkqPYS+460dwS7zVj0KBjbxjXfvoZZJ87imjHXsLhm\nMZf9x2X0KetDiBBhCXPM4GOyelOloepZOPpaEgO1q25ropW4iqcYjULRFG9Ky54L6UTAjZYW/YJp\nt9cEy99dTiL0BaGworxcCJfpXkFpCSJlPjFBgiH9EX3ArBpN8kGTlyrISByN6vbhsP52rwq97pl3\n3pm+gh8/3mEYLS165Tt2rHNNL4whPRbTnylT9HmJhCawtbXB55rzx46Fq65yrtPY6IwB9Pgefzxz\nHO5rGzWJcS6oqdEEa9w4h3G6VU7hsGYm2XJkmbE99lhw2vtQSH9MGhP3/2CwZUvm9pQpwUzLqNAM\nEgnNWNz3777fXr2KzyjMOAyzbW3VDDHovwx6FvOF9/8rpb3DShY7EYJSjbhjN+rW1AVHd7cR9752\nL7NXzs5IiujATVGSxnJJAAo57iLGffkMJhweZdWqEFu26NXviBGacGjVhnKd28rooz/l3DP3TKX3\nNvYEdyI+Pz2y1zDtZiJl5XESCsrKwR1HIqJTsffv7/TpVz7VDT9Dsdt7zBB5vziObJl1o1FHKjD9\neN12/Vayfvry2tr0bL81NdmDAt0wBDCo3vsZZ8DBB2faetyELhZLD+wsL9dz7J6nsrJ0puV1m/bm\nzzJ2pGLEJmRTHUWj6a7i8Xjwit/PplGIWqojYy0ss7BIYyKmUJLBkD2G8N5n79ESb0FEqN63OsMQ\nLgiD9hjE25+87du/QjH5wcksWr+I/rv1Z8R3JxO+rZp43OSYUtB7K+zY03VSGaw4HxVK8KWJ/+bC\nCzN119XVMGMGvPaasP6NBPF4gooKYea1e6ZemunToalZkYgL8bhi1izJqis3v9MMqH9bi6q5EDYc\niRq6jBHfvYmKudUpBmTcc8NhOOmk7O66XuPqjh16xdyrV3Yib87NllnXy+z8CLHfSvaKKzKJjOmr\nri59Xz7EyOs67K2P8sEHznhNRmGzPXu2NoLvsotj4xCB731P/y4vz15S16SS92bmdf8Pue6jvXER\nkUh6KWC3lOqFn6E831os7jF2hKutZRYWaag5rCbNc+qdT9/hoshF3Bi7kdZEq6/HlIgEMgqDuIqz\n8DVtGykP3cGX//tKNs+7QjMFBHb0dffo2CcSirv+vB/GNbe5WXH55cKwYaSkjOOPJ03qcGOrbCCR\nGIxWUQlKaQK9alWwEdrrFbVgUSPxAU+j9nuSuIRprHyAxYurUyv8225z1B+jR2sPHD9i4zWugiai\nixbBH/9IhiTkJnKxGJx3ni5fa+CXWdcdL+JXg6NQN03jglxIPiwv01q0SBfBMit9r9QU5GBgoJSu\nnWISKE6c6B9l7bUrtWXFXYy4CKMSvPnm/Nym3XOQLbNyIWMsBSyzsEjB5JiKVEVSqqiWRAv1b9Vn\nxF0YCJIee5EHWhItbP7aL+HwL8GKiUCyfmiaOslsZ6ZEf+oplXS7zTwmou0JRoX0+18MBhV2tVAo\nJcyZExwwVlmZHn09fP8qlsYr/GuZx9KLNQWt9LwShXu13drqjKO+PlPqicXgmGPSvYlCoWADupeY\njBjhHwMSRMTcq/v2RlnHYjrCOR530mMERbQvWODfl/GkMvPl5wrtR8Dbor8PYgTulB8bNwbXS2kv\nIc+HmXekUdsNyywsANJyTHnx/ufvB3pHBe0fvMdgNm/bnN0t97A6WH0WtFbgMAyAOIhyEXkT9OdS\nWwXU1TC2gro6TVQSrWFPWz3e5hbF5MmS8sQx6R0g0yuqnxqqAx8fWA8N34LNg6FKtw2yc2STKEIh\nTWzcgVzuhHjGtmJQX59ZH2TECP3tF23uJiZBHkdBxMW7ujfjNF5H+cIQ1+XLHQO5cWcFf0I4YQI8\n+mj6Pr+58huHm8iGw/q6V19dWH4obz/mf5k0ScecGAcEMya/YM+2EvKgypH52jk6ApZZWADpOaaM\nZ5Qh9G4VU1jCjP/aeN799F2ef/f5QGaxedvm/N1v62vhzbFaJSVx+MrjcNA/4aGbU3EVSAtaPSVo\nxuI1iruheODJd/lBzQ7Ky4fS3OwdRxyVgHhSNeUt+Wq8oozrY2UlsDnC3EsiNDXB7TdonbRb3751\nK1x5pSaIvXunZz91SxTe2AWTluSOO5w2psjR3Llayti4URNAt6dPNBqcXddr6A3Kj+UXC+BNK2+Y\nVEuLVo+de25woJ979W1UaV4j95FHwnHH+RPCiRO1ysqMwczVhAlabbhunVYhrl2budpvbNQSTH29\nbmtiLaAwou1n9/G6/5r/yE/C8WM2boaeT5Cht3JktjGW2qjthmUWFgAZ6c9nHjeTBesW8Nibj6UR\n/YRKMHrAaKJDoqnqfO4qfAZKKcTlxzmk3xAG7TEo09Oq6lmI1sLbR0NcQbhFb1c9C/u+BGuS7i6H\nJS2tDVHo8yGsPRPePoZMlZUex+ZX92XGVc2MPvUJXnp0NNu37orxltLf6QzmySf1CtJ413z3u3Df\nfU6iuHPOcYh5IgGTJ0MonEilGlEJx93YRH8DjPl2nKYmzeRCISEU0oxl7Vpgn7XcdsdBxFsc6ccd\ngbxjhx4TOLEcSmkj77Zt2aWHmTOdaOmbbvK3gfglZBR/gY1EQq/WlyfLpfgFOfoFKXoxbFh2QnjZ\nZTry2ox3woRMgr18uU7meNNNmWo9rzFdxD8FSjYjtpG8jP3Ay/BCoewr+iAju1/uLUhfTBipOBcj\n6Cijths2gtsiBWOzcKc/j86NpqmmeoV7seSsJanjdWvqMlKJCEJIQiildJqQJI4ZfAzLNgakT990\nhGYEQ55Eqp7NLpVsOgLmPOkqxKRA4oz+z6d4Y11fPnp5eNJwbhhDyHVyHC2Z+KmzdD8hCZNIpFPN\nY46BZ55xq4OMWiyE19YSCsHTT8OMP21h4V+/lLxeK4MPaOLt9bumzg8ddB+JV07ErYIbPRrWrHFU\nHqk5dQX9mbxHDz7oEHjTNhTShNxtR/Hz/Jo+XcdoGFuCt+/t2zWBfsrD2wG++tV0ScxIT48/7khP\nRhIyfScS6Z5s2eCWGBYscPr1juGtt/yrMZr5Ki/XRbgefNDJH/aDH8Duu+eXwtzLUE84Qe8PKtrl\nDcY78URt2Dfz8I1vwMqVetvkLJs7N53hhcNOITG/sZUitYeN4LYoGN44jEhVhPqz6qlbU8eWz7bQ\nf7f+1BxWk2oTqYpQt6YulbRQEI4edDSxzTFaE60ZBD9r/Eay4JJOQRIKNKgDmqkkkt5SSQLP9yax\nfNjtsPsR8OpizSdw2zwAErDLh7B9X1dnCRxJQ0CFSfhcdulSTWjmz3cTKDfDcU469FDt0nvf/XuT\nYiai+KJlO7Brql1i275akkomXgyFhP32g+gpG/jXv4Q3VuyfVLs5xM+46Br3XCOFpPpM6JTv7hxO\nq1ZplYkbXh2/2yZgku/FYrpmujfp44YNmii606O7o9l79dLSmEnhHaTfD0IkQirnl9uw7capp2rJ\nwlzXSBTGnnDOOU6kvTsr7F13pfeTTUWVzRXZ6zQA6V50iYRWhYVCzrVXrUo3jJvruz3jlHIWCt6x\ntSXKv5iwzMIiK7LVDI9tinHn6jtTRL0iXMGwLw1j2aZlue0VARARlNJ1wkf29y8Ty5B6KGvWDCGU\ngBOmwKjb9bGUHeSXsGEc+hF3rfy37538rUBak0Z0r+4lUxejFPz971rn7nhitabaSkgxqKqMjRsV\nq1fr9CVpEowq48NNlWl9hr+ylMTIOajYT5DGg0gkYOFCBfcNhK89CDIAVAUgiDjutRs3asIRpBRo\naEjfDlpFu11rwT9Z3pIlev/LL8Pf/uYQNLeqzaRtNzCSjCGaQfr9IJiIdq9R3+Dkk+G662DoUIeh\nhMNw8cVOgkJzLXeciBcmhbnXruCGVyWVzWnAG4xn4mXcVSLPPddxdwYtWbhVbEa686ZX986Je/47\nynZhmYVFm1HfUE88oZfZgnD28LNTOahM5b4jBhzBi++/mJYKXRDKw+Wc8NUTeL3xddZ9uC51zKio\nEomEP6OAtLxUDKnX297j0au1TaMV7Vm192vw4YEp9VTfYSvYtmc9LPs/MqUDfxfelpYES5929ksI\nFHFIhFEJ2LgpgVLiOc/pKxEX9toLPvooea/LLiIUVqiWcFoyeBIV8OpJrnM1kVq1ShP2GTP87QF+\nMOk0ID2dhNe1dtUq//PdxNJtDwiHHULmJpCJhFYdTZjQ9sjk+vp09ZIJQHRLPrGYvo7JkKuUZhRe\ne0hNTXocDDjqnvPOy7Qr5FNNERyJxxsdfsstuHKWaZSVOYzFK13lW1+9vj5TLRlUUrZUsMzCos3w\nGsWNisqdPgQgOjeaOkcQTjrwJC77j8tSdo9j/nJMXvU3QoQIhUK6bVJt5QdBUF6GAsksudqIvst3\nZrDtpa+jbRhu6cMwDj/DeQKVcKQFlTD2kHByG/CopMyITF+GUWimEE4SALdrsEE4OTZS5z75pI4P\n8curBZnGXS/KyjSBcRtUm5q0sd4QU1MlDtKz7Ho9xNzR0xdfrBmYwWOPabWdm+i5U8l7VSi5EiJe\ncomWJsx41q5NN3pnMzhHInosOtIf1q93bAYjRmiGk49x2R3Rfscdznx5XYqN4d99rzNn5mbGuVKp\nRKP6Wua/D4V0nx0Zb2GZhUWb4VcT3Ow3v6cvnZ6SPkDHZSxav4jL/sMpNHDiASemoruDcGb1mexe\nsbsu8EQwYxGEP5/4ZwB+u/S3vF11rXPQxTy27PksDDnCUWelDOFug7UzaiQOg56Gt7+VulIQQ4EE\n9NsIWweRoQZLtff0jwAJKvZ6l+aP93W5CDvtX33VywycjVBIckobxx+vbQlugmOS+Rm4EyWadoaB\neNNSGNVNv36Z6hd3lHyQCsXYRbyrY2+cy7Zt6atv4w7sVt1ceGF2Q/A99zhGfaNGmzw5XVUETkZb\nw9AgvR/3Ct+byNBg4kTHrbqyUs+DidMIqjPiDmL0U4lFItoOY1KzmzF0ZLyFZRYW7YLXpuH1qPKW\nfQVSmWj779Y/Vd0vRCjNcyokIc445Aw++PwDJgybQPU+1dTW12YUcHIjJCF++h8/pXF7I5W7VPLe\nZ++lHZeq59hv2Cbe+fQdvcMtfbzzDXj1lFTb3b/0MZ9+0BcnpkNB9V2wKeLEfoRaCGGkAzdDEPhk\nEOEyIR5vTRJ+45GVRdUlcZo/2Vu31/64yetrI7wmjm7GFIf+a5FwC4cO/Dqrn9sjNf7+/dOztpaX\n629HKlEkEoqhB25nw6u7paX83rIlXXppanJSnV9xRabXz8UXO1KHm2HMmuWkZHEzMrcKyy+Izayi\nTQ4oI025U4+7pSiltDF9aGQtjZUPUNl4ItN+UJ2hnolG0+NV3ExSqfRtE3vj9irz1ng3aiU/GELv\nNv6DnoepUzUzKTSNR01N+ngKSe5YDJSUWYj8//bOPUyK8kz0v7e6h0FUboNyneGyAkpCYJRFRtSg\noEFQ5FlysjHuQhSdmCMJiAkb92x2PXGfwzmuBowSIt4CWY2bhCwoAl6ACUSHmwKiXARh5A46CIjI\nTHfVd/74qqqrarqnZ2CGufD9nmceuuv6VVXzvfXeZSTwBPoX/6xS6v9G1t8H3I/Wt08CxUqpLSKS\nB/wJ+Fvgt0qpSfU5TkPdEMwCbxFrwbLxy3ztY8rSKSEfxKsfvRqKeJKIU1kpxZ+3/pll4/Xr3fB5\nw32B45mjbMcO7T+mzxieXPMklXalburk2KFjxq04f9v1b9m/bX9qoWfKOl4AViU4cYgl+OLqabB4\nViDqSuCrDnDXMD/3o23Ldpwo/XuqRkXFQCktKK56Fk52DAki2n8ERy8jpMVYNvR5FbaPQf+3tF1h\nY6PrZAEhjcQd05GvoZTF5iOpaKl4jk3B4E0cenWg3lcUEyemd+Lv2HIhQQHmKMVrr8WI8uabsHy5\nfisuL09NgI4Djz+uw22PflXOyjfaucJRC7cFC/S4vAKAYjkU/Y/VPDqrJ53mdc5YATgYzptIpCZb\nkVS01WOPBSu7Ku7/9R9xui+HkkGoiq+jHKniUwi+nVeHUjqQIWiiKilJ9XgPTtBeeZRx48KJmp4g\njJ4rUxXaaCfC6DbpkvFKS3XAg2eia5JmKBGJAbOAm4B9wDoReUUptSWw2UtKqd+4248BfgmMBE4D\nPwe+7v4ZmgDBLPBKu9KvM1WUX8TMkTMZNncYCTuhczAiiXxVkvpQ/jEAP/nPW9cnrw9bP93qbx+3\n4nS6qFMqC11ZWGL5DvOYxHig6AFOnD5BjpWT0lD8jn8twErCoDk6AdATIotn6Qk3Vplyprvrju0d\nAqvHAC3wcilSmog7mbfZo4+3Y5TfVZBey+Hzv3EnVRu6rkNGPqjHu3Mk2IqcFsIDD5e5IbQ9XIFh\n4wsjAHH0chXHsW1uu+MgnbpW8vyxCayzExB70z9n68FvMPbysTzzjINtBwVH0DRmYSdtHFH+8m7d\nYN8+PYElk9p5e8cd4QnQtpXOupbWrtwJC6ZEAu67D2j9Cc+U/oGVv5/iBhroPiWjRwMXHaLTNW9A\nt96U/GdRKCzYiw7ych06ddI+DC8ayrYhlpMk2fIw6rdvuOVj8GtRHTuWMu2MH69NQeF8mei90Of1\nOjB6eSPBxL50xQ/feEMLRdtRxOJJvnPXYZAuiAWWpUDFfBNX1GRUWhrukWJZ4fMFzWqeEz/aRMwz\nF9aXwKhPzWIwsFMptQtARF4Gbgd8YaGUOhHY3n/FUUp9CfxVRC6rx/EZ6oCg2Snq8PYc3JDK2Sgp\nK+FYxTFmlM4A9CQ/sONA1h9c7xckjImeDIPHsCwLx32NVCi2fJp654hJjKdGPaW3c3M0LMtiatFU\nTpzWP7HWLVszo3SGFiRipboBlg3TgkLFESXQZp92joMOx+34QfVRV54Z64LPtOZxujWUPhgSMJK/\nBuuum7B3X5tytm+ckMpYH/kAKn81DuIfz+n5V55IrqPyG1fBhtSkz8jJcOhKfYxO78HSJyAJCodF\nJx/hnusUyfdWoZQTGttjL3Vk/+B22GoI4CUzegRMY1YSKxYDJ06LFjq3JOi8dhzFiy8FgwAC5jQV\njxwvdY7W3T/m1WP/B/vt2fiVhtGCZOFChYq3xmo5h7lH3+NHsc3A3/i5JdGKvBWVimeeTzD03j8w\n+jvD6XRxZwq/tY37f92RpNcXXpTv23j00VT+x8yZ6bQKLbQHDtvDyS8VO9f1IJjIOWiQTpR85hk9\noXs5HEVFqa6IHomEvjeObfHi0x21KdFyoOgJbiv4Bzpd3Nk3XQV9E9EIMNvWgsgr+RIs0f/kk6kQ\n6kTAKlvfTu76FBZdgb2B7/uAq6Mbicj9wFT069mNtTmBiBQDxQAF0awjQ72TzuyUzuHt4X0fPm84\ntmNjWRZP3vIk/S/tHzrOzJEzKT9VHjrGrFGzmLR4UtpkP4CPP/+YJ9c86WsMSSfJzNUzKZlQAsB1\nL1znaxlKBbSaHiV6UrcVykrQ/vJNHA0euJqoq4zrL38F65MbcbqvcNcJ5JdidXsn5ZdJE/qrUP7x\nbNzJI/+d6sOEAV77NTgxkot+yZK/uQuntZMa2+Gvw+JZOI7Fi295vpN0yYSulnPRYYaPLadXq0IO\nHYp2bnP3UZHv/ufUcS5qV8HJzy8ABMtSPL7sBWx1aTiZ0t1eKYFkDs7u66gAHp/b3Z04FYmkDZdu\n5aHi/qHeJNgWK2d/B5RFvEUlvSuW8fXcW9gcB+UoLNEO/6Cv4HSF4qFffI7ttEtzDyw2Jf6AaqGA\nfwqMD1q2FJLJVBhysG7Xe+9VfRze8VJl9pPYb09m4dtxWuaGw3WjDbmCSX2gv8+cmdIeKiu1Yx5S\nIcWewKhvJ3eDO7iVUrOAWSLyPeBfgAm12HcOMAd0uY/6GaEhE+nMTg9d91DGJL7gPg4OooTyU+UZ\no6qCFF9VTP9L+zNv0zxe2PgClXalP+Hbyuaxdx4jWrqm0q70mzkFS4wI4mtA0RDbo5dkFgwWFn07\n9GX7Z9tDzvgq5K9GCtbRL68vWz/TGkyVEidZhJCnXdnKrn7bQ1em3tRti09WXg+3vqzX7R3i+l0C\nZVFCqMhnC07k88a8fG2Sc6JVfyGc8R4N9wXEJqeF4vT1P4HXHtMaUdzG7u6GFsWSYOt9ewzYz8Ft\n3UkkHRxJID1XYn0yHDuZOq9jC/f/+o/0v+okw4YVIZbt7i/u+GIkK5Js/e2PAEUsDsX3Cq1ba6e3\nZ8oS0aHNR/d7QQDRUGmFevvBgDKUEoJvvx2OGvN8CvPng6OCAkfcj+699O+PHqvC4vRpeOKJlG/C\nthW/eRpycx2+/f2DLHm1FUf3t/PPr5SOggsSHMegQdClS+YSJHWJlX2TM2Y/fiFnALq5yzLxMjC2\nHsdjqGM8s1NMYlXMTrXdpyi/iIeu08bY6aumU7q3atPiovwiZt86mxUTVvCDq37gT6hAlcKF1TGm\n7xhWTFjBTb1u0o51t5d4dZO3IOTGc/lm92/WKDvdVjbbPtuWdVvPsR908AvCvVfey4PXPFij68lI\n2bDIm3zUdBSYzKKmo6Bj39/Ghq5rIVYBktABAblHU/uJQ7+rD1A47SfYhb/RQvjGf8X5xxuq3ttY\nkoJxs5k8ewHWjQ8jE24iVrCWqXdcSU5OQBBZSZKfd2HKM//F5sObA5Fl3p8bMeb6buykcOiLg8x4\nIkEi6SCWzZ337aPL5Qe8EwPQNv8gd963H3Hb9+rri5HK6E/dG9sOm3tAT9THTh/FsW13DNp5H4vb\n2j8R0rpSIdmeLyQV2QYooeI0vPh0J47u95qAKf886ZzxXj2w9et14cX6FhRQv5rFOqC3iPREC4nv\nAt8LbiAivZVSO9yvo4EdGJoMNdEIarNPpmiqdMcoyi+isHMhkxZPwlY2cStOvw792Hg41ckvbsUZ\nP0AbiD1txBKLA18cYPORzTw87GFW7VkVCusFPVl3vKgjR7484h/n7oF3+8d65r1nQpqChUV+m3wq\n7AoOn0z1/qhW+wByrBweKHqAx995PHS8uBWnsHMh87dk6AYUZMA82HBXyqfhVecFUmVRvIkfqpid\nLMedgBWonMj6wD2xHJRVCSMfgMP9YfWPdUZ8RfvUca0EW772HYi5giGqEZUN09FmxMBWrJw7jL/e\n8O9wbSlKOSgV48TpEwy5bQsrP9yuj7ljFLx7D2s3VrJu4O9QyX5priU1XhEdaWdX3g3KwrGTvPTm\nh3Rq1wbo4m99bN8l/MEegRrdV5vxqgiJIFV9MI4Da1e2C2/VZwHOtY8T+3QA8tpTOLYX7hzV6lTo\nWCmzlbdtkvZdT/D5gfbpBYWluDjvBCc+bY0TifiqT+pNWCilkiIyCXgdLc6fV0p9KCK/ANYrpV4B\nJonICCABfE7ABCUiZUBroIWIjAVujkRSGRoB1dWOqu0+maKpMhE1TW06vMlfJwj3FN7j779iwgoe\nfftRFmxfwNoDa1l7YC1jLx/rl2J/a/dboY5/R786ypg+YwBCBRTTaTwKxZ7je8iJ5XD75bezYFv1\nCYaDuwzmys5XMn7AeErKSqp0Gkw4Cd8/k5X81fD9G9L7NFwTW+5bs6n4ZEB4P0lCrJIr/nEOhz9N\n0r5yIDvfuIFwrxBXoFy+kK5XHKLDFR+w6YiFWvxkKtfEn/QcKHyhev+O5x/yijx+PAJn941I0Qyk\n5Qnkos95bun3SFQCsR4wcK4WLiqufUo4ocKLmqC5BxCF3aLcjRTT0Wlq13AOiqcBpDSmxK6hcN10\nve9rswMCwwkcO2p2C/4bvH4FX3RBdXsHp9tqir81BN4fz2+edlK+i5AmR2B/FV4visu+8SnrDrQN\nnBv/PEolOfHZBf73eFzOSQOkevVZKKUWA4sjy/418HlyNfv2qL+RGRoj1UVTZaIov4iSspKQ41sQ\nWsZb+pqAt92peW0NrgAAGTxJREFUxKnQvgu2LeD1na8zc+TMkIbhhe16WeWWWMzdNNfXiKK+Ee+8\nlXYlKK0ZZJroc6wcZo6cCWjhmNcqLxWZFaC65EMgnMRYnU8jfzUVI34Iz68MRCElodcyBn9vCWtj\nT0B3OLp3CFh/iZitbIhXwND/YF/+avYBbPy1Kygib/exyrBW49Lt4m7s+2JfapzRIo+OQr09DVDY\nlmtyUZaOFPOO60WNDZin/zaN56LKy6jYNpxk0kFE+zZAUDY6Gs2x3Mtw9HUrL7kRfV2xSqyeK/US\nrwilFyZtJfX+Khg1prL8Cxy8EvYOwclfTeHg0xT/ELbsPsbK1z3tK2oCzICKs3ZJH1Ll9COajVUJ\ndiv/eL0Gb4duR4H6VS0a3MFtMHiciVkLwkImZsVCJqPpq6b7xxrXbxxv7Ar37axIVlB+qly3TU3j\nPAfd8MnTdIb1GEZuPJeKZIX2W0a0gk4XdWJq0VQefTsVb3pn/zvZUb6DLq27+GVOgua2a7tfW335\n9gC+j0MEUVWFTFryV8Po/xnKGbl+/ApOdy6FA4FtRt0fnjALXwjnnFRBAQ5cvhAZ+ngq5DjA4S8P\nh/Na8lfDFfPh428RjUZSfm14OywcolpT/mpOAld8dTd9T/6APt3yePRfO0MyB92O1wLiWrOwbFdG\nWPgTrygYORmn29upgQ56Fjp+SLej/8C+9r+DpTNg/9Wp8eUehz6vwKlLodUR/W+njVyyfyKflnXQ\n2zmW3q/zBpZceIz+l5bSfsQqeHNyRBNz75t//VENxru3wVIxpD7brUL3eOtXKxg+b2pGs21dYYSF\noVFxpmatqJBJ5/8ovqqYJTuXhMxEDg5LP17KnuN7GD9gPIWdC3nuvefYcGgDtmPj4GCJ5Ws6wXPl\ntcrjR0t+5DeHyrFyfNOS9+YvCBe3uJg1967xzzl91fSQua19y/ahNraZiEkMhcJRDo7Sb9Se8IgK\nrSpEckZK5V2cg06126QVEkEfiThaCA16NqPISjgJruhwBbmx3JQ/6asOZC7g6H4fOTkkHNKx9YLn\n2XrB8/Rr3Q/Gt9H90S/4TOee+Dksbl7Ku/cQzO/QY0ghCC17buTnP/w+9y9eT7LwOVdYuONJtILB\ns0NjscTi5rYX8+KDd6X6yO+/GvZfzYINFbz20bdIdl0Fd/03vP1T2H6b9g+JK7AcnV1PzyWw+ya8\nRMUUQZNV1G/iBQAkYMDcGpltzxYjLAzNgqiQyeT/mHbNNBZ9tChkJlr5yUpWfrKS5zY8hyW6qm3M\nilF8VTGFnQur5HwEz+X5TABfm1m7P1XWRKF4YeMLoaZRUU1oyc4lKKWIW3Guyb+GVZ+sQqEQJNTf\nY2rRVJ5c8yQVyQptglL4y2eUzqjWdHVJq0soL1iL4052iUyypSZ5JZl8JBnY+tlWYhJLmdt6/AXJ\nSaDctupaE/D8EO7bdWQyr44tn27RcZf5rj+p4we03HcLp7stSY2v03tVs/EDeJWQQRe2fEU9j7Pz\nFtg2Vo9Nib7mwPU6yuHlY5Nhwovwp9/B8V74k7ndgsSua6DrSr3Pd8elukEeL4D196K1HwW7bw5c\nuw0dtkL55a7pLBpwEPneZxGSvwaRGHmtwv1S6pr6DJ01GBqM6kJ0Z42aRY6VQ7QeVcJJ+ALGqysV\nFRRBvOz18QPGM/vW2fq8c4exYPuCUCRU0klSUlZC6d5Spq+aDsCy8ct45IZHuHvg3SSdJA4OSin6\ndehHy3hLt2Og+AmEjuOw8eBGZo6cyYheI1IlU5SibW5bJhZODIXhXl9wvT9Bt4i1YNwV4zKayXOs\nHHJjuXjtcG/udTM5Vk7mbXtswLruUWIF60LhyxYWPdr2qHJfAT9iLSYxLui5kcunToLhP4fRPwTx\nypiA/zZ9wWcZz399wfXpL8QjfzWni/4tLMgGPQt3fRNu/Ln2m0Q0hE4XdmLepnncMPcGFm5fqDW1\nof8B8dM6TDiWqCJgvOsSBI73CCx1NYfo9l6I9oB5OgrNu1aF1tIkoX1EPVa5OwSd3kkdstx2d/iY\nX3RB7b0a27GZsnRK2gCMusJoFoZmSXX+j2AUVbB/eI6VE9IsvIq4QT+Id5x0Zq6SshISdvjt3pus\n81rlVdn+oeseonRvqd8syusJ4oUEe057QXBweGv3W6zas8p3yEcDAYLHufMbd7Jm/xpdVBGhdcvW\nxKyYXzIlOL7RvUcDOuRUoVhetjytWcsSy6+vBXDoy0Ms3LYwdSwR9hzb44856E/JsXJ4atRTbDio\nGzt8UfkFWy8I1BVd9BtSkUp2Rs3CUQ79LunHnuN7KDtelv7hZyKD1iQIz214rmp1gGxNtly6H5tA\nWdSMds2j1QYdhPxDsUptLvuqA+nLwUTW/XaFNgNiwYFBMHcZasJwKgvW1aspyggLQ6MkWur8TKjO\n/+GtGz9gfBUzUklZCXuO7/HzKWzb5ul3n/YjorwIrKiZa1iPYeTEckI+jNG9R9Ppok5sOLghY5HF\nqFDzwmm9Cru92vVi17FdvqPdc8hH748XBjyu3zjKT5X7k1/CTvi5HNFJ3BKLJTuXhJ36rnkrVBYF\nPVHPKJ2BoxxiVixU9deryeWNuW+HvpxKnGLP8T0AfsLk3E1zU2Y0Fxn0nD5KNWai4HiDAv5MiN4D\nW9mZ/UVZzHIWFnvazYP4P0ASxLJod+NzHL32n6sfRDb/UHVC6vs36IiyXSP8sGIpu4EWPTfVKILw\nTDHCwtDoiL61p6sVVVekEyieg3zuprlVwmm9ST5dmK9XLNETPoWdC5mydIrvm4hbcXBIW2QxOIbo\nsX869Kf+cYLnCmo5XiRXwk6wvGw5U4um+seAVLkThdKOcqWwLItb+9zKq9tfDYUd58ZzmTlyJkt2\nLOGV7a/4E7tXxddRji6dHiAqWLZ+tjW03nZs5m+Z75d6Ce2Lyjh5xiTmCyFBuKz9ZVWOXVN6tOnB\nyMtGsuijRalw3rNEoVDd3oYJw5GyG7h9ZFsWffW/IEu8AZA15LnadV7bYFf7uHzQYZ4z0VCG843g\nW3tFsoJJiyfhKKfarO66xnvj9ybhpJOs4vuoSZdA7zpw4N4r76WgTUFWoZfu2P0v7V9t1nswC91x\ntAbw1KinKD9VztoDa0MRYH3y+vDN7t/0NaklO5bg2A5xK87Ewon+8ilLp1RpSBWTmB8cEHwbD2kg\naWbKuBVnXL9xrNqzytcsquSXRCbIfh36MXnI5JCgbBFrUeXYNSHHyuGlcS8BsHrf6jMSFp6Q9SLk\nQgIyfzVWwTrIvw17W3otJSYxhhYMpV+HfrRu2ZrH33lcR7W5ZsZaETGR7bzwXeCeWl9TbTDCwtDo\nCL5Zi4j/NnsuwgODBE1V6SbqbGG+mXqU1+bc2c7lCdZovoWtbF8bW7t/rW8mAthevp2yY2W+UPC1\nChF/jNNXTde5JAEc5XBP4T0UtCkgr1UeP17yYyrs8DbpEIRbLruF8lPlvpZ4rOIYJbtLQqXpg8Qk\nxrNjng0JymMVx0L5K9no3qY7e4/v1VqJCJuPbA6FOgNc0eEKtpdvr6IZeWPwBGLQpFjYuZAX33+R\nlXvCuTH9L+3P4h2LM+a+2Mrmnb3vcGf/Oym+qpixfcf6v6vNRzYzc/XMKlpTuoRNn4BwTTpiQmcN\n5x/RXIaoCaYhxnMm/wnPNMkQqvfZzHl3ju+biApWb9LLjeWS1yqPYXOH+ZNjMCfDq8i76/Ndvm/D\ndmx/wslrlVelpFFMYiGB1//S/kxZOoV1B9aFzFgt4y35uyv+jt9/8Hud0R6Ls2TnEl796FXfrPjI\nykd8YRR9S/cKKUavu2R3Sei75QZziugormBeTG4sl1suu4Vn3ntGm81cM1g0AGHn0Z062Eh0VJI3\nhrF9xzJt6DTmbZrHoZOH/PHHrBijTo5i1Z5VRHn/yPshwefd7+Bkn3SSTFo8if6X9g/9roryiyg/\nVc4/Lw/7Oq4ruI51B9ZRkazw/T6ez0gQ33cTt+L1/n/DCAtDoySay3C2zu6G4kwETXUFFee8O4cf\nLNKt2d7Y9QZP3/p0SCAB/udodJZCkWPl+JON5+OIJh6W7i1lytIpflkTQYhZusFU9Fo2HNrgT4be\n2zfAxS0uZvbo2Ww4uIH3Dr7naxCVdiXzt8wPObljxBjTdwxLdi7xzX2e1hO8F9Gqwj8Z+hPa5rYN\nXXdeqzxfo4JwhNi4fuMo+aTEF56e5uDgYCmLuBX3zZ3Thuqci4I2BRz68pCvvdm27ZeBiZJOQ4pb\ncV+IedgqJZSjzcOivehPJ0+HfHbB6/R8Sp7mVN8YYWFo9Jzpm31TpbqCitFKtPO3zKf4quIqJiuP\nYHRWbiyXX93yK9+PsXDbQj96aUTPETw87GHfBOVNjhYWI3ql1gWZt2leKCqpqFsRi3cuDkWDWWKF\njuVN2svLloc6Hw7uOphpQ6dVeSkI3osYMcb2HcupxCnG9RtH8VXFofGk+42k8/1kCkAYddkov2gk\nUMUXVFviVtwPFw5WKs6xcnyhHA3kiIY3rz+4ns1LN1fx1UXHFtQK6wsjLAyGRkZ1BRWj9a3G9RuX\n8TjR6Kxg5dyH//JwSiOI5YSEQfT86QRFOk4nT4c0mSrhraLDe/tf2p9be9/Kqx+9qlvgikVeq7y0\nLwV5rfL8BETvjT/TWNKZ7rL5foJViz0zmeejCvZ99/CSEIMmv7gVZ0jXIfx1z19T2pLb6rf4qmI/\nEVPfAuGugXeFhLL3UjB/y/wqZqyor650bykPlzxMhZ0am5fLY8xQBsNZUBf5GueabAmFgO+ziL5d\npztWMMR2+qrp7Dm+x89QD05eNTl/kPEDxvP8xudJ2AlyYjlMvHIiGw9vDGkWjnJCUVMbDm7w3+ZF\nRJtdlMOUpVN8O76HZw7zWvDOHDmzWkGRrRdKJmHiVS2O5swE+7571+NFmEVNQlOWTgG0kLit721M\nu2ZaRuHraS7R5V60mKfpCBKKwvOu0TPhWWKFeq3U9+/bCAtDs6WmzZQaI9WZ3oqvKs4qJKIE70U0\n5yNYyr0m5w9uUzKhJKOZZ/yA8Ww+stlvUJUbywXw36ZFpRzA6SLdgi14cbSAjAqU6LaZeqFU91vI\nlDMT7Pvu+WzSmb48DcHBIUaMwV0GhwR0SVlJ2lyhbCHS3nV564LniZoOzwVGWBiaLbVtptQYqC9N\nKHgvapPzkY1sZp50E6DndE739hzEm8S9N2mv3Ek6oZ+tF0p1v4VMmpRXFibb88h07kwCKvqMs4VI\nl5SVpD3PuRQUYISFoRlzJs2UGpL61ITOJufjbIlOgJmit6Lj8Sbxh0se9jsZRif64MRbnemsugnd\n28frAV/d2DNdX7pzpxNQQI2rE6T7PaQ7z7kytRphYWi2nE2eQ0NQn5pQY7oX6d6mq9vW65Vekzf3\ndBO+d5ya9Dw50/uSTqikE1C1qU4Q3PZ08jTzNs1j9q2za2xeq2uMsDA0a5pS2G19a0J1dS/OddBA\nbd7cswmeTJNxfZgpM427ptUJhvUYpgs22rpgY7Qvyrm4hiBGWBgMjYTG9PafiYYKGqjpm3ttOBdm\nynRaVE2rExTlF3H3wLt5+t2nUSi/L0pUoJwrU6tEm883VQYNGqTWr1/f0MMwGJotXoy/5z+ISYxH\nbngko+nnTI5fW0F5tlpOQ4dWZzv/mYYE1wYReVcpNSjrdkZYGAyGbKSL8c+N5daZZtGUw5zrm/oW\naDUVFladn9lgMDQ7gjkPXox/XU7omSKHmgteQmR9tj2tb4zPwmAwZKW+Y/ybWphzbTgbrakxaVxG\nWBgMhqzUtfM9XWJaY3funylnE7HUmBJLjbAwGAw1oi5Db9O9LTelMOfacDZaU2PSuIywMBgM55TG\n9LZcV1TnhD4brakxaVxGWBgMhnNKY3pbrgtq4lc4G62psWhcRlgYDIZzSmN6W64LmqOmlA4jLAwG\nwzmnsbwt1wXNTVPKhBEWBoPBcBY0N00pE0ZYGAwGw1nSnDSlTJgMboPBYDBkxQgLg8FgMGSlXoWF\niIwUke0islNEfpZm/X0isllENorIX0WkX2DdQ+5+20XkW/U5ToPBYDiXNMVaUfXmsxCRGDALuAnY\nB6wTkVeUUlsCm72klPqNu/0Y4JfASFdofBf4GtAFeEtE+iil7Poar8FgMJwLGlO9p9pQn5rFYGCn\nUmqXUqoSeBm4PbiBUupE4OuFgFcv/XbgZaVUhVJqN7DTPZ7BYDA0aZpqhd36jIbqCuwNfN8HXB3d\nSETuB6YCLYAbA/uujuzbtX6GaTAYDOeOppqX0eChs0qpWcAsEfke8C/AhJruKyLFQDFAQUFB/QzQ\nYDAY6pCmmpdRn8JiP5Af+N7NXZaJl4HZtdlXKTUHmAO6U97ZDNZgMBjOFU0xL6M+fRbrgN4i0lNE\nWqAd1q8ENxCR3oGvo4Ed7udXgO+KSK6I9AR6A2vrcawGg8FgqIZ60yyUUkkRmQS8DsSA55VSH4rI\nL4D1SqlXgEkiMgJIAJ/jmqDc7f4AbAGSwP0mEspgMBgaDlGqeVhvBg0apNavX9/QwzAYDIYmhYi8\nq5QalG07k8FtMBgMhqwYYWEwGAyGrBhhYTAYDIasNBufhYh8CnzS0ONoIDoAnzX0IBqQ8/36wdwD\nc/1nfv3dlVKXZNuo2QiL8xkRWV8TB1Vz5Xy/fjD3wFx//V+/MUMZDAaDIStGWBgMBoMhK0ZYNA/m\nNPQAGpjz/frB3ANz/fWM8VkYDAaDIStGszAYDAZDVoywMBgMBkNWjLBoAohIvoisEJEtIvKhiEx2\nl7cXkTdFZIf7bzt3uYjIr9we5u+LyJUNewV1g4jERGSDiCxyv/cUkTXudf6XW90Yt1rxf7nL14hI\nj4Ycd10gIm1F5E8isk1EtopI0fn0/EXkAfe3/4GI/F5EWjbn5y8iz4vIERH5ILCs1s9bRCa42+8Q\nkRr3CkqHERZNgyTwoFKqHzAEuN/tU/4zYJlSqjewzP0OcAu6rHtvdHOo2VUP2SSZDGwNfP9/wAyl\n1GXoqsUT3eUTgc/d5TPc7Zo6TwBLlVKXAwPQ9+G8eP4i0hX4MTBIKfV1dBXr79K8n/9vgZGRZbV6\n3iLSHvg3dIfSwcC/eQLmjFBKmb8m9gcsBG4CtgOd3WWdge3u56eBOwLb+9s11T90A6xl6Na7iwBB\nZ6zG3fVFwOvu59eBIvdz3N1OGvoazuLa2wC7o9dwvjx/Ui2a27vPcxHwreb+/IEewAdn+ryBO4Cn\nA8tD29X2z2gWTQxXpS4E1gAdlVIH3VWHgI7u53T9z5t6D/OZwDTAcb/nAceUUkn3e/Aa/et31x93\nt2+q9AQ+BV5wzXDPisiFnCfPXym1H3gM2AMcRD/Pdzl/nr9HbZ93nf4OjLBoQojIRcB8YIpS6kRw\nndKvDs0yDlpEbgWOKKXebeixNBBx4EpgtlKqEPiSlAkCaPbPvx1wO1podgEupKqJ5ryiIZ63ERZN\nBBHJQQuKF5VSf3YXHxaRzu76zsARd3lt+583doYCY0SkDN2r/Ua0Db+tiHjdHoPX6F+/u74NUH4u\nB1zH7AP2KaXWuN//hBYe58vzHwHsVkp9qpRKAH9G/ybOl+fvUdvnXae/AyMsmgAiIsBzwFal1C8D\nq17BbUXr/rswsHy8GyUxBDgeUF+bHEqph5RS3ZRSPdCOzeVKqTuBFcC33c2i1+/dl2+72zfZt26l\n1CFgr4j0dRcNR7ccPi+eP9r8NEREWrn/F7zrPy+ef4DaPu/XgZtFpJ2rnd3sLjszGtqJY/5q5Oi6\nFq1yvg9sdP9Goe2wy4AdwFtAe3d7AWYBHwOb0VEkDX4ddXQvhgGL3M+9gLXATuCPQK67vKX7fae7\nvldDj7sOrnsgsN79DSwA2p1Pzx/438A24APgd0Buc37+wO/R/pkEWrOceCbPG7jbvQ87gbvOZkym\n3IfBYDAYsmLMUAaDwWDIihEWBoPBYMiKERYGg8FgyIoRFgaDwWDIihEWBoPBYMiKERYGQxZExBaR\njYG/n2Xfq8bH7hGsLGowNFbi2TcxGM57vlJKDWzoQRgMDYnRLAyGM0REykTkURHZLCJrReQyd3kP\nEVnu9hZYJiIF7vKOIvLfIrLJ/bvGPVRMRJ5x+zW8ISIXuNv/WHQPk/dF5OUGukyDATDCwmCoCRdE\nzFB/H1h3XCnVH3gKXRkX4ElgrlLqG8CLwK/c5b8C/qKUGoCu7fShu7w3MEsp9TXgGDDOXf4zoNA9\nzn31dXEGQ00wGdwGQxZE5KRS6qI0y8uAG5VSu9xCj4eUUnki8hm670DCXX5QKdVBRD4FuimlKgLH\n6AG8qXRDG0Tkn4AcpdS/i8hS4CS6vMcCpdTJer5UgyEjRrMwGM4OleFzbagIfLZJ+RJHo2v+XAms\nC1RYNRjOOUZYGAxnx98H/i11P7+Dro4LcCewyv28DPgh+P3E22Q6qIhYQL5SagXwT+gy21W0G4Ph\nXGHeVAyG7FwgIhsD35cqpbzw2XYi8j5aO7jDXfYjdFe7n6I73N3lLp8MzBGRiWgN4ofoyqLpiAH/\n6QoUAX6llDpWZ1dkMNQS47MwGM4Q12cxSCn1WUOPxWCob4wZymAwGAxZMZqFwWAwGLJiNAuDwWAw\nZMUIC4PBYDBkxQgLg8FgMGTFCAuDwWAwZMUIC4PBYDBk5f8DAkVpn8pWhMcAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ctawd0CXAVEw", + "colab_type": "text" + }, + "source": [ + "This graph of _mean absolute error_ tells another story. We can see that training data shows consistently lower error than validation data, which means that the network may have _overfit_, or learned the training data so rigidly that it can't make effective predictions about new data.\n", + "\n", + "In addition, the mean absolute error values are quite high, ~0.305 at best, which means some of the model's predictions are at least 30% off. A 30% error means we are very far from accurately modelling the sine wave function.\n", + "\n", + "To get more insight into what is happening, we can plot our network's predictions for the training data against the expected values:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "i13eVIT3B9Mj", + "colab_type": "code", + "outputId": "afc103e2-0beb-4a26-fe18-c0cccc6d3d2a", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + } + }, + "source": [ + "# Use the model to make predictions from our validation data\n", + "predictions = model_1.predict(x_train)\n", + "\n", + "# Plot the predictions along with to the test data\n", + "plt.clf()\n", + "plt.title('Training data predicted vs actual values')\n", + "plt.plot(x_test, y_test, 'b.', label='Actual')\n", + "plt.plot(x_train, predictions, 'r.', label='Predicted')\n", + "plt.legend()\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJztvXmcVNW16P9d1c3kiLQYvaLigANK\nBMXGUkB8GjDRi6hPkwhB41AgmheTFxm8zye5MSDo517yokj3zwnSSJKnVxxeEohDi9oVCEaMEYyi\nYsCIYCMIyNi9fn/sc7qrq6uqq7rmqvX9fM6nhrPrnH1OVa299lprryWqimEYhlFeBPLdAcMwDCP3\nmPA3DMMoQ0z4G4ZhlCEm/A3DMMoQE/6GYRhliAl/wzCMMsSEf4EjIhUiskNEjs1k2wz062IRWZft\n8+QCEakUERWRvt7rh0Xkzhyc9yYRqc/2eQoBEdkgIiMyfMw235uRGib8M4wnfP2tWUR2Rbwem+rx\nVLVJVQ9S1X9ksm0uKTYhp6o3qeqMjtqJyGsicn0OupRzSvnaDEdlvjtQaqjqQf5zTzO+SVVfiNde\nRCpVdX8u+lYuiEiFqjblux+GUciY5p9jROQeEfmNiCwSke3AOBEJisifRGSriHwqIv9HRLp47aNN\nEnXe/t+LyHYRCYvI8am29fZ/U0TeE5FtIvJLEXk9nrYnIgeIyK9E5AsReQc4O2r//xKRD73zvCMi\no733BwAPAMO82c/n3vujRWSViHwpIv8QkbsS3LOLRWSdiPxvEWkUkY9E5DsR++tE5EER+YOI7PTO\n1V1E/kNE1ovIZyIyV0S6R3xmqohsFJFPgOuizlcnItMjXl8Z0de1IjJSRGYBQWCed11zvLb9ReQF\nEdkiIu+KyFURx+ktIs97x/kTcDxxEJE/isjEqPf+5t23gPe9bvK+u7+KSP84x7lJRNZ438sHInJT\n1P6krk1EThIRjfpsy+xARPqJyMvedX/u/VYOjXd9Ecc4X0Q+EZFAxHtXi8hfvOdx/xsxjtVmtiJR\nM84OvpvLIu7TBhH5UUd9L3pU1bYsbcA64OKo9+4B9gL/iht8ewDnAENwM7ETgPeA27z2lYACfb3X\ndcDnwGCgC/AboK4TbY8AtgOXe/t+DOwDro9zLfcD9cBhwHHAamBdxP5rgKO8a7oW2AF8zdt3E1Af\ndbz/BpzutT/T6+dlcc59MbAfuA/o5n32K+CkiOv8AiewAl6bXwJPe/09BPgd8DOv/WXAp0B/4EDg\ntzHu23Tv+XnAVuAi79jHAKd4+16LvF/AQcAnwHjvuzgbaIxo/ySwCDgA+LrXh/o413wD8ErE6zO9\nY3UFLgVWAId6feoPHBnnOP+K+02Jd992AV/vxLWdBGjUsVvaACd7x+nq/bZeB+6PaLsBGBGjf4L7\nn1wY8d7TwE+856n8N6L73PK7S+K72Qyc5z3vBZyVb/mR7c00//zwmqo+p6rNqrpLVf+sqstVdb+q\nfgjUAhck+PyTqrpSVfcBC4GBnWh7GbBKVZ/x9v0nTgDH4xrgHlX9QlU/xmnzLajqb1X1U++ansD9\noQfHO5iqvqSq73jt3wJ+3cE1NwN3q+oeVX0J+ANwdcT+p1U1rKrNuEHsZuB2r79fAjMBf7ZwDfCI\nqq5W1Z3A9ATnvRH4/1T1Ra+v61X173HaXg68p6oLvO/yDWAx8N89bXUMcJeqfqWqfwV+leC8TwHn\niEgf7/W1wFOqute7vkOAUwG869gY6yDe7+xDdbwEvAgM68S1JURV3/OOs1dVN+F+T4m+T/9zivvu\nvwsgIj2BUd57dOK/EY+43423fx/QX0QOVtUtqvqXTpyjqDDhnx/WR74QkVNF5P95ZogvgX8HDk/w\n+cg/+lc4rSbVtv8S2Q/vT7ghwXGOiur3x5E7ReR6EXnLm55vxQmmuNfgTefrRWSziGzDaWmJrrlR\nVb+KOv+/RLyO7NuROO0/sj/P4zRSiLr26GuJ4hjggwT7IzkOON8/p3feb+Pu3deAimTPq6rbcAPc\nt0VEcAPXQm/fUmAe8BDwmYjME5GDYx3HM2cs90wdW4GRtN7nVK4tISJypIj81jPhfAk8TuLvM5In\ngKu8AfIqYLmqbvCOm+p/Ix6JvhuAK4DRwD+83+WQTpyjqDDhnx+iU6nWAH/DmTEOAf43bjqcTT4F\nfK0ST8AcnaD9Rpyw8GkJJxWRE3CC6BagSlV7Au/Seg2xUsf+GqfdHqOqhwIPk/iaq0SkR9T5/xnx\nOvIcn+FMa6eoak9vO9Q7D7hrj3ktMVgPnBhnX/R1rQdejDhnT3XRV7d5fWpO4bzgTETfBYbi/qvL\nWk6sOkdVzwLOwJl9fhz9Ye9+PYmb9XzN+16W0nqfU7m2nd4xD4h478iI57OAPcAA7zd8PUn+hr1Z\n0Eacxn8tbjDwSeW/sRNnUovVv0TfDd7sYjROQXgeb+ZRypjwLwwOBrYBO0XkNGBCDs75PHCWiPyr\niFQCPwR6J2j/W+BOEekpbh3BbRH7DsIJi824ceRmPJOEx2dAnyhH3cHAFlXdLSLn0mqSiUcAmC4i\nXcXFi38TJ9jaoS7S52FgjudkFRHpIyIjI67lBk+rPBC4O8F5HwFuEpELPUdrHxE5JeK6Toho+yxw\nuohcKyJdvK1aRE7xTGuLgZ+KSA8ROQP4XgfX/BzQDyfwfu3NzvCOWe19bztxA11zjM93w9ngNwNN\nInIZzi7fmWvb6G3jxK0nCeG0aZ+Dvb5sE5FjgJ90cG3RPAH8COe3ifxeU/lvrMLNIHqIyMk4v4lP\n3O/Ga3+tiBzifU/biX0/SwoT/oXB/8RFnGzHaTq/yfYJVfUz3LT3P3COrxOBN3HaWyzuxmnM64Df\nAwsijvVXnIN1hdfmFGB5xGf/CLyPM1H4ZqhbgJniIp7uxAnkRGzACZdPgfm4ENr3E7T/nzizygqc\n8FiKE6So6nPAg8ArOAfiH+MdRFUbcP6D/+Md52Vatfc5wHc9M8J/eKaaUcA4r58bcVp3t4hrPgwn\nWB8BHkt0waq6GzdgXExbbbin9/mtuO/jU9z3GP35rTiB+jSwBWfffr6T16Ze2ztxvqGTaPsd3w1U\ne8d5FjerS4UncA7pP6rqFxHvp/LfuB+nhGwCHsU57v1r7ei7uQ742DMt3ei1K2nEUyaMMkdEKnBm\nlP+uqq/muz+RiMjFwMOq2jfffTGMUsE0/zJGRC7xzDjdgLtwEQ8r8twtwzBygAn/8mYo8CHOJjwK\nuEJV45l9DMMoIczsYxiGUYaY5m8YhlGGFGxit8MPP1z79u2b724YhmEUFW+88cbnqpoobBsoYOHf\nt29fVq5cme9uGIZhFBUikmjFegtm9jEMwyhDTPgbhmGUISb8DcMwypCCtfkbhlGa7Nu3jw0bNrB7\n9+58d6Wo6d69O3369KFLl5i1bTrEhL9hGDllw4YNHHzwwfTt2xeXTNZIFVWlsbGRDRs2cPzxcQvC\nJcTMPoZh5JTdu3dTVVVlgj8NRISqqqq0Zk8m/EuUcBhmznSPhlFomOBPn3TvoZl9SpBwGC66CPbu\nha5d4cUXIRjMd68MwygkTPMvQerrneBvanKP9fX57pFhFB6LFy9GRHj33XcTtnv88cf55z//mbBN\nIurr67nssss6/flsYcK/BBkxwmn8FRXuccQI9360KchMQ0Y5s2jRIoYOHcqiRYsStktX+BcqJvxL\nkGDQmXp+9rNWk49vCrrrLvdYW9v2tQ0ARiGTaUVlx44dvPbaazzyyCP8+tet5XpnzZrFgAEDOPPM\nM5k6dSpPPvkkK1euZOzYsQwcOJBdu3bRt29fPv/8cwBWrlzJCE+7WrFiBcFgkEGDBnHeeefx97//\nPTOdzRJm8y9RgsG2dv5oU9BTT7V9vWCBa1NVBY2NbrZgfgKjEMiGD+uZZ57hkksu4eSTT6aqqoo3\n3niDTZs28cwzz7B8+XIOOOAAtmzZQq9evXjggQe4//77GTx4cMJjnnrqqbz66qtUVlbywgsvcOed\nd/LUU6lWs8wdJvzLhKoqCASguRlEYOBAePVV94eqrIRHH4X9+93+QAC6dTNHsVEYxPJhpfu7XLRo\nET/84Q8B+M53vsOiRYtQVb7//e9zwAEHANCrV6+Ujrlt2zauu+463n//fUSEffv2pdfJLGPCv0gJ\nh92fIBkNPRyG2293wl3V/Yl++UuYM8dp+StWwDPPuH3gBoBM/ckMI118H5av+fs+rM6yZcsWXnrp\nJd5++21EhKamJkSEq6++OqnPV1ZW0tzcDNAmzv6uu+7iwgsv5Omnn2bdunUt5qBCxWz+RUi0/b4j\nO6ivOfnCXdW99s07v/996z5wmn+iP5k5io1cEsuHlQ5PPvkk3/ve9/j4449Zt24d69ev5/jjj+fQ\nQw/lscce46uvvgLcIAFw8MEHs3379pbP9+3blzfeeAOgjVln27ZtHH300YBzEhc6JvyLkFRDOX3N\nyV8TEinc6+vdjADc/jFj4J574v/JUh14DCMTBIMwbVpmZqKLFi3iiiuuaPPeVVddxaeffsro0aMZ\nPHgwAwcO5P777wfg+uuvZ+LEiS0O37vvvpsf/vCHDB48mIqKipZjTJ48mWnTpjFo0CD2+3+qQkZV\nC3I7++yz1YhNQ4Nqjx6qFRXusaGh48/U1Kh26aIqolpZ6V77x+rWzb3frVvHx5oxw50X3OOMGfH7\nOGNGcn0zyovVq1fnuwslQ6x7CazUJGRsRmz+IvIocBmwSVXPiLFfgF8A3wK+Aq5X1b9k4tzliD8N\nTtbmD87E09zszDuq7rVPpDmoIxLZX30/RFWV8zHYCmPDKFwy5fB9HHgAWBBn/zeBft42BHjIezQ6\nSXQoZ0eMGOEWfTU3u0dfaNfXO/ORqjP/TJ/utnjHjjfwRIbjBQLumOY4NozCJSPCX1WXiUjfBE0u\nBxZ4U5I/iUhPETlKVT/NxPmN5PBt/pH5oHxNfs8eJ6xfeMGFgCbS1mMNPJF+CD9cNHqFsU8qkUqG\nYWSHXDl8jwbWR7ze4L3XBhEJichKEVm5efPmHHWtPPAdu76G7zuJfU3+4otb1wF0Jh+QP7MAd45A\nAG6+uf0gYg5jwygMCiraR1VrVXWwqg7u3bt3vrtTUsTL9wNOOE+f7hZ2xdPWOyIYhBtuaJ1VNDfD\nsce2F/zTp7tZhiWdM4z8kqtFXp8Ax0S87uO9Z2SRaPNKIidxZ5zI0YwfD/Pnx3cGX3RRq3mpo7UE\nhmFkl1xp/s8C48VxLrDN7P3ZJZZ5paNY6XRjqRMtxvF9An56icGDY5uEbPGYkQsqKioYOHAgZ5xx\nBldffXXLwq7OEJmy+dlnn+Xee++N23br1q3MnTs35XNMnz69Zd1BpshUqOciYARwuIhsAO4GugCo\n6jzgd7gwz7W4UM/vZ+K8pUyk1g6pa+TZyIeSDPGikHyfgB9Z9NZbbfdbARojl/To0YNVq1YBMHbs\nWObNm8ePf/zjlv1+LHwgkJp+PHr0aEaPHh13vy/8J02a1LmOZ5CMaP6q+l1VPUpVu6hqH1V9RFXn\neYIfb+3Brap6oqoOUNWVmThvqRKptV94oROcd90FB5/Xn+ZAAI44okP1OJGNPx9E+wQinc6QeNWy\nzQiMbP4Ihg0bxtq1a1m3bh2nnHIK48eP54wzzmD9+vUsXbqUYDDIWWedxdVXX82OHTsA+MMf/sCp\np57KWWedxX/913+1HOvxxx/ntttuA+Czzz7jiiuu4Mwzz+TMM8+koaGBqVOn8sEHHzBw4EDuuOMO\nAO677z7OOeccvv71r3P33Xe3HOvnP/85J598MkOHDs1OeuhkVoLlYyvXFb4NDaojR6oGAm45lojb\n3uI0bQaNWKeletxxrUt14xwr36tsI/sQvTK5pib+Pr/PnVnNbBQ2Ka/wzcKP4MADD1RV1X379uno\n0aN17ty5+tFHH6mIaDgcVlXVzZs367Bhw3THjh2qqnrvvffqT3/6U921a5f26dNH33vvPW1ubtar\nr75aL730UlVVfeyxx/TWW29VVdVrrrlG//M//1NVVffv369bt27Vjz76SE8//fSWfixZskRvvvlm\nbW5u1qamJr300kv1lVde0ZUrV+oZZ5yhO3fu1G3btumJJ56o9913X7vryPsKXyMzxHKKdukC1U1h\nTt+/BoA2JZs//hgmTHDPQ6F2x/NNML7SlOu4+limHN+pHGsVcCyHc77MV0YBkYUfwa5duxg4cCDg\nNP8bb7yRf/7znxx33HGce+65APzpT39i9erVnH/++QDs3buXYDDIu+++y/HHH0+/fv0AGDduHLW1\nte3O8dJLL7FggVv3WlFRwaGHHsoXX3zRps3SpUtZunQpgwYNAlyRmffff5/t27dzxRVXtKSXTmRK\n6iwm/AuISKdoIOBi7++/Kkz/20Ykts9Nm+YeYwwA+bSlx/rP+g7lmTPj74sk0+l8jSIkCz+CSJt/\nJAceeGDLc1XlG9/4Rrsyj7E+11lUlWnTpjHBV+I85syZk7FzxKOg4vzLnUg7fbduLiZ+wJsLqNi3\nFyFK649kyxY3Axg0qMUm6mv7Cxbkr5h7Ir9Dsj6JTKfzNYqQPP0Izj33XF5//XXWrl0LwM6dO3nv\nvfc49dRTWbduHR988AFA3BrAF110EQ899BAATU1NbNu2rV166FGjRvHoo4+2+BI++eQTNm3axPDh\nw1m8eDG7du1i+/btPPfccxm/PtP880zCWPwHx8HChW0/EAiw48Svs2/zVg7evYnK3REhaqtWwdCh\nfPCTh7jol6GWKl3+yttca86J8gDV17cWk+nIHJVqHiOjBMnDj6B37948/vjjfPe732XPnj0A3HPP\nPZx88snU1tZy6aWXcsABBzBs2LA2At3nF7/4BaFQiEceeYSKigoeeughgsEg559/PmeccQbf/OY3\nue+++1izZg1B79oOOugg6urqOOuss/j2t7/NmWeeyRFHHME555yT+QtMxjGQj60cHL4J/VjV1a2O\nXX+rrta/1jS0fOaWypq2DmBvawLdyBE6g8laUaE6cWL+Hb8+5sA1LKVz5kjH4WtmnzwSN7xx1ChX\nWzESEZgzh+cbgy2fqdUQbw8c2+64AhzBJqYym/k6jvHjM1cII11SLURjGEZ2MOGfR2LavWtrYenS\n9o2vvRaCwXaf2Tm3DmpqoH//lqaR/oFrmxcSHHUIjBuX1WuJFYYd671CW39gGOWKaDIVPPLA4MGD\ndeXK0l8LFg47p+zGjXDkkXD/H07nwHWr2zaqrobly9t8JuaK39pamDgxflWWfv1c8p0MTwFiRRRB\n/CgjS+lc3qxZs4ZTTz0VkbghDEYSqCrvvvsup512Wpv3ReQNVR3c0edN8y8AHnsMFi+G/fNqaV63\njjaiu1evNoIfEuTgCYXg9ddh4MCWpbT+sRTg/fdh6FA3SGSQWKacROYdv/9gK3fLke7du9PY2Eih\nKp7FgKrS2NhI9+7dO30Mi/bJM76QnMEUpjK7fYOZM1M7YDAIb74J4TA7xt/CgWvfQokIE21udmGh\nH3wAs2al13mP6DDsqirXhURRRonWH9jMoLTp06cPGzZswGp2pEf37t3p06dP5w+QjFc4H1s5RPuo\nuhQHIanRJqQlcqcZVHv1Spi6IRlmzFD9HSPbp4Xwt4MOUp08OSPX4adxqKlpjebp1s1FGsWK6Jkx\nozWFRSDQWgjeooEMIz2waJ/8kkweqtpa+NWkMA/qLQiKQKuWPnNmzBW7qTBiBFzVYwkTpYYmArSb\nZO/YAbNnw5QpaZ0HWk05jY2t5p79+9sXdPGpqnKTEHCPVVXuuUUDGUZuMOGfBToqVRgOwy23wKRJ\n8KOm2VTQ3GKWEYDhw9MW/NC6yKrvz0OsqXkNGT48dsMHH4QhQzLiC0g2mqex0aWwAPfY2Bj781VV\n5hcwjKyQzPQgH1sxm31mzHBmC3CPvklD1ZlFKivdvnNp0L1UtDX3BALZtXWMHdve/BO5HX102udP\nJptoIvNOLBOSmYAMIzkws09uiTTzxNN+w2G47TZnDjmXMHczHfG0fgUkEICHHsqul7POWxfQty9E\nJLFq4ZNP4Lzz0loXkExFsETpWmKZkMwEZBiZxaJ9MkBHqYt9oVVf3yr4X+ZCurIXQVEEqaxw5pcM\nmHs6JBRyWzgMF1wA+/a1b7NwIWzeDEuWZK0bHaVrsYyehpE9TPPPAPHSjY8Y4XLW+7b/rVudbeVB\nJtGNPQQ8J69UnwPLlhEeEMqtfTsYhFdegaOPjr1/6VLo0SPrq4PjYRk9DSN7mOafAeJpqNGDwqpV\nMJ9xDCIqH/hZZxEmmJ+8+8EgbNjgBPz//b+uA5Hs3u1mAStWZGV1cDLdM6FvGJnHNP8MEE9DHTHC\npVQWcY+160cxDpei2bfzAzB+fP5DHOvqXAmxkSNj73//fecLyEBYaDRWo9cwco9p/p0g1grUeBqq\nv4J98Z5RHLvGJWxrE88/dqxL2EaB2LeXLHECfnaM1cbg3v/Tn+DeezOiksdb6WurfA0ju5jwT5FU\nyiLW1ztNfoiGGUVbwa/A5pFjOaKuDohf+CQvzJoFY8bAddc5jT+aZcvg/PPhjjvSThERb8aTr9KT\nhlEumNknRVIxz/i+gOvEFXGONPUsYSSPjKhr0z6ZEMmcEQzCe+/B5Mmx96u6WcAxx6Rlr4kVFpt3\nE5hhlAEm/FMklXz0wSCsGzaOkM4DWgX/HxjJVT2WFEfo4qxZresCYrFhQ1rrAmL5Syznv2FkH8vn\n3wmStkcPGdKuItenYybyePVDLZ+NdayCtXd3VC9g4ECYOzdjvoCCvAeGUeAkm8/fhH+2GDWqfUUu\nEZdv35NmqRZBKQjCYbjmGqfxx2PsWBc9lIOuLHAWNcaPL7D7ZBh5woq55JMpUxKWYvRJtQhKQRAM\nwvr1TsB37Rq7zcKFMGhQVmM3/cXJ8+a57cILLVTUMFLBhH+mCYfhvvvav19d3U4bjmXbLhp7t78u\nYGz7AvKAW9GWYYkcuR5gwYK2WSkKcqA0jALGQj0zzYIF7W3iI0fGzJETL7yzYEI+k6GuzqWHiLUu\nYM8eZ4+54460cxZFm8hGjWq7PxAo4IHSMAoQE/7ZZvjwhMnRYi0OK7qUBrNmwYknwvTp8Omnbfet\nXevKRv7+9y5stJMXFm0OO/LI1kVxFRUZ8zMbRtlgDt9M4YenVFXBD37gbBJdurRmeSsXfJvM88+3\ndwp36eJ8ATfemPJMIJ5zvGhmSIaRIyzaJ5fU1rpE/U1N0K0bzJnjktGXs1SqrXUafzyqq2H58pQO\naeGfhtExOY32EZFLROTvIrJWRKbG2H+9iGwWkVXedlMmzlsQ+LHv+/a5YrR79kBjI+ER05hZHyzf\nCJRQyC0Oq652Gn80K1bAYYel5BAuqBXQhlHkpC38RaQCeBD4JtAf+K6I9I/R9DeqOtDbHk73vAWB\nr91GzZ7erhqRsIZv2RAKOe3+lVdcrqBotm51q4OvuKKMb5Jh5IdMaP7VwFpV/VBV9wK/Bi7PwHEL\nmrdrwzRPmECk2Fdgc9UpzH0zWNix+rkmGISnn3azgFgsXgxDh2akgLxhGMmRCeF/NLA+4vUG771o\nrhKRv4rIkyJyTAbOmzfCYdgyYaqrwuW95w8C/+vz23n0UZe/v+Bj9XPN8uVw3HGx9zU3u1nUySfb\nLMAwckCuFnk9B/RV1a8DfwTmx2okIiERWSkiKzdv3pyjrqXOq7PDnBlRjcsX/LOZTK2GaGqC73/f\nyg/GZN26+AVjwKWQHjo0KwOAFY0xjAhUNa0NCAJLIl5PA6YlaF8BbOvouGeffbYWJA0NuqeihzYh\n2gwt22cjx2qPHqoVFao9eqg2NOS7owVOQ4PqmDGqzmPSfquuzuhNbGhQ+36MsgBYqUnI7kxo/n8G\n+onI8SLSFfgO8GxkAxE5KuLlaGBNBs6bH2bPpkvTLgIozQiN9OKTsZM5YkmdFRtPBd8PUFMTe/+K\nFTBsWErO4ESafcHnTDKMXJPMCNHRBnwLeA/4APg3771/B0Z7z2cC7wBvAS8Dp3Z0zELR/BsaVGfM\n8DTFkSNVI7T9PXTRYZUNpkWmS0ODar9+8WcBIqqTJ3d4iESavWn+RrlAkpp/RoR/NrZcC/82Qj7i\nPV9ghKnW5giB1AwaplorKtznjAzQ0KA6caK74bEGgeHD40rtGTNaPxbvO4n1HRtGqZGs8LfcPsSv\ny+ubCp5tGsUQWouy+A7ex+RGi+bJJH5So0GDYheN8WsHX355uzxBfjZU/zv0v5PIVcGGYbRiwp/Y\n9mC/nOBMpvDNiOLr/uOW6pH0HRPixRFm3884ft6fWAOAqlsXsHhxm2ypsTKkRg7qFRXuo/v3l2fK\nJcOIxoQ/8bXGIGGGNLnc/BL5gepqei1fwrQc97OsCIVgwACYOtVp/LFYuhSOOAKeeQaCwXbZUCMH\n9aam1vf37nW550z4G+WMFXMhdhFxAKZOJYC2FfwjR6ackMzoJMGgSw1RU+MS9sdi82Y47zw+mFLb\nLtInsjBORUVOemwYRUPZC38/PBCcsKiv9wTIuHGwbFmLfV+hw9z8RpYIheC112LnB8J9N8fPnkDg\nziltcilFDupz57qEqyLucfz43HXfMAqRsk7pHG0TFnE24ZDU8uB+l45YcMKlGWF1zesMCJmtIK/E\nSRXt/4r3UsnfB36br7/ZvoC8pYQ2yoFkUzqXlc0/+s8faRNubnZtVOFyngJaBT/A/dxBc2OQHSZA\n8ovvC7jmmpZiMUrrd9WV/QxYtRBGbW43Syu6CmmGkUXKxuzja/mRaZYjbcJdusD5gTAvcwHneGGd\nvuD/FWP5aY9ZVFW1P4aRB4JBWL/ehXv27Nki+CViY+lSOPRQZ76LgeX5McqdshH+8cI5fZvwb24P\n83LTeVzAMg5jKwAf0ZdbK2oIT3SpGxobLUVAQTFrFnzxBYwc2dYp7/Pll7BwIfR35SV8gV9b2zqI\nX3gh3HKLDQJG+VE2Zp+44ZyeKWD7gZdTQdsUzT17VfK950NtTAWxjmHkmSVLYMoUeOAB+Oqr9vvX\nrGHvoYfzq69mUKshRJyZr7nZDeQ1NTB/vuVkMsqLsnL4xnX4jRqFLm1dyOXfEZk82WmXyRzDKAzC\nYZg0CVatavO2/52GqWZYYDnp9iXJAAAdxUlEQVQVFc657//8Kyrg5pvh2GPtuzWKGyvgnixR0SMt\ngr8TBcaNAqJ/f1jTNnms/90ulZF8PG8Jb74Jjz7qtP/IaK/IFB+GUWzktIB70RIOO4Ovhy8cvjru\nNBP8xc7q1e2KxvgmvZG6lNBPDuGh7eOor3c+nxtucILf/DlGuVCWwt93/H06e0FrjKfHe/Sj96bV\n5gAsBZYscQb9Qw5peaslGmj7dli4kOBlVUyrqmX8+NbIL/PnGOVA2Ql/P+QzcOcUeix+os0K3iYC\nXM980/xKiVAItm2LXzpyyxaYMIHg1AtYPidsxXiMsqHshH99Pdy9awqTmc2hfNm6I1DB/6h8iD9X\nBE3zK0WWLIGGBjjzzNj7ly1jwK3DmTYiHFPw27oAo9Qom1BPnxEj4AQeByIie3r1Qp5/nu8R5Jh6\ni/YoWYJBFwU0apRbBBbN/v1w4418esoF/PHI8fQbH2yXGtqcwUapUHaa/zEPTuEINgGtDl5uuqkl\nJfC0afbHLnl8X4C0XRqmgK5Zw5GL5zFu3nm8NmxKS2ivLe4zSo3yEv61tfzLwrb5+bf2PK5dLL9R\nBoRC8PrrLlNoINAa4hux/aRpNgdOGtcmDYiZBI1SoSyEfzgMv7uiFr3lFsTLz+//2beE7sxn14x8\nEgzC00/Da6/xZvVEp/l7u3zlYMCqhQRvOp23f1BrzmCjpCh54R8Ow4dDx3HJ4gnQ3NwmRfO9TObF\nE0P57qKRb4JB9sx5iEWBsQAtg0BLWOjq1Zw4ewLT6keZ4DdKhpIX/jp1Ctc2L2z5IzvBH2Ai87iT\nWTz1VJ47aBQEwSCc8Fod4eGTaepxUOxEcUuXwvHHu1XhhlHklLzwP+cv7o/qC34FJvIQD+M0/quu\nylvXjAIjGITzXplF5VfbnUO4d+/2jdatc+lAjjrKBgGjqClt4V9bS5cdW9u8ta/XkXw+JkR1tft/\nh8zqY8QiFHKF4ePVDt640Q0CceoFGEahU7rCv7a2JW+PP4UXoNvMn/L00y51jwl+IyHBoKsdPHw4\n9OwZu83ChXDBBbb6yyg6SlP4jxvntLLIvD0irvKTSXwjFYJBeOUVVzRm7NjYbZYtg/POczUFDKNI\nKD3hP2WK08Yi0ECA310+j/AYi+c30qCuDqqr4++fPRsOOshMQUZRUHrC/4kn2rxU4NbAQ4x+LmR1\nd430Wb7czQC6dYu9f+dOp3yMGpXbfhlGipSe8D/hhDYvNx45kFoN2dJ8IyEpJW6rq4Pdu+ObgcCF\nhZqmYRQwpSf8773XrcMHqKjg85/OtaX5RkL8xG133UVqs8O6OudH6t0bunRpv/9b33IVxSwk1EiS\nXGaPLb2snsEgvPpqS6HdAcEgLw6wurtGfGIlbkv6dzJrltvCYef0jWTrVrdNmOCcwnV1Ge65UUrk\nOnts6Ql/cHcs4q5FvTSMNviJ2/w/Xadmh8GgWzgyaZIbRaLxgxBsADDikJYS0glKz+xjGCkSDDot\nK1bituhpeG2t8+X6lpw2+0MhN+scMyb2iRYuNDOQEZecZ49V1bQ34BLg78BaYGqM/d2A33j7lwN9\nOzrm2WefrYaRT2pqVCsrVQMB1R49VCdPVoXWbfJk935FhXtsaIj68JFHtv1A5DZ2bN6uyygsGhpU\nZ8xwj5HPOwuwUpOR28k0SngAqAA+AE4AugJvAf2j2kwC5nnPvwP8pqPjmvA3ckn0n66hQbVLl1ZZ\nHQionnRSW/l90klO8IN7nDGj/XE/GzlWm0GbYw0Aw4erTpyY3j/dKGoaGhIoEJ0kWeGfCbNPNbBW\nVT9U1b3Ar4HLo9pcDsz3nj8JXCQiMRMnGkauiRXtU1/f1nQfCMCVV7b93JVXJp6mh8PQ99U6JkoN\nq6V/a+U4n2XLYN48GDbMTEFlSrSdf8GC4or2ORpYH/F6AzAkXhtV3S8i24Aq4PPIRiISApdu89hj\nj81A1wyjY2I52kaMcOu49uxxwv2BB5xJ/8QT4amnXDbYUMiZ9+NFkvnHrdUQj1SE+MuAcXx91cLo\n07sTe3moLP1IeTFiBFRWukw0gQA89pgrJZ2LaJ+Ccviqaq2qDlbVwb1jpdM1jCwQy9HmO4Hvucel\n9vFlcijkSgD7rxPVfY4+7s65dS4iqLra/eMjaW52IaGHHmrpIcoM9aaEzc2wb1/uakVnQvh/AhwT\n8bqP917MNiJSCRwKNGbg3IaRNvGifRIJ9k4fNxRyKSKWLXPThmjr55dfuqigQw4xU1AZ4JsXfUdQ\nRUXuon1EtZ0lMrUDOGH+HnARTsj/GbhWVd+JaHMrMEBVJ4rId4ArVfWaRMcdPHiwrly5Mq2+GUbB\nU1sLt97q5vqxOO00WL06t30yckb0wq45c6CxMb0FqSLyhqoO7qhd2jZ/z4Z/G7AEF/nzqKq+IyL/\njvM6Pws8AvxKRNYCW3ARP4ZhhEIwYIBbHLZqVfv9a9bA4YfDjBnmDyhB/NlhPjIQpK35ZwvT/I2y\nY8gQWLEi/v7Jk10qCaNk8SPNcqH5F5TD1zDKmuXLnUP4gANi758926qGlTCdTjDYSUz4G0YniE77\nkLFsjKGQqwkwcmTs+sHLlrmykjYAlBThMEyf7kKLcxXtU5qJ3Qwji8Ry0t1+e4azMS5Z4k40bFj7\nRHH797sOXHmlJYorAfzf0549rfH+uYj2Mc3fMFJkwQJXy8XX0J56qv0isYzgpycfPrz9vl27XEho\nt24WElrk+IsBfcF/8cXZX+AFJvwNIyXCYXj00daFOZWVbrVv1rIx+gXk/cVh0UVj9u51i8OOOsoG\ngSIlcjFgt27O/JOLqB8T/oaRApE5f0Tg+993Zvp4KaEzhr847Jo4y2M2bnSDwJQpWTi5kU0SpRTP\nJhbqaRgpkOtqSzE56ign7ONxxBFw/fUWFlqmWKinYWSBfGlpbfj0UxcN5NeqjmbTJhcWevLJFhVk\nxMWEv2GkSLo5fzLCkiUu6qemBo47Lnab9993dYXNFGTEwIS/YRQzoRCsW+cGgXjMnm0DQIGQsfUg\nGcDi/A2jFPDz/kyYEHv/7NnOW718ec66ZLSlIPxFEZjmbxilQigEDQ2x1wWAyxvUr19hqJ1lSH19\n6wrePXvc63zOBEz4G0YWyfmf218XMHly7P1r17pVw1dcYYNAjqmqcgu5wD1u3ZrbXD7RmPA3jCyR\n60RdbZg1y/kBDjmk/b6mJli82DmDR43KYafKm8bG1nRNgYDL4J2VleFJYsLfMLJErNrAOSUUgm3b\nXFho167tq4YBLF0KgwbZLCAH+HWh/ZW8WV0ZngS2yMswskQsBx/kp3AH4NI/TJrUPlEcuIHh2mst\nUVyWic7Xn4n8/dEku8jLhL9hZJHIPze0DgaVlS41xPjxOR4EwmG47jq3BiAW1dUWEdRJsiHIO4MJ\nf8MoMGbOdPb/yNxA3bvnKeRv3Dh47jlXMD6agQNh7tw8r2IrLgopjNPSOxhGgeFnb/RN76p58gWA\nM+9s2wZjx7bft2oVnH8+nH66ZQpNgnwUYskEJvwNI0f4eYEmTMivo68NdXXO1BONKqxe7TprEUFx\n8TX+F17IbSGWTGDC3zByzLHHwi9/CTff7MzveWf58tgzAJ+lS+GYYywiKAapFGIppNQOYDZ/w8gZ\nkXbhigqnXO/f7zTFl18uABN7OOzSQCxeHL9Nv34wf34BdLYwSNbWn0ufgNn8DSPPRGt6kXH/+/a5\nTdXZihcsyGtXHcEgPP104iRxfqbQQlFf80yyKb7zvuYjBpbYzTCyQCxNz3f47t3rhH6BTrrd4rAB\nA1zVsA0bYreZOhUuuST/cY0FQDDY8S2I/O4LxSdgmr9hZIFYmp6vJd58c9s6LJWVLt6/oAgGYf16\nlyPogAPa71+2DO6800UFWURQhxREEaAoTPgbRhaILModqekFg87h6yf4EoGbbioMYRCTWbNg587Y\nEUHgpi9WOzgpCqIIUAQm/A0jCyTS9CIHhu7dC1Drj8Xy5c4XMHKk63Q0s2e7tJUlPAsotGiddLFo\nH8PIA6mkAiiUtAEtjBsHCxfG3z92bMnlCCqkFbwdkWy0jzl8DaNAiCXkC1Lo+IL9qadg9+72+/2B\noYQGgHg+nGLGzD6GkWNi5fmPl/s/Uujs3l0gIaHgBPuuXc4MFIuFC+Hgg+GCC0rCThLPh1PMmPA3\njBwTS4uMFwc+YkRrZJAqPPZYgcnSJUtcRNBBB7Xft2OHiwoqgXUBhRitky4m/A0jx8TSIhNFB91w\nQ2syuP37C2OBUBtmzYLt2xOniLjwwqJ3BhdatE66mPA3jBwTS4tMpFmOH+8CbAre5FBXF7928J49\nLiS0f/+iHwSgNCJ/0or2EZFewG+AvsA64BpV/SJGuybgbe/lP1R1dEfHtmgfw2il4CJ+EhEOw7e+\n5SqUx6OII4LiOeEL5TvKVbTPVOBFVb1XRKZ6r2Ot9tilqgPTPJdhlC3JpBAoGIJB+OILGDIEVqyI\n3WbhQnjnnaIpGhMp2OP5ZwouKqsD0jX7XA7M957PB8akeTzDKHtqa10K/aK3jixfDg0NMGYM9OzZ\nfv+qVTB0aMFfaHQkVlVVe/9MISZu64h0Nf+vqeqn3vONwNfitOsuIiuB/cC9qhozZ6yIhIAQwLHH\nHptm1wyj+KitdaZxcGn0weVZK1r8TKHhMAwb1r54fHOzu+BlywrSDBRZpau52Qn2xkan2UebeAot\ncVtHdGjzF5EXgCNj7Po3YL6q9oxo+4WqHhbjGEer6icicgLwEnCRqn6Q6Lxm8zfKkVGjWoU+uDD6\nJUvy15+MEg7DpElO449FdTVs2QJXXukiiPKMr/H7gj8QgG7dEufsLyabf4dmH1W9WFXPiLE9A3wm\nIkd5JzwK2BTnGJ94jx8C9cCgFK7FMMqGq65K/LqoCQbhzTddjqD+/dvvX7EC1q51eYKGDMl9/6JI\npUoXFF8oaLo2/2cBvxDddcAz0Q1E5DAR6eY9Pxw4H1id5nkNoyQJhVrzp9XUpG/yKciQxFDIOXsT\nrQtYsSKvpSPDYfjHP1y67YoK6NIFTjgB3n67AO9nZ1HVTm9AFfAi8D7wAtDLe38w8LD3/DxcmOdb\n3uONyRz77LPPVsMwOk9Dg2qPHqoVFe6xoSHfPYrB5MmqJ52k2qePX9+m/TZ2bE67FHnfunZVHTPG\nPQYCrjuBQAHfT1UFVmoSMjYtzV9VG1X1IlXtp848tMV7f6Wq3uQ9b1DVAap6pvf4SDrnNAwjOYoi\nAmXWLFca8re/dbaVWCxcmNMcQZH3rakJvvrKPfo1GHzHb0HezxSwFb6GUaIUVTKyYBBeew369Im9\nf9mynA0A0fftqqvcoz82BQJFcD+TwPL5G0YJUygRKCmRqF5A376udvD48Vm9IP++VVW50M7ox0K+\nn8lG+5jwNwyj8AiHXZH4cBj27Wu/v0sXeOWVrA8AxbZqFzIY6mkYhpFzgkEn3B94IPb+ffvgG9/I\nqimoKHwmaWDC3zCKlIIM48w0fuxrdXV7h/DOna31Ampr074f0Z8vKp9JJzCzj2EUIcVqkkiL2lq3\nQjg6RQTQDLzFQH4QmMufK4PccENqboFCz9SZCmb2MYwSptRNEjEJheDVV2H48Ha7BBjIKl5pPo/p\ne6dQU9O2HGZHxLufxbZqNxVM+BtGEVLqJom4+L6AqNrB4m0BYCqz+blOSWlQLMf7aWYfwyhSitEk\nkVFqa+Huu2HjxjZvK84MtJdu7Kq+gF7Lk8uMVyr300I9DcMoD2IUjfGlmgCcdhqsLp90YmbzNwyj\nPFi+3NUOjigY45uBAFizBo46quCLxuQaE/6GYXRIwYeVzprlSkc2NEC/fu33b9zoisZMiVVltjwx\ns49hlBiZtl0XZVhp//5O448mEHA5hAr+AjqPmX0MowyJrjebCU29KMNKV6+OXy9g6lTo1QsOO6ys\nZwIm/A2jhEhWUKdixinaMMi6Orc6uG9fEHFafyDgVgV/8QVs3eqqhpXpAGBmH8MoIZIx0XTGjJOP\nMMiMntM/2IMPwieftN/fp4+rKVAC5qBkzT6VueiMYRi5IRh0wjyR0Kyvby1KvmePe92RzAsGcysX\n0/UztBs4/Avwtf1oNmxwOYIaGkpiAEgGM/sYRonRUUqCqqq2VamqqnLXt2SJNF/t2QPTpyfvv0jo\n95g1K3Ht4GuugSuuKOCwpsxhwt8wyozGxrZVqRob89ufWPh+hkDADVAvvJC8A7tDv0ddndPwY1UN\n27ABFi+G888v+XUBJvwNo8yoqmr1fXbrln0HbmfWCPjmq4svbh0Ako00SspBHQzC+vUuVbTfMBJV\nty6gb9+SHQTM4WsYJUg8Z6lvEtmzx8m8Bx5wyTKzdW5I33bfmc+n7CyurXXCPh41NZm/UVnCHL6G\nUaYkEpi+SaS52UU/ZtrkE33u665rb4JJVXh35MCORcoO6lAIPvggtjMY4Lbb3PRl2rSiGQQ6wsw+\nhlFiJLJ5ZztmP/rckPz5fPNQbW1bhy2knlO/U+koZs1yvoAY9QLYtw/WrXOzg1GjUjho4WKav2GU\nGL6A97XvSIGbTChoqkRq6dHnHj/ebR2dL3LGIOJmJpF2/lyYioDWegG1tfCLX8D777cvIL90KYwb\n5xzHRYwJf8MoMToS8JmM2Y8WtHPmOFMPtC2j2NH5ImcMgYCbKYh0bnYSa+YT6/wJ/QKhkNumTIlt\nClq40DmM+/dPrV5kAWHC3zBKkFQEfDoraaPj8W+91QXK+Fp/skTPGObMcf6IzvQp0czHJ+nZwaxZ\nbkXwwoXt9y1b5raaGnj99aIbAEz4G0YZk240TVVVq6ANBNwgEM9ck2iQyaQ5KtlVzkk7ouvq3Kg2\naRKsWtV+vypccAH86EdusCgSTPgbRhmTkhD0iGXqaWx0A8Htt8fWuDsaZDKdO6ijmY+/1sGfpXRo\nWgoG4c03nS9g0iR3wyLZt8+Zhx5+2HmaiyAiyIS/YZQwHQnVZEwk0UQPGI2NLhoHYMCA2OdLNMjk\nul5AOOwGKd+/MGdOCucLhdxFXnedcwZHs2VL63qBAh8ATPgbRomSjFCNNpGAU1wTaeAdRROlOsjE\nC03NVhbRtNc6BIPw3nsu4uepp2D37vZtJkyAn/2soDOFmvA3jBIlWZOOL7DDYbjwwlYB/fLL8dun\nap9P9Bl/YNizxwnjrVuzu6q3M7OdmNTVuW3UKBf+GU2hZwpV1YLczj77bDUMo/M0NKj26KFaUeEe\nGxoSt584UdVZwd02cWLnzjljRsfniqamRrVLF9VAoPURXN9nzEjuvKlca2f7GZeaGtX+/dveQH/r\n3dvtq6nJ0MkSA6zUJGSsaf6GUaJkY0FXItKx3Tc2ti7sAmeLTyXOP1XHdcbrE/jrAoYMgRUr2u7b\nvNltEya4FBIFEhGUVnoHEblaRN4RkWYRiZtISEQuEZG/i8haEZmazjkNw0iejnL7RzJ+vBO2vtBN\nJU4f0qv1G5l2ols3V3DrZz9LfgDpKG1Fp9I9dIbly+NnCgUXEXTBBQVRLyBdzf9vwJVATbwGIlIB\nPAh8A9gA/FlEnlXV1Wme2zCMDBIMOoHdmZlCOAz/+AdUehKlstK9DoeTC+lMdpbSmc/nOpqI5cvd\nY7xMocuWOV/AwIEwd27+/AHJ2IY62oB6YHCcfUFgScTracC0jo5pNn/DKA4i7e1du6qOGeMefft7\nTY2zr9fUxLbLJ2t/T9Wu7zNjhvtMKj6EjDF2bGw/QOQ2cmRGT0kB2fyPBtZHvN4ADInVUERCQAjg\n2GOPzX7PDMNIm0hzD8BXX7nnfsqH225rDauMTtgGyWvlnVmQBhmM7ukMdXUuS+jMmS4raCyWLnUR\nQ0uW5LBjSdj8ReQFEflbjO3yTHdGVWtVdbCqDu7du3emD28YRhaIrAzWtStcdVWr/T0QgP37WweD\nioq2dvlYAj2efb6z6ah9k1AqPoSMEgrBRx+5HEBHHhm7zdKlOa8d3KHmr6oXp3mOT4BjIl738d4z\nDKPIibVa1l8Eu2ABrF7tTNzgbBw/+hH07NnWLh+plVdVxZ8JpBO9lPHons7gRwTFWxeweDE88wzc\ncUdOIoJyUczlz0A/ETleRLoC3wGezcF5DcPIMpGrZVVd+puZM+Htt2H+fHj11da2gYAT/NHRR9dd\nBzff7AR7Y2PrTGD3bjeARJJK9FLBsmSJmwX06tV+n6qLCDr55OzPApJxDMTbgCtwNvw9wGd4jl3g\nX4DfRbT7FvAe8AHwb8kc2xy+hlF4RDtno5293bq555ELtUBVpL2TNpYDt6HBHcf/XLduGVyIVYjU\n1LibE8sRXFHRqYsnSYdvWpq/qj6tqn1UtZuqfk1VR3nv/1NVvxXR7neqerKqnqiqP0/nnIZhZI5U\n4t/9kEm/vKIfxunb02+4oa193y/K0rWri3iMtrfHc+DecINzDoM7XirrBYqOUAjmzWu94EiamrJ6\n8bbC1zDKlFTj3+MJ68jcQPPnJ1+QJV4UzvjxbY+T0+icfBAvU2hFRVYv3oS/YZQpqYZOdhQymapD\nNl77XKelKAj8TKFTpsATT8AJJ8C992b14sWZiAqPwYMH68qVK/PdDcMoWTqz8jXTRVeMzCMib6hq\n3HQ7Pqb5G0aZ0tnUzJkW+jag5AcT/oZRxvjC1vcr5lr4JltDwMg8JvwNo4zJRtKzVDT5BQtcCghw\njwsWmPDPFSb8DaOM6Wy+nHjkPIOm0WlyscLXMIwCpbP5cuKRak7/dGsIGJ3HNH/DKGMyHVYZHQ5a\nVZW4IHw6NQSM9LBQT8MwMopv86+qcknfzASUW5IN9TSzj2EYGcVPvhaZpC3Vso5G9jHhbxhG2sTK\nEZRpf4KRWczmbxhGWsSL8CnLNA1FhAl/wzDSIlG4aEEUUTFiYmYfwzDSwsw7xYlp/oZhpIWZd4oT\nE/6GYaSNmXeKDzP7GIZhlCEm/A3DMMoQE/6GYRhliAl/wzCMMsSEv2EYRhliwt8wDKMMKdisniKy\nGfi4kx8/HPg8g93JB8V+DcXefyj+ayj2/kPxX0M++n+cqvbuqFHBCv90EJGVyaQ0LWSK/RqKvf9Q\n/NdQ7P2H4r+GQu6/mX0MwzDKEBP+hmEYZUipCv/afHcgAxT7NRR7/6H4r6HY+w/Ffw0F2/+StPkb\nhmEYiSlVzd8wDMNIgAl/wzCMMqTkhL+IXCIifxeRtSIyNd/9SRUReVRENonI3/Ldl84gIseIyMsi\nslpE3hGRH+a7T6kiIt1FZIWIvOVdw0/z3afOICIVIvKmiDyf7750BhFZJyJvi8gqEVmZ7/6kioj0\nFJEnReRdEVkjIgWV9LqkbP4iUgG8B3wD2AD8Gfiuqq7Oa8dSQESGAzuABap6Rr77kyoichRwlKr+\nRUQOBt4AxhTZdyDAgaq6Q0S6AK8BP1TVP+W5aykhIj8GBgOHqOpl+e5PqojIOmCwqhblIi8RmQ+8\nqqoPi0hX4ABV3ZrvfvmUmuZfDaxV1Q9VdS/wa+DyPPcpJVR1GbAl3/3oLKr6qar+xXu+HVgDHJ3f\nXqWGOnZ4L7t4W1FpSSLSB7gUeDjffSlHRORQYDjwCICq7i0kwQ+lJ/yPBtZHvN5AkQmeUkJE+gKD\ngOX57UnqeCaTVcAm4I+qWmzXMAeYDDTnuyNpoMBSEXlDREL57kyKHA9sBh7zTG8Pi8iB+e5UJKUm\n/I0CQUQOAp4CblfVL/Pdn1RR1SZVHQj0AapFpGhMcCJyGbBJVd/Id1/SZKiqngV8E7jVM4kWC5XA\nWcBDqjoI2AkUlA+y1IT/J8AxEa/7eO8ZOcSzkz8FLFTV/8p3f9LBm6q/DFyS776kwPnAaM9m/mvg\nv4lIXX67lDqq+on3uAl4GmfWLRY2ABsiZoxP4gaDgqHUhP+fgX4icrznYPkO8Gye+1RWeM7SR4A1\nqvof+e5PZxCR3iLS03veAxdA8G5+e5U8qjpNVfuoal/cf+AlVR2X526lhIgc6AUM4JlLRgJFEwGn\nqhuB9SJyivfWRUBBBT1U5rsDmURV94vIbcASoAJ4VFXfyXO3UkJEFgEjgMNFZANwt6o+kt9epcT5\nwPeAtz2bOcCdqvq7PPYpVY4C5nvRYwHgt6palOGSRczXgKedLkEl8ISq/iG/XUqZHwALPUX0Q+D7\nee5PG0oq1NMwDMNIjlIz+xiGYRhJYMLfMAyjDDHhbxiGUYaY8DcMwyhDTPgbhmGUISb8DcMwyhAT\n/oZhGGXI/w++6U8tCYD1ygAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Wokallj1D21L", + "colab_type": "text" + }, + "source": [ + "Oh dear! The graph makes it clear that our network has learned to approximate the sine function in a very limited way. From `0 <= x <= 1.1` the line mostly fits, but for the rest of our `x` values it is a rough approximation at best.\n", + "\n", + "The rigidity of this fit suggests that the model does not have enough capacity to learn the full complexity of the sine wave function, so it's only able to approximate it in an overly simplistic way. By making our model bigger, we should be able to improve its performance.\n", + "\n", + "## Change our model\n", + "To make our model bigger, let's add an additional layer of neurons. The following cell redefines our model in the same way as earlier, but with an additional layer of 16 neurons in the middle:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "oW0xus6AF-4o", + "colab_type": "code", + "colab": {} + }, + "source": [ + "model_2 = tf.keras.Sequential()\n", + "\n", + "# First layer takes a scalar input and feeds it through 16 \"neurons\". The\n", + "# neurons decide whether to activate based on the 'relu' activation function.\n", + "model_2.add(layers.Dense(16, activation='relu', input_shape=(1,)))\n", + "\n", + "# The new second layer may help the network learn more complex representations\n", + "model_2.add(layers.Dense(16, activation='relu'))\n", + "\n", + "# Final layer is a single neuron, since we want to output a single value\n", + "model_2.add(layers.Dense(1))\n", + "\n", + "# Compile the model using a standard optimizer and loss function for regression\n", + "model_2.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Dv2SC409Grap", + "colab_type": "text" + }, + "source": [ + "We'll now train the new model. To save time, we'll train for only 600 epochs:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "DPAUrdkmGq1M", + "colab_type": "code", + "outputId": "34ad91e0-229b-479c-bd65-12ad1ed1c660", + "colab": { + "base_uri": "https://localhost:8080/" + } + }, + "source": [ + "history_2 = model_2.fit(x_train, y_train, epochs=600, batch_size=16,\n", + " validation_data=(x_validate, y_validate))" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Train on 600 samples, validate on 200 samples\n", + "Epoch 1/600\n", + "600/600 [==============================] - 0s 422us/sample - loss: 0.5655 - mae: 0.6259 - val_loss: 0.4104 - val_mae: 0.5509\n", + "Epoch 2/600\n", + "600/600 [==============================] - 0s 111us/sample - loss: 0.3195 - mae: 0.4902 - val_loss: 0.3341 - val_mae: 0.4927\n", + "...\n", + "Epoch 598/600\n", + "600/600 [==============================] - 0s 116us/sample - loss: 0.0124 - mae: 0.0886 - val_loss: 0.0096 - val_mae: 0.0771\n", + "Epoch 599/600\n", + "600/600 [==============================] - 0s 130us/sample - loss: 0.0125 - mae: 0.0900 - val_loss: 0.0107 - val_mae: 0.0824\n", + "Epoch 600/600\n", + "600/600 [==============================] - 0s 109us/sample - loss: 0.0124 - mae: 0.0892 - val_loss: 0.0116 - val_mae: 0.0845\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mc_CQu2_IvOP", + "colab_type": "text" + }, + "source": [ + "## Evaluate our new model\n", + "Each training epoch, the model prints out its loss and mean absolute error for training and validation. You can read this in the output above (note that your exact numbers may differ): \n", + "\n", + "```\n", + "Epoch 600/600\n", + "600/600 [==============================] - 0s 109us/sample - loss: 0.0124 - mae: 0.0892 - val_loss: 0.0116 - val_mae: 0.0845\n", + "```\n", + "\n", + "You can see that we've already got a huge improvement - validation loss has dropped from 0.15 to 0.015, and validation MAE has dropped from 0.31 to 0.1.\n", + "\n", + "The following cell will print the same graphs we used to evaluate our original model, but showing our new training history:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "SYHGswAJJgrC", + "colab_type": "code", + "outputId": "efcc51f6-f1f1-490a-ffba-ed283586f83e", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 851 + } + }, + "source": [ + "# Draw a graph of the loss, which is the distance between\n", + "# the predicted and actual values during training and validation.\n", + "loss = history_2.history['loss']\n", + "val_loss = history_2.history['val_loss']\n", + "\n", + "epochs = range(1, len(loss) + 1)\n", + "\n", + "plt.plot(epochs, loss, 'g.', label='Training loss')\n", + "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n", + "plt.title('Training and validation loss')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('Loss')\n", + "plt.legend()\n", + "plt.show()\n", + "\n", + "# Exclude the first few epochs so the graph is easier to read\n", + "SKIP = 100\n", + "\n", + "plt.clf()\n", + "\n", + "plt.plot(epochs[SKIP:], loss[SKIP:], 'g.', label='Training loss')\n", + "plt.plot(epochs[SKIP:], val_loss[SKIP:], 'b.', label='Validation loss')\n", + "plt.title('Training and validation loss')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('Loss')\n", + "plt.legend()\n", + "plt.show()\n", + "\n", + "plt.clf()\n", + "\n", + "# Draw a graph of mean absolute error, which is another way of\n", + "# measuring the amount of error in the prediction.\n", + "mae = history_2.history['mae']\n", + "val_mae = history_2.history['val_mae']\n", + "\n", + "plt.plot(epochs[SKIP:], mae[SKIP:], 'g.', label='Training MAE')\n", + "plt.plot(epochs[SKIP:], val_mae[SKIP:], 'b.', label='Validation MAE')\n", + "plt.title('Training and validation mean absolute error')\n", + "plt.xlabel('Epochs')\n", + "plt.ylabel('MAE')\n", + "plt.legend()\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzt3Xl8VOX1+PHPyQ4JEAhRtmBAEQg7\nRDQiJYgiasUflVpwQayI0rpUy1epK0WtuFQRS61LRVEUF6qioNSyiGhklUU2QQwS1hDWsGQ9vz/u\nzWQIWSaQySTMeb9e88q9zzxz73nuTObM89xNVBVjjDEGICTQARhjjKk5LCkYY4zxsKRgjDHGw5KC\nMcYYD0sKxhhjPCwpGGOM8bCkYKqUiISKSLaItKzKuoEkIueISJUfuy0il4hIutf8BhHp7Uvdk1jX\nayLywMm+vpzlPi4ib1T1ck3ghAU6ABNYIpLtNVsXyAEK3PnbVHVqZZanqgVATFXXDQaq2rYqliMi\nI4AbVDXVa9kjqmLZ5vRnSSHIqarnS9n9JTpCVf9XVn0RCVPV/OqIzRhT/Wz4yJTLHR54T0TeFZFD\nwA0ikiIi34nIfhHZISITRSTcrR8mIioiie782+7zn4vIIRFJE5FWla3rPn+5iPwoIgdE5EUR+UZE\nhpcRty8x3iYim0Rkn4hM9HptqIg8LyJZIrIZGFDO9nlQRKaVKJskIs+50yNEZJ3bnp/cX/FlLStD\nRFLd6boi8pYb2xqgR4m6D4nIZne5a0RkoFveCfgH0NsdmtvjtW3Her3+drftWSLysYg09WXbVERE\nBrnx7BeRuSLS1uu5B0Rku4gcFJH1Xm29QESWu+W7ROQZX9dn/EBV7WEPVBUgHbikRNnjQC5wFc6P\niDrAecD5OD3N1sCPwB1u/TBAgUR3/m1gD5AMhAPvAW+fRN0zgEPA1e5z9wJ5wPAy2uJLjJ8ADYBE\nYG9R24E7gDVACyAOWOD8q5S6ntZANhDttezdQLI7f5VbR4CLgaNAZ/e5S4B0r2VlAKnu9LPAfKAh\ncBawtkTda4Gm7ntynRvDme5zI4D5JeJ8GxjrTvd3Y+wKRAH/BOb6sm1Kaf/jwBvudHs3jovd9+gB\nYIM73QHYAjRx67YCWrvTS4Ch7nQ94PxA/y8E88N6CsYXC1X1U1UtVNWjqrpEVRepar6qbgZeAfqU\n8/oPVXWpquYBU3G+jCpb99fAClX9xH3ueZwEUiofY3xSVQ+oajrOF3DRuq4FnlfVDFXNAsaXs57N\nwA84yQrgUmCfqi51n/9UVTerYy4wByh1Z3IJ1wKPq+o+Vd2C8+vfe73vq+oO9z15ByehJ/uwXIDr\ngddUdYWqHgPGAH1EpIVXnbK2TXmGADNUda77Ho3HSSznA/k4CaiDOwT5s7vtwEnubUQkTlUPqeoi\nH9th/MCSgvHFVu8ZEWknIjNFZKeIHATGAY3Lef1Or+kjlL9zuay6zbzjUFXF+WVdKh9j9GldOL9w\ny/MOMNSdvs6dL4rj1yKySET2ish+nF/p5W2rIk3Li0FEhovISneYZj/QzsflgtM+z/JU9SCwD2ju\nVacy71lZyy3EeY+aq+oG4M8478NudziyiVv1ZiAJ2CAii0XkCh/bYfzAkoLxRcnDMV/G+XV8jqrW\nBx7BGR7xpx04wzkAiIhw/JdYSacS4w4gwWu+okNm3wcuEZHmOD2Gd9wY6wAfAk/iDO3EAv/1MY6d\nZcUgIq2Bl4BRQJy73PVey63o8NntOENSRcurhzNMtc2HuCqz3BCc92wbgKq+raq9cIaOQnG2C6q6\nQVWH4AwR/h2YLiJRpxiLOUmWFMzJqAccAA6LSHvgtmpY52dAdxG5SkTCgLuBeD/F+D7wJxFpLiJx\nwP3lVVbVncBC4A1gg6pudJ+KBCKATKBARH4N9KtEDA+ISKw453Hc4fVcDM4XfyZOfrwVp6dQZBfQ\nomjHeineBW4Rkc4iEonz5fy1qpbZ86pEzANFJNVd9//h7AdaJCLtRaSvu76j7qMQpwE3ikhjt2dx\nwG1b4SnGYk6SJQVzMv4M3ITzD/8yzg5hv1LVXcDvgOeALOBs4Huc8yqqOsaXcMb+V+PsBP3Qh9e8\ng7Pj2DN0pKr7gXuAj3B21g7GSW6+eBSnx5IOfA5M8VruKuBFYLFbpy3gPQ7/JbAR2CUi3sNARa//\nAmcY5yP39S1x9jOcElVdg7PNX8JJWAOAge7+hUjgaZz9QDtxeiYPui+9AlgnztFtzwK/U9XcU43H\nnBxxhmaNqV1EJBRnuGKwqn4d6HiMOV1YT8HUGiIywB1OiQQexjlqZXGAwzLmtGJJwdQmFwGbcYYm\nLgMGqWpZw0fGmJNgw0fGGGM8rKdgjDHGo9ZdEK9x48aamJgY6DCMMaZWWbZs2R5VLe8wbqAWJoXE\nxESWLl0a6DCMMaZWEZGKzswHbPjIGGOMF0sKxhhjPCwpGGOM8ah1+xSMMdUrLy+PjIwMjh07FuhQ\njA+ioqJo0aIF4eFlXfqqfJYUjDHlysjIoF69eiQmJuJcnNbUVKpKVlYWGRkZtGrVquIXlMKGj4wx\n5Tp27BhxcXGWEGoBESEuLu6UenVBkxTStqbx5NdPkrY1LdChGFPrWEKoPU71vQqK4aO0rWn0m9KP\n3IJcIkIjmDNsDikJKYEOyxhjapyg6CnMT59PbkEuBVpAbkEu89PnBzokY4yPsrKy6Nq1K127dqVJ\nkyY0b97cM5+b69ttF26++WY2bNhQbp1JkyYxderUqgiZiy66iBUrVlTJsqpbUPQUUhNTiQiN8PQU\nUhNTAx2SMcZHcXFxni/YsWPHEhMTw+jRo4+ro6qoKiEhpf/OnTx5coXr+eMf/3jqwZ4GgqKnkJKQ\nwpxhc3is72M2dGRMNaiOfXibNm0iKSmJ66+/ng4dOrBjxw5GjhxJcnIyHTp0YNy4cZ66Rb/c8/Pz\niY2NZcyYMXTp0oWUlBR2794NwEMPPcSECRM89ceMGUPPnj1p27Yt3377LQCHDx/mmmuuISkpicGD\nB5OcnFxhj+Dtt9+mU6dOdOzYkQceeACA/Px8brzxRk/5xIkTAXj++edJSkqic+fO3HDDDVW+zXwR\nFD0FcBKDJQNj/K869+GtX7+eKVOmkJycDMD48eNp1KgR+fn59O3bl8GDB5OUlHTcaw4cOECfPn0Y\nP3489957L6+//jpjxow5YdmqyuLFi5kxYwbjxo3jiy++4MUXX6RJkyZMnz6dlStX0r1793Ljy8jI\n4KGHHmLp0qU0aNCASy65hM8++4z4+Hj27NnD6tWrAdi/fz8ATz/9NFu2bCEiIsJTVt2CoqdgjKk+\n1bkP7+yzz/YkBIB3332X7t270717d9atW8fatWtPeE2dOnW4/PLLAejRowfp6emlLvs3v/nNCXUW\nLlzIkCFDAOjSpQsdOnQoN75FixZx8cUX07hxY8LDw7nuuutYsGAB55xzDhs2bOCuu+5i9uzZNGjQ\nAIAOHTpwww03MHXq1JM++exUWVIwxlSpon14oRLq93140dHRnumNGzfywgsvMHfuXFatWsWAAQNK\nPV4/IiLCMx0aGkp+fn6py46MjKywzsmKi4tj1apV9O7dm0mTJnHbbbcBMHv2bG6//XaWLFlCz549\nKSgoqNL1+sKSgjGmSgVqH97BgwepV68e9evXZ8eOHcyePbvK19GrVy/ef/99AFavXl1qT8Tb+eef\nz7x588jKyiI/P59p06bRp08fMjMzUVV++9vfMm7cOJYvX05BQQEZGRlcfPHFPP300+zZs4cjR45U\neRsqEjT7FIwx1ScQ+/C6d+9OUlIS7dq146yzzqJXr15Vvo4777yTYcOGkZSU5HkUDf2UpkWLFjz2\n2GOkpqaiqlx11VVceeWVLF++nFtuuQVVRUR46qmnyM/P57rrruPQoUMUFhYyevRo6tWrV+VtqEit\nu0dzcnKy2k12jKk+69ato3379oEOo0bIz88nPz+fqKgoNm7cSP/+/dm4cSNhYTXr93Vp75mILFPV\n5DJe4lGzWmKMMTVYdnY2/fr1Iz8/H1Xl5ZdfrnEJ4VSdXq0xxhg/io2NZdmyZYEOw69sR7MxxhgP\nSwrGGGM8LCkYY4zxsKRgjDHGw5KCMaZG69u37wknok2YMIFRo0aV+7qYmBgAtm/fzuDBg0utk5qa\nSkWHuE+YMOG4k8iuuOKKKrku0dixY3n22WdPeTlVzZKCMaZGGzp0KNOmTTuubNq0aQwdOtSn1zdr\n1owPP/zwpNdfMinMmjWL2NjYk15eTWdJwRhTow0ePJiZM2d6bqiTnp7O9u3b6d27t+e8ge7du9Op\nUyc++eSTE16fnp5Ox44dATh69ChDhgyhffv2DBo0iKNHj3rqjRo1ynPZ7UcffRSAiRMnsn37dvr2\n7Uvfvn0BSExMZM+ePQA899xzdOzYkY4dO3ouu52enk779u259dZb6dChA/379z9uPaVZsWIFF1xw\nAZ07d2bQoEHs27fPs/6iS2kXXYjvq6++8txkqFu3bhw6dOikt21p7DwFY4zP/vQnqOobinXtCu73\naakaNWpEz549+fzzz7n66quZNm0a1157LSJCVFQUH330EfXr12fPnj1ccMEFDBw4sMz7FL/00kvU\nrVuXdevWsWrVquMuff3EE0/QqFEjCgoK6NevH6tWreKuu+7iueeeY968eTRu3Pi4ZS1btozJkyez\naNEiVJXzzz+fPn360LBhQzZu3Mi7777Lq6++yrXXXsv06dPLvT/CsGHDePHFF+nTpw+PPPIIf/3r\nX5kwYQLjx4/n559/JjIy0jNk9eyzzzJp0iR69epFdnY2UVFRldjaFbOegjGmxvMeQvIeOlJVHnjg\nATp37swll1zCtm3b2LVrV5nLWbBggefLuXPnznTu3Nnz3Pvvv0/37t3p1q0ba9asqfBidwsXLmTQ\noEFER0cTExPDb37zG77++msAWrVqRdeuXYHyL88Nzv0d9u/fT58+fQC46aabWLBggSfG66+/nrff\nfttz5nSvXr249957mThxIvv376/yM6r92lMQkQHAC0Ao8Jqqji/x/HDgGWCbW/QPVX3NnzEZY05e\neb/o/enqq6/mnnvuYfny5Rw5coQePXoAMHXqVDIzM1m2bBnh4eEkJiaWernsivz88888++yzLFmy\nhIYNGzJ8+PCTWk6Rostug3Pp7YqGj8oyc+ZMFixYwKeffsoTTzzB6tWrGTNmDFdeeSWzZs2iV69e\nzJ49m3bt2p10rCX5racgIqHAJOByIAkYKiJJpVR9T1W7ug9LCMaYE8TExNC3b19+//vfH7eD+cCB\nA5xxxhmEh4czb948tmzZUu5yfvWrX/HOO+8A8MMPP7Bq1SrAuex2dHQ0DRo0YNeuXXz++eee19Sr\nV6/UcfvevXvz8ccfc+TIEQ4fPsxHH31E7969K922Bg0a0LBhQ08v46233qJPnz4UFhaydetW+vbt\ny1NPPcWBAwfIzs7mp59+olOnTtx///2cd955rF+/vtLrLI8/ewo9gU2quhlARKYBVwPl98mMMaYU\nQ4cOZdCgQccdiXT99ddz1VVX0alTJ5KTkyv8xTxq1Chuvvlm2rdvT/v27T09ji5dutCtWzfatWtH\nQkLCcZfdHjlyJAMGDKBZs2bMmzfPU969e3eGDx9Oz549ARgxYgTdunUrd6ioLG+++Sa33347R44c\noXXr1kyePJmCggJuuOEGDhw4gKpy1113ERsby8MPP8y8efMICQmhQ4cOnrvIVRW/XTpbRAYDA1R1\nhDt/I3C+qt7hVWc48CSQCfwI3KOqW0tZ1khgJEDLli17VPRrwBhTdezS2bXPqVw6O9A7mj8FElW1\nM/Al8GZplVT1FVVNVtXk+Pj4ag3QGGOCiT+TwjYgwWu+BcU7lAFQ1SxVzXFnXwN6+DEeY4wxFfBn\nUlgCtBGRViISAQwBZnhXEJGmXrMDgXV+jMcYc5Jq2x0ag9mpvld+29GsqvkicgcwG+eQ1NdVdY2I\njAOWquoM4C4RGQjkA3uB4f6KxxhzcqKiosjKyiIuLq7Mk8JMzaCqZGVlndIJbXaPZmNMufLy8sjI\nyDil4/ZN9YmKiqJFixaEh4cfV273aDbGVInw8HBatWoV6DBMNQn00UfGGGNqEEsKxhhjPCwpGGOM\n8bCkYIwxxsOSgjHGGA9LCsYYYzwsKRhjjPGwpGCMMcbDkoIxxhgPSwrGGGM8LCkYY4zxsKRgjDHG\nw5KCMcYYD0sKxhhjPCwpGGOM8bCkYIwxxsOSgjHGGA9LCsYYYzwsKRhjjPGwpGCMMcbDkoIxxhgP\nSwrGGGM8LCkYY4zxsKRgjDHGI2iSwsKF8PDDkJcX6EiMMabmCpqkkJYGjz8OOTmBjsQYY2ouvyYF\nERkgIhtEZJOIjCmn3jUioiKS7K9YwsOdv9ZTMMaYsvktKYhIKDAJuBxIAoaKSFIp9eoBdwOL/BUL\nWFIwxhhf+LOn0BPYpKqbVTUXmAZcXUq9x4CngGN+jIWwMOdvfr4/12KMMbWbP5NCc2Cr13yGW+Yh\nIt2BBFWdWd6CRGSkiCwVkaWZmZknFYz1FIwxpmIB29EsIiHAc8CfK6qrqq+oarKqJsfHx5/U+iwp\nGGNMxfyZFLYBCV7zLdyyIvWAjsB8EUkHLgBm+GtnsyUFY4ypmD+TwhKgjYi0EpEIYAgwo+hJVT2g\nqo1VNVFVE4HvgIGqutQfwVhSMMaYivktKahqPnAHMBtYB7yvqmtEZJyIDPTXestiScEYYyoW5s+F\nq+osYFaJskfKqJvqz1js6CNjjKlY0JzRbD0FY4ypmCUFY4wxHpYUjDHGeFhSMMYY42FJwRhjjEfQ\nJAU7+sgYYyoWNEnBegrGGFMxSwrGGGM8LCkYY4zxsKRgjDHGw5KCMcYYj6BJCnb0kTHGVCxokoL1\nFIwxpmKWFIwxxnhYUjDGGOMRNElBBEJDLSkYY0x5giYpgNNbsKRgjDFls6RgjDHGI6iSQliYHZJq\njDHlCaqkYD0FY4wpnyUFY4wxHkGVFApDjvH9ttWkbU0LdCjGGFMjBU1SSNuaxs4jW1mxbQ39pvSz\nxGCMMaUImqQwP30+GpKLFoSRW5DL/PT5gQ7JGGNqnKBJCqmJqUhoAWg4EaERpCamBjokY4ypccIC\nHUB1SUlIoW18Nhpdl8nD5pCSkBLokIwxpsbxqacgImeLSKQ7nSoid4lIrA+vGyAiG0Rkk4iMKeX5\n20VktYisEJGFIpJU+Sb4LjY6hpb1zrGEYIwxZfB1+Gg6UCAi5wCvAAnAO+W9QERCgUnA5UASMLSU\nL/13VLWTqnYFngaeq0zwlWWHpBpjTPl8TQqFqpoPDAJeVNX/A5pW8JqewCZV3ayqucA04GrvCqp6\n0Gs2GlAf4zkplhSMMaZ8vu5TyBORocBNwFVuWXgFr2kObPWazwDOL1lJRP4I3AtEABeXtiARGQmM\nBGjZsqWPIZ8oPByys0/65cYYc9rztadwM5ACPKGqP4tIK+CtqghAVSep6tnA/cBDZdR5RVWTVTU5\nPj7+pNdl1z4yxpjy+dRTUNW1wF0AItIQqKeqT1Xwsm04+x6KtHDLyjINeMmXeE6WDR8ZY0z5fD36\naL6I1BeRRsBy4FURqWin8BKgjYi0EpEIYAgwo8Ry23jNXgls9D30yrOkYIwx5fN1n0IDVT0oIiOA\nKar6qIisKu8FqpovIncAs4FQ4HVVXSMi44ClqjoDuENELgHygH04+yz8xpKCMcaUz9ekECYiTYFr\ngQd9XbiqzgJmlSh7xGv6bl+XVRUsKRhjTPl83dE8DucX/0+qukREWuPnoR5/sKRgjDHl83VH8wfA\nB17zm4Fr/BWUv9jRR8YYUz5fdzS3EJGPRGS3+5guIi38HVxVs56CMcaUz9fho8k4Rw41cx+fumW1\niiUFY4wpn69JIV5VJ6tqvvt4Azj5s8gCxJKCMcaUz9ekkCUiN4hIqPu4AcjyZ2D+YEnBGGPK52tS\n+D3O4ag7gR3AYGC4n2Lym/BwUIWCgkBHYowxNZNPSUFVt6jqQFWNV9UzVPX/UQuPPgp3L+FnvQVj\njCndqdyO894qi6KahLkH4NphqcYYU7pTSQpSZVFUk+1HfgZg4eYlAY7EGGNqplNJCn69IU5VS9ua\nxqTv/w7AoKlDSduaFuCIjDGm5ik3KYjIIRE5WMrjEM75CrXG/PT55IceAiA3J4T56fMDG5AxxtRA\n5V7mQlXrVVcg/paamEp4xA/kAuGF9UlNTA10SMYYU+OcyvBRrZKSkMIT/Z0bu/2z/+ukJKQEOCJj\njKl5giYpAHRNaA9AmwadAxyJMcbUTEGVFKKinL/HjgU2DmOMqamCKinUqeP8PXo0sHEYY0xNFVRJ\nwXoKxhhTvqBKCkU9BUsKxhhTuqBKCkU9BRs+MsaY0gVlUrCegjHGlC6okoLtaDbGmPIFVVKIjHT+\nWk/BGGNKF1RJISTESQyWFIwxpnRBlRQAwiPzWbBpqV0l1RhjShFUSSFtaxrZ7OK7zavoN6WfJQZj\njCkhqJLC/PT5EHEIzY0mtyDXLp9tjDEl+DUpiMgAEdkgIptEZEwpz98rImtFZJWIzBGRs/wZT2pi\nKhJxBHLrEREaYZfPNsaYEvyWFEQkFJgEXA4kAUNFJKlEte+BZFXtDHwIPO2veMC5fHbXlmfTKroD\nc4bNsctnG2NMCf7sKfQENqnqZlXNBaYBV3tXUNV5qnrEnf0OaOHHeABoFteARqFnWUIwxphS+DMp\nNAe2es1nuGVluQX4vLQnRGSkiCwVkaWZmZmnFFRMDGRnn9IijDHmtFUjdjSLyA1AMvBMac+r6iuq\nmqyqyfHx8ae0LksKxhhTtnLv0XyKtgEJXvMt3LLjiMglwINAH1XN8WM8gCUFY4wpjz97CkuANiLS\nSkQigCHADO8KItINeBkYqKq7/RiLR1FSUK2OtRljTO3it6SgqvnAHcBsYB3wvqquEZFxIjLQrfYM\nEAN8ICIrRGRGGYurMln5WygogK82fefvVRljTK3jz+EjVHUWMKtE2SNe05f4c/0lpW1N499r3wMm\ncPm/hzL3j+/YUUjGGOOlRuxori7z0+dTELMFgNx9Z9gZzcYYU0JQJYXUxFTCG+4EIOxQazuj2Rhj\nSgiqpJCSkMJHIycCMOrcv9nQkTHGlBBUSQFgQOfzqFMHQg+1CnQoxhhT4wRdUhCBM86AUzwx2hhj\nTktBlxQAoupls3jTJrufgjHGlBB0SSFtaxobjy5iQ9o5pI6ZYInBGGO8BF1SmJ8+n8LIfQDkvvOe\nHZZqjDFegi4ppCamIgVRx80bY4xxBF1SACC3XqAjMMaYGinoksL89PnHXQxvysopgQvGGGNqmKBL\nCqmJqYQPut0z/9rSN2xnszHGuIIuKaQkpHBlz3ZwxR8AyD/UkKe/8eutoY0xptYIuqQA0CSmCcT9\n6Mzsac+nP35qvQVjjCFIk8KwLsMIabLWmdnViUIttENTjTGGIE0KKQkpjO5/I0Tvgm09UZT9OfsD\nHZYxxgRcUCYFgNjIWGj3Cay7BjZexjPfPMP4mVO55hq7h7MxJngFbVJITUwlpP0nUBAJU79Ad7fj\ngTGh/Oc/8MkngY7OGGMCI2iTQkpCCgNSGxUXHGyB5tYFIDo6QEEZY0yABW1SAHio/x+KZw4mQG4M\n4Fxe2xhjglFQJ4WUhBRGfHiXM3MgAfKcLsKKXzYGMCpjjAmcoE4KAL/vORRidsCa38HheAA+XvVl\ngKMyxpjACPqkkJKQQpchH8OetrC/NQArf/nJTmYzxgSloE8KAC890hXOLu4daF5du/SFMSYoWVLA\n6S1ceufH0PEdpyA3hhkbZlhvwRgTdCwpuP76m2GEDL4R6mZCbgyFFFpvwRgTdPyaFERkgIhsEJFN\nIjKmlOd/JSLLRSRfRAb7M5aKpCSkMLDdQIjIhpz6oFhvwRgTdPyWFEQkFJgEXA4kAUNFJKlEtV+A\n4cA7/oqjMu678D4IPwKrboRPXrfegjEm6Pizp9AT2KSqm1U1F5gGXO1dQVXTVXUVUOjHOHyWkpDC\n2ef95MysuBmw3oIxJrj4Myk0B7Z6zWe4ZZUmIiNFZKmILM3MzKyS4MryxqR46PC+M5Pj7FsYMWOE\nJQZjTFCoFTuaVfUVVU1W1eT4+Hi/ruuixBQuuMI9o/mn/gCs3bOWPm/0scRgjDnt+TMpbAMSvOZb\nuGU13t9GXOJMvD8dMnoCkFeYZ/sXjDGnPX8mhSVAGxFpJSIRwBBghh/XV2X6nns+vxv9jTOzbpCn\n3PYvGGNOd35LCqqaD9wBzAbWAe+r6hoRGSciAwFE5DwRyQB+C7wsImv8FU9lTXumF22Tt8H6QaBO\nWSGFjPnfCUfWGmPMacOv+xRUdZaqnquqZ6vqE27ZI6o6w51eoqotVDVaVeNUtYM/46ms+0Y1h6y2\nsHqop2zBLwu4/3/3BzAqY4zxn1qxozlQrrsOuvQ8CB+/Cdt6eHoMz3zzjA0jGWNOS5YUyhEVBfNn\n1yeybh68uhT+7exnUJTrpl9nicEYc9qxpFCB2Fj4w63ObTrJuBDmPQoL/4/0A+n0ntzbEoMx5rRi\nScEHDz4I4RHuSddfjYX/PQ0KBVpgh6kaY04rlhR8EBcH27eV2FQHWgLwyYZPeGXZKwGIyhhjqp4l\nBR81bgzbt3sVzH0MCsJQlNs/u90SgzHmtGBJoRKaNoW8PAiNyIFVw+D73wNYYjDGnDYsKVRSWBi8\n9vF6Z+aHIbC9O2CJwRhzerCkcBKGX96F7hdvhvS+8Moy+No5y9kSgzGmtrOkcJJefqp18cycJ+Fg\nM8ASgzGmdrOkcJKSk2Gr990intsGK26EQkFRbvvsNrschjGm1rGkcApatIDCQjin2w6n4OMpMK4Q\nVjnXSnr6m6fp+q+udoKbMabWsKRwikRg8ZymjH51BsT+7BSuHOZ5fuWulfR6vZcNJxljagVLClWg\nYUN4ZsRAXvxsLjRfBLs7wdsz4ccrAGw4yRhTa1hSqEJ39LqFP4+Kh0PNYdMV8M5MJzns7Aw4w0lt\nJrZh1GejbEjJGFMjiaoGOoZKSU5O1qVLlwY6jDIVFsJzz8F/VswhbWq/4if+kARnrPPMhkgIL135\nEiN7jAxAlMaYYCMiy1Q1uaJ0dm+9AAAVTklEQVR61lOoYiEhMHo0fPt2P56b82bxE5O/hr2tnF7D\n4TgKtZDbPruNPm/0sV6DMabGsKTgR/dcfBOvfbqKxFtHw9E4mLgZ/rUS3v3UU2fBlgVc+PqFlhyM\nMTWCJQU/u+XXnfn5lWfp+muvL/yMFPji7zB5vufchgXpTnJo+vemDHpv0AkJIisLLrwQNm2q3vir\n2+LFzhFdK1YEOhJjgpMlhWqy7JMU3p33PYmXfOEUfHcvbOlTfG7Dp6/A8pvZ+dWv+Xj9x1z4+oW0\neqGV51DWadMgLQ2eesp5eVYW1LLdQT756CPn78yZgY3DmGBlSaGahITAkNRu/PzlAL5YtYQ6Tbcc\nX2H5rTDjdfj0VZj5IqweQvr+dG777Dbino5jzKwnAPh+79eMnvYSjRvDv/7lvHTmTHjggWpu0Gnu\nhx+cHss33wQ2jn//Gx57zJneuhW+/z6w8VSHjz+GKVMCHUXwsqQQAJd1Oo/sjLP4Zksa5z15HQy6\nAWK8btaw5A6Y/i68vAQW/4G9R/eSnRkLwLKtq/j7jFkA3PXs1zT9e1MG/ymNJ5/Ko+fLKX4/Se7w\nYaeXUpUWL4bQUOdLr6j3I1K166isop7K9OnVt87CQucLsbCwuGzECHjkEWe6dWvo3r3i5ezcCR98\nUPXxbdwI8+eX/pwqTJwIGzac+noGDYKbbjr15VS1os9mTo5vvfQjR8reXr748kvnR0F1C6v+VRpw\neg4Xtkxh8ZgU0ram8ddPH2JZxmr2vDALjsQ7lXYkO4/dHSE91Sk71NRz17f8zb3Z+a/X4efuUBjO\nkg0ZLNl5G6P/O5rQTQMJT1hJaL09J6w7KiyK2KhYcvJziI+Op1FUI5rENKFb025kHckiNTGVC1qk\nIOIcXnvoEDz6qPPa4cPhww8hM9O58VBlZGfDV1/BlVceXz5xovNF+OWXxV+IeXmVW3ZVWrYMvv3W\nmY6IKLve4cMQEwOTJzvbpTIee8y5TMrNNxeXvfoq3H576cvLyYH8fGc6P9+5hHtZBg2C776D3bsh\nNxfeew/uucdJtA88AJ06wdChlYsX4Nxznb+qTuI54wzncwzw3//C3XfDgAHw+eeVX3ZlrF7t9OTK\na8Mvvzh3TIyOLn9Ze/dCo0bOdF4e/OMfzntQp87x9XbudO6n8uKLcOedzmf2zjvLX/aYMU791auh\nY0enLDe3/M+Ut/79nb+33OJb/api5ynUMG9/uZKxz2/hQNfH2DPzTudmPr7qNR7OXOXsyF58J7Rc\nAJEHoctb0GYmRB6GbT1g/lj47e+gMAxUICwHwo85y8hsB5Oc8ynajR3E+rHOIH+TZ5sCsHO0c52n\nelc/TN0L3+Dgx48T3et1wpr8eFwoBYcaE1J3HxJa4Ck7NHs0h7/8M/EPnEdoowxP+f63/8mxFYOo\nP3g0R5dfQ97mFKJbr6Aweid1+48nvGnxz8/8rJYUHm5EREtnT3T+znMpzIkmvNlaoqKE+tKM1X+e\nS/2rH6Huhc4YRFES3Hd0HzkFOaVuuoJ9zQltuI2osCjS//Szpzwk6hAtxiWTG3LwuPpRYVFEZiWz\n4bEPkDr7OPOxJE98BVlnEd5sLVLnABKa76mfUC+RY7ubsjvqW7bckw5A24eu5WDeXvJDD3Dwk8fI\nWTOAMy6bTJ0Bf+XYMdj1F6fe2Y9ezk9/db5t4x9MJrThNk8s2XPv4PDcOznjsbaIwK6H16FHY4+L\nN+zMDTS87Voyx60EIHFCq+O2SVRYFC0btASFn77twqFFgwk561uiUl8AoPBILLsfcT4XzR/qw7bH\nvyK82Voiz1rB2cOeYfPUuzm0cDgRbb6i0W1DSt3GAFooIIpI6T9Ofl7ZgpXjXwQg4W+dyPxwLPUG\njHc+L1sv5JyO+1l46xwAer/Wl18O/ex5T4uWl3XgCFvv30BUu69o96c/kZOfQ2RY5Al/9yy/kIxX\nJtHk7kFEtVpB2LI72fTWvcT+eryn3Y3qNKJbk27Mn9WYbf+eULw9m/1A43svLbWNRXGsnzCBY+v7\nULfPS0Sn/hPS+5D55j84b/wQdkd+d0Lcx/Kc7XB48TWEH0lg0ZRBALQZfyHRsUfJyc+hbeO23Hfh\nfaQkpJS5jcvi63kKlhRqsLStadz36kyWfnApOXuaoe3fg68fOrmFhR+G9tOLk0ziPOd+EN7i10Bm\nh+L5ix+Euc6+DK68HVosgtfSoCDKKQvJhcII55pPf+gAB86CuA2QVxeezC5ezpWjnLLvb4ZM9ydT\nvzFOjyg0FzZeDru6Qp0s59Ddknr8Czq+ByF5MHmhU/ZICBxqBs+7yaXr6xCdCd/cX9zeZkuh3nY4\nkAA3DIDDZ0Kjzc7zm/rD3nOgzSz45SL46C0Y1s/plX3xwvHrP28SnD8RGh+f+NhwJbz7mTN9wXNO\nIi4MP77OgLug5yQIKYS0u2H2BBhyNUz75MR2Fmm6zEni6/8f7HbOhueGy+Dt2e50fzjny+L6Y93/\n4bvOdtr3VCYcLaUbF3YU8t2fwI8KZLaHsGNQb4fzo+CT1+DwGfDjVcWvOX8CXH4PLL0VPnOHJkNz\noCCyuM5FT8LCvzjTIXnQfzScsRri10K9XVAQCiEFsKM7vDkHukyBK+526heEAQqhBc70Y15dxMv+\n5GyvDu8563h5BaQ8C2mjnefbfQTbe0CHDyCrDeQ0gPb/gfoZ8L477ndLCix4EK65Hva0gxaLoVDg\nWCx8McH5f+j7MBxtBN/dU7zu+xtCnf1wpCG8O8Mp23pR8fNnfeV8ruM2Qmg+rB4CaffCTX0hJN/5\nofXOp7Dx1ye+D4OvdT4nEYcgcb5TtrcNvPVfaPgTbO95fP2bL4KzvoH9CVA/g/CwML4a/lWlE4Ov\nSQFVrVWPHj16aLB6eenLmvzPFG1+7VPafGyynjk+QSPPe1O57B6l/QfO48Kn1Ong+/NRcGJZVJbz\nN/xQ1a2n7u6yn7v0z0riHN+XFb3D+XvxX5QR551cPNcOUvrfq3SZrDT/Tolb79vrovYqiXOL52O2\n+b7OJsudv96vR5Ub+znxtPm0uKzbq05sviw35ZnS37/SHpfcp4TkVH57RRxQhvdW6m9xlt9w4/HP\n19mjhB5VWi5wtmv9LVXzuQk7cmJZg3Tnb7NFSt1dFS8jJFfpf4/S98Hy68VsV4ZeWTzf/RUlcr9y\n9udK9M4ylu21LaN3KmGHy19Hp7eVK/6ghGcrV4xSxqJ/W/C3Sn9/AEtVK/6OrbBCTXsEc1Ioy7e/\nfKt/W/A3ve/L+7T9P9rrmc800TPGdtCGI6/VMx4/W5s820QbjzlfG/3xKq1z2WPKpaOVbq8pI3oq\nHd5VOr2lnP+8Muh655/3/OeVnhOVRhuUzlOcD2bsZiXpPeeL595mzoc/9GjxF2N4tpLyrCJ5xf+Y\nt3d2llX0hSP5Ff8ztvnM+Rv/gxOP559vW/EXzXFfAIeVGy4t/Usr9qfyk1STZU78JesULavXk0rT\nJb5/GXWceuKXLao0XuPEUtprQo86ybxo/vbOSs8XlF7jlSFXKQ9FKO3+U/x8ZeKJ2e5s+zp7ir9w\ny/qiQp0vssvuVvo86sx3/ffx2/KONkrbj4rLuv7bibFo3rsdZT3a/efkEkx5jzNXnPhF3/fBstfT\ncJNXG153PrPxq5WR3YuTcGmPq29ytmnC18Wfec/7eMx5lHzNpX9W7m+g/P5CZ/kl68RuLn1dceuV\nRj8qPf5VXNZ6tnJPcw0fF67f/vJtpb8nfE0Kfh0+EpEBwAtAKPCaqo4v8XwkMAXoAWQBv1PV9PKW\nGUzDR/6StjWN+enziasbx/c7vmdn9k72Ht1L5pFMIsMi2Xd0HyJS4Th8kdyfUghr9gMhdQ6Rn9ma\nkJhMJOLIcfsTCo80oPBwHIWHGxFSfyd6OI7Qxj+jOTFI+FGnUmgeIVHZRIYW7wM4tKYXEW3nA8qx\n5YOJ7PQZmhuN5kVBfiSE5hEW9wtHsxpzYFN7Z/iq4WZiw5oS1eAQuQW57N2XD8tuhZYLiT7agZCD\nLQlPWEnE2c4JgloYQv72DhQei6H+uStpWLc+Ow7sYtdR9z4ZBWHEFrRFMjtRmN0YaT2X/fnbYOMV\nEJpL7LlrCTl0lmc/x7HVVxDWbDWhDbehR2PJj3K2L0cbOMM1e9tQN6IOR9jjDHNF7yY2P4mohntL\n3f+RfTCE7FWXOLF0nUL0wa6E7+tESN0DhDbYTsG+BCTyMHnbOhLZbi6aH0FovUwIP3rcUVyFR+sh\n4Tkc+akbh7Y3hYIIqLedmHbfkf1TJzj7vxBagCBo1tnQaBPRP44g5HAzorpNJ7RelrM994Q4Bzw0\nWe0seM01ELeRhi13EpLZmdBGW5zPQlZLctZcRkTiYrQwnLAzNhFSdz+qcOTHCzgUsgUa/QjLboM9\nbeHcmdBuBuSHw4KHoelyGpz1C2G5cRSeuZx9x7Jg23nOMGD4Eee9DsuhUeNCCvY158CPnaDVXMhu\nQr2zfuLQj92cYaGWC53PSnZTZz9awnfOvrW8upD4NRxuTGx0NAf4BS0IgbxoYsOacXR1f3JyC6HT\nO1AYSr1Gx4iOcPZca0EYhOSjRxpybOVVhJ+1HKSQ7G+vJ+fcd+CHa6HOXhpeMYHISHG229G9sKML\nZLWF/Cganj+LyPAwQrOSkMNnkhG6wLmYZmguNPE6e3P9/4OCcEj6kF+16s34fuNr5z4FEQkFfgQu\nBTKAJcBQVV3rVecPQGdVvV1EhgCDVPV35S3XkoIpS1GyS01MPe6fpqzyk13eySwzbWsaU1Y6O76H\ndRlGSkJKpZbxyrJXmL52OtckXVMlF1EsuW7veaDcuCpT15cY4urGkXUk64S/pb2PU1ZOYWf2TgCa\nxDTxbMvy2lTW8kuup7T342S2e3mfw5KfgZKvK2pf0Y+0to3bcvk5l5e6PSqrJiSFFGCsql7mzv8F\nQFWf9Koz262TJiJhwE4gXssJypKCMcZUXk24SmpzwPsuxhluWal1VDUfOACccPiJiIwUkaUisjQz\nM9NP4RpjjKkVZzSr6iuqmqyqyfHx8YEOxxhjTlv+TArbgASv+RZuWal13OGjBjg7nI0xxgSAP5PC\nEqCNiLQSkQhgCDCjRJ0ZwE3u9GBgbnn7E4wxxviX3659pKr5InIHMBvnkNTXVXWNiIzDOV52BvBv\n4C0R2QTsxUkcxhhjAsSvF8RT1VnArBJlj3hNHwN+688YjDHG+K5W7Gg2xhhTPWrdBfFEJBPYUmHF\n0jUGTryWdO10urTldGkHWFtqKmuL4yxVrfDwzVqXFE6FiCz15eSN2uB0acvp0g6wttRU1pbKseEj\nY4wxHpYUjDHGeARbUvDvDYyr1+nSltOlHWBtqamsLZUQVPsUjDHGlC/YegrGGGPKYUnBGGOMR1Ak\nBREZICIbRGSTiIwJdDwVEZHXRWS3iPzgVdZIRL4UkY3u34ZuuYjIRLdtq0Ske+AiP5GIJIjIPBFZ\nKyJrRORut7zWtUdEokRksYisdNvyV7e8lYgscmN+z73WFyIS6c5vcp9PDGT8JYlIqIh8LyKfufO1\ntR3pIrJaRFaIyFK3rNZ9vgBEJFZEPhSR9SKyTkRSqrstp31SEOcOcJOAy4EkYKiIJAU2qgq9AQwo\nUTYGmKOqbYA57jw47WrjPkYCL1VTjL7KB/6sqknABcAf3e1fG9uTA1ysql2ArsAAEbkAeAp4XlXP\nAfYBt7j1bwH2ueXPu/VqkruBdV7ztbUdAH1VtavXMfy18fMFzu2Lv1DVdkAXnPenetviy42ca/MD\nSAFme83/BfhLoOPyIe5E4Aev+Q1AU3e6KbDBnX4Z5zanJ9SriQ/gE5xbtNbq9gB1geXA+ThnmIaV\n/LzhXAwyxZ0Oc+tJoGN342mB8wVzMfAZILWxHW5M6UDjEmW17vOFc+uAn0tu2+puy2nfU8C3O8DV\nBmeqqnsneXYCZ7rTtaZ97rBDN2ARtbQ97pDLCmA38CXwE7BfnTsHwvHx+nRnwQCZANwHFLrzcdTO\ndgAo8F8RWSYiRTdSro2fr1ZAJjDZHdZ7TUSiqea2BENSOO2o87OgVh1LLCIxwHTgT6p60Pu52tQe\nVS1Q1a44v7R7Au0CHFKlicivgd2quizQsVSRi1S1O85wyh9F5FfeT9aiz1cY0B14SVW7AYcpHioC\nqqctwZAUfLkDXG2wS0SaArh/d7vlNb59IhKOkxCmqup/3OJa2x4AVd0PzMMZZokV586BcHy8NfXO\ngr2AgSKSDkzDGUJ6gdrXDgBUdZv7dzfwEU6yro2frwwgQ1UXufMf4iSJam1LMCQFX+4AVxt436Xu\nJpyx+aLyYe6RCBcAB7y6mgEnIoJzM6V1qvqc11O1rj0iEi8ise50HZx9I+twksNgt1rJttS4Owuq\n6l9UtYWqJuL8P8xV1eupZe0AEJFoEalXNA30B36gFn6+VHUnsFVE2rpF/YC1VHdbAr1zpZp24FwB\n/Igz/vtgoOPxId53gR1AHs6vh1twxnDnABuB/wGN3LqCc3TVT8BqIDnQ8Zdoy0U43d1VwAr3cUVt\nbA/QGfjebcsPwCNueWtgMbAJ+ACIdMuj3PlN7vOtA92GUtqUCnxWW9vhxrzSfawp+v+ujZ8vN76u\nwFL3M/Yx0LC622KXuTDGGOMRDMNHxhhjfGRJwRhjjIclBWOMMR6WFIwxxnhYUjDGGONhScEYl4gU\nuFfaLHpU2RV1RSRRvK56a0xNFVZxFWOCxlF1LmFhTNCynoIxFXCv1/+0e83+xSJyjlueKCJz3WvZ\nzxGRlm75mSLykTj3XVgpIhe6iwoVkVfFuRfDf92zohGRu8S538QqEZkWoGYaA1hSMMZbnRLDR7/z\neu6AqnYC/oFzhVGAF4E3VbUzMBWY6JZPBL5S574L3XHOtAXnuveTVLUDsB+4xi0fA3Rzl3O7vxpn\njC/sjGZjXCKSraoxpZSn49xcZ7N7cb+dqhonIntwrl+f55bvUNXGIpIJtFDVHK9lJAJfqnOjFETk\nfiBcVR8XkS+AbJzLGnysqtl+bqoxZbKegjG+0TKmKyPHa7qA4n16V+Jcw6Y7sMTrSqXGVDtLCsb4\n5ndef9Pc6W9xrjIKcD3wtTs9BxgFnpvyNChroSISAiSo6jzgfpzLUp/QWzGmutgvEmOK1XHvqlbk\nC1UtOiy1oYiswvm1P9QtuxPnLln/h3PHrJvd8ruBV0TkFpwewSicq96WJhR4200cAkxU514NxgSE\n7VMwpgLuPoVkVd0T6FiM8TcbPjLGGONhPQVjjDEe1lMwxhjjYUnBGGOMhyUFY4wxHpYUjDHGeFhS\nMMYY4/H/AZN6yxQ6gTLNAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzsnXl8VNXZ+L/PvVlARWij1oVAcGeT\nLaIpImHR4q7V1rVBQBHcSm1fK77S8qoVpVqpSy2oUOJS608/UjfckJFtQHapC4IaSEQUUkEQSTJz\nn98fd+5kZjKTTDKZbJwvn3yYuXPuveeee+95zrOc54iqYjAYDAZDQ7GauwIGg8FgaN0YQWIwGAyG\nlDCCxGAwGAwpYQSJwWAwGFLCCBKDwWAwpIQRJAaDwWBICSNIDM2OiNgiskdEujRm2eZERI4VkUaP\nrReRESJSEvF9g4gMTqZsA871hIjc3tD9aznu3SLyj8Y+rqH5yGjuChhaHyKyJ+LrAUAFEAx9v05V\nn6nP8VQ1CBzU2GX3B1T1hMY4johcA1ylqoURx76mMY5taPsYQWKoN6oa7shDI95rVPWdROVFJENV\nA01RN4PB0PQY05ah0QmZLv4lIv8Ukd3AVSJSICLLRGSniHwlIg+JSGaofIaIqIjkhb4/Hfp9nojs\nFhG/iHSrb9nQ72eJyKcisktEHhaRJSJydYJ6J1PH60Rkk4h8KyIPRexri8iDIlIuIp8DI2tpn/8V\nkeditj0qIn8Jfb5GRD4OXc9nIW0h0bHKRKQw9PkAEXkqVLcPgQExZe8Qkc9Dx/1QRM4Pbe8NPAIM\nDpkNd0S07ZSI/ceHrr1cROaKyBHJtE1diMhFofrsFJF3ReSEiN9uF5GtIvKdiHwSca2nisjq0Pav\nReTPyZ7PkAZU1fyZvwb/ASXAiJhtdwOVwHm4g5X2wMnAKbha8NHAp8CNofIZgAJ5oe9PAzuAfCAT\n+BfwdAPKHgbsBi4I/XYLUAVcneBakqnjv4GOQB7wX+/agRuBD4HOQA6w0H294p7naGAPcGDEsb8B\n8kPfzwuVEWAY8ANwUui3EUBJxLHKgMLQ5/sBH/AjoCvwUUzZXwJHhO7JFaE6/CT02zWAL6aeTwNT\nQp/PDNWxL9AO+BvwbjJtE+f67wb+EfrcPVSPYaF7dDuwIfS5J7AZODxUthtwdOjzCuDy0OcOwCnN\n/S7sz39GIzGki8Wq+oqqOqr6g6quUNXlqhpQ1c+BmcCQWvZ/QVVXqmoV8AxuB1bfsucCa1X136Hf\nHsQVOnFJso5TVXWXqpbgdtreuX4JPKiqZapaDtxby3k+B/6DK+AAzgC+VdWVod9fUdXP1eVdYD4Q\n16Eewy+Bu1X1W1XdjKtlRJ73eVX9KnRPnsUdBOQncVyAK4EnVHWtqu4DbgOGiEjniDKJ2qY2LgNe\nVtV3Q/foXlxhdAoQwBVaPUPm0S9CbQfugOA4EclR1d2qujzJ6zCkASNIDOmiNPKLiJwoIq+JyDYR\n+Q64Eziklv23RXzeS+0O9kRlj4ysh6oq7gg+LknWMalz4Y6ka+NZ4PLQ5ytC3716nCsiy0XkvyKy\nE1cbqK2tPI6orQ4icrWIrAuZkHYCJyZ5XHCvL3w8Vf0O+BY4KqJMfe5ZouM6uPfoKFXdAPwW9z58\nEzKVHh4qOhroAWwQkfdF5Owkr8OQBowgMaSL2NDXGbij8GNV9WDgD7imm3TyFa6pCQAREaI7vlhS\nqeNXQG7E97rCk58HRojIUbiaybOhOrYHXgCm4pqdOgFvJVmPbYnqICJHA48BE4Cc0HE/iThuXaHK\nW3HNZd7xOuCa0L5Mol71Oa6Fe8++BFDVp1V1EK5Zy8ZtF1R1g6pehmu+fAB4UUTapVgXQwMxgsTQ\nVHQAdgHfi0h34LomOOerQH8ROU9EMoBfA4emqY7PAxNF5CgRyQF+X1thVd0GLAb+AWxQ1Y2hn7KB\nLGA7EBSRc4Hh9ajD7SLSSdx5NjdG/HYQrrDYjitTr8XVSDy+Bjp7wQVx+CcwVkROEpFs3A59kaom\n1PDqUefzRaQwdO7/wfVrLReR7iIyNHS+H0J/Du4F/EpEDglpMLtC1+akWBdDAzGCxNBU/BYYhdtJ\nzMB1iqcVVf0auBT4C1AOHAOswZ330th1fAzXl7Ee1xH8QhL7PIvrPA+btVR1J/Ab4CVch/UluAIx\nGf6IqxmVAPOA4ojjfgA8DLwfKnMCEOlXeBvYCHwtIpEmKm//N3BNTC+F9u+C6zdJCVX9ELfNH8MV\nciOB80P+kmxgGq5faxuuBvS/oV3PBj4WNyrwfuBSVa1MtT6GhiGu2dhgaPuIiI1rSrlEVRc1d30M\nhraC0UgMbRoRGRky9WQDk3Gjfd5v5moZDG0KI0gMbZ3TgM9xzSY/Ay5S1USmLYPB0ACMactgMBgM\nKWE0EoPBYDCkxH6RtPGQQw7RvLy85q6GwWAwtCpWrVq1Q1VrC5kH9hNBkpeXx8qVK5u7GgaDwdCq\nEJG6MjQAxrRlMBgMhhQxgsRgMBgMKWEEicFgMBhSYr/wkRgMhqalqqqKsrIy9u3b19xVMSRBu3bt\n6Ny5M5mZiVKt1Y4RJAaDodEpKyujQ4cO5OXl4SZdNrRUVJXy8nLKysro1q1b3TvEwZi2DAZDo7Nv\n3z5ycnKMEGkFiAg5OTkpaY9GkBjqxO+HqVPd/w2GZDFCpPWQ6r0ypi1Drfj9MHw4VFZCVhbMnw8F\nBc1dK4PB0JIwGomhVnw+V4gEg+7/Pl9z18hgqJvy8nL69u1L3759OfzwwznqqKPC3ysrk1u2ZPTo\n0WzYsKHWMo8++ijPPPNMY1SZ0047jbVr1zbKsZoao5EYaqWw0NVEPI2ksLC5a2Qw1E1OTk64U54y\nZQoHHXQQv/vd76LKqCqqimXFH0/Pnj27zvPccMMNqVe2DWA0EkOtFBS45qy77jJmLUN68Zf6mbpo\nKv7S9DnjNm3aRI8ePbjyyivp2bMnX331FePGjSM/P5+ePXty5513hst6GkIgEKBTp07cdttt9OnT\nh4KCAr755hsA7rjjDqZPnx4uf9tttzFw4EBOOOEEli5dCsD333/PxRdfTI8ePbjkkkvIz8+vU/N4\n+umn6d27N7169eL2228HIBAI8Ktf/Sq8/aGHHgLgwQcfpEePHpx00klcddVVjd5myWA0EkOdFBQY\nAWJIL/5SP8OLh1MZrCTLzmJ+0XwKctPz0H3yyScUFxeTn58PwL333suPf/xjAoEAQ4cO5ZJLLqFH\njx5R++zatYshQ4Zw7733cssttzBr1ixuu+22GsdWVd5//31efvll7rzzTt544w0efvhhDj/8cF58\n8UXWrVtH//79a61fWVkZd9xxBytXrqRjx46MGDGCV199lUMPPZQdO3awfv16AHbu3AnAtGnT2Lx5\nM1lZWeFtTY3RSAwGQ7PjK/FRGawkqEEqg5X4SnxpO9cxxxwTFiIA//znP+nfvz/9+/fn448/5qOP\nPqqxT/v27TnrrLMAGDBgACUlJXGP/fOf/7xGmcWLF3PZZZcB0KdPH3r27Flr/ZYvX86wYcM45JBD\nyMzM5IorrmDhwoUce+yxbNiwgZtvvpk333yTjh07AtCzZ0+uuuoqnnnmmQZPKEwVI0gMBkOzU5hX\nSJadhS02WXYWhXmFaTvXgQceGP68ceNG/vrXv/Luu+/ywQcfMHLkyLjzKbKyssKfbdsmEAjEPXZ2\ndnadZRpKTk4OH3zwAYMHD+bRRx/luuuuA+DNN99k/PjxrFixgoEDBxIMBhv1vMlgBInBYGh2CnIL\nmF80n7uG3pVWs1Ys3333HR06dODggw/mq6++4s0332z0cwwaNIjnn38egPXr18fVeCI55ZRTWLBg\nAeXl5QQCAZ577jmGDBnC9u3bUVV+8YtfcOedd7J69WqCwSBlZWUMGzaMadOmsWPHDvbu3dvo11AX\nxkdiMBhaBAW5BU0mQDz69+9Pjx49OPHEE+natSuDBg1q9HPcdNNNFBUV0aNHj/CfZ5aKR+fOnbnr\nrrsoLCxEVTnvvPM455xzWL16NWPHjkVVERHuu+8+AoEAV1xxBbt378ZxHH73u9/RoUOHRr+Gukjr\nmu0iMhL4K2ADT6jqvTG/ZwPFwACgHLhUVUtEZCAw0ysGTFHVl5I5Zjzy8/PVLGxlMDQdH3/8Md27\nd2/uarQIAoEAgUCAdu3asXHjRs4880w2btxIRkbLGsfHu2ciskpV8xPsEiZtVyIiNvAocAZQBqwQ\nkZdVNVKvGwt8q6rHishlwH3ApcB/gHxVDYjIEcA6EXkF0CSOaTAYDC2GPXv2MHz4cAKBAKrKjBkz\nWpwQSZV0Xs1AYJOqfg4gIs8BFwCRnf4FwJTQ5xeAR0REVDXSyNcOV4Ake0yDwWBoMXTq1IlVq1Y1\ndzXSSjqd7UcBpRHfy0Lb4pZR1QCwC8gBEJFTRORDYD0wPvR7Msc0GAwGQxPSYqO2VHW5qvYETgYm\niUi7+uwvIuNEZKWIrNy+fXt6KmkwGAyGtAqSL4HciO+dQ9vilhGRDKAjrtM9jKp+DOwBeiV5TG+/\nmaqar6r5hx56aAqXYTAYDIbaSKcgWQEcJyLdRCQLuAx4OabMy8Co0OdLgHdVVUP7ZACISFfgRKAk\nyWMaDAaDoQlJmyAJ+TRuBN4EPgaeV9UPReROETk/VOxJIEdENgG3AF7ymtNwI7XWAi8B16vqjkTH\nTNc1GAyG1snQoUNrTC6cPn06EyZMqHW/gw46CICtW7dyySWXxC1TWFhIXdMJpk+fHjUx8Oyzz26U\nPFhTpkzh/vvvT/k4jU1aY9BU9XXg9Zhtf4j4vA/4RZz9ngKeSvaYBoPBEMnll1/Oc889x89+9rPw\ntueee45p06Yltf+RRx7JCy+80ODzT58+nauuuooDDjgAgNdfb9tdVot1thsMhv2LxlzS+ZJLLuG1\n114LL2JVUlLC1q1bGTx4cHheR//+/enduzf//ve/a+xfUlJCr169APjhhx+47LLL6N69OxdddBE/\n/PBDuNyECRPCKej/+Mc/AvDQQw+xdetWhg4dytChQwHIy8tjx44dAPzlL3+hV69e9OrVK5yCvqSk\nhO7du3PttdfSs2dPzjzzzKjzxGPt2rWceuqpnHTSSVx00UV8++234fN7aeW9ZJHvvfdeeGGvfv36\nsXv37ga3bVy8xV3a8t+AAQPUYDA0HR999FG9yi9dqtq+vaptu/8vXZp6Hc455xydO3euqqpOnTpV\nf/vb36qqalVVle7atUtVVbdv367HHHOMOo6jqqoHHnigqqp+8cUX2rNnT1VVfeCBB3T06NGqqrpu\n3Tq1bVtXrFihqqrl5eWqqhoIBHTIkCG6bt06VVXt2rWrbt++PVwX7/vKlSu1V69eumfPHt29e7f2\n6NFDV69erV988YXatq1r1qxRVdVf/OIX+tRTT9W4pj/+8Y/65z//WVVVe/furT6fT1VVJ0+erL/+\n9a9VVfWII47Qffv2qarqt99+q6qq5557ri5evFhVVXfv3q1VVVU1jh3vngErNYk+1mgkBoOh2UnH\nks6eeQtcs9bll18OuIPn22+/nZNOOokRI0bw5Zdf8vXXXyc8zsKFC8MLRp100kmcdNJJ4d+ef/55\n+vfvT79+/fjwww/rTMi4ePFiLrroIg488EAOOuggfv7zn7No0SIAunXrRt++fYHaU9WDuz7Kzp07\nGTJkCACjRo1i4cKF4TpeeeWVPP300+EZ9IMGDeKWW27hoYceYufOnY0+s94IEoPB0Ox4SzrbduMt\n6XzBBRcwf/58Vq9ezd69exkwYAAAzzzzDNu3b2fVqlWsXbuWn/zkJ3FTx9fFF198wf3338/8+fP5\n4IMPOOeccxp0HA8vBT2klob+tdde44YbbmD16tWcfPLJBAIBbrvtNp544gl++OEHBg0axCeffNLg\nesbDCBKDwdDspGNJ54MOOoihQ4cyZsyYsDYC7mj+sMMOIzMzkwULFrB58+Zaj3P66afz7LPPAvCf\n//yHDz74AHBT0B944IF07NiRr7/+mnnz5oX36dChQ1w/xODBg5k7dy579+7l+++/56WXXmLw4MH1\nvraOHTvyox/9KKzNPPXUUwwZMgTHcSgtLWXo0KHcd9997Nq1iz179vDZZ5/Ru3dvfv/733PyySc3\nuiBpW5nDDAZDqyUdSzpffvnlXHTRRWETF8CVV17JeeedR+/evcnPz+fEE0+s9RgTJkxg9OjRdO/e\nne7du4c1mz59+tCvXz9OPPFEcnNzo1LQjxs3jpEjR3LkkUeyYMGC8Pb+/ftz9dVXM3DgQACuueYa\n+vXrV6sZKxFz5sxh/Pjx7N27l6OPPprZs2cTDAa56qqr2LVrF6rKzTffTKdOnZg8eTILFizAsix6\n9uwZXu2xsUhrGvmWgkkjbzA0LSaNfOsjlTTyxrRlMBgMhpQwgsRgMBgMKWEEicFgSAv7g9m8rZDq\nvTKCxGAwNDrt2rWjvLzcCJNWgKpSXl5Ou3b1WqkjChO1ZTAYGp3OnTtTVlaGWQuoddCuXTs6d+7c\n4P2NIDEYDI1OZmYm3bp1a+5qGJoIY9oyGAwGQ0oYQWIwGAyGlDCCxGAwGAwpYQSJwWAwGFLCCBKD\nwWAwpIQRJAaDwWBICSNIDAaDwZASRpAYDAaDISWMIDEYDAZDShhBYjAYDIaUMILEYDAYDClhBInB\nYDAYUsIIEoPBYDCkhBEkdeD3w9Sp7v8Gg8FgqIlJI18Lfj8MHw6VlZCVBfPnQ0FBc9fKYDAYWhZG\nI6kFn88VIsGg+7/P19w1MhgMhpaHESS1UFjoaiK27f5fWNjcNTIkwpggDYbmw5i2aqGgwDVn+Xyu\nEDFmrZaJMUEaDM2LESR1UFBgOqWWTjwTpLlnBkPTYUxbhlaPMUEaDM2L0UgMrR5jgjQYmpe0aiQi\nMlJENojIJhG5Lc7v2SLyr9Dvy0UkL7T9DBFZJSLrQ/8Pi9jHFzrm2tDfYem8BkProKAAJk0yQsRg\naA7SppGIiA08CpwBlAErRORlVf0oothY4FtVPVZELgPuAy4FdgDnqepWEekFvAkcFbHflaq6Ml11\nNxjSgd9vtCZD2ySdpq2BwCZV/RxARJ4DLgAiBckFwJTQ5xeAR0REVHVNRJkPgfYikq2qFWmsb1KY\nzsDQEExkmaEtk05BchRQGvG9DDglURlVDYjILiAHVyPxuBhYHSNEZotIEHgRuFtVNfbkIjIOGAfQ\npUuXFC/FxXQGhoZiIssMbZkWHbUlIj1xzV3XRWy+UlV7A4NDf7+Kt6+qzlTVfFXNP/TQQxulPmam\nu6GhmMiy5DATS1sn6dRIvgRyI753Dm2LV6ZMRDKAjkA5gIh0Bl4CilT1M28HVf0y9P9uEXkW14RW\nnK6LiMTrDDyNxHQGhmQxkWV1YzT+1ks6BckK4DgR6YYrMC4Drogp8zIwCvADlwDvqqqKSCfgNeA2\nVV3iFQ4Jm06qukNEMoFzgXfSeA1RmM7AkApmcmvtGPNf6yVtgiTk87gRN+LKBmap6ociciewUlVf\nBp4EnhKRTcB/cYUNwI3AscAfROQPoW1nAt8Db4aEiI0rRB5P1zXEw3QGBkN6MBp/60Xi+KnbHPn5\n+bpypYkWNhhaOiYqsmUhIqtUNb+ucmZmu8FgaDEYjb910qKjtgwGg8HQ8jGCxGBoBkyYq6EtYUxb\nBkMTY8JcDW0No5EYDE2MmdhqaGsYQWIwNDFmlruhrWFMWwZDE2MmthraGkaQGAzNgAlzNbQljGnL\nYDAYDClhBInBYDAYUsIIEoPBYDCkhBEkBoPBYEgJI0gMcTEzrw0GQ7KYqC1DDczMa4PBUB+MRmKo\ngZl5bTAY6oMRJIYamJnXBoOhPhjTlqEGZua1wWCoD0aQGOJiZl4bDIZkMaYtg8FgMKSEESQGg8HQ\nBmjOkH1j2jIYDIZWTnOH7BuNxGAw1BszYbVl0dwh+0YjMRgM9aK5R7+Gmngh+949aeqQfaORGAyt\ngJakATT36NdQEy9k/667mkewG43EYGjhNLUG4PfXPoeouUe/hvg0Z8i+ESQGQyNTV0dcX+JpAOnq\nMJIRWmbCqiEWI0gMhkYkHdpDU2oAyQotM2G1cWjsQUdzYQSJwdCIpEN7aEoNwJitmo62FLRgBInB\n0IikqyNuKg3AmK2ajqY0WaYbI0gMhkakLXTExmzVNLQl7c8IklZGW7GptmVMR2xIhrYw6PBISpCI\nyDFAmapWiEghcBJQrKo701k5QzRtyaZqMBjazqAj2QmJLwJBETkWmAnkAs+mrVaGuJiJYAaDoSWS\nrCBxVDUAXAQ8rKr/AxyRvmoZ4mFWLjQYWi4tKftAU5OsIKkSkcuBUcCroW2Zde0kIiNFZIOIbBKR\n2+L8ni0i/wr9vlxE8kLbzxCRVSKyPvT/sIh9BoS2bxKRh0REkryGVk9zp0EwGAw18fthwgQYOhQm\nT3bNz/ubMEnW2T4aGA/8SVW/EJFuwFO17SAiNvAocAZQBqwQkZdV9aOIYmOBb1X1WBG5DLgPuBTY\nAZynqltFpBfwJnBUaJ/HgGuB5cDrwEhgXpLX0eppKzbV5sQELBgaC89vuW8fqLrbWnsob0NISpCE\nOv+bAUTkR0AHVb2vjt0GAptU9fPQfs8BFwCRguQCYEro8wvAIyIiqromosyHQHsRyQZ+DBysqstC\nxywGLmQ/EiSG1DABC4bGxPNbekJEZP80Oydl2hIRn4gcLCI/BlYDj4vIX+rY7SigNOJ7GdVaRY0y\nIR/MLiAnpszFwGpVrQiVL6vjmF6dx4nIShFZuX379jqqmhh/qZ8JjxUz4feb9zt1tS1iAhYMjUms\n3/K66/bPwUmypq2OqvqdiFyDG/b7RxH5IJ0VAxCRnrjmrjPru6+qzsSNMCM/P18bcn5/qZ/CuydR\nOet1CGYx+6EgC96197uHpC3RliaBGZqftjQXJBWSFSQZInIE8Evgf5Pc50vcMGGPzqFt8cqUiUgG\n0BEoBxCRzsBLQJGqfhZRvnMdx2w0fCU+qj4bBMEs0AwqKoJMmQJTpjTNA2Ns+Y2PefENjY3xWyYv\nSO7EdXgvUdUVInI0sLGOfVYAx4Uc818ClwFXxJR5GTcSzA9cAryrqioinYDXgNtUdYlXWFW/EpHv\nRORUXGd7EfBwktdQb3IOyMHq9jpBuxICgNq88w4sWtQ0a0IYW356aIsvvhl0GJqTpHwkqvr/VPUk\nVZ0Q+v65ql5cxz4B4EZcAfQx8Lyqfigid4rI+aFiTwI5IrIJuAXwQoRvBI4F/iAia0N/h4V+ux54\nAtgEfEaaHO3+Uj8T35iIdl6KffXP6HHqV1iW4DhNY1s3tvyWTUuaM+ANOvbX0FND85NsipTOuCP/\nQaFNi4Bfq2pZ4r1AVV/HDdGN3PaHiM/7gF/E2e9u4O4Ex1wJ9Eqm3qngK/FRGazEwcHO9XN6/wV8\nsa6oyWzrxpbfcmlp2mJbyiJraJ0ka9qajZsSxev0rwptOyMdlWoJFOYVkmVnURmsxLZsyPUz/dl+\nrPEfDHnvQefjgPS9rcaW33JpaR23GXQYmhtRrTugSUTWqmrfura1VPLz83XlypX13s9f6mfakmm8\n8ukrKEqGlYEgBJwAWXYW84vmU5Brevj9jZamkXh1aqpBh/HH7D+IyCpVza+rXLIaSbmIXAX8M/T9\nckLRVW2dVze+SlCDAFQFqwBQlMpgJb4SnxEk+yEtQVuM7cybKoCgJQpRQ/OTrCAZg+sjeRBQYClw\ndZrq1GLwlfgIOsHwd0XJtDJx1CHLzqIwr7D5KmdoVpoz8qs5O/OWYNYz2lfLI9kUKZuB8yO3ichE\nYHo6KtVSKMwrxLZsAk4AAEEY228sXTp2oTCv0GgjhmahOTvz5vbHNKUQNdpX8iSb/TcetzRaLVoo\nBbkFPHr2o2RamUjo38ItC8k5IKdZhUhLCj1tbvbHtmjO5QSaOwN1U4bFmxD85Ellqd39In37uAHj\nAJjw6gQcHD7a/hHXvXpd1G9NrWo3xiipLajs++uIsbl9NM1p1mtKjai5ta/WRCqCpEH5q1oj5XvL\ncXCitr340YuMGzCuyTuzxjBrtJUOuCXY65uLtjg7PxmaUog2t8BuTdQqSERkN/EFhgDt01KjFkhh\nXiEZVkbYVwJwcQ93Yn9Td2aNMUpqKx2wGTHunzSlEN1fBXZ9qVWQqGqHpqpIS6Ygt4CFVy9k2pJp\nbN29lbH9x4bNWk3dmTXGKKmtdMBmxGgwtAySmpDY2mnohMRE+Ev9+Ep84cit1uhvaI11Nhjqi3nO\nU6OxJyQaQvhL/QwvHs6+wD4ABncdzL3D72XSpNb1lBqV3dDWaSu+wNZAKuG/+yXF64r5IfADGvq3\ncPNChvxjCP7S/Sj+1LDf0JrDq034btNhNJJ64C/1M2vtrBrbq5wqky7F0OZo7SP6+voCjRms4RhB\nUg9iU6Z4WGKRc0DsUvMGQ+umtUf31ScYoyUJzdYo0IwgqQeRqeVFhKM6HEXpd6WoKje8fgNQPUnR\nYEgHTdnJtIXovmR9gS1FaLYkgVYfjCCpBwW5Bcwvmh+O2PKV+Ljj3TtQlIAT4PrXrmfNV2so6lPU\nYDNXaxyN1Jf94RpTJV4bNXUnsz+FV7cUodlSBFp9MYKknhTkFoSFxPpv1rtTM0MR1EENMmPVDOas\nm9OgtUpmzoQbb3Qfouzs1jMaqQ+tdcTVlCRqo+boZPaX6L6WIjRbikCrLyZqq4GE13SPmYejKBXB\nCnwlvvodzw833ABVVeA4UFHRNqNMTCRN3SRqo+ZM1rg/UFAAkyY1r+Bs7qSYDcVoJA3EW9NdUQRB\nPbWk9FTYPJyck86t3/F8rgDxsO222VG01hFXU5KojVrKqNmQXlqjFmgESQOJXdNdECpLBqBz3kad\ndkxcYtG7HiOKwkLXnFVRAZYFjzzS+h6mZDCdYd3U1katsZMxtH1MipQUiEyVAjDl7greeXIITlCw\nbbj2t5vpcu6z5JSfS/nHvZMKQTQdrCFZzPNiSDfJpkgxgqQR8ATKzk3deXDCOQQDGWRmOmjRcAJO\nAGfOW1hOe7KzpFXZPeNhOq+WQbqCFhJFizXHPTfPWnKks51Mrq0mwsu9VRGowMFBfvVT7M3DOWVw\nJYucheii30MgC0clLZE2rXFJ2+7WAAAgAElEQVRRrbZCc3Z0jbUuTWT9491faJ57bp615Ggp7WQE\nSR3U1ln4/TDlHxVUOP1xOi8BQDsvxem8jCUacsDn+cCuxFKbrCxpVOdya1xUq63Q3C9wqkEL8eqf\nKFqsOe65edaSo6W0kxEktVBbZ+H9VlE5BMd6Cyk6A81diiUWllg46oZgSe5yLpj6CIdvvxTy3oPO\nxwGNc6db46JabYXmfoFTCVrw+2HKFDeww3Gq65/o/jbHPTfPWnK0lHYygqQWaussvN+coGDRnhH2\nn7j43E9Z89Uatu3ZxrxN8wg4AWzLhs5+Zu/7A4HtAeYUZzVosmI8WuOiWm2FlvACNySCKzwACgkR\ny6quf6L72xz33DxrydFS2sk422shGY0kyp7c2c/QOUOpDFaSYWVwznHnMG/TvPB8EwBbbO4aeheT\nBk9qlGszDsnmozW2/dSpMHmyOziyLBgxwtVOWkv9DU2LcbY3EqNGuf8XFdWM548dCUx4tZiKYAXg\nppZfuXUlVU5VWIgIQpadFQ4X9ohdcbE+mHkFzUdrbPtYTaqlCJHWKJQN1RhBkoBIjcO2q7fXZ3JY\n2e6yqO8iwvSR06OEhRf1VRmsJMtuPLPX/ojpjOqmpZhCImnuwAVD6phcWwmI9Y/MmOE+7LWtFFfU\np4gsOyvh7446zNs4j6mLpoZXVPRSrQQ1SGWwst45uloSzbmantcZTZ5c932q7RitdTXA+tASckpF\nYvKvtX6MRpIAzwSwbx+oun+RDvd4o9+C3AIePuthrn/teoLqLoAVlYcLeOXTV3h5w8tYlsWjZz8a\nlWolntmrJVFXKHRzjipTjaJq7vrvz7SEwAVDahhBkgDPBFBcDLNnQyBQ/ZDX1umU7y2POk6kf0RE\nwgLGcRxufP1G3rv6vag1TlqqWauujra5w2FT7Yyau/77M8mY24zZsmWTVkEiIiOBvwI28ISq3hvz\nezZQDAwAyoFLVbVERHKAF4CTgX+o6o0R+/iAI4AfQpvOVNVv0lF/zwdSVBT9EE+dmrjT8TSMfYF9\nYSFiYZF/ZD5rtq0Jzy8Bd/0SX4mPSYMntVgB4lFXR5uoI2+qDiBV239jjIpNZ9dwavM37g/aYmt/\ndtImSETEBh4FzgDKgBUi8rKqfhRRbCzwraoeKyKXAfcBlwL7gMlAr9BfLFeqavqSZ8UQ+5DX1ul4\nqygWrytm9trZBJwAWXYW/Y/oz6qvVoXLCUKGlcGWXVvwl/pbrCDxHvCcnNo72ngdeVMv1JVKFFWq\nE/xiNde22Nk1F21dW2wLgjKdGslAYJOqfg4gIs8BFwCRguQCYEro8wvAIyIiqvo9sFhEjk1j/RpM\nXZ2Ot4piUZ+iqOzAc9bNoSLghgd36dSFsu/KmLFqBrPXzmbBqAUU5BakFArc2MQ+4NOnQ3l54o42\nsiP3FuoKBNzv3kJdqb4g6Ry51SWIalv+1vOlQfo7u9Y+eq0vbd2HUlxc/fy0VkGZTkFyFFAa8b0M\nOCVRGVUNiMguIAfYUcexZ4tIEHgRuFtb6KzKyGV5AaaPnO464recTMmiQjcPV+4yKoIVFK8rBuD0\nu35P4PPTyDj69yycfF/KwiSVTid2JOgJES+qprbj+XyNv1BXU43c6rNeus/nCsnIJ1DE1eDSQaJ6\ntGXh0hJDlhsLvx9mzap+fjIyWqegbI3O9itV9UsR6YArSH6F62eJQkTGAeMAunTp0qgVSKZDi/di\nl+8tJ7hlIMx5B4JZYFfCqOGQuwyAaf9aRGD2GxDMIvBeJdOOf4SXftfwtybVjjd2JJiTk/zx0rFQ\nV1OYOOq7XnpOTrTAtCz3+8SJ0Lt349cvUahsazeN1EVrnPyZDD6fey/BHYCMHt06rzOd80i+BHIj\nvncObYtbRkQygI64TveEqOqXof93A8/imtDilZupqvmqmn/ooYc26AISUVfce6I5DYV5hVgfjIJA\nNmgGBDOhpBBbbIr6FLF1/fGugAn9tnZZp6g5J7H4S/21/l48dzP7KpwGx+d7I0Fv/ejy8uTj/b19\n774bFi6EceNSn6eRzjXLvboVF9dvvfTycld4gNsRqEYnQmxs4tXDzMNovUTez3bt3MCe1kg6NZIV\nwHEi0g1XYFwGXBFT5mVgFOAHLgHerc1MFRI2nVR1h4hkAucC76Sj8rVRl8027ovd2U/xqxuRtWNx\n5beCFcTqtohBXQYxbck02h1zAthnQlDBrqKk02zuWPA+2XZ2eMa750PJOSCHiW9MTDgj3l/qZ9bO\nSaj1OmgmGZkWhYU29SV2JFgfW3WszyTVUXO6TByxWQwyQm9F5DUmOreneXn7ikSHijc2ierRkn0I\nbdnslioNfaZbWpumTZCEfB43Am/ihv/OUtUPReROYKWqvgw8CTwlIpuA/+IKGwBEpAQ4GMgSkQuB\nM4HNwJshIWLjCpHH03UNiajr5tcwCXVfz/Di4exb8Bs0ACCIKCeeuZxNXd5n4eaq6p1HLYSSQshb\nALnLcBT2BfaFfSheOhURwVEHR53wjPhIQeIr8RE8arFrOls3ij5HDgT6p/W6a6OxzFLpMHFE1g3g\n2muhS5fk1kuPbRPveOl8wWPr0ZI7o7YQkZRu6vtMt8Q2TauPRFVfB16P2faHiM/7gF8k2DcvwWEH\nNFb9UqG2mx/7YvsCr7oZgPPeBWsyOBaZmRZDztvMJ98EonfO9WN1WY5q9Xx4RXl89eNs+35bOJ2K\npRZS9lPki9Oxj1lSY0a8N5+lQmycdUWsXNOe4W+n/tA1tCNvyZE3sXXzzAvJBBXEzXBQUG0qa6oR\nY0vtjNp66G5z0BLbtDU621sMtY3ool7s0ohOHW+WOxy8axDW4tsJdp0fdrgDqGqN1CpBDfLKhlfI\nsDLQoLpC5Kn5UJWBLFHWn/URvpKp4bBhbz7LlLsreMdpjxNMz1K/ydKSI2/iaRXJdLK1RVC1tBGj\nV9/w4MbXNJ1RSx5ANCWNqf21xDY1gqQeRD4MkHxnEdWpk42jQlUVPPiHY1DnLix7Ms6ZN8EPOaGQ\n4OU4ODWO46hD90O688E3HxD8YjBUWqAWVVXKDX/7f+hp95BlZzF95HTK95ZTmFfIlKsLWfRU3Q9d\nU8xfiR01R052rG1+SlMQWbfaMhdEkqgzTuT8bk4hGm9OUFN0Ri15AFEbjdnxN/bAoiW2qREkSRL7\nMIwaVb8RXUFuAVOuJtypi7j7Oo5gaTbWG4/hBNVd3/3qn7lrv8cIE0VZ+/Va90veArArEUewMxyC\nXd/F0SA/BH7g+teuBwg74efPL4g7L6J47ma2OR/CD4fwetUkgkctbrJU9vFW6ktl9nsqL37svsmO\n+BKVSyVsOl3ECrfy8upccukm3aG7je3raWh4fyLSof21tHBoI0iSJPZhgPqP6CJHEjk57lwDT6g4\njg0K4ojrbO+8FFtsBnUZxJadWyjZVRJ9sNxlriO9ZCi/PP8ont/1fng+g5cYsjJYSfG6Yrp09LHz\n8O5M/DscueRTzjr2LG6+vAcV+44CuoAEwX4dRg2nssuKGo77dOC1p1fnyJDZhgiChnbUifZNZsSX\nqFzs9pYwczmR0Jszx902Z056BFy6Hfr1uffJ1qWujj82ym/MmJoL30XSEk1RjY0RJEmSk+OOmlWr\nF7qqK2VIPCJHEr171xQqVoZDMG8BDg6WWvhL/VQ5VfEPlrsMzV3GM9+6fpdYghrkiTVPENw8EJ3z\n6/AkyH/3fQqt6I57+zU0b0WRkqFkdVuXVCr7hnYQsbm74q0dnnDfBOa3VEZ8ifZNdsSXqJy3vaXM\nXI4n9JI14TWUpvAVJXvv61OX+oT3B4PuWkW1CeJkByYtLaS3PhhBkgR+v9vRB4PVk84efzz+Ou71\neRC8Mj5ftVDK6f4JEz9cTWXQjko7XxeRjvlIAk4ASk6PmOioKI47qz4AbhS1A1aQk3+6l+l1mLX8\npe58mNm3XEmgyiYjM8jovzxD0bnH1anFJMrdlYyPpLaVJBO9+Mn4fdI1WvTOveXVKwgGu4a39+nT\nOMdvCLFCryHX3twmnVi8a6ioqJmapqHBBcmG9ydaqyjRMeuK/mtu82cqGEGSBJFmGM+3AdGO1Mjs\nr8mouxDtJ7AsePRRGHdhb3oPmF9j0qFt2Zx97NkcftDh9DuiH/M2zmPuhrnJXUCezxUcoYmO9Cl2\n/9YVweox4LiPQWG3wvAKjfESSHqd+b4Fv0ErFBSCjsOMFz/h8e1jGNRlED0O6UFRn6IakyO9TrWy\nsmuUnX7SpCTvQYmvxkqS3jkKCmD6s+t58qXPOLL3p9B5MP5SogRPZACCt5+/1I8v4GP6s+dS/nHv\nxrOxRwg9e+ebZGTOR9XGcWDlSveez58PdE4+wCHZYIh6BU109jPqgY1QMoSiC7vWee317ezSbdLx\nBMVNN8GDD7rv3vXXw7x5cNZZ7uDPe7duuaXhk2nj/RZvraKcnIaHfDeF0E0nRpAkQeSoJzKvUkZG\ntSM1Mvurp+7OmlW7QPH5qo/pOG7KdTc/U3Wyx96H9Y7bMYwbMI7fv/N7pi2ZFt4mCIO7DGbxlsVR\njvrDTvycb0YND0109FWHGpcUgtqADao88OwqGDwV27I59ahTWVK6BEXDM+u9zlzz3gX7f8OCSfMW\nENQgCzcvZOHmheFsxkBUOn2vUwU7qZc5chb/ll1byLAywAHbssPp971zPLn+SaqOroLvYd6cbEb3\nHR0WPBWBCm58/UaCTjC8MmXvw3ozvHg4FSX9sTb/wKPXQ0FB77oehaSIFHoctZhr//IMn88t4p13\nqn1BxXM3M+fgmhpWPEFQmzYW217JlKtR9uAsijrPByKEf4zm4ffDlCnVz2tkZ5eozo0ppGsMaiKE\nmje4U3X/nzsXXnnFraeXsubBB918b40VHegJGm+tokjzdJ2+mjjtlVCrbiXmLiNIksAbgUyZQrgz\n8BKsefmnYhO7eOpubfbTwsLqJH/gvgSxI5HYDMKR3DfiPgDuX3o/KGTYGfQ4tAdXnnQl8zbOY0P5\nBjb9dxPbv98Oud9EzVUBOLzXBnYsdghUVoFVRbDdNlj4PwTzfCwMLgyX2xfYxxTfFC7ucTG2ZRMM\nO/oLowVTCM/JP2fdnKgFvrxOtcvOIvdF6exn6qL4o2evo6sIVODghNdvOe/485i3aR6Pr36cWWtn\nIYgr3CJMexXBCl759BUssVBVEMJ+Jm9lyrH9xlJR0h/nH2/hBLO4/j2HeX+axuEnfhHWqBoaEh27\nfHLRucdBX1i0qLqjIO89KrdHa1jrv1nvCjwNRqXFqU0biyTZcrFlvcwJYSEQxwTpje5j/VmR9ylW\nSLvXfxfTR07Ht7ccShO3Y21tHaXhWTZj+o6BRbeFtVvLqjY5e3jbIwd39dGAIwcxsZpsJJ5ASdbf\nlEjYxzOntSZzlxEkSVJQ4AqSyM7AmwHtjSRsG84+21WtPeGi6morxcU1H4KCAtecFbn4U33V//tG\n3MeFJ1wYHvk/vvrx8APqK/Fxx7t3RPtPSk8NC4BtuS8hvxoKXwyBfR3g9UdBrRpZiRXl7c/fZv4X\n8xEJOfVzlyG5y0O/R5NhZbD6q9VUBCuilhrOsrPoN3Af5XunMvf7nTww+wEcdci0M/GN8kWNZqf4\nplARrAhrVopS5VTxafmnYcERDCb2H325280PGhYmEXh+J2vzMJyQ7yhYVcXcN76F7//O7LWzeeis\nh6LMimP6joky2c1cNZMXP3qRvkf0pVN2p6iOJry42auu2YiyrtXmkLmbIe89+g3cR9Yb1cJmZ8VO\n7lhwR3gFzYpgRVgQxAqmRMEQyZbzytqWTTAYRFFmr50dvj432Wcu6lhUVsKLL1abdi0LRoxw34WC\nApjw2Eb2LZiI5i3AyV3G9a9dz4AjBrj3Tp244ehRWktJ3XnjIoVeMBhkxqoZZO78OEq7vekmeOAB\nCAbde52ZpfxmosWDDyb/bsXWxxvEWGKR8eVgxnSak9AEGKlRZGQG2dLpGfylNf2GdZloI4/dmsxd\n0kKX8mhU8vPzdeXKxllQMdFaFbEjieJiePJJqAoFXGVnw4IFiSNKEqmvyaq2UxdNZfKCyQQ1iC02\ndw29i8K8Qk7/x+muwx1cITJnfs0U9qWnwuz3wMkEBCQAwybD4HujBE+s5hGJhcVpXU/jx+1+zLxN\n86gKVkWZ17zfl5ctr/EbwPgB43ns3MeYuWomN75+IwEn4Aqh2s4f8ZvdZQXnHX8eS0uX8s3e+Csv\nW1gghEf761cdxI2XnUigykKtinB7CMIZR5/B/C/mRwU72GJz3gnncXzO8TVMiu0y2kV3kp7/q1Kx\nMwI88twn9B6wJ2pk7fm8Dm53MPcvvT9qGeZMK5P3rn6vRqdbHx8JEHefmXPX8+K8cvZ1foNFzjQU\njXpmCu+eROWs1yGYSXa2xUN/teOabfx+GDosSEWF1hh8xMMWm2v7X0uXjl2ihEdk3rjYMuV7y8Nl\nI7VbW2yuPXRWWLstKHCv6/p7lhHcfQh2h+387fYCev+kd+J3K6at4uWxA8LvjTjtaJdt1ZrpoHju\nZmbtHJVwTlZd5seGTnpOFyKySlXz6ypnNJJ6Es8JF7st8vuMGa5WEggkHlEkcuzVK2Qxzmi0ILeA\nR89+NGwusbacQdDJRtV2/Rslhe6LX1IIjgUIockskOfjoG1nsGfO3Lhrp8SiKCOPGcn7W9+nIlhR\n43cHh4WbF8bZ02X1V6uZuWomN7x+Q92CL85vwVEjOP6nx/PqxlejjmuLHSUMzj/hfG796a0hkyH0\nXuC+/E98+ysCR7nHzrAyuLjHxfg2+6K0nqAGmfvJ3Bqh1opSEahgim8KUwqnuOYonytEnKDgODDh\n0ec4/5qPokbW/97wbzLtTIJOMEqIWGLxm4LfRAU+JDJxxg40vHLxTE7jBoxj5tz1XPfLYyDQHeyB\nZI7243ReEn5mopJ9lhRyeJ8S1hzZienPXh/l6/CX+pn496+prDwvHA1IydDw/YlN8SMItmWH/WXg\nZmpQ3KANW2z3L1TGG2xYYpFtZzN95HTWfLUm/JuI0G/gPsYN8JZT8LElYwvaZz384y2CwSwmXBrg\nsX+tZ9Kkmr6v2PY59/hzo/LY2ZY7r8vBQUqGosEsVK2wf8sXeLaGgC4oAF/gWYILFsfVODzB5V1L\nvHsZ+77XMHeFoiaTDZBoKowgSZF42oj3vaioesJXQ6JW6hWyGDKnxI5Axw0YF3bY55x0LhMX21RU\nKmIrg05X/JJJVZ4PMkLhwJYDZ98IucvYs+i2qLDhsOCJpfRUKBnG3ODXvG8nGUkWw4qtK1ixdUW0\nGa6kMPH5a/w2hD8vuS/qmBeecCGHH3Q4f1/1d8DtFF779DVu/emt1e1WAAUFXTn4nQL+vGQxiuKo\nw2fffkbfn/Tl/a3v16hrvFBrB4e3P38b32YfY/qOoV/367EzTsQJCojitPsmnCvNCbodqKJUBaui\njmeJxe9++jseXv5wXLOav9QfzgTdL3A9E6/oHbqfVZz3p+mcNbQT5XvL2bJrS9g04zgOE16bAMCL\n8453hUio3Y777hquGjoy6pkJ+8Fyl7EZ+PsqyLajl4MunFNIZWZ/sM4EzQS7CqvbQmwr09UsLBtB\n3CALz68BPL768bgh7ZZYjO03lm3fb+Pfn/w73CZeduvyveU8du5j9DuiX3hgNPGNiQBRJkgpuTX8\nXDih1EG9B+yJa2KKbB/v3njBHJERkmsOa8fsJUKgyjVbzdo5iuCCmhqHv9QfDgrRoCIi5ByQE/7N\nE1wigiUWjjrMWTcnfAzXpNgZddx31OcTJk2C9VkzmfLRi/T9vi/T/9/ykLaYxeyHgix4146eLNlM\nS3UbQZICiZyStY0o6kN9wycTjVojt/eeDz6fUFiYRUHBvUx4dRcznBlohPPc6vI+qoLGhg3n+QB3\n9AjuS05pATrnbTSYxfvvVcKo5bWaNwA6ZXfiu8rvcNQJj1zjdc7SbSHqzXcRhfbVKzBLXui3iLpF\nHiPbzubWQa7AeGLNE+FRcFCDFK8rjjJpFK8r5vHVj4f3D2owynQVD1tsBhwxgONyjuPZ9c+Gr6Ey\nWMnfV/2dLHsWv/ztWzxzb4Hrd3rjrwR/8iEDTglyZIcjeW3ja1Q5VTWu+3c//R2dsjuFOznPJzBn\n3RxuOuUmHlj6QLWPZ3EOTkUPcGxwhLlvfMvc73+PIGTama4/K3R4R90gg9+c+jxvzaput40HP0Fh\n3r0ATHjVFTZnH3c2cz+JHhBELgc9xTeFqmBVdXaFddWrMY3tNzb8ud8R/cIj76I+RczdMDdK84rE\n2/76xtdraDIAcz+ZGzZ1eWanfYF9/HXZX8MmLw0qJwzYyob3AjgB9/qCXeYzxbecKYVTAKKiAGPb\nZ3Tf0Wzbs41XPn2Flz99mWw7m6I+RYybUEC/I9zw8m8OeZ7NBy9EVdkX2Bd+TrzAlqAT9CqOow4T\n35gYHsiFTXManX1i2pJpbN29lZX/zUStt0AzcaSKnO6fMXOVn+tevQ6Atz5/Cz6rHtxVVgajBpf+\nUj+Fd0+i6rNBZB4zCd8dU5tMmBgfSQpMnequghgMuo52b36A9/2uu6qjRFKdCZ6u8L/w3JAI+7OF\nq9o76uCUnoJ+cXqUj8IWOzxyZ9EkePdOd4QrVTDsD65vpbFYeU3cIABB0NJTXWd2rP+ktIAeeyZw\nfP5XHH7iFxzc7mAe9D/omi1CI0FVJcPKwBKrRtRXPAThqA5H8eXuL1Hc7MwXnHABW3dvjau1AHRe\n9yhf/vs61LHDbWOdPo0MK4OgE4w7Mr9n2D3srNgZV5DFmouqzXuuRhBp+hOELh27sGXXlur7Khbj\n+o/jvcWVfLzyJ5C3IOxbennDy2G/lS02llg1MirYYpNhZUS3V4yJse+tv2N99kwUt31V3SCJGnWP\nwMIiOyObUX1GRWks8fa5ddCtTF82ncpgZfxjiYVVNgj9Ykgoq7YbIp5hZWCLHa674GoFkffg1kG3\n8hf/X8KDDq+9AJ5c82TiDBO1YGGRf2Q+lcHK6jx5tRHh9xt4isPW3Vsp210W/XvEPb9w6iPceulg\nCnILmPBYMX+/+ZLwvbhw6iMMPCWYknaSrI/ECJIUiNVIvIlRXpRIY6YVT6cg8swlnv06cgJfPEen\nJ2iCWwbC2l/hrBnljopDnZnV5X3yj8hn5Vcrw2HJZx97Nlt3b61pvqqLRbfBu3e5gsoKYA/7Pxg8\nFREJrdnidlhnH3u26+TfnI/zj7ei/Cp2lxX89qe/5bt93zFz9cyQJpVcEIF3vdkZrp3+pnk3JezE\nauC99E4WYlehRcOqhWCcNsi2s3norIei/UTJnCOJ65DQP4Rwu1likWFlEHACNTQFzyz4xqY3auZ5\niyTFgYQgnHzkyRzZ4UiAuIEakZx59Jkc/aOjmbFqRsLnyBab844/L/GE3ThtJgjH/OgYPvv2syiH\nfg3B2UhEPQP1eBZjy0vucjLtTMb0HcNHL53PwtlnRNyLPyKD760RFVmvehpne/qJl4TRi1+fPj06\nBUoqYXyJUovUJVSSFWCe6auoT1Hc2Pneh/WuIWhuOvJZHvzTOQQDGWTaDv3OWUfhhVvodOz5FOb9\nJeHM+MI5heGOOFLz8SZBLlxSFa1lRJjXsrMsHrrhl6zJ2OE6XZ0qLMvi4bMeZtyAcW7Y8N0VvB0T\nUBDMXcaD/gcZ229stRCJ48S/8MQLOTDzQJ5Z/0xU+4w4ekTYiT5v07waZp+ERMy30YhOwouSCpuo\nQhFtPQ7pwbxN85IXIt45akSzFcTV1BycsClHEPKPyGdP5R4+2vFRaL/qDurTQz7l1kG3sm3PttoF\nSSgLdaz5Mxk8QbZm25qwVmdh0aVTF0p2xj9n3yP6cuEJF9acoxRDwtF/gnuvKJu+3RQuJgiH/vd8\nvv7PiWhotdLGQhC6durqXmNMfTqMu4g9h71du+CKuOcKYXOqWB+APSTiXiwIm1sj5wmlAyNIUiR2\nQpI3WXHNmup0CammiogURBUV7rwTx6kWDl6Z2JxV9RVg4Vm2ceyskYKmMK8Q39MFOAFwgqCOTf8j\n+3Pf6P7AhdUHLCuAxQXuU5brHt83yhe2tRf1cW3rxeuK2bZnG5QVkPH0RAKVrhnLvvpn/Payn/Ld\ngBciolR6M3VRF3cUjYOoUL63PFz/KVfDe8VBKiqqojo2r9POsrOojOPEl9zlHH7g4Xz+7edRbWKJ\n5UZwlfiYu2EuL7+zHb64LdxJe53hmL5jOLjdwbyy4RU+2fFJdUcQp6MXhGv7Xxv+3u+Ifkx8YyKL\nNy9OOBJPmtICrOJ3cQKuBke/2dDnKTRk4om8rjXb1lSba2I6tI8Yzmk7TqueN5SIOianxqPHIT34\n9am/DgcFzFg1I/ybg8PmnZsT7vvw8oc55kfH8LNjfsYrn74S1zwY1GBCQVRrAEcI12x6CtvmPA2B\nrFAAyg2Q/0S4zOldTmdJ6ZLEufDq0DLKviuLW5/dG/rDYW8lvP5ILKwo/6J2Xlrve9FYGEHSSEQK\nC9uOzsHjOd0buvZD5LGr1zEJpdoodiPD4q3r0RABVvzqxnBUSOV7lRT3fYGCCdUT7cKjmkL3Or3U\nFLNnR6eCiacNebmlIif2+Uv9zFo7y9VSFp2Iu1hXBuII1/74ae4b0RVGxLRHLRPvCgpgwbs2xXPL\n+OjAx1jCChQ3hLTfEf0Ywxjm7tjAtvdqOupnr53Nr0/9tevUDHF5r8urJ6eVDozqbE+ffCcjh3aM\nskF3yu4Uns+TiAzLfe28dpi6aCqVwcooIeL5YA4/6PCwfd7T4MJzbGIQBGvzMIKBjFDHZMPKcbB2\nVI3Q7UFdBrFo86LqneN0sE7uspqzTT28jrL9DvjhUKxuC9HOy8PFvYCMeOl2njj/iaiJnfGuJVJj\nizQDeRMcw6HDsfWpqwONE0Di+c08lJCACWQBGeCo66f7yX8gdxmZViY79u6oXYjE0Xq8wAFFcRx3\nkbq9fUrYsrAKDdSh0fxLaigAAB5fSURBVMWYs9zUq3EGHXEGLplWZnjQli6MIGkkIs1cW7a42YGD\nwepZ7ZGhwMmu/RDp34i3jklWllsu0boekyY1IGqsZEiNsNpE1ztmTOJ5MrHaUKLcUr4SnxsBBOGX\nXByhXbZN0YVd4587QahzZN0KCroC9+IvvaBmAsxDbOyrf0bwi8FIt/cg1AEGnACdsjsx49wZvPjR\ni1zc42LK95bz3H+ec1/amM62x/cTmDQ4uo6RQk5E6H94fwq7FfLdvu/Ytmcb//3hvywpXcLM1TPD\noZ/ePpEzqb2os1hNEKrzl0Xa7jOtTDd89sCDmeurhIAAFmC7jtmYkXePQ3qwrGxZ2MyYefRSWAxV\nlVW1dmiWWIzInMz8p24nWGWDWogodpaDjBoRnogXmyQzdvLf1EVTyTkghxc/erGGz8i2bG4puIVO\n2Z3IOSCHNV+tiXJ21+jAa5lvdHrX06OXYojRoCR3OadxK0sWZxJstw354VCk20KcPJ+riTiK61iy\nkJJhDD4tC3+pv9ocGAcpGYZGPCd5O0cz8sK+Yc3Tu88byjeQ/eMS/mfGmzzw7MoaS24nur5DJlzO\n9pyXE54/koFHDmT6yOlpj94ygqQRiV2DInK0DvUzM8Ub0XsRYN46Jp6GEamRiESn0052XQ2Pogu7\nMvuhIJWVQbKyrISdOUQLR9t2Bajf754vVhuKl1vKS/+RaWe6HVruMjJHn8XYHxXXOdmqthxk8cp5\no/6gBsGBay/oRZeOHcg5YBQT31hTYyLnuAFutI6/1F/dyUeMZjOzJG7bxBNyfj/4VkC/7uu5YWP/\ncEe4r6QfU+6uYMrVheF94uV2ir3W2vxZM4+YyVwvJHfNaAhm1AidzrQyAXj4rIerw3PHFMHoDIrn\nbo7S5DKsDPr+pC8rv1qJs2UgWjKMnQeeHxKo7vFUBSdgcW2nOXQZWnOiXuQ1hKMEv+iHlpwOebuB\nU5CSoVjdFkGuH0cdHl7+cI1Z34kc7FIyFHWy3QSkDmGhaWHRzm4XpW0IgoZG7YKQufV0lhf/iWCF\ngAoqQTQUfbbunJvQ1x4GtbAzHf5242WU5xwUrcnhdtZj+48Nt2W/Ppdx8xIJv0PP/nZc+FnufVhv\nJj7+L1YsPQAnbwGVXVbQ6diPWTTrHG6bHWDRwmEhn0y1KfLQb37J9gjBdOg3l0QLkhhtLDM0l8cT\n6E0RAmwESSMRGx0VO1qH+pmZaozoi+P7QaBa69i5szpq7KabXD9NXansY/FMQ8loMQUFruP/ySfd\ncz3+eLS2FalFrflsCPbO0yA0Yi3MKwy1WQEP91rJmoy/AW6HVpCbWHg1lBqJFCPMa4kyLEO0YMg5\nIIc1Uf6a+BPACnILoKwA39OwPmrRshMJXjUQcpdC6anonLd5R9uz6CmYP7+ASYOTv1GJBGn53nKs\n3Pdds9Tha7Dm/Q11Msh4+zEKug5hxw5l48FP8Lg+XjNFR25NTc7TIArvnkTlnNfRYBZrMoWMjOrM\nul4SR7dNJuH3w9Sn4z8/vhIfFSX90TmhyDrLfTlUM2FREC0ahtN5SY1Z4UV9iqpNoMQEahyzFFlC\naMKgoMf4CYpNlp3FxT0uZtGWRdUTFmMnSe69jcer7JBQ1HBnvX3ZWSDL4OwbkB8O49qLT2DchUX4\nS/dUD3wgPOs+Ns3J6KvdzzXev7IC1v15oLsMg12JPeZst43LClhxbwFUKCJBrHNuhgEzybKzuHvM\nCG5+u1ow/fqyvtyw3o22q6mNjUDyVnFtv2trLOeQVlS1zf8NGDBA08nSpart26vatvv/0qWJt91z\nj/t/fY6ZlaWana1qWW4aSMuqPmYk99zjlvfSRYrEL9fY1y1SfU7bdusR7zqy2wV0/N/m6NItS+O2\nT23nqavdlm5ZqvcsvEeXbklcKJky9Tn30i1Ltf3d7dX+P1vb390+fNzIa8vIiLhvtqMZZ0xWa4ql\n1ojbVaxgjTarzzOS6Bq9OmWcMVkt2wk/M5mZ6p4z43tl7Klq/5+t9yysvlm1nXv8rSVR9R0/3i07\nY0b0PnXd16VblmrGGZMVqQo9M4HQX3X7xLZnuA6vjFeZIsoU1P4/W8e/Mj58PyPrHnufI7/X+G2p\n+1wiAXXtWFWKVKqdEXDrmPG9Zo0bElWXpVuW6vhXxuv4V8bXqGNd1x/5jooV0PG3loS3e88JqNoZ\nwfC7Eu/eeHXocWmxYrnth1Qqw28L39dUnyVVVWClJtHHNnsn3xR/6RYkkQ9Ho3YKof3Hj48WEPE6\nbK98oo69MR6qWJIRXJEviGVV1zlRm8Vrg7oETqIOPVVmzHA734SCe+E9av+fHe7YvE458tq8Dtyr\n/4yXPtB7Ft6jM176IKnBh9cG9bl3XmcZeY5IgQYBJf+xhMIv9lqXLnWfwezsugV/Mvd1xksfaGZ2\npVp2UO3MKs3MCtZon6iOO3T9M176oNHv84yXPlD75Jkq+Y+pddp9atlVijjVz3REZx+P2HtT2/XX\n1o5Ll7r3yDtv5LtSG959s2xHyfxerWsGafu728d9vhqCESRNKEjqM7pO5fh1aSRe2QsvdOvilZsx\no3HqV2NUFKM1jR9f89gzZkQLwBkzau6baseUqENPhWRe7GQ0Eq/94wmCGTNUzzyzuk3iXWuqz1a4\nE57h3qOQQUrtzCqd8dIH4XK1DYbqusex50umvlEaRC2CskZbxhE0DWXpUtXM7Mqw5iH5j4W1rmQ0\n+mQtEbFla3tXahu4JGqnSEHrtU2yA7W6SFaQGB9JI1DXGs91Udfs81h/Q12TEd98030VvImR3uJb\nqaxrkGhyY13XXf7/2zv3WDuK84D/vvuwoaQCYiJABdegoEZUTgxxKW5pZdJgQagiSyARGhWKrKAL\nlFKpqgOKVKVVFbf80RTHNDWkvJSoiQIlINLyMtwKyVcG8zA4cdJA6xIQLuAGIqricn2nf8yOz5y5\ns7Ozu+dx77nfTzo65+zZszvf7O58M99j5mBn8a6xMfs9lCklS04Ic+46HDlLADimp7tXwxwfn3/u\nsuixHNlmZjq+k6eesgEUMVnbJrP6ASBr1sAzz1jnOHMTHNy3+kjaT1k9++cHWLmy2m8Wk90trXDg\nAJx0kvUd+ItMlS2f8Oqr3fIf3Lc6OptvE6anKUKlBQ4bxmSciWWG2Q/ylsuOXZuySMmcerz66u5A\nmq7JGBPJxZ2AmtXFC1jf32WO55GjbRb7q98jkia4Ye7GjXkmg1z60astO24OvTh3ro9k6u/uNlOb\n92f1bFO9R3//sTE7MnGjhl6Raw7tVf3ljGhj9dyr83dGQ/a1fHleT9/5B/sx2vfNQpPL/89sv//F\nxn7MHD9fm3ps8vypj2TEFEl4Qbdvn+/zaNJAxx76lA22rb+m6YPQD/9M3fLFHsTQl7Fhw/z6jDmU\nB1HecN8256+SM6esbc/v++2c2ajsXg+v1caN3SbAXuGelypzXdUx6iiepvVYdb/06xlTRbJAFEnY\ns928udv2nmOPzeml5thge6FM+q0Q6uLK5AckpAIRYnWW6qn30/81qOvRTxlyyrIQRySDqBP/XL14\nblI+kn7JoopkgSiSMKxvbGx+72xycn7D7/eGw5ukqned25BWMUjFUXWuKrNLToNTdowNGzrXKKy7\nrnBNsddpWPjlr3s9Q8d+nfNUmhUzyuJ6/xs35o0AcjsITemVM9ova5nc/VZY/YoaNUYVyYJRJDt3\ndo9ARDqRGW7YHl5oP3qjKw9hrNMY1LH3G1PvwUmZyPpB1WiqTLZQJpfbUHcklqq7sDed6kn3gtxe\nZ24DW/daNhkN9LJRTpXHv9fbNpC9auB7+dzFjt10xNkr+XIViUZt9Zl16+DWW+2MvW6dktQ08DMz\ncN11nWx4Y2CiuEpzc/D44zbKJzzGzAxceaXdLxZpkhP95CJr7rzT7meM3d400isXP6Ll8GE7I4Cf\nIT893ZkC5tChTllCmWJy50yln4qyqppTLEXdNWRSZQ0jhMDuc+hQ95Q4seO9/37+tfTP46LWjEn/\nL3Vv5dZB2X7htYH2a/v4x60zkWqsjFVRdU1n/q6zdEQY1RmLduvn8wvoiGRQ5PYuQlPY5GTHLBEz\nv9TpceYMwWNO0UGNSMoy5MtyUapkMqZ9tEvT0VmTHmHKRBErQ2XCpHe83Gvp+43Gx7uTKav+1zTi\nq05dtYkebBORVrZv1Wg2FayRundTSa2xc4SjkF75llDT1sJSJLn4D7IfdpoavqamKMkl1ug4M1Ps\nQei1/yTVYPvKtcpPEZYr1QCUJQm6xtmfmqbMDFl23qYKrE4QRR2/WE4yYVkdNI1qyq2DumbXJr6+\nHNNo6rypfZsoqTq/j493nvGwg5Eyd5aZeuuQq0j6atoSkQuBW4Bx4BvGmL8Kfl8O3AN8EjgIXGaM\n2S8iK4B7gV8D7jLG/KH3n08CdwFHA/8M3FAIPBK4iRDvuw8uucQmKbntofnlmmu6zRYizZOPwvVU\nXDIWRNYViWxrOmx25jSw57viivnmg/XrrXnPmdvCtU/8Y+UkTZbtF5oVP/igU7cA3/8+bN4clyE8\nXpVJI2YmqTJRQHciW9U5mibKHjxozVpzc/a8VUmIvizQKf+rr3bMsq58ZfvmmF3d/2LXsyyJL2Xm\nqWN2Su0bm2G7yuRV9btverv99s59ODERTxb1zZ0pU2/fyNE2TV5Y5fEKcDqwDNgDnBnscy3w98Xn\nzwHfKT4fA5wHTAHbgv88DZwLCPAvwEVVZVkMIxJ/GOz3MlLDYt8JHIv8aloG/xhNIsRyzzU1Zcud\n48iemor3ynxy8yXKyh+aFV1vsGr+o7Lz5jrOU7/HTBShOaNOhFUOTU0+sclF/RFNSq5Urk5qVJnK\nm8ox85SNJqpMUbGRbx1zaG4dT0117j+wo+LUMXptKWDYpi1gHfCI9/0m4KZgn0eAdcXnCeBtQLzf\n/8BXJMDJwI+875cD26vKstAViX9DhFFaZbbRqrDUOg9ITtl8U1sT80LsmHWS1HIeGr+sVRncKXu3\nL2uVH6LOeR1VijgVjdbEVt+E3HslvA/Da+qX3ze9+PvWMSu5e73KrFtVh23CdcN9/M7fxIR9LzML\n1lX8oSIpe877kTRrzMJQJJdizVnu++9HRhd7gVO8768AJ3jfQ0WyFnjc+/5bwEMl578a2A3sXrly\nZW9rt8eUOdZ8pRKzy6acfL1s/GONadXDmMoYDv0xuaG1/nE3b47b8XfuNOacc+Y3UmEeRU6vM7Ut\nVrZUPkq4b1P7ednIrJ8huClyRiSuZx76nMJOUuqahCHY4YzYYSBBU4Wb4wsJfREbNsTv57CD1+QZ\n3LnTyitS/nz0uhPhs+QVif9aTCMSf5hfNWtv2YMXi/zyb/6602TkNlI7d9qht3/u2M0fNj51ktT8\nnn8suqx7llt7/s2bu/ft9VQbMblyo5xSpstQGcca1F6MSKoUf87/w162kytsdDduLO8ApMrvK1AR\ne5yqQILY81FlIi0b+fqmqphZLjbCDq9RLyPOfPrZiVgIikRNWzWo00POOVY4BfrUVD3zS3i8nOF+\nOA2Gb64K5aiTae0oG8n4pg2/d+h6hBs2dO+3YUOezE1MBXX+FyrUUGmEo8oyc2Yb80ZKObUlbIBj\nkUWpEVWooMJy1g2rzZU1PG/MhBZGRPmylpnbUte7bT33y1eSq0j6GbX1DHCGiJwGvI51pv9esM+D\nwJXADHYE80RR+CjGmDdE5Ocici6wC7gC+Fo/Cj9oytZWr7vmuvtPmATpIqK+/GWb1Dg314kWgerp\n1auif1wESciyZTYqJ0yuCqdQz5Fx/XobUeYimMBOTT8+buVZtsxGuj31VCf6DOz06Y8+2vnPJZek\nz5OK6vIjzMqmPk9FSPn7pBIx/STMuTl7Lbdtmx+Vk5NwmWJ62kanOXKS16qipMLEwfFx+MIXOlGA\nd9/dHf00MzM/wiu8Z3bsmJ8Y6hL03D2cishzsrqIPBG46KLOf93vYeSWuw5+qzQx0ZHFP/e6dXa7\nS+qdne3IsmVLJ+LM/R4uTV2H8F7sR9JmLXK0TdMX8Bng37Amqy8V2/4C+Gzx+Sjgu8DL2Gis073/\n7gf+G3gPeI0i4gtr3tpbHHMb3gim7LUYRiS9JNVDdb0ylyfiTEHue46Zoeyc4WjhzDM7ZSmzKVeZ\nylKmCd/xWtY7dDJs3pw/AirrHfu92bGx7ryS3FFbzEnr92B9mWILa4Wy1s1vCH+LmQKrTHI5vofU\nFC6xHn/YSy+7zmVObleWVH2Eia3ORxPz4/jnCKc4ipWlbFRUt4xVuLpJRTv20tTFsE1bC+m1lBRJ\nzg3uO+82bux+uNyU3TlO4xA3Pb6bT6ws7LNqDRb/YfEVXI58jrJInxxfQOwcX/lK2gae8/D6viun\nGLZvn+/zmZy0x8xJDMxt2MPORNgg5/pIUqHTYZRU3etUprT9evaVUNl/Q5NgrP79qLGyz36ghh9s\nkpppIiTmk2nSUfOvW1W0Y9Pjx1BFskQVSd3Q0nPO6b4pXehiXT+KIzYaKHNYphrGsMEOo19yoqj8\nXtvkZH7OSuwcZT4g53PJWc44NtVLTEH5DUNZfabKakz6PmjbI46NIJoqpzKlHY5ucx3jrp5jIdth\n2WOjkLKck3CEkbvsdSo4InUPV13TmJL1/9t2nRVHriLRSRtHjKps3fD3TZtgzx77fWzM3ppuWdxP\nf9r6VOr4Atwki7Oz85cg9bdDd8a0n4V86FBaxly/kb9U7uxst527yhcQnsPJdfPN8OCD3ccum0gz\nJLbssMva9/0UExO2HmZmyuszLFvoq4hllTtyM7pT2fcxO7+fle98YM6Xkzp2zP/mJqR0dVUnc9/P\nzPfrKzYBZNXncDlbfzaJ1DPijuHudRG46qrY8rjxujn//E79PflkJxN/YsLKNTFht73/vn2G/efI\n94/E6r4fqCIZEaoeTEc4DcfBg7B1q30PG4AyJVLlyCtrqMq2+8cU6W6koRMsUIfp6W7FIdLdYIdO\n0BzFtG4d3H9/x9H53HOwe3enwTp4sHsd8pD1660sofwi9n1sDM47D3bt6m6g60zlcdttnSCLiQnr\n5D7rrPlO4XAqFvebo2qN8Jhyc+V6/vn09B+xY4frt4f3aNk1ijXIYX2tWGEVAHTWig+V5MxM+piu\n3Hfc0bmvJifLlUisHLn38D33dDpThw7Z7+4c7tzG2PtkdhZeeqkTsFI19Uq/UEUyAlQ9mCHuxoo1\nFKtX50Vn+Teq2+7+Uza/1fR0vNfuH9M1qmA/X3DB/Ic1JzrKn58LbOTQtm22kQPbuPpK080hVnVc\nV3+xCKGqOc5i9bJlS3ev9aij4qO5nDmzZmbg2ms7x3MRSqGcfkPpR1a5+dXKGiS/fmKNtX+sstGQ\nO0aVoqkT/VbVWbr++s59cOedtuPk14kfRZiKckqNMGI0ness5MCB7vMb04n0M6b5/GE9Jcf+tdhf\no+4jaWLzbpMcVRU5U/WflMO4avrrOo7EVPJZU+dwTLY28fqp+qybaxBOp+HkyvGVON+M79Oqus5l\njm+XMJiKGKvKn6kKxMidOyv0QYnMjxqsE0XY1omd69/zI8X8QIOc56RJjlYZqLN96SiSJjd4m4ei\nKnImJGcf30GYctTWUYCpiKbcRLlBEHPsN1kDJTbBX070ViqBLvc6pxzLMTnLZKwKZ64qb0zZxRIZ\n63aGyq5VFaHMuZ2Esk5QeLxYeH+vIraMUUWypBSJMc16xm170+4YbUck4T5Vs7XWeVBijXRO1FHb\nB7BOmWI0GTG6RjMWMp0aHeQorZz6SY0Aq6KzykJtU1FL4fxaZXUWi2KK3Re9yARP1VnV/GCp/+aW\nq5c5JMaoIllyimSY5A7XU/uEppGq+ZCaPvRNE/h6RW4D0bQhcTLUnS6lF9cwVeYcpeGH1oY5IOHx\ny2bWHURnIPc+KTOfxhJQ25wn/I+OSFSRLFlyRyS9PE+/Rx4x6prmmii2YcpYVuawTDGlkWsCrZt/\n0Uvq1G1M5qmpzsSVuTlNTcrYqzrIVSQataUsCFIx/r0MX+xVJE1T6kTVNJlnDYYXAgrpOeP8endl\nnJuzEVAukq+qbqrqpGmd5VKnbsPoMT8y7OKL4YEHrCqZne3tNep3HcRQRaIsGGJJgIM4zyAZhCIb\nWghoBWG9h2UctpLPoW7dOpm3bOlWQCedZEO9F9o1aorY0ctos3btWrN79+5hF0NRBkZOLsawWQxl\njNGk3LFcL1j48ovIs8aYtZX7qSJRFEXpP4tRceYqEjVtKYqiDIBhmlT7zdiwC6AoiqIsblSRKIqi\nKK1QRaIoiqK0QhWJoiiK0gpVJIqiKEorVJEoiqIorVgSeSQi8hbwnw3+egLwdo+Ls9BRmZcGKvPS\noK3Mv2yM+UjVTktCkTRFRHbnJOOMEirz0kBlXhoMSmY1bSmKoiitUEWiKIqitEIVSZrbhl2AIaAy\nLw1U5qXBQGRWH4miKIrSCh2RKIqiKK1QRaIoiqK0YkkrEhG5Q0TeFJG93rYPi8hjIvKT4v34YruI\nyFYReVlEXhSRs4dX8uaIyKki8qSI/FBEfiAiNxTbR1ZuETlKRJ4WkT2FzH9ebD9NRHYVsn1HRJYV\n25cX318ufl81zPI3RUTGReR5EXmo+D7S8gKIyH4ReUlEXhCR3cW2kb23AUTkOBG5V0R+JCL7RGTd\noGVe0ooEuAu4MNh2I7DDGHMGsKP4DnARcEbxuhr4+oDK2GtmgT8xxpwJnAtcJyJnMtpyHwI+ZYz5\nBLAGuFBEzgX+GviqMeajwM+ATcX+m4CfFdu/Wuy3GLkB2Od9H3V5HecbY9Z4+ROjfG8D3AI8bIz5\nGPAJ7DUfrMzGmCX9AlYBe73vPwZOLj6fDPy4+LwduDy232J+AQ8AFywVuYFfAJ4Dfh2b8TtRbF8H\nPFJ8fgRYV3yeKPaTYZe9ppynFA3Ip4CHABlleT259wMnBNtG9t4GjgX+I7xeg5Z5qY9IYpxojHmj\n+HwAOLH4/EvAT739Xiu2LVoKE8ZZwC5GXO7CzPMC8CbwGPAK8I4xZrbYxZfriMzF7+8CKwZb4tb8\nLbAZmCu+r2C05XUY4FEReVZEri62jfK9fRrwFnBnYcb8hogcw4BlVkWSwFiVPZLx0SLyIeA+4I+N\nMT/3fxtFuY0xh40xa7A99XOAjw25SH1DRH4XeNMY8+ywyzIEzjPGnI014VwnIr/t/ziC9/YEcDbw\ndWPMWcD/0DFjAYORWRXJfP5LRE4GKN7fLLa/Dpzq7XdKsW3RISKTWCXyLWPMPxWbR15uAGPMO8CT\nWNPOcSIyUfzky3VE5uL3Y4GDAy5qG34T+KyI7Ae+jTVv3cLoynsEY8zrxfubwP3YTsMo39uvAa8Z\nY3YV3+/FKpaByqyKZD4PAlcWn6/E+hDc9iuKqIdzgXe9oeOiQUQE+AdgnzHmb7yfRlZuEfmIiBxX\nfD4a6xPah1Uolxa7hTK7urgUeKLo1S0KjDE3GWNOMcasAj6HLf/nGVF5HSJyjIj8ovsMbAD2MsL3\ntjHmAPBTEfmVYtPvAD9k0DIP21k0ZEfVPwJvAB9gNfsmrG14B/AT4HHgw8W+AtyKta2/BKwddvkb\nynwedpj7IvBC8frMKMsNfBx4vpB5L/BnxfbTgaeBl4HvAsuL7UcV318ufj992DK0kH098NBSkLeQ\nb0/x+gHwpWL7yN7bhRxrgN3F/f094PhBy6xTpCiKoiitUNOWoiiK0gpVJIqiKEorVJEoiqIorVBF\noiiKorRCFYmiKIrSClUkitIQETlczDLrXjdW/yv72KvEm5VaURYyE9W7KIpSwv8aO+2KoixpdESi\nKD2mWBPj5mJdjKdF5KPF9lUi8kSxDsQOEVlZbD9RRO4Xu17KHhH5jeJQ4yJyu9g1VB4tsvIRkT8S\nu57MiyLy7SGJqShHUEWiKM05OjBtXeb99q4xZjWwDTsTL8DXgLuNMR8HvgVsLbZvBf7V2PVSzsZm\nZYNdM+JWY8yvAu8AlxTbbwTOKo4z1S/hFCUXzWxXlIaIyHvGmA9Ftu/HLqT178UEmQeMMStE5G3s\n2g8fFNvfMMacICJvAacYYw55x1gFPGbswkSIyBeBSWPMX4rIw8B72OkwvmeMea/PoipKEh2RKEp/\nMCWf63DI+3yYjk/zYux8SWcDz3gz+irKUFBFoij94TLvfab4vBM7Gy/A54Gnis87gGvgyAJcx5Yd\nVETGgFONMU8CX8RO+T5vVKQog0R7MorSnKOLVRcdDxtjXAjw8SLyInZUcXmx7XrsSnZ/il3V7qpi\n+w3AbSKyCTvyuAY7K3WMceCbhbIRYKuxa6woytBQH4mi9JjCR7LWGPP2sMuiKINATVuKoihKK3RE\noiiKorRCRySKoihKK1SRKIqiKK1QRaIoiqK0QhWJoiiK0gpVJIqiKEor/h+mPrdO7d3H3QAAAABJ\nRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzsnXl8VNXZ+L/P3CQssmnUgiQQ6goY\nEYhoqmAQtWBdsNiK2gbceKvCW2zVV21VXFqs2hb3AgolVUGrPyMWECwSQAhCICCCCwiBhE2M4gZk\nMnPP74+75M5kkpksk0zC+X4++WTu/txzzz3PeZZzriil0Gg0Go2mNnzNLYBGo9FoEh+tLDQajUYT\nFa0sNBqNRhMVrSw0Go1GExWtLDQajUYTFa0sNBqNRhMVrSwSHBExROR7EenRmPs2JyJykog0es62\niFwoIiWe5U9FZHAs+9bjWi+IyL31Pb61ISJlIpLTyOd8SUQmNeY5NfUnqbkFaG2IyPeexfZABRC0\nl/9HKfVyXc6nlAoCHRp73yMBpdSpjXEeEbkJ+JVSKsdz7psa49yaxkFEXgK2KqUmNbcsrRWtLBoZ\npZTbWNs915uUUv+taX8RSVJKBZpCNo1G03AivbN1fY9b4nuv3VBNjIg8IiKvishsEfkO+JWIZIvI\nKhE5ICJ7ROQpEUm2908SESUiGfbyS/b2BSLynYgUikivuu5rbx8hIp+JyDci8rSIrBCRsTXIHYuM\n/yMiW0XkaxF5ynOsISJ/F5FyEdkGDK+lfP4gInPC1j0rIn+zf98kIh/b9/O53euv6Vyua0RE2ovI\nv2zZNgEDw/b9o4hss8+7SUQut9dnAs8Ag20X35eesp3kOf439r2Xi0i+iHSLpWwiyPyIiMyx68f3\nIrJBRE605dsvIjtF5ELP/l1EZKb9TMpE5CER8dnbThaRJSLylYh8ad9/57Dy+Z2IbLTrwGwRaVOD\nXLWey+Zs+9l8LSIvOucSkeNFZL5dd74SkWWe8/YVkaX2to0i8rMarn+TiBR4lt26LiK3AlcD99pl\n9qa9T5qIvGmX23YRua2Wcm8rIn8TkVIR2Sciz4lIW3vbhSJSIiL3isheYHqkdfa+0erBrSKyFfik\nJlkSFqWU/ovTH1ACXBi27hHAD1yGpazbAWcBZ2NZej8GPgPG2/snAQrIsJdfAr4EsoBk4FXgpXrs\nezzwHXCFve13QCUwtoZ7iUXGt4DOQAbwlXPvwHhgE5AGpALLrKoX8To/Br4HjvKc+wsgy16+zN5H\ngAuAQ8AZ9rYLgRLPucqAHPv3E0ABcDTQE9gctu8vgW72M7nWluFH9rabgIIwOV8CJtm/L7ZlPBNo\nCzwHvBdL2US4/0fse7rQPvYVYDtwt718C7DFs//b9vXaAz8C1gI32ttOAYYBKfbzXgE8EVY+q4Cu\n9nP5DMsSjiRXLOf60H7Gx9rndcrncSyFm2wfP8Ren2Lf2132tgvtcj8pQhmHPAMi1/VJnu0+YD1w\nr32dk7Dex2E13N/TwJt2/egEzAce9tSrAPBn+1ztalgXSz14x75Gu+Zun+rcnjW3AK35j5qVxXtR\njrsD+Lf9O9JL8Q/PvpcDH9Vj3xuA5Z5tAuyhBmURo4zneLb/P+AO+/cyPI0QcAk1KAt7+yrgWvv3\nCODTWvb9D3Cb/bs2ZbHT+yyAW737RjjvR8DP7N/RlMUs4M+ebZ2w4lRp0comwnUfARZ4lq8EvgF8\n9vLR9vk6AN2xFEsbz/6/Bt6t4dxXAWvCyme0Z/lvwDMxPv9I5/I+48ud54bVoP4/4MSwcwwFdgHi\nWfdv4I8RyriuyuJcYFvY9e4Dpke4Fx9wGOjpWTcYWynb9eowkOLZHmldLPVgSCzlm4h/OmbRPJR6\nF0TkNOCvWK6R9lgV64Najt/r+X2Q2oPaNe17glcOpZQSkbKaThKjjDFdC9hRi7xg9aavsf9fa/93\n5LgU66U/Geslbw+siXI+sKyGGmUQy/12O5bVgS37sTGcF6z7W+ksKKW+FZGvsRpzp0zq8sz2eX4f\nAvYrpUzPsiNfT6ANsE9EnP19WJ0URKQr8BRWw9nR3rY/7Frhch0TSaAYzxVevifYvx8FHgQWi0gQ\nqwPzuL19p7JbVs9x3SPJUEd6Aj1E5IBnnYFlXYbTFascN3jKUcL22aeU8kdZF0s9CHn3WxI6ZtE8\nhKeNTsXqyZ6klOoE3E/1ytrY7MHq8QAg1ltS20vaEBn3AOme5Wipva8BF4pIdyw32Su2jO2A14HJ\nWC6iLsCiGOXYW5MMIvJj4HksF0+qfd5PPOeNlua7myolg4h0xLIAdsUgV0MoxW7glVJd7L9OSqkz\n7O1/wcrGy7Sf2VjqX69iOVd4+e4Gq9FUSt2ulMoARgL/JyLn29vTxdNC28dFKrcfsDoGDl3Dtoc/\no1Isy6CL56+jUuqyCOfeh+UaPtWzb2ellDcmE6kOhK+LpR602Gm+tbJIDDpiuRp+EJHewP80wTX/\nAwwQkctEJAn4LXBcnGR8DZgoIt1FJBX4v9p2VkrtBd4H/onlythib2qD5R/eDwRtK2NYHWS41w4I\n98CKozh0wHqJ92PpzZuB0zzb9wFpYgf0IzAbuFFEzrCDupOxXHw1WmqNgVKqFFgKPCEinUTEJ9YY\nliH2Lh2xGtlvRCQdy3VYX2I513jPM74HK0aGXcdOtJXCN1iuGROrFx4Afi8iySJyAZaL8tUI594A\nnCEimXan4YGw7fuwYlkOhYBfRH5vB68N+9iBYcehrJTzF4ApInKcWKSJyMUxlo1Ds9SDpkIri8Tg\n98AYrIDzVCK/LI2KUmofVgbJ34By4ESgGKv32NgyPg8sBjZiuYxej+GYV7D8wq4LSil1AMtV9CZW\nkPgqLKUXCw9gWTglwAIgz3PeD7ECnKvtfU4l1MX2LrAFy93jdds4x78DPGTLtQerd3xdjHI1lF8B\nR2EF7L/G8vk7ve4HgEFYDfRc4I0GXCeWc80G/gt8DnyKFasAqzzfwwperwCeVEotV0pVYCUsXIGV\niPEUVqxqS/iJlVKb7fMV2OdeFrbLC0A/OxPrdWWlpV5iy1xin38qVhwhEr/HcoGttu9xEZarM2aa\nuR7EHQl1F2qOVETEwDKjr1JKLW9ueTQaTWKhLYsjGBEZbrtl2mAFjSuxelYajUYTglYWRzbnAduw\nfPU/Ba60XQMajUYTgnZDaTQajSYq2rLQaDQaTVRazaC8Y489VmVkZDS3GBqNRtOiWLt27ZdKqdrS\n5oFWpCwyMjIoKipqbjE0Go2mRSEi0WZUALQbSqPRaDQxoJWFRqPRaKKilYVGo9FootJqYhYajaZp\nqKyspKysjMOHDze3KJo60LZtW9LS0khOrmmKs9rRykKj0dSJsrIyOnbsSEZGBqETxmoSFaUU5eXl\nlJWV0atXr+gHRCCubih7OolP7c8M3h1h+xARWSciARG5KmxbUETW239z4ymnRqOJncOHD5OamqoV\nRQtCREhNTW2QNRg3y8KemO5Z4CKsr2itEZG59uyRDjux5sWPNN3xIaXUmfGST1OdwkIoKICcHMjO\nbm5pNImMVhQtj4Y+s3i6oQYBW5VS2wBEZA7WVMSuslBKldjbzEgn0DQdhYUwbBj4/ZCSAosXa4Wh\n0WiqiKcbqjuhnxAso26fS2wrIkUiskpERkbaQUTG2fsU7d8f/oVHTV0oKLAURTBo/S8oaG6JNJrI\nlJeXc+aZZ3LmmWfStWtXunfv7i77/eFfPo3M9ddfz6efflrrPs8++ywvv/xyY4jMeeedVy1WcOml\nl9KlS5eQdU888QTt27fnu+++c9f997//pXPnzu49nnnmmSxZsqRR5KoLiRzg7qmU2mV/8vI9Edmo\nlPrcu4NSahowDSArK0vPiNgAcnIsi8KxLHJymlsijSYyqamprF+/HoBJkybRoUMH7rgj1JOtlEIp\nhc8XuT88c+bMqNe57bbbGi6sh44dO7Jq1SrOOeccvvrqK/bt21dtn9mzZzNw4EDy8/P59a9/7a4f\nOnQo+fn5jSpPXYmnZbGL0G/yplGHbxIrpXbZ/7dhfR2rf2MKpwklO9tyPT38sHZBaRqfwtJCJi+f\nTGFpYdyusXXrVvr06cN1111H37592bNnD+PGjSMrK4u+ffvy0EMPufued955rF+/nkAgQJcuXbj7\n7rvp168f2dnZfPHFFwD88Y9/ZMqUKe7+d999N4MGDeLUU09l5cqVAPzwww+MGjWKPn36cNVVV5GV\nleUqsnBGjx7NnDlzAHj99de56qqQnB4+++wzAoEAkyZNYvbs2Y1ePg0lnspiDXCyiPQSkRRgNNbn\nGKMiIkfbH+RBRI4FzsUT69DEh+xsuOcerSg0jUthaSHD8oZx35L7GJY3LK4K45NPPuH2229n8+bN\ndO/enUcffZSioiI2bNjAu+++y+bN1ZuRb775hvPPP58NGzaQnZ3NjBkzIp5bKcXq1at5/PHHXcXz\n9NNP07VrVzZv3sx9991HcXFxjbJddNFFvPfee5imyauvvsrVV18dsn327NmMHj2anJwcPvroI778\n8kt325IlS0LcUCUlJfUonYYRN2VhfwN3PLAQ+Bh4TSm1SUQeEpHLAUTkLBEpA34BTBWRTfbhvYEi\nEdkALAEeDcui0mg0LYSCkgL8QT9BFcQf9FNQUhC3a5144olkZWW5y7Nnz2bAgAEMGDCAjz/+OKKy\naNeuHSNGjABg4MCBNTbEP//5z6vt8/777zN69GgA+vXrR9++fWuULTk5mXPOOYc5c+YQDAZJS0sL\n2T5nzhxGjx6NYRiMHDmS11+v+lT90KFDWb9+vfvXHDNsxzVmoZSaD8wPW3e/5/caLPdU+HErgcx4\nyqbRaJqGnIwcUowU/EE/KUYKORk5cbvWUUcd5f7esmULTz75JKtXr6ZLly786le/ijjOICUlxf1t\nGAaBQCDiudu0aRN1n2iMHj2aX/ziFzzyyCMh64uLi9m2bRtDhw4FoKKiglNOOYXf/OY39bpOPNBz\nQ2k0mriSnZ7N4tzFPDz0YRbnLiY7vWn8nN9++y0dO3akU6dO7Nmzh4ULFzb6Nc4991xee+01ADZu\n3BjRcvGSk5PD3XffHdEF9cgjj1BSUkJJSQm7d+9m+/btlJWVNbrM9SWRs6E0Gk0rITs9u8mUhMOA\nAQPo06cPp512Gj179uTcc89t9GtMmDCB3Nxc+vTp4/517ty5xv19Ph933nkngGudKKV49dVXWbx4\nsbufiDBy5EheffVV+vXr58YsHB544AGuvPLKRr+f2mg13+DOyspS+uNHGk38+fjjj+ndu3dzi5EQ\nBAIBAoEAbdu2ZcuWLVx88cVs2bKFpKTE7IdHenYislYplVXDIS6JeUcajUbTAvj+++8ZNmwYgUAA\npRRTp05NWEXRUFrnXWk0Gk0T0KVLF9auXdvcYjQJOsCt0Wg0mqhoZaHRaDSaqGhlodFoNJqoaGWh\n0Wg0mqhoZaHRaFoUQ4cOrTbAbsqUKdxyyy21HtehQwcAdu/eXW0SP4ecnByipeBPmTKFgwcPusuX\nXHIJBw4ciEX0Wpk0aRIiwtatW0OuJSIhMq1fvx4R4Z133gk53jCMkPmjHn300QbL5EUrC41G06K4\n5ppr3NlbHebMmcM111wT0/EnnHBCyLxLdSVcWcyfP7/adynqS2ZmZsi9/fvf/64239Ts2bM577zz\nqs1M265du5D5o+6+u9qXrBuEVhYajSbuFBbC5MnW/4Zy1VVXMW/ePPdDR870GIMHD3bHPQwYMIDM\nzEzeeuutaseXlJRw+umnA3Do0CFGjx5N7969ufLKKzl06JC73y233OJOb/7AAw8A8NRTT7F7926G\nDh3qzuOUkZHhzhD7t7/9jdNPP53TTz/dnd68pKSE3r17c/PNN9O3b18uvvjikOt4GTlypCvz559/\nTufOnTn22GPd7Uop/v3vf/PPf/6Td999t0Hf1K4rWlloNJq44nyy9777rP8NVRjHHHMMgwYNYsGC\nBYBlVfzyl79ERGjbti1vvvkm69atY8mSJfz+97+ntlkqnn/+edq3b8/HH3/Mgw8+GDJm4k9/+hNF\nRUV8+OGHLF26lA8//JD//d//5YQTTmDJkiXVvla3du1aZs6cyQcffMCqVauYPn26O2X5li1buO22\n29i0aRNdunThjTfeiChPp06dSE9P56OPPmLOnDnV5pBauXIlvXr14sQTTyQnJ4d58+a52w4dOhTi\nhnr11VfrVrBR0MpCo9HElXh8stfrivK6oJRS3HvvvZxxxhlceOGF7Nq1K+IX6RyWLVvGr371KwDO\nOOMMzjjjDHfba6+9xoABA+jfvz+bNm2KOkng+++/z5VXXslRRx1Fhw4d+PnPf87y5csB6NWrlzu3\nU23ToEPVR5Ly8/Orzf/kfPPC2c/rigp3Q4UrmoaiR3BrNJq4Eo9P9l5xxRXcfvvtrFu3joMHDzJw\n4EAAXn75Zfbv38/atWtJTk4mIyOjXq6a7du388QTT7BmzRqOPvpoxo4d2yCXjzO9OViB6JrcUGB9\nm/vOO+8kKyuLTp06ueuDwSBvvPEGb731Fn/6059QSlFeXs53331Hx44d6y1brGjLQqPRxJV4fLK3\nQ4cODB06lBtuuCEksP3NN99w/PHHk5yczJIlS9ixY0et5xkyZAivvPIKAB999BEffvghYE1vftRR\nR9G5c2f27dvnurzA+pb2d999V+1cgwcPJj8/n4MHD/LDDz/w5ptvMnjw4DrfW/v27fnLX/7CH/7w\nh5D1ixcv5owzzqC0tJSSkhJ27NjBqFGjePPNN+t8jfoQV2UhIsNF5FMR2Soi1ULzIjJERNaJSEBE\nquWyiUgnESkTkWfiKadGo4kv8fhk7zXXXMOGDRtClMV1111HUVERmZmZ5OXlcdppp9V6jltuuYXv\nv/+e3r17c//997sWSr9+/ejfvz+nnXYa1157bcj05uPGjWP48OFugNthwIABjB07lkGDBnH22Wdz\n00030b9//3rd2+jRoxkwYEDIutmzZ1dzS40aNcp1RYXHLBo7GypuU5SLiAF8BlwElGF9k/sa7+dR\nRSQD6ATcAcxVSr0edo4ngeOAr5RS42u7np6iXKNpGvQU5S2XhkxRHk/LYhCwVSm1TSnlB+YAV3h3\nUEqVKKU+BMzwg0VkIPAjYFEcZdRoNBpNDMRTWXQHSj3LZfa6qIiID/grlsVR237jRKRIRIr2799f\nb0E1Go1GUzuJGuC+FZivlKr1A7RKqWlKqSylVNZxxx3XRKJpNJrW8oXNI4mGPrN4ps7uAtI9y2n2\nuljIBgaLyK1AByBFRL5XSjVuxEaj0dSZtm3bUl5eTmpqKiLS3OJoYsBJs23btm29zxFPZbEGOFlE\nemEpidHAtbEcqJS6zvktImOBLK0oNJrEIC0tjbKyMrTrt2XRtm1b0tLS6n183JSFUiogIuOBhYAB\nzFBKbRKRh4AipdRcETkLeBM4GrhMRB5USvWt5bQajaaZSU5OplevXs0thqaJiVvqbFOjU2c1Go2m\n7iRC6qxGo9FoWglaWWhaFI051bVGo4kdPZGgpsXgTHXtTEjXWPMMaTSa6GjLQtNiiMdU1xqNJja0\nstC0GJyprg2j8aa61mg0saHdUJoWgzPVdUGBpSi0C0qjaTq0stC0KLKztZLQaJoD7YbSaDQaTVS0\nstBoNBpNVLSy0Gg0Gk1UtLLQaDQaTVS0sqgBPVJYo9FoqtDZUBHQI4U1Go0mFG1ZRECPFNZoNJpQ\ntLKIgB4prNFoNKHEVVmIyHAR+VREtopItS/dicgQEVknIgERucqzvqe9fr2IbBKR38RTznCckcIP\nP6xdUBqNRgNxjFmIiAE8C1wElAFrRGSuUmqzZ7edwFjgjrDD9wDZSqkKEekAfGQfuzte8oajRwpr\nNBpNFfEMcA8CtiqltgGIyBzgCsBVFkqpEnub6T1QKeX3LLZBu8s0LYTCQj13laZ1Ek9l0R0o9SyX\nAWfHerCIpAPzgJOAO5vSqtBo6oPOoosNrVBbJgmbOquUKgXOEJETgHwReV0ptc+7j4iMA8YB9OjR\noxmk1GiqiJRFpxvDULRCbbnE072zC0j3LKfZ6+qEbVF8BAyOsG2aUipLKZV13HHH1VtQjaYx0Fl0\n0dFp6S2XeCqLNcDJItJLRFKA0cDcWA4UkTQRaWf/Pho4D/g0bpJqNI2AzqKLjlaoLZe4uaGUUgER\nGQ8sBAxghlJqk4g8BBQppeaKyFnAm8DRwGUi8qBSqi/QG/iriChAgCeUUhvjJatG01joLLra0R+w\narmIUqq5ZWgUsrKyVFFRUXOLodFoNC0KEVmrlMqKtp9OSdVoNBpNVLSy0Gg0Gk1UtLLQaDQaTVS0\nskB/u0Kj0WiikbCD8poKPUhIo9FoonPEWxZ6kJAmnmirVdNaOOItC2eQkGNZ6EFCmsZCW62a1sQR\nryz0ICFNvNBzRWlaE0e8sgA96lYTH7TVqmlNaGWh0cQJbbVqWhNaWWg0caSlW6362xMaB60sNJoj\nkFiUgA7QJybNpcC1stBojjBiVQI6QJ94NKcCP+LHWRzJ6DEARyaxji3S355IPJpzXJi2LI5QtIvh\nyCXWLC0doE88mjPDTiuLIxTtYjhyqYsSaOkB+tZGcyrwuCoLERkOPIn1pbwXlFKPhm0fAkwBzgBG\nK6Vet9efCTwPdAKCwJ+UUq/GU9YjDT0G4MhGK4GWS3M9u7gpCxExgGeBi4AyYI2IzFVKbfbsthMY\nC9wRdvhBIFcptUVETgDWishCpdSBeMl7pKFdDA1Hp5VqjiTiaVkMArYqpbYBiMgc4ArAVRZKqRJ7\nm+k9UCn1mef3bhH5AjgO0MqiEdG9y/qjYz6aI414ZkN1B0o9y2X2ujohIoOAFODzCNvGiUiRiBTt\n37+/3oJqNHVFz1asOdJI6NRZEekG/Au4Xillhm9XSk1TSmUppbKOO+64phdQc8Si00o1RxrxdEPt\nAtI9y2n2upgQkU7APOAPSqlVjSybRtMgdMznyOVIjVXFU1msAU4WkV5YSmI0cG0sB4pICvAmkOdk\nSGk0iYaO+Rx5HMmxqri5oZRSAWA8sBD4GHhNKbVJRB4SkcsBROQsESkDfgFMFZFN9uG/BIYAY0Vk\nvf13Zrxk1bRO9Ah1TWNT31hVa6iLcR1noZSaD8wPW3e/5/caLPdU+HEvAS/FU7ZE50g1dRuLI7kH\nqIkPhYWwcyck2a1mrLGq1lIX9QjuBKS1VK7mRI9Q1zQm3nfSMODmmyE3N7Y61VrqYkJnQx2p6LTM\nhqOzlTSNifedDAahR4/YG/zWUhe1ZeEhUVw/eiqOhtNSs5USpQ5qQmnIO9lS62I4opRqbhkahays\nLFVUVFTv4xPN9aMbjSOPRKuDmlBa6zspImuVUlnR9tOWhU2i+RV1WuaRR6LVQU0oR/o7qWMWNq3F\nr6hpueg6qElktGVh01r8ipqWi66DmkRGxyw0mhZEa/Wba5oPHbPQaFoZOgAeP7QSjo5WFhpNC0EH\nwOODVsKxoQPcGk0LQQfA44MeBBsbtVoWItJJKfVtDdt6KKV2xkcsjUYTjg6Axwc9CDY2ormhCoAB\nACKyWCk1zLMt39mm0WiahiM91z8eaCUcG9GUhXh+H1PLNk0D0ME1jaZ50Uo4OtGUharhd6RlTT2o\nKbimFUhioJ+DRmMRTVkcLyK/w7IinN/Yy/qj141ATcE1nZ3R/BwpWTKJqhATVa4jlWjZUNOBjkAH\nz29n+YVoJxeR4SLyqYhsFZG7I2wfIiLrRCQgIleFbXtHRA6IyH9ivZmWSKQMF52dkRgkwnOI9xfW\nHIV4333W/0T5kluiynUkU6tloZR6sKZtInJWbceKiAE8C1wElAFrRGSuUmqzZ7edwFjgjgineBxo\nD/xPbddp6dQUXNPZGc1Pc2fJNIVlk6hjNxJVriOZOg3KE5E+wDX23wGgtiHig4CtSqlt9rFzgCsA\nV1kopUrsbWb4wUqpxSKSUxf5GkJhaSEFJQXkZOSQnd60tTI8uKazMxKD5n4OTdFg1qYQm9MNFE9F\nrd1b9SOqshCRDKoURCXQE8hyGvpa6A6UepbLgLPrI2Qtso0DxgH06NGj3ucpLC1kWN4w/EE/KUYK\ni3MXN7nCCEdnZ4TSXC94cz6HprBsalKIzR2viZeibu77aslEG5RXCHQC5gCjlFJbRGR7DIqiSVBK\nTQOmgTWRYH3PU1BSgD/oJ6iCHA4cJm9Dnru+OSwNTShH6gveVJZNJIWYCG6geCjqRLivlko0y2If\nloXwI6zspy3EnjK7C0j3LKfZ6xKO1Pap7m+FYvpbHzH9yfmojCW0yXg4ISyNI5nW+oLHYi01l2XT\n3PGaeNFa76spiBbgHikinYGfA5NE5GSgi4gMUkqtjnLuNcDJItILS0mMBq5tDKEbk8LSQia+MxFT\n2WGT0nMIzloIwRQw7qFi7MUUlBRoZdGMtMYXPNGtpeaO18SL1npfTUHUmIVS6htgJjBTRH4E/BL4\nuz03VHotxwVEZDywEDCAGUqpTSLyEFCklJprZ1S9CRwNXCYiDyql+gKIyHLgNKCDiJQBNyqlFjbs\ndqvjuKCUYzCVDLUUhUqCoMK34wJyMnIa+7KaOtAaX/CWYC211rhZotxXSwu01ykbSim1D3gaeFpE\nesaw/3xgfti6+z2/12C5pyIdO7gustWXnIwcUowU/EE/hs/gnCEmy5b6IajAqOR31wxo0VZFS6uQ\nNZEoL3hj0RqtJU3sJLplGYloAe65UY6/vBFlaRay07OZMnwKb2x+g1F9RlF+sJz3Sy/G3D4YX6/l\ndDnpZ8DI5hazXjR3hWwtiioetEZrSRM7LcGyDCeaZZGNlf46G/iAVjh5oBOz8Af9LN+5nCnDp9Am\nYx3+9FWkGCnkZDze3CLWm+askM2tqFoCsVpLWulGp6WVUUu0LKMpi65YI7CvwQpOzwNmK6U2xVuw\npsKbNusP+ineU8yYfmMAyO2X26JdUM1ZIVtizykR0Uo3Oi2xjFqiZRktGyoIvAO8IyJtsJRGgR2I\nfqYpBIw3TsyiIlABwIvFL2IqkxQjhdx+uc0sXcNozgrZEntOiYhWutFpqWXU0uJwsYzgbgP8DEtR\nZABPYWUwtQqcmMX4+eMJmAFOJlLKAAAgAElEQVSCKghARbCiVaTMNleFbIk9p0REK93o6DJqGqIF\nuPOA07Eymh5USn3UJFI1MeUHyzGVWZU+C5jKDBmsp6k7La3nlIhopRsdXUZNgyhV84Bse4K/H+xF\n744CKKVUpzjKVieysrJUUVFRvY515oY6HDjsKgwfPsYNHEePzj3iMuVHSwvIaTSa1omIrFVK1TYp\nrLVfbcqiJdEQZQGWwsjbkMfM9TMJmAEMn4EgVAYrEREuO/Uy7vrJXUDD54xqiQE5TdOiOxOapiJW\nZVGnQXmtmez0bLLTs8ntl0tBSQE7v9nJtLXTMDFBQf4n+cz7bB4+8REwAw2anbalBuQ0TUNzdSa0\ngkpcEuHZaGURhqM0pq2dZjvbqrZVmpUIgkLhD/rrHQBPTQWfD5TSATlNdZqiMxHe+GhrN3FJlGej\nlUUEnIF64S46o+w8KMlBZSwhJWNdveaMKiyEiROthsDngylT9MCspqQllGNjZvdEut9IjY+2dhOX\nRHk2WllEIHxywd7H9ubUQ2NZ8NLv8fsFSfoDE/7xTjWrIpaGyHnwpgkiUF4eXZ5E6Vm0dFpKOTZW\ndk9N9xup8dHpp4lLojwbrSwikJORg+EzCAatMRdbv9qKbOpGhR8wfSh/Ek/8+ShOPHoj40ZmArE3\nRPV58M3Rs2gJPfC6kig9tFhojLTjmu43Uh3U6aeJS6I8G60sbEIbx2xuOPMGpq6dikJRaVayucNz\n4BsFZgpgYH4+lFuvNsksANIKmfTPCir852MGpdaGqD4Pvql7Fi2lB15XEqWH1lTUdL811UE9LiZx\nSYRno5UFkRvH3H65zNowq2rsRfoqGDMMCh6AbReCSiJYWcljL69mYbdhVJgDMH2L8NEWX1KQ1N6f\nAJkRr1fXB9/UPYuW1AOvC/Utx5ZqZdV2v4nQ+Di01PI90tDKAsjLg8OHrewkp3G8555sFucuJm9D\nHi8Wv0ilWWkpjJwHYccQ93sXu495BX/Qj5m2Asm9CHYMJZhRwMRN68gc2HifY23Kl7s1Z2vVtRxb\nupWVSEohEi29fI8kfPE8uYgMF5FPRWSriNwdYfsQEVknIgERuSps2xgR2WL/jYmXjIWFMGOG1TAC\nJCV5zPX0bJ6/9HmeueQZfE5RORbGBQ+QdP1wbryiDylGCoYY+Hp8gDrvz5hpK9zU2pZGfbO1mpLC\nQpg82fofbyJZWZrGQ5dvyyFuykJEDOBZYATQB7hGRPqE7bYTGAu8EnbsMcADwNnAIOABETk6HnIW\nFFgV1bouXH999cax/GA5IlWf8vD1WI0MfhSjx2oyj89kce5iLjvlMpRSbgZVki+pRX6O1ZutpVT1\nbK2mbKgj4fRE77vP+h9vORy/v2FYfzt3Nt+9twbC64+3fFubFdvaiKdlMQjYqpTappTyA3OAK7w7\nKKVKlFIfAmbYsT8F3lVKfaWU+hp4FxgeDyGdyurzWRW2f/8I+9jTmBtikOSzPHcKRcAMuNbD25+9\nbY32BgTh+jOvb5Ez1tb28jZ1Qx2Jpu6JOn7/m2+2OhPTpzfdvTe3Ym5sItUfp3wfftj6D63rnlsT\n8VQW3bG+sudQZq9rtGNFZJyIFIlI0f79++slZHa25WoxDKs3PXFiVUV1XlbKrPjFw0Mf5tlLnqWN\n0QZDDPtLejnkbchzpzYH8Ikv4rcwCksLmbx8MoWlifsmhL+8XisrEVwGzdETzc6GHj0gEGi6e28M\nxZxoyqam+pOdDffcY/1u7s6IpmZadIBbKTUNmAbWRIL1PU95uaUoTDO0EocG3rK5Z7DVcmYen0lB\nSQEHtvZm0iMVHE7rHHK+y065rPqAPXtmW3/Q36B5peJFeEZKY40RaWyaK+c8J6eqQ2EY8b/3mhrW\nWO87EQPH0epPa83C89KSM7/iqSx2Aeme5TR7XazH5oQdW9AoUkUgUiWureJmp2ezcW0H7v2fEyGQ\nAsYgjLErMNNWkGKkMOLkEUxePpnU8ksp/zjTOl8g9POtifRhpVgbltoa6qZ8CRojw6c+8jphK6Ws\nDDpHlngQXidTU+vW+Dd1wxtLeUZT9InQGYkn9VHgCaVclFJx+cNSRNuAXkAKsAHoW8O+/wSu8iwf\nA2wHjrb/tgPH1Ha9gQMHqoawcqVSf/6z9d9ZbtdOKcOw/jvrHS4et0QhlQqUQvzquMv+pn7z9m/U\n1KKpqt0j7ZTvpnMVyT8on2Gqdu2Umvrmh6rdI+2U8aCh2j3STq3cubK6EM3En/9s3SdY///857od\nH62sEo36yOstI1BKJP736q2TdX1GTflMGvNa4e9hayJRnyFQpGJo0+NmWSilAiIyHlgIGMAMpdQm\nEXnIFm6uiJyF9YnWo4HL7G9791VKfSUiDwNr7NM9pJT6Kl6yQvXearRe0JnnHGDRDL873mL/8a8x\nc30xgDXuYvtgCKRgKmtEd/nHVtZUQ7+FEQ8i9ejq0qNpae6D+sjrlJEzHsc7Jide9xpeJ+vS625K\nd11jPv9EHxfSEOpqOSXaexXXmIVSaj7WJ1m96+73/F6D5WKKdOwMYEY85YuGU3GdQKE7nXNpIVN2\nXQ1jBkBJDmQUQPoq/EHLT5FipFDRazlmkh+faZCSItax9vTnhYUw+aXGe4kLSwsbpITCGxaom7kc\nPogvNTW0vBKN+ro7xoyBvXthwQIr2N2UrpK6NP5eRe8EjuNJa3cfNRZ1VeCJVq4tOsDdFESczjlQ\nQGXQHtGdvsrdN8VIIbdfLv279eeNzW9w5k8W8e0n/SFjKaSdDGS756vwK4ykAM/M+cSdjDDkujEq\ngMYKnHt7dJMnx96jCR/EN2GCtZxIgdVw6vrShteBp56ykiIiHRtPH3Msve7mCGwnykR3LYG6WE6J\nVq5aWUQhkimY86scko1k/EE/YH2v+/LTLnc/uzrxnYn4g34WsxjVXmF+YfLCP5N49pJnKS8YR4Vf\nYQYF04Rbn32V4qTnyO2X6zbyXgVg7DqPS5Ifo2vfT8i99ORqisCZTr0xA+d16dF4B/GJwPr1iWU6\n10RdXtrwOlBebvXYq1mcCZCB1Fyui9bsPmpOEqlctbKIQsTpnNOzKRhTQN4GKyXG29Df8p9bqiYf\n9BAwA4yfP57bk4cBGSBWrCPYczFT137ArA2zXKvAVQA7zyI4az75wRQwTmfG+kso+OPkEGWQk5GD\nses8zM/PxThxRaOMGq9Ljya8fEaNguXLG246N1UWSE0fB/Kuqymmk4gfEGpo/OlIoznrWUtDK4so\n1Didsx1/8FJYWsiM9TNQpWeHxDIcKndk8dd/paOUD6QShk+E9FUooCJQwaSCSUzKmeSOGD9cMhQV\nTAGVBEFF5efnVrccyrKRvMXgF9RyRZ7PgNyGV8hYezSRyiczs2EvRlP10CNdByJfO/weI7nqEsHH\n3ND4U31JpMYwVlniUc9i/TJhc5dRfdDKIgZibTgLSgoI7DgLZi2CYAoYfmvSQUdhlJxPsNIHShCf\nAYeOc+0PE5N3t73L8p3LWZy7mCnDp3Bryb8IGlUZV8knriAnY3LoNQsgUGmgTKj0w9SpMGtW01bI\nSJlkDbl2U/XQI10HIl87/J4izcybKD7m+saf6ksiNYZ1kaWx61ldvkyolUUrJdaeSk5GDr4dhzA9\n1gAlOVXKIqPAUiBBhfJVQsaSkOMVisOBwzy24jEOVh5Epa+EMcOQkqGc9ZODTLl5cjVrpq4pnXUJ\nnDdWmq9TfqmpNQeGI91TvHvoNV0n2rVrm5k3kXzM0DRlmUiNYV1kaeyyqenaiWBxNgZaWYQRrhjq\n0lPJTs/m2Vs7MH6ZIhgwEUPh+/FKgvgwSwdZimP4b+HQsdVcVA4KRf6n+QiCQuHrsZo2vTYwJdfy\nkdzyn1sA6N+tP+UHy8nJyGHx4mzy8mDmzNpTOmPNnGrMqUnc7K8KKwju80GbNlHKMY499PDnG+k6\n0a4dHtSP5TvqzUVTWDtN2RhG67jVRZbGLpuarh3JNZjIqeU1oZWFh8YIWo4bmUnmEmc6iBT6//QZ\nFmxdQP6s8RFdU45SCEeh8OEjq1sWA7oNYOMXG5mwYIKbgeUc2zapLYtzF/P889nk5to9+N4bKQj8\nB0pz2Li2A28sKGfUiFTKU2PLnGrMDCtvwwqh82/VVo6ReugR/cF1sIBqUvzh14lmHbS0nmK8rZ14\nKySvZRopLTuWDkBtsjeWvLVd27lOIrns6opWFh4iKYbaGobaejmzZtnHzMok86JKS1GEuaZ84nOm\nNwGsFFzTM1u7iFC8t5g1u9fgEx+mCp3JXaGoCFa4jXl2NpBWZRVI2U8IzHwHAr1ZNF1xUjYYpy+E\n7u+7M+ZGwgmwO5ZFLBlWNTXaTvl5LYv6NLARg9FpsVtKBSUF7PzPtfj9PRvsLomr5dMMbsKQ89Yz\nUB0vheR97iKxTPjZvK7AaNdOJJddXdHKwkNOjvWlPNOs+mKe0zA4E8c51NZDCK8QJ3TsBkl+CFiB\n6iFDFMecMpJ5W+ZRqSoBaGO0YcTJI8j/JN+9RlAF3anPvVOgezHECGnMvWm3FNxrTXRIEijF1pWn\nk7Tmv9z85CsRx2w4ZKdnV5uapLbGqTa3lbdhDY9Z1KXBixiMPi+6BRQyZuXAQpKSFwNGiMKqT8Mb\nq+VTF+LlJoxZAdWx1xsPxRa+r/e5+3zWjL9gKY7U1NB6cfiw9Z7G023ZUFqaVepFK4swnI5+MBiq\nIBxLwck0qq2HEF4h7rqtGyOu2ui6gzIHXsGkgkmuAhCEESeNAFWzWyoSgnB79u1uY563IY+93++F\n0myYtdBWFAagAGsqkkClj235uXAmoXMCU/1FjThI0Gdww5k3hIwtiea2itiw1tLgRWpcUntvBONU\nRBkkJUNOjgFp0S2ggpICKkoGYG4fjOq1nHF/e5keB3JDFFZjxGfq0tDW1HjG6v6ri5vQub+KQAU+\nn49nL3mWcQPHRdy3Lr3eeCi2SPWsf+9bSUnJdMt1wgT4+98tGSdOrPoWTTBovbszZ0JuHVLHa+0E\nxcFlVJNVmkipxzWhlYWHgoKqShcIVKWhjhlTN/dUxApRmEl5BsBG9+U1MfGJjyRfEm8v/pLg9vMg\nY2/EwHckFIonVjzBqrJVFJYWUmlaVopsv8dye5EEBLAUhc8+Slj0bpCly2DJe0aI79/7Uk8ZPsUN\noHsbp2AwyNS1U0MGEaa2T7VcaqiY3Vbec1YEKpj4zkRO6HgCAAu2LiBgBqoajG79mfDRBIK/tubi\nMn+8EtIeDbGAUtunUlBSwMYvNrpyZ6dnk1p+Keas31qTOvoCbP76cxiW506/UlvDW5cecV7+Dg5X\npKNMH4crFLm/3c6d9/5QbSqX2hrPWN1/seznut6+2cnh7f1RJUMwMwoYP388mcdnRryfuvR646HY\nItWztkmzmPLKBxQvtMrx229DXVHl5XDDDda76ry3XiVXX4sYGu4yquna4Z2nlhLH0MrCQ01pqFD1\nEjnfYYba/dbegNYtt1RlKvmSTiP46wGYaSvw4ePCXhfSft8w8v8ZOQDuw8epx57Kx19+HFFmE5Nl\nO5ZVrSg9B/VNOvgCYFpur2N+/hBfbesJe/rD7ixQSVRUVJKXX0Z2dk8KSwuZVDCJimAFpjKpCFQw\nfv54TGW6iiPFSHFHpjspvs4I9onvTCRoBvH5fEwZPiXiSxn+0jgNnqM0V3/gg5LT7CyxCgC3wfCJ\nz7LC7Lm4KsEdwOicz6uABcHwGdb0Kh+Pw2cqTCUQNFj2xmkse6uXOxo+JyMHw2cQDAZRSrF692r3\nS4ax9oin5W9k+pJVKPk1kIQyDbYW9eB/fumH1zaSOfB79/4LSgqqytkTb4LI7r9IhO8HMHn55BCX\noTdupWa969atwNiLQhViHYLDjvUKVjaeV2Gltk+1vuHSPjVEWdcl/uUORg2rZwu2LmDhrEz3/fMZ\nJqZSVRYmnhhhmHuxVmXgUU5OfY4Ub/O+99OmWQoqtfdGylP/U+NzqotF1VLiGFpZePDGJ5zG3fGR\nTpkCxcXW+unTq9xRtc3q6fQYHOUDoEjCt+MCJH0VKUYKk3ImkffMCSEBcN+OYSRnFBMwA6QYKZzf\n83w+Lf/UDXCHB8JdSs+BWYutc/kCMPAF6JfHV+mroI93u6VEyFhKYenJ5MzKcbOsBAGxpidRKPxB\nPwu2LCCjSwaHAoco/aaUoAqiUMxcPxPAmpIdE1FC+cFy997z8new97hXWXD4fvdenJfGafAmFUxi\n0dJvq+QOU5YKFTFes2jbIt7d9i6Dew6mz7F9XBmcY5zpVZ7JzKZNSqbnGRgQTHZHw+dk5LhJBiYm\n+Z/ks2DLAq4/8/paGxKHafkbueXqkzEre1tl3r3IVcgEFA/nLWf/R79z7//nvX/uPkdTmaS2T42o\nTCNZSd7GOrdfLvcMvidio+RtBGXbYDDbgDIgqDB2DHMb7Gn5Gxk/+jQClQZJyUF3Usuaxud460my\nL5mfnfwzunboSv9u/Zn4zsQQa7mN0cZ91rUpNi/Ovnkb8nix+EUqzUoUircXfodZYaJMHwoTGfAi\nSgUJiI+N+7IZNzIzopKryapxyvtAxQH32grFtLc2sved1dx13SC3s+dtD6ZNs9OlfQplnIhvzDyS\nek6q5pat7dpueXqUdCSLLl4JDA1BK4swnEqSm1tVSaZPtx7imDHVv8McS+aDoyhEoE2KMOXWX1Ce\n2q6qIoyEmU8F8fuD+JIUz912NZkDfxbygs3aMCvERbRgywLyP80PuV7bshEcdpSOqaDzzlCXVvoq\nqyHekItPDPp3yyZvw3PV0nGVsnp1UvYTVEkO+Tvfg/SP3e1OXKUyWMm6PetI8iWhggoRsRq/Qhh6\nQZCKiu5gjIcxb9pTuFd/YUf1GcW7s3ZUTWsSENiQW6Mr7vj2x/PFwS8A6wVftmMZK3auIMmXhBk0\nQ+I9ATPAG9/dwZRXnqB4YSYvzlBUVgZCRsPnbchz3XcOFcEK1u1Z51o0CsX0ddPp361/iL+/sLSQ\n255bgFl5f1WZdyuGfWe4CrnsmJcgaFlKhwKHeGXjKyFlXbyn2J140nm2kRreKcOnhKROT183ned+\n9hzlB8urNUre3rxx4gpkBVRWKowkeObWX5CdnunKHvDfD8pHpT8YMqklZdnk5e+AjKXkXnoyBSX2\nTMs2lWYlb336Fm2T2rL3h70h86GZyuRw4DAT35nIgG4DalVskZInnMZx6tqpKBRmz/dQvntAJWNK\nJXRdAwumEAymcOvVJsVT8si99GTuuSf0ZYxk1XhjOCEdrtJzMGctIj+YwoIXg66L1hmBHQg46d8K\nZYoly/bB+NNWVHPL1nRtt95EcDuFjMOIIcuvOZSJVhY14K0kjnKAumUyhJuxN9zgBN8ygSpfdna2\nFT+wKothbyekEoS7J8YNHMe0tdO45T+3uJXen74IjDs8lkNBZMHWj0GZbbj16iDp47+BTlWbnHNJ\n6U+Qfy3GrEwC4w+WkgFUSQ5Gr/dRaSsxMVmze41rjQTNILfNv41L952J35/l9mYpyUHSP6j2wjov\nw7WXP87LBQEIGoAPiq+HfnkRFcbXh7+ulgQQVEEuO+ky3v7s7ZCkAYXiv9v/y3LjbBbfu5jc3Gwe\ne7mI3ce8Qs6Qs90ebCSK9hRZ9+W5xvj54wEo3lNcVV49N4Jxtx0aUtB1HTLmJY7edyVfd30TlVYY\ncl6v3Ek+6/XzWjAvrnsxxEoylYk/6OeNzW+ENNaOPM9c8ozrznOUNcCYfmMAyL0+F8Y6dSvZrVsF\nJQWYPd+zZA9WTWr5j7WreGHuJtSsdwlWdgfjKl4oHs6lF6SS5EsKUayOm+jtT9+ulpShsFx6q3ev\nZub6mSwZsyRibxuIGIDP7ZfrdpCk52qCYy9CbR+CZCxDdgx1Z0kI+oP8Y8ZBZpTnuD18wLXAvLG3\n7PRsJi+fHFK+LiU5rnXv9wd57OXVHNx6H6P6jCInZxxJyUGCQQUYIIGQ98uxwJ37cWJoY/qNYe/3\ne+naoSsbv9hY5YosyK7mdrrnHiDNjjNt2OlmNB4qGcpjbZbz5h0eq6QRB83WhbgqCxEZDjyJlZLz\nglLq0bDtbYA8YCBQDlytlCoRkRRgKpAFmMBvlVIF8ZQ1EuHmYW4uVQPfUqvyvBtjJHK0/OxIExeO\nGziO4j3FVT2wtBVWo14y1JpKxBP3cHzAzkuhlEGw0qRkQ08YHOGCJTmYgeSqBn9DLqwfA8EUfCnQ\n/647WG08WXVeu60ImAHyD09EjPesY+2Xqvdxvbn0lEvdoKu3gfyAKZw07FS2LhoKGGAa1j3YY1EE\nwVSW1RA0q7ukkn3JfHX4qxB3VfeO3dn9/W63sbVeVFjYbRiHA4dZvaJ6xlnvY3sjCB9/+TGmMvHh\nCxnfEjAD3DrvVvc6PnwYPQzM4RNh/jOgfPDOkyTfcAmTJ3Vi4jvF+IOWHzPclWaIwTOXPEPm8ZnM\nWD/DipmgKN5b7FpqTvwF4HDwcLXGOqiCFO8p5qcn/tRSlGaQW+fdiogQNIOkGCnWSP9AATm/CnVl\n7f1+L8k9i6gYcyGUnB8yo0Bg27lQabhu0UDxNeRv24lkfEGXkzdz4HCV+0YkevZeRbCCvA155PbL\ndS1An/hYvWs1+Z/ku1aJaZrcMs+aoWDcwHEhyQsTFkygMq2QZCOZiefeyV+XQrBSYXUuxuLvN4up\nwanWRJ5KueXUxmjDkjFLaoyV+cR+xr2WY9pT8fiSTPIP/xa2rWLRtkXcde7n9LuzgtUr20O7/dVm\nYBAEEWHT/k08UPBANUvV6bgIQoqRwm+7zwHjkpDMvmlrpzF+/niCKmh1IpyMxmAK+Uv9/F9qPn+5\nfiTQuINm60LclIWIGMCzwEVAGbBGROYqpTZ7drsR+FopdZKIjAb+AlwN3AyglMoUkeOBBSJyllIq\ngqM+ftTW2NeUvRAeMIz3ACGnB+a6AdJX4UtfjeEzMJXhujaK9xQzc/1M/BnLUJ7JCckosGIZYQpG\nZbwHvj+ASrb2A7fnFagMUrEtG05+MrJQ6YWo3KEhM+9u3g+b92923VhOp12h2Pr1Vki/j6Q2SwlW\nQlKy8LPhXSBtJJRlQ0kO8yvvInDCcus4T9vU59g+/Pac33LrvFtDRPjx0T9m3w/7ADB8Bqt3real\nD1/iUOBQNXGdkfATz5nIhAUT3MbP8Blkp2Xzfun7KGW52byNvokJJsih46yZhFUSYgo3dJnFuIE9\nyTw+k7wNeawqW8X6fetDrnfzgJvJPD6Tx1Y8Ruc2ndl/cL91TmVyY/8b6dG5BwcqDvDXlX8lqIIs\ne9+Pb8ddnDnoKza2mYZCkeRLsp5p0O/KHFRBt3wqghVuuaQYKUw4e4J7PrCU7KCzg6xJ/0tog59R\n4M5hhi9oWXpmEsrwc8COJzlJBL/L/h1TVk0JcWVG4sXiF9n7/V5X6VealdXcqM79ezO2HAXnKE1B\nOPGMLxg44kNWv90flNidixxU+ioqg5Uh9+IP+snbkMdjKx5j93e7uXHAjW5cxOn1u9bIgNfZXHQc\ny+ThEKv28RWPowwVuVNFVYzs5Y0v17jd+V8RrOCxnVfCr89xM/s2plzHbfNvI2AGXJmPKvspP3ji\nmE+8UsTIC39ULWnA8Bns/GYnhaWFcVcY8bQsBgFblVLbAERkDnAF4FUWVwCT7N+vA8+IiGCFY98D\nUEp9ISIHsKyM1XGUNyLerCZnPpeashea5StldlDwsVeX8/bC71AZS2iTsa6a+Q2WYplUMIlFeHqT\nEDm47MQ3nAYfsS0LaxLEDW2nhMgRYr1Ata8IOoRbIi7pqxhw112MbDeF1N4bmbhpEhWLB2D+czxi\ntsFIXoT8ehhm2gprKhTbl//C5S+QtyEvpBH34eODXR8QNINuLztSw+RlwtkTKD9YHmK5mMrk/Z3v\nVwXOVfUetIlpKVnjDxC0MnT6Z3/rZgbNWD+jWkMqInRq24nBMwdXk9vwWZaIkzllKtNNTDCDKXy4\nLMA1j/dl/9FzaZ/SPqILKEQ+u3GuCFTwxMonQmYBCJgBTuh4Akm+JDehwXkW7rP/pgesvdmOJwEF\nD0DOg5x1tskJHU/gsy8/CykXR4lc3fdqlpYspey7MoAalYP3OK/C8/aWC0oKXPkqg5WMnz+ewHFn\ngfEuYrZBkkxUxjJAqjLnnDIVH9PXTXfXrd69mrvOvSskBugGpy+FIV8OAbvRdoh13FOd8GT2Pbnq\nS0wztB/8Q/d5YPzO7dSpjCUUlBwVkhzy2IrHePuzt5m2blq1mEk8iKey6A6UepbLgLNr2kcpFRCR\nb4BUYANwuYjMxho6NtD+H6IsRGQcMA6gR48ecbgFi3AlMGVK5NiFV4nEYzRpjZRls/D+bJT9qdYp\ncz5h3MDqn2rNTs9mUs4klu8cxuH0VdZLsPzuiFORACEN/pCeQ1hGdZcFWC/685c+HzHoDpbLxWm0\nauPGK/qQebwnjXf7YNdlFvCbyPbBqLT33ZTjSTmT2PjFRqavmx5yrctOvYy5n8x1M7SiXVeh+Hvh\n313/v9O4xyKzVU6FyNiLMHYM4/ZrBjBx07WWr91WVNWup6zxMeF+824du/HlwS+Zvm46szbMYsLZ\nE6xG1ONPNysVL88twzfkv/jKzoXt9+DLWIKvh6UcXXlLz0FKhuLrtRwzbQUiUqUoSs+xOwHLmOeb\nR9AMug38nI/mhKQpG2XnEVw/xlIUGLDtQtgxhLXyU1anVX/WIsLVfa/mtU2vuT3lWAaa+sTnlrlP\nfG7spbC0kJ3f7LRcM6Z1/oAZcGdkViU5qIylkG7FhryKQhAGdhvI6t2hfcx/rPmHa4kfDhx207AL\nSgoiPq+Q+7MV0nFHHWcNgK2FJF8SF2RcwKJti2rdb/OXm6379xaRrbClZCgqo4CUnuvIyXgi5Dhv\njK62jL3GIlED3DOA3kARsANYCVR7ikqpacA0gKysrDiof4twS6K42MqMgtDRojk5DRtNGk6sozod\n+cygICRT/nEmjIy8rww1L44AACAASURBVDeV8UDFAf5a9r77zQxfsolhz5Lr8/m4uu/V7P9hP6P6\njCLz+ExyynLwpxdWO+fgHoMpP1jOiJNHMH/r/JA03GQjmXO6n8OK0hU1TlniEx93/OQOMo/PDM1U\n8bpDjEqk1zJ8YrgpxwC3zb8tJKh984Cb6d+tvzttikJhiFHt2pGC5OUHy0NSN2uSNxIn9N7BWRdu\n5DM2ug2RT/lCerqOKyXEAvPgfKpXoTgUOGS5P1DVyoGMAsydgzBnvQNmG3xJf+Dqx1/ktW9ut/zl\nToq02Ybg0goYcyG+nkX4xEfljoGuJakMP5WOW0kJfY/ry80DbnZjYIYYXDbsWN7mpwSX/BG2DXNd\nbcHtgyHt/Wr3oJRi9kezXcUkCGedcBbFe4upNCtxPkF8SuopVfdnl3/Pzj0p+7YMU5lMfGcin3/9\nOX8v/Lvrx7/slMsATyNZg/XqkGKkkNMrh6LdRSGK+Vv/t1Xyoli09Dve+9cCLhjqi3SaapjK5MuD\nX5LsS64Wn3AQhEtPuZTd3+6O+ZxASF01eqxBpa+2pUwO2T/cmnZS2cNTeBuTeCqLXYROKJFmr4u0\nT5mIJAGdgXJl2bW3OzuJyErgszjKGhHvbJferCbvVOC5uVX7Z2fXPpq0rteuzaUVLU+7NhxTdvLy\nyVaPbMwwpOQCxo06ldxLH62WkldYCAUvwdOnF1Gc9Bx7v9/LvC3zCJgBknxJfLDrA1aUriDFSGHi\nORP5e+HfCZgBd6LE5TuX19qzFIQubbq4gTsTK7jcrU8puzyusMuHdWVQ94dDMlu85nuSL4ncfrkU\nlBS4gWmf+Lh5wM2s27PO7WF6s5yc5TZGG1Lbp5K3IY91e9aFNPBXnHoFXTt0DXFnhLPru13s+iS0\neicbyTw14ik3e8oZjxDps7uGGJQcKAlZV90tNNS16mT5PXa6sYFZafLK3N0w2HaflAwFsw3KNKyY\nU8n5BNM/QERCsn68mWqO77t/t/60TWrrumi6dugK6W9DziTYMRgxBV9SEHotr9Z7c1xQ3t55ki+J\nGwfc6GaSmZjM+2weu7tWb0R3fLPD/R3uNvMH/cz9dK4b9I/FWjn6y0t4/NEkVMbZiGNJh1N6Dsz6\nL4FgCouW+GHMwloVkHOOgBlg5KkjGdR9EKt3rw6Z083Z761P3nLdirFiKpOenXsiIuz8Zqer5AJm\nwM22yvvPFpYt/Ql0WB/6JU6zMq7B7ngqizXAySLSC0spjAauDdtnLjAGKASuAt5TSikRaQ+IUuoH\nEbkICIQFxuNOJNdTebk1inP69OpfV3Ma7tzcyKNJ60pNcZFIsoXnaceinELM+x5rSOm1gdxLF1fL\nugq9ViaLFz9P9qWh00k4jag/6Gf9nvWu+8ZUZmRXjusGKQhJqQVCctPvP/9+JhycQGX6ByQbydx1\nbuiLkJORQ5ukNm7a5TOXPONub2O0cQOAADcOuJGNX2x01wlSfUqRsCngnbjIXefeRXZ6Nrn9cl1l\nsmb3mqgKcMRJIyg/WB7S23OC3jPXz6QyWInP5+PSUy7lrU/eqv2BeXrR12VeR6mYLFvq9/i038Px\nYxi9llsWhZOckLEUxO69RrBSjm1/LAcOH2D6uukYPoNLTrokJPA7a8Ms/D3WINcPR20/H5WxhKSe\nazi3+xBWlK5wg+1OOTrjRJxnUn6w3HVJgdWohbuGIhRgtVmWvYMuaypzrxtu76yX7FjcH1BjLsTo\nsdpS9m79W2q5VSO4YY9pewwHKg7UPhC2LBu238WI3htZmLTQ7QQ4cigUSilGnjqS3d/tpnhvMaYy\n3frnTUxwUKgQpem9t9W7VvPAv96hcuYC+75GhQxgDZ9UtLGJm7KwYxDjgYVYqbMzlFKbROQhoEgp\nNRd4EfiXiGwFvsJSKADHAwtFxMRSNL+Ol5w1Ed5Yl5dbudCFhaHKIDW14Q13JGqzFvLyqkaFe/O0\n6zJ5mnfCtpsH3Fyj+RpeDnl5zr1lc89gK1PFGywc1WcUy3cuD2mUQ14K7yhzw88Vk5/hrqsHu9cO\nH0+SeXxmzYOPyrIZ8+3H7sAx73Ynx33elnlMXTuVZCOZp0c87Qb9gZDzTl4+OWQcA8CPu/yYO8+9\nM2Q6Dic7xzuaORIKxbwt83j7s7erjVx3FI930OX8LfMjns+Hj2OPOpYvfvjCXbf/h/207bXfHWDp\nRRAGDvJTxMVWzCdjKb4eH1QNtEz/AOVNXEhfxZcHPcHlkrPIL+hNyokrye0HG9d2IPOzVzgh8zO6\nXr6d6esmY6ogQdNg+EnDefTC6pao88ycqT9S26e6LrZweh/bm61fbXXdOYYYKKWqKYracKw/xwUa\nHuexZkW4gN+PHszb//2ST//1HCqQTFJykF/+fiWvva8IVAbtr1cWAHDTwJt4+oOn3diTqcyQmELy\nriEs+NfvebvS6kRNeeUDylP/Q2r7VDfzsDJYiYgw4uQRjBs4LmQgHRAyUt3F05EKT33P/zQftoXG\nGKXkArAtQ29nKR7ENWahlJoPzA9bd7/n92HgFxGOKwFOjads0aipsQ5Pp41kAdSl4Q7H614KVzqF\nhVZj/eKLVaPCfT7L2iksjP2a4TOx9ji/R42VLHxg4f9v79qj5CrK/O/r7plJlF2EwQcKIaCsGg9I\nIDs6iybR4CwqSHbDCugxEQLjCHHJHg8jkaMnKE509Wh4yU4WwjKrKz4wLnJ4GUiA3c4BA4GExypJ\nDCFKNjBr8ICazOPbP+re7uqaqlt1X909PfU7p0/fvl236qu6VfXV96ivZBWcUI1NjGckT/AAKivp\n0fFR4LkFGAvCTxS4iK6RfnRLykpVstHtLwnbSTDpY9DevhiLgyi6MiMEIEK175qPgzM3YssLW3DD\nGTfU5F2p58z5Eya0nft3Yvndy3HCG6ob2Sqhs5dsrImTtOWFLRMkjnAS0EbiVeqlyw8QHmzb9m3D\nZ+74TCXtolmLAAD3PnBzZe8LHl+Cwqd70DHzsUCKWo4DR20CCBVGUUABpx13GhadsSigdxybXyjU\nGr4DRn7wgYO4vO2neHDN3wOjIpTJ3IU7UOx8puY8FN37qexpuGoFRnacira33oNre6/FXRv2o/xg\nG/a94UeViXDeMfNw08duqtR97yt7a5wkTOqmtkIbClSohFF50yFvqqi/CASS9k2gOILzzjwS1z58\nGf68+Z/AI0WACxgfLeBdh8zDAxvEONv/pnvxeOkvsWjWIHpP6cXCty+sML1wl30oieKPl2PNwSLG\nx8VZLcPPnIAVK6pOJbOPnF3ZN1HpQ3u6gf/qBkrBpt/Q2y3AMX84F8/dclNlIVX89N/izAVH4PZf\n315Np0iGZ51+KLre87W67ORuVgN3w+Fy6lWIrOLTr1kDLFsmGE949GgYe0oXZwoQIQjkWFUuDKMm\nEmvpIDo/usOYVm4HnQquuxs1gwBH6yf8cCXdeeIZWP7fxaC9KHM1nbxhSWxsWl8ZfHvfcR1wxsS8\nBIPurthkHnvhMfzykRLGfzMXB459CENPDNVIT7KUIOeBh57D4/uXYPTND9WoLAp7TsXuOz6BTaWI\nDZwGphj+BwC3PX0bFs1aVAk5cte0k/AzifGeVrgKKxd3VNIvu3NZxeU0VKnJwRdVCfNtr1yMp6VV\n687ybBHmnksiCONPZ6Gj4z5c9O3vY/EZxwN7urHqe3oJeuiOZ3Fw7Z0VxrN6ZA12/uDzGBkpAPQZ\nYMlp6Jj5WEWiDWkKjw0OEdqLgIlMFEDNoqQmJE7fYtz19uvwu21/haV/91YMd/4Bt244KNR1xStA\n44T29iI6O6tHESyevxDf6K56hsh0qRLumt/Vnv7Y2Vlb/+E/DlfUsAfHDmLojmdxy+e7azQQakiQ\nD5e+jjU8HeNMoHHCRYd/DzPe8h+1HoaKS/uHP3C+MeR85mDmlviccsop3CiUy8wDA+I7TR6lUhjr\nlpmIua+v+v/AAHOxWP0fYC4URDpA/Dcw4FbWwABzoTgu8iiOW58rlwUtCxcyd3SIsqZPF/fLZXFd\nLDK3t4t0tnbIqr3CckNamJnLu8s8/arpXLyyyMXTrmDQiGgvOsh9/buc8hlct5XR9qp4tu1VXvjN\nb3DxyiJjJbh4ZZEHHhww5tExbZS7vnQpF64sMFaCsbSbi+0HJtCZBUxtwMw88OBAhebCygL3DPVw\neffEwsu7yzzw4ACXd5e5XBb0U2GUO6aNcv+q7aIdMMoi8FW1n0WVzczc17+rpu3x1rsrvwvFce7p\n3WCkp+OrHUwriTu+2qFNY2wPqS4qBtdt5dKHvsSFC0/l0kXv567F67h/1XZub+egbuPc1j7m/H4G\nBsT4C8ehOobKu8vc3juPacEXub13Hvf176qMX3lsq+2vtml5d5nbv9ou+pLmo/bFJIAwC1jn2IZP\n8ll9Gs0s+vrcJkoT5M4Xfjo6pElQ6kilkvgOGUWhEG8isg10Na0YUOLT1lZbT5WJESWbFF0ZiDq4\ndM+EaQbXba2Z/HR56wa9ykz7+ndVGND0q6ZPmIzkNigWuSZ96UNfquQlM/QsGKYpn3JZ0NDeO89I\ns2t+g+u2ctdZj3Jb+1hNf1HrPGGyDBgPCiOM0quMMy5klF5lKoza+5xh0k/aZmF/LxTHudh+gNsu\nmlt5N6AxacyNahcUUXmaxlCV8Y5xx7RRHhysHUfy2LbVsby7zH0/7+O5a+dyYWWhwijiMlMTPLOo\nE9TJ1NQJop4fGGAeHBSdLmQAukEYpu3rqw7UQoG5pye7yVm9PzBQSxPRRJpc6TbR6Mq8ZKnBdQK0\nlT04WMugBwf19EStWqPSD67bOvE/k1SUscTVMW2U+757S2JGEfWfyztTmVa4wk5SvzgLHBUyY6PC\nKNOCLwqJ68JTmUp/rkgWKP6J+757SyyaTO2lY6Z9fck0ATVlBoyj7+d9mTAKZs8sckPcydSWlzwA\nBgdFh1JVPbbnslJtaCc9B2YYSlY6ul1ota1SK+kk1YpOHZQEJnVC3Ik7Kv3goGDog4PVMtX6pn2n\nuoWEqS1N0kjc8k356FbGJkbrCtc+YqJz+nTxfoulMS6d9dnKgqN/7Tou/vUapjk3cHvvvMwmYNNY\niquyrQc8s8gBWv32YK0aRp1M5cGjYzQ16ou+qpRhekaXb1YwDUhXNZuOJpdBnrVkEadtspqk40hN\nuntZTIayitKkmjTVN035urxdJsOosRGVd5L3NDgoVKiFwkSJy8bM0qi/tCpChwVhHDrSzgWeWeQA\n3eReWbEUhQFYt1oL/29ri15pRBmPszaO6pBHec6MwLHDuwxsU3lZD7Y0UpOrWseFNp3zQ6lUlWRc\n6Un77uPYr2x9X4c0k6JpYWbtbzmMiSSMOap/pKXPlVl419kYUPdeAEFMpnGxB6GrS9yTo9MeOFB1\nsRsbqz4T7seIcksN07qezOcaS8qEOOdvuJbtmqdrKPcoF1MgWUTgpGHko3bZh4jar2NrJ9coxmEZ\nNcf3sthIakpr2z+UpD3mzwdKpWo/Zza3i9x24fiISh/SmDR0zu7dgjbAtF9I/6zLO1bLsrWhLTyP\nLg+jq3hM+lLBhaNMhk8jbBY6m4P6W3aHjVptuaorouiqpxRS77KdJY8c1SyuZSWlXUUcmlX1RpQa\nKA8VZpivbN+K8tLTSRaFglkiSkOTqhpzsevons9yDEZJuXEkiHpKFg2f5LP6NNIbKrQz9PRUjaVh\nJ5R1pboBrNPbutgsVMgTS1IPqaTIYyKWEXdA6NpMfg9RE1jcSTSviTfMO4nROYlOPIt6xO2Dcpku\n7yctTUmdCXTj0rWsLOiV6dDZDr3NYpIwC+Za24RuRRVnFeHSkXX52WjIE1lKFrq6pR2EctuYVq71\nlMziDO4kE0Hc9sqq7mnyyWvBkcYW5JKPjCwYXlQ5efVRV2bhbRYZINQbjo+LWE2nnQasXGnXiev0\njYBZ575xowgrsHz5RD12qHNeuRJYv17QkrsOE9X6ZRE80aSjV3W8nZ1Vu5BLWfL7IdLr8uul+417\nmmISPb1ryPqwT+3e7V73KJ18mn6gozmtDS6Kpqh2jWMzkJ9Zvlz8XyiIKNVZ2w1N80XaNnKGC0eZ\nDJ9mkCzicnzdSsQmbZRKE1VdaWnJU5USp6yoTUuyKiBJ/fJcFcZpvzxVdjqVZpQdJYk3Up7Sl0p/\nI2xwSW0DeatidTQkGQs6wKuh6oukYq1ONaLmpeqCVRfcpLQk1W8nhU3EdtkJn3RQRrWJ/C50LtBJ\n65Q0fRLmHZfZJXElzWovRlwbXBbvOYtyXfpQHJdt3f9x0mbFoDyzaHLEedHqRGAztLkg7NxRYTqy\nRpQBVP5PDaKoozsvv/e48a2S+szrbE5pJCfZ604X2E73TB4SWlbPZ0lfPco1GcBt+blKeKY+U0/J\nIlebBRGdDuBqiMOPbmTmryv/dwAYAnAKgGEA5zDzLiJqA3AjgJMhAl8PMfOqPGmtN0Kf9PFx8R2l\nV1Z1ob0ZRCQO9Z8c+OUTpQ+xbkOolw73nqxfDzz0UBCueX6tznrxYn0eoU43DCstI66OWz02N9yn\nwBZ/f12d4oSoV/Xlsh2DSLRNHJvTxo3VvQqA2EdgPVo3oh1tz6htHPeseBfbSBL7RxZ7EeKWa7NB\n2cp22W9iKiMrW6EzXDhKkg8Eg9gB4DgA7QCeADBLSXMxgH8Jrs8F8MPg+hMAbg2uXwNgF4CZUeVN\nFslCXoW0t4tVbHu7eVWQdOepS/mq/3neKqgoF2M5TRJVTlp1kGtcrqi6JW2/tGrGKJWmje60Lsl5\nr9pd6mCzc+Rp/7BJlq6ShSnKg0sZaYFGq6EAdAO4R/q9AsAKJc09ALqD6xKAlwAQgPMA/Dy41wng\n1wAOjypvMjALuePYDNW6Z1wNkS7l69RZaSe9qHJlxmhTtdjo0Kmz4my0UvNIwrCyhO29yHSZ2i4J\n3XE3/enKdWnHODp5XbmmZ02LhjQ2i7hwYUS2skM1M5HeZhannknQDMzibAjVU/j7UwCuU9I8CeAo\n6fcOAEcAaANwK4AXAbwKoNdWXjMyC/WFRq0go+wQYT5xJ0QVNuNdXquvvj6u2cUeSjGue09UyKux\nsC3jMlKTJ1q9GUWIqLKTLDJcy3R951FMIUrKi/teTJKCLp96eSDZFlRZSJbyWTZtbdHMLuux6sos\nmnWfRReAMQBvBnAYgIeIaD0z75QTEVEvgF4AmDFjRt2JjIJOz6jqt1evFj7/pr0TIUL95KZN4vhU\n1Re9cizk4to9GaoeM0q/HhVTyaQTTeMHH7X3JLRpHDig1y93d4u2++Y3gR07RNqxMeCii4AZM+z0\n6OxAgNv+hyx8/3WI8vuX302hIGwSWdiY4ui8TXtdOjuBJUtEmtmzJ+7bcI37BEwcM0uWROeTxF4U\nByo9q1dXx2mxCFxwgajz8PDE9ovTT+bPF+9VjiGntpPcP1atqmM8KBkuHCXJB+nUUNcD+JSUbi2A\nj0eV12ySRRw1R1x1gLzCUN1Nk6p44toBXFc35bKQJEIx23Y4lHoYUX//xBAHOskijgpA194u70Bd\n6SYJ2Z40jU1NVQ+oqjCTZCfHejLp4XVQvdLmzrVLKHlKg2qf6Omppc8UAyvJyt/V9TkcT1m6u6MJ\n1FAlADsBHIuqgftdSppLUGvg/lFw/QUANwfXrwXwNIATo8prNmYRp8MkFSsHBiYevCR36LiiedgR\nZTWRKS9ZdDa5asadXNV8Qx2uuvfC5IKr0m9q17iMUabN1cU2quwk6oRGqshUqO0QtoW8oXLhwuiY\naDqUyxMDb/b3J7N32NImZdKqu7nO7TzrPSI6erJ0Smk4sxA04CMQxukdAK4I7n0FwMeC62kAfgxg\nO4BHABwX3D8kuP9UwCgus5XVbMyCOdvObXomjmQRpzPKg0OXl+44UhVJ9yDI+nndKYSmyV5tiygb\nTxLdc1iuy94UXd1Vum02qGZiEDJ0kp268k9qX+vqqu1XPT3J6IvyNHRh5HJa1Wah0qguGrK2KYTI\ny0bjyixytVkw850A7lTufVm6/jOAf9A894ru/mRDnLg+SWIAhT7kqs3ihBP0vvA2nbxqtxgeNvvV\n33ab0JszC31rnLMTbHUKy+zsBD73OfE8IPaj7N4trlW6Vq0CRkaq+YTPmMqX7UBynClbnKZwb4J8\nHoKuXrq6q+1roi+0Q6lnLgDp7CU6+1YSqO8o1NnL9AET7WsuWLoUeOSR6u9Fi+LTNzRUbd+DB8Vv\nlxhLCxYIOxkRcOaZQH+//tyR1atFfUZGhO3iwgtr2zOv/Q9522iscOEok+HTjJJFI6GuiOLq5G3q\nlXB1bYuwm3Z1HKqWFi6M1tPqJAub6iLJClDW28e1R5gkIl0aVXoJT2VM4zbtEk7FRHsSmPKw5a2e\nWR4XOu87tXy1LVWPJFsbubZPFv0/ajxl8Z7QDGqoen48s6giSk1j2wRo63zqoCKqDmqdGisrNYor\ns3M5K9wlT5NKIumEHcVkZLpl9Y2s4sjCbVqn0jPRGqeervr/qH0iWaJcFhM9kbCb6PqDbtLVHVRW\nL9fkJM9npe5yZRbN6jrrkQI6MXv+fCFeA+J72zbzMag2F8owH0AMqy1bJpZ74ACwbJlwBzSpvWxq\nEdn90EUEl1VLoWohjjuo7Ip8881CzVAoANdfL0KsJA1jrnPBDOkL6QjVJm1ttcd/XnBBNfSJSa3j\nepRnW1ut+sukxohTTxf1pi2cCZBMPRbVf84/H9i7F7jrLnFc8S23TDxKN6xr+Pv664GLL64eCxs3\nVLqaLml/CTE0VA1BYwsVUhcXWheOMhk+XrKoQjZAhuEfTBsCk3hVLFxYuwLr6qpKL66bx2xqEReV\njZw26YYl3bOqt0u4SSpO3nK+UZsx5bDscnu6rIZN7RRVPxfJKyrPJOpN22ZUF/WYThIwqR2j+qBN\nwlHbyPWdm/pr0pX/4KDeE9D1PcUBvBqq9REl/usi1eoGkc0F1FRuR0f1ed0uaJu6waYWcfX8UAdM\nGnWNziVUpc1F5WJq+0JBfGQX076+2klPZwdypVmnSkurNrO9Uxdmpe4LkFVxqn1Bp/ox2Rh0/cfG\nmFwXMy7t65LO1l9MCwBZJWaKwhxX7WqCK7PwaqhJCpv4PzxcK+7Lnk3hjvEkUVYBkWbDBv2pfCtW\niDQbN1Z3qOvEd1UtUigIuuT/XTw/TB5GoVdLZ6e7GkEuE6iqI5iFh86mTXY13aZNwCWXCC8mQNAx\nPCy8ur71reou3UKhGl138WKhcnjsMWDz5ngRZ5PsyneBrNIzqZCGhsSOedN7lp8tFsUOe1ldtGkT\nsHZtbbm6d21Sq5rUarooCao3mutOeNd+aEpnO5FPN4Y3bqyNIlwqTYzCrD5ritKcKVw4ymT4TDXJ\nwrbiSbLiy8IAF2c1OzjI/M53VlfccfeFhHmEu8NDlZu8sk8am6ivL5n0pToAtLUJelTjaaFQu+HM\n1J4uiFLPpVVTRKkvbe1q66OqJBeqM3X1c1UZRXmqqfm4OmC49MM46UKYTobU9WkVWe65gFdDtTZc\nJgKXzhu3g9uej6s+SnP4kpqHznNI3VkcZzdtEvrC51R7keqWGXrquGwMM5Xjan9I0wdME6yLus9l\nwZLEBmT632WjXZz+nheTCJ+JsrnI/cekqsrKq8wzixZEmo6fRXmuz7gYXE0uonEkC52NoVCo3ZOQ\nJqx7UulL957UvSlposdGTTRZvy9dfVyfMz0b539X6BYpaSbUOPVLUoZJqlLrEbW/Jqu288yixZC2\n4+cxicQtT85TDTynrpBdJzGd95K6GstaekqTR7gyT7PfQGcUtm3aM9XBNMnmtaKOizi06Izg6sTr\nSq9JRaQiqTpIliBkpwa1X6TdX+MCzyxaDGk7ZZqVT9zyTAMyzuCNo87q6aldtbuoivKe5FzgqkpS\nn1G9p1ziYKmqDfU/ncdQGvWma12SqJZMUHd+mxYnLiov193uaRdVulMj1YWOlyw8s4iFRkz6cY2B\nLiqoODrqRqfNG0loMakvovLSGd1Nk05ax4mousY15sdxRzXlF6aNs0KXyzW5rUbVLbyXdk+LLX9v\ns/DMwohGqZNcVSYujEk3aUQZV7M2RmbpRZIWSWhJstIsl2u9saKkL1t/iTuRmvJ07Suu3nZZMrkk\nY0aVBvKOw+W9oTyzyAVpxdW4Hk6ug6wRq/zJLlmEz8V9n66H69jyd5kI1eeTGJ/lhYqLt13SFXqS\nNtClVe0MrnG4ksJLFp5ZNCXirspcXTXzNN5lJbHkRUO9acmyrCjjr6s0EKWmsUlPeevw40Jni4sj\nWSRFVvX1zMIjU2TVMeXBnsat1bWMRkkPzUBDXoiqm4udIY6qy6RSahYvrDBtHGbYbHBlFrmG+yCi\n0wFcDaAI4EZm/rryfweAIQCnABgGcA4z7yKiTwK4TEp6IoCTmfnxPOn1MCPJ4Uw6yCEXABECYsaM\nbA+JqXs0zgbS4BrKJEtEHe5jiuSrHlQV1Ta2EBtZ9UUTXCLpqvTI7QGI6zQHTDUlXDhKkg8Eg9gB\n4DhUz+CepaS5GLVncP9Qk88JAHbYyvOSxeRAPVbczbCqnyr11MEmRWRtX8gaaYzHpvo2g6u2CWgC\nyaILwHZm3gkARHQrgLMgztQOcRaAlcH1TwBcR0QUVCDEeQBuzZFOjzoiryMn611GM9DQDBKUDvLK\nXydFrFhhb5u8pYcopDm+VH0nQ0O155DYpJRmRp7M4i0Anpd+7wHwHlMaZh4lopcBdAJ4SUpzDgRT\nmQAi6gXQCwAzZszIhmqP3FGPiaCRk029aGj4mcwOSBKNtdFIw+jV+gLNydCToKlDlBPRewD8kZmf\n1P3PzGsArAGAOXPmsC6Nh0erohkkKBsmA406JGVmOvuF6YTDyYY8mcVvARwt/T4quKdLs4eISgAO\nhTB0hzgXwA9ypNHDY1KjmVfoISYDjVlCre9kZJY65MksfgngeCI6FoIpnAvgE0qa2wEsAbAJwNkA\n7g/tFURUAPBxyA05dgAABsVJREFUAO/PkUYPDw+PXNEqzDI3ZhHYIJYBuAfCM2otMz9FRF+BsL7f\nDuAmAP9ORNsB/B8EQwkxF8DzoYHcw8PDw6NxoFrHo8mLOXPm8ObNmxtNhoeHh8ekAhE9ysxzbOkK\n9SDGw8PDw2NywzMLDw8PDw8rPLPw8PDw8LDCMwsPDw8PDytaxsBNRC8CeC7h40egdtf4VICv89SA\nr/PUQJo6H8PMr7clahlmkQZEtNnFG6CV4Os8NeDrPDVQjzp7NZSHh4eHhxWeWXh4eHh4WOGZhcCa\nRhPQAPg6Tw34Ok8N5F5nb7Pw8PDw8LDCSxYeHh4eHlZ4ZuHh4eHhYcWUYBZEtJaI9hHRk9K9w4no\nF0T0bPB9WHCfiOgaItpORFuJ6OTGUZ4MRHQ0EW0goqeJ6CkiujS438p1nkZEjxDRE0GdrwzuH0tE\nDwd1+yERtQf3O4Lf24P/ZzaS/jQgoiIRbSGiO4LfLV1nItpFRNuI6HEi2hzca9m+DQBE9Doi+gkR\n/Q8RPUNE3fWu85RgFgD+DcDpyr3LAdzHzMcDuC/4DQAfBnB88OkFcEOdaMwSowA+z8yzALwXwCVE\nNAutXecDAD7IzO8GcBKA04novQC+AeA7zPw2AL8HsDRIvxTA74P73wnSTVZcCuAZ6fdUqPMHmPkk\naW9BK/dtALgawN3M/A4A74Z43/WtMzNPiQ+AmQCelH7/CsCRwfWRAH4VXA8COE+XbrJ+APwngA9N\nlToDeA2AxyDOfH8JQCm43w3gnuD6HgDdwXUpSEeNpj1BXY8KJooPArgDAE2BOu8CcIRyr2X7NsQJ\nor9R31W96zxVJAsd3sjMLwTXewG8Mbh+C4DnpXR7gnuTEoGqYTaAh9HidQ7UMY8D2AfgFwB2ANjP\nzKNBErlelToH/78MoLO+FGeC1QD6AYwHvzvR+nVmAPcS0aNE1Bvca+W+fSyAFwHcHKgbbySi16LO\ndZ7KzKICFuy35XyIiegQALcBWM7Mf5D/a8U6M/MYM58EsdruAvCOBpOUK4joDAD7mPnRRtNSZ7yP\nmU+GULdcQkRz5T9bsG+XAJwM4AZmng3gVVRVTgDqU+epzCz+l4iOBIDge19w/7cAjpbSHRXcm1Qg\nojYIRvF9Zv5pcLul6xyCmfcD2AChgnkdEYXHB8v1qtQ5+P9QAMN1JjUtTgXwMSLaBeBWCFXU1Wjt\nOoOZfxt87wOwDmJh0Mp9ew+APcz8cPD7JxDMo651nsrM4nYAS4LrJRB6/fD+4sCj4L0AXpZEvUkB\nIiKI882fYeZvS3+1cp1fT0SvC66nQ9honoFgGmcHydQ6h21xNoD7g9XZpAEzr2Dmo5h5JsT59fcz\n8yfRwnUmotcS0V+E1wB6ADyJFu7bzLwXwPNE9Pbg1gIAT6PedW608aZOBqIfAHgBwAgEl14Koau9\nD8CzANYDODxISwCuh9B3bwMwp9H0J6jv+yBE0q0AHg8+H2nxOp8IYEtQ5ycBfDm4fxyARwBsB/Bj\nAB3B/WnB7+3B/8c1ug4p6z8fwB2tXuegbk8En6cAXBHcb9m+HdTjJACbg/79MwCH1bvOPtyHh4eH\nh4cVU1kN5eHh4eHhCM8sPDw8PDys8MzCw8PDw8MKzyw8PDw8PKzwzMLDw8PDwwrPLDw8LCCisSDC\nafi53P6Uc94zSYqG7OHRrCjZk3h4THn8iUUYEQ+PKQsvWXh4JERwrsI/B2crPEJEbwvuzySi+4Oz\nBO4johnB/TcS0ToSZ248QUR/E2RVJKJ/JXEOx73BDnQQ0T+SOJNkKxHd2qBqengA8MzCw8MF0xU1\n1DnSfy8z8wkAroOIAAsA1wK4hZlPBPB9ANcE968B8ACLMzdOhtiBDIhzB65n5ncB2A9gUXD/cgCz\ng3z68qqch4cL/A5uDw8LiOgVZj5Ec38XxIFLO4PAjXuZuZOIXoI4P2AkuP8CMx9BRC8COIqZD0h5\nzATwCxYH2ICIvgCgjZmvIqK7AbwCEd7hZ8z8Ss5V9fAwwksWHh7pwIbrODggXY+hakv8KESMn5MB\n/FKKJOvhUXd4ZuHhkQ7nSN+bgusyRBRYAPgkgIeC6/sAfBaoHNR0qClTIioAOJqZNwD4AkQ48QnS\njYdHveBXKh4edkwPTuALcTczh+6zhxHRVgjp4Lzg3ucgTjW7DOKEs/OD+5cCWENESyEkiM9CREPW\noQjgewFDIQDXsDinw8OjIfA2Cw+PhAhsFnOY+aVG0+LhkTe8GsrDw8PDwwovWXh4eHh4WOElCw8P\nDw8PKzyz8PDw8PCwwjMLDw8PDw8rPLPw8PDw8LDCMwsPDw8PDyv+H54gjB3Fee3GAAAAAElFTkSu\nQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f86dWOyZKmN9", + "colab_type": "text" + }, + "source": [ + "Great results! From these graphs, we can see several exciting things:\n", + "\n", + "* Our network has reached its peak accuracy much more quickly (within 200 epochs instead of 400)\n", + "* The overall loss and MAE are much better than our previous network\n", + "* Metrics are better for validation than training, which means the network is not overfitting\n", + "\n", + "The reason the metrics for validation are better than those for training is that validation metrics are calculated at the end of each epoch, while training metrics are calculated throughout the epoch, so validation happens on a model that has been trained slightly longer.\n", + "\n", + "This all means our network seems to be performing well! To confirm, let's check its predictions against the test dataset we set aside earlier:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "lZfztKKyhLxX", + "colab_type": "code", + "outputId": "b792a12e-713d-4b07-9f8e-de0d059d5cdb", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 298 + } + }, + "source": [ + "# Calculate and print the loss on our test dataset\n", + "loss = model_2.evaluate(x_test, y_test)\n", + "\n", + "# Make predictions based on our test dataset\n", + "predictions = model_2.predict(x_test)\n", + "\n", + "# Graph the predictions against the actual values\n", + "plt.clf()\n", + "plt.title('Comparison of predictions and actual values')\n", + "plt.plot(x_test, y_test, 'b.', label='Actual')\n", + "plt.plot(x_test, predictions, 'r.', label='Predicted')\n", + "plt.legend()\n", + "plt.show()" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "200/200 [==============================] - 0s 146us/sample - loss: 0.0124 - mae: 0.0907\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJztnXmYVMW5/z9v9yzgEpVR44KIMRhj\nnJ+Ak+iJim3QuMS4EaOJZhSJjQtRkmvQyY0JuS4ImlyMIDIKyFwTjHEUl2gkoq2irTgoCRE1oBcR\nl6ijeF1glu76/VHnTPf0dPf0TPdMb+/nefrpPnvV6XO+VfXWW2+JMQZFURSlvPDlOwGKoijK4KPi\nryiKUoao+CuKopQhKv6KoihliIq/oihKGaLiryiKUoao+JcwInKWiCzLdzo8RGSoiDwgIh+LyJ/z\ncP2AiGyKW35JRAL9OM8RIvJqThM3iIjIuSKyIt/pSEfif5XD8xZ83gcLFf8MEJEfikiLiHwqIu+I\nyMMicni+09Ubxpg/GGO+ne90xPE94ItAjTHm9HwnxhjzNWNMqLf9RMSIyJfjjnvKGPOVAU1ckSEi\nI937VJHvtCiZoeLfCyLyM2A2cC1WuEYANwMn5zNdvVGgL+HewL+MMZ3ZnqhA86coxYMxRj8pPsAO\nwKfA6Wn2qcYWDm+7n9lAtbstAGwCpgHvAe8ApwAnAP8CPgR+EXeu6cDdwJ+AT4AXgIPitl8BvOZu\nWwucGrftXOBp4L+BVuBqd90Kd7u4294D/g9YAxwYl88m4H3gDeCXgC/uvCuAG4CPgP8Fjk9zP74K\nhIDNwEvASe763wDtQId7TyclOba3/G8ALgf+AbQBFcAeQLOb9v8FLonbfyhwu5vutcDPgU0J5zva\n/e0HfhF3f1cBewFPAgb4zE33Gd7/2lue3W23A3OBv7jnfQ7Yt7f/JMm9mQi87J7jdWBy3LYA9jn7\nD2LP2cS47TXA/e41VgJXec9Fimv9GXgX+NjN/9cS7ulv3efkY/fZGApsdO/Tp+7Hcf/PO+KOHenu\nU5FpnlKkbx5wQ8K6+4CfZfierEiWHnddCPhx3PJ5bho/Ah4B9u7rf1eon7wnoJA/wHFAZ/zDkWSf\n/wKeBXYFdgGeAa5ytwXc438FVALnY0Xqj8D2wNeALcA+7v7TseL4PXf/y7CCVuluPx0rdj6sCH0G\n7O5uO9e91k+wojg04UE/FitoO7oP7lfjjm1yX57t3RfiX7ji7J6jw027H7gQW8hJkntRCazHimgV\n8C33BfxKXP7uSHMve8v/BmA1VpSHuvdhlXt/q4AvYUXkWHf/64CngGHuMf8ktfj/3H2Bv+Len4Ow\n5imwAvHluOMC3nkyyPPt2ML4G+7/8gfgzt7+kyT35jvAvu5+RwKfA2MTnrP/ctNzgrt9J3f7ncBd\nwLbAgcBbpBf/89xnwavYrI7bNhcrkHu6z8M33f1G0lNIu/3fiftkkKdU4j8OeBP3GQR2wr5He2T4\nnmQk/tjW/Xr3f6nAVoqe6et/V6ifvCegkD/AWcC7vezzGnBC3PKxwAb3d8B9KP3u8vbuw3ZI3P6r\ngFPc39OBZ+O2+bC1uCNSXHs1cLL7+1xgY8L2+Af9W1hRPxS3Vu+u92Nr5AfErZsMhOLOsT5u2zZu\nHnZLkp4jsDXG+PMvAabH5a838U+Zf6xYnxe3/ZAkeW4AFrm/XweOi9sWJLX4v+rdyyTpSif+veX5\nduC2uG0nAK+k+08yfDaXApcmPGfxIvaee14/tkDdP27btaQR/4Tr7Ojmfwf3/9hCXGssbr+R9FH8\nM8hTKvEXbEtjnLt8PvBYmjwkvieZiv/DxLVQ3fx/jjVf9vu/K5SP2vzT0wrs3It9eQ9sE9jjDXdd\n1zmMMRH39xb3+99x27cA28Utv+n9MMZEsc35PQBEpF5EVovIZhHZjK3F7Zzs2ESMMY8Bc7A1t/dE\npFFEvuAeX5kkD3vGLb8bd57P3Z/xafbYA3jTTXeqc/VGyvwnbse+hHt498O9J7/A9s10pSchLanY\nC1uQ95VM8vxu3O/Pce9dmv+kByJyvIg8KyIfuvk8ge7/favp3pfiXWcXbK01o/sgIn4RuU5EXhOR\n/8MWkLjX2hkYQv/uU7Jr9ZanpBirxHcCP3BX/RDbovLO29t7kil7AzfGnedDbMGzZ1/+u0JFxT89\nYaxt+ZQ0+7yNfUg8Rrjr+ste3g8R8QHDgbdFZG/gVmAK1hyxI9aMIXHHmnQnNsb83hhzMHAAsB/W\n1PEBtmaYmIe3+pH2t4G93HT391xJ8x+3PT6PbwL/a4zZMe6zvTHmBHf7O/Hnc9OSijexJoi+klWe\nU/wn3RCRamy/xg3AF93//iG6//epeB9rEsr0PvwQa+44GlvbH+klA/usbCX5fUr27H2GbSl67Ob9\nyDJPYFtX33Pfi0Pcc5HhexKfPlKlEftMTE54voYaY56BzP67QkbFPw3GmI+x9uS5InKKiGwjIpVu\njWWWu9sS4JcisouI7Ozuf0cWlz1YRE5zWxtTsYXPs1h7rcG+zIjIRGyNJiNE5OsicoiIVGIf+q1A\n1G2V3AVcIyLbuy/Pz/qZh+ewNc5p7n0KAN/F1tIyJVX+k7ES+ERELnfHEPhF5EAR+bq7/S6gQUR2\nEpHh2P6QVNwGXCUio8Ty/0Skxt32b2x/QjL6nedU/0mSXauwdvX3gU4ROR7IyIXX/X/vAaa7z+8B\nwDlpDtkee89bsaJ4bdy5osBC4Hcisod7vx1XyN930x5/n1YD40RkhIjsgDXJZZ0nNy0vYguj24BH\njDGb3U0ZvyfGmPexhfTZbl7Oo3vBdgv2+fmae64dROR093em/13BouLfC8aY32LF8JfYB+pNbK1i\nqbvL1UAL1gNlDdZD5eosLnkftpPqI+BHwGnGmA5jzFqsl0UYK0a1WO+eTPkCtkb0EbbZ3wpc7277\nCfYBfh3rvfFH7EveJ4wx7VjhOx77Yt4M1BtjXunDaZLmP8X1IsCJwGhsx7AnBju4u/wGm9f/BZYB\n/5Pmur/DFhbLsN4bC7CdymBt14vd5v/3E9KQTZ7T/Sfx1/gEuMRN30fY2vn9GZzfYwrWBPQutg9i\nUZp9m9y0vIX1lEkseC/DPufPY80gM7E278+Ba4Cn3ft0qDHmb1jPrX9g+7YezGGewD6nR7vf3nn7\n+p6cj62xt2IdMJ6JO9e9bv7udE1g/8T+z5Dhf1fIeL3lSgEgItOxHYtn5zst+aDc868og4nW/BVF\nUcoQFX9FUZQyRM0+iqIoZYjW/BVFUcqQgg2OtfPOO5uRI0fmOxmKoihFxapVqz4wxuzS234FK/4j\nR46kpaUl38lQFEUpKkQk3Uj2LtTsoyiKUoao+CuKopQhKv6KoihlSMHa/BVFKU06OjrYtGkTW7du\nzXdSipohQ4YwfPhwKisr+3W8ir+iKIPKpk2b2H777Rk5ciQimQbxVOIxxtDa2sqmTZvYZ599+nUO\nNfsoijKobN26lZqaGhX+LBARampqsmo9qfiXEOEwzJhhvxWlkFHhz55s76GafUqEcBjGj4f2dqiq\nguXLwXHynSpFUQoVrfmXCKGQFf5IxH6HQvlOkaIUNkuXLkVEeOWV9FMv3H777bz9dv8n5wuFQpx4\n4on9Pn6gUPEvEQIBW+P3++13IGDXJ5qC1DSkKJYlS5Zw+OGHs2TJkrT7ZSv+hYqKf4ngONbUc9VV\nMZOPZwq68kr73djYfVkLAKVYyHWl5dNPP2XFihUsWLCAO++Mzbg5c+ZMamtrOeigg7jiiiu4++67\naWlp4ayzzmL06NFs2bKFkSNH8sEHHwDQ0tJCwK1prVy5EsdxGDNmDN/85jd59dVXc5PYAUJt/iWE\n43S38yeagpqbuy83Ndl9amqgtdW2FrSfQCk0BqI/67777uO4445jv/32o6amhlWrVvHee+9x3333\n8dxzz7HNNtvw4YcfMmzYMObMmcMNN9xAXV1d2nPuv//+PPXUU1RUVPDoo4/yi1/8gubm5uwSOoCo\n+JcwNTXg80E0CiIwejQ89ZR9iSoqYOFC6Oy0230+qK7WjmKl8EjWn5XtM7pkyRIuvfRSAM4880yW\nLFmCMYaJEyeyzTbbADBs2LA+nfPjjz/mnHPOYd26dYgIHR1Jp54uGFT8i4hw2D74mdTQw2GYOtWK\nuzH2xbnpJpg929byV66E++6z28AWALl6sRQll3j9WV7N3+vP6i8ffvghjz32GGvWrEFEiEQiiAin\nn356RsdXVFQQjUYBuvnZX3nllRx11FHce++9bNiwocscVKiozb9ISLTf92b79GpLnrgbY5c9887D\nD8e2ga35p3uxtKNYyRfJ+rOy4e677+ZHP/oRb7zxBhs2bODNN99kn332YYcddmDRokV8/vnngC0k\nALbffns++eSTruNHjhzJqlWrALqZdT7++GP23HNPwHYSFzoq/kVCX105vdqSNw4kXtxDIdsiALv9\nlFPg6qtTv1h9LXgUJdc4DjQ05KZVumTJEk499dRu6yZMmMA777zDSSedRF1dHaNHj+aGG24A4Nxz\nz+WCCy7o6vD99a9/zaWXXkpdXR1+v7/rHNOmTaOhoYExY8bQ6b1ghYwxpiA/Bx98sFFiPPOMMUOH\nGuP32+9nnun9mPnzjamsNEbEmIoKu+ydq7rarq+u7v1c115rrwv2+9pr06fz2mszS59Snqxduzbf\nSSgZkt1LoMVkoLE5sfmLyELgROA9Y8yBSbYLcCNwAvA5cK4x5oVcXLtc8Jq+mdr8wZp4olFr3jHG\nLnvEm4N6I53NNb4fAnSUsaIUC7nq8L0dmAM0pdh+PDDK/RwCzHO/lT6Q6MrZG4GAHfQVjdpvT6BD\nIWs+Msaaf6ZPt59U505V8CS64J1zTu69MhRFGRhyIv7GmCdFZGSaXU4GmtwmybMisqOI7G6MeScX\n11dS49n842NAeTX5tjZbMDz6qHUBTVdTT1bwxPdDtLXBCy/YQgbStxC0QFCU/DNYHb57Am/GLW9y\n13VDRIIi0iIiLe+///4gJa108Tp2vRq+10ns1eSPPjo2DqA/8YC8QsQ7R0uLLWTOP797QaIdxopS\neBSUn78xphFoBKirq8vAGq2kIxCAGVzOGfyBDdF92anmOggDoRDO5s3cvSnEi2YIrQzjfXbDqakH\nMq+We4XI9Om29RCN2kJmxIjuwj99eqyVoeYgRSkMBkv83wL2ilse7q5TckljI5/cuIAPPxvCdnsP\nw3nnJQ6NrANgL/MWctER1i7T0QHGsD1whHdsBOSSRVD7eJ+U2XGsuHsjhxODyo0fHxP+3sYSKIoy\neAyW2ed+oF4shwIfq70/xzQ2YiZPZru1KxnxxpMMe3IpZt06BLo+RCJdwu/RbXs/Y0GnGoTj9Ql4\n4SXq6pL3K+gAMmWw8fv9jB49mgMPPJDTTz+9a2BXf4gP2Xz//fdz3XXXpdx38+bN3HzzzX2+xvTp\n07vGHeSKXLl6LgECwM4isgn4NVAJYIy5BXgI6+a5HuvqOTEX1y0lEl0m+9w56o40TDu3j99vP15A\nn0R6G+KbJlHJOoQ9byPPs+jvf09+WnUPVQaboUOHsnr1agDOOussbrnlFn72s591bfd84X2+vtWP\nTzrpJE466aSU2z3xv+iii/qX8BySK2+fH/Sy3QAX5+JapUi8AFZUxGLx9EkMJ0yAZctI7CjpKgwO\nOgjmzbO/QyHYvNl+DxkCw4bBbrtBfX36Ib59VGjHgfPOg/nzu3c6p4s8Gr9dPYSULgbwYTjiiCP4\nxz/+wYYNGzj22GM55JBDWLVqFQ899BCvvvoqv/71r2lra2Pfffdl0aJFbLfddvz1r39l6tSpbLPN\nNhx++OFd57r99ttpaWlhzpw5/Pvf/+aCCy7g9ddfB2DevHn8/ve/57XXXmP06NEcc8wxXH/99Vx/\n/fXcddddtLW1ceqpp/Kb3/wGgGuuuYbFixez6667stdee3HwwQfnNN8F1eFbrjQ1wdatViC9CrkX\niyfjztFgEIFuNv+aYSQX9b6+PH0Mqxj/ntbXw+LF3fsD4renGkCmLQKliwF8GDo7O3n44Yc57rjj\nAFi3bh2LFy/m0EMP5YMPPuDqq6/m0UcfZdttt2XmzJn87ne/Y9q0aZx//vk89thjfPnLX+aMM85I\neu5LLrmEI488knvvvZdIJMKnn37Kddddxz//+c+uVseyZctYt24dK1euxBjDSSedxJNPPsm2227L\nnXfeyerVq+ns7GTs2LEq/qVGOGxDK3tm+IoKax/3av7drDC91X6CQbYPBtk+g137RB/CKiZ7T+MH\niEH67Yn9BTpgTBmIh2HLli2MHj0asDX/SZMm8fbbb7P33ntz6KGHAvDss8+ydu1aDjvsMADa29tx\nHIdXXnmFffbZh1GjRgFw9tln09jY2OMajz32GE1Ndtyr3+9nhx124KOPPuq2z7Jly1i2bBljxowB\n7CQz69at45NPPuHUU0/tCi+dzpTUX1T884w32has6E+aZGvLPYS7sREuvtg2DXoJvJ/zilK6Ib7u\nw+21LpK9p/EBuWbMSL/dI9dhfJUiZgAehnibfzzbbrtt129jDMccc0yPaR6THddfjDE0NDQwefLk\nbutnz56ds2ukQqN65pn4uXeHDIlZaLoJYjgMU6bEOmrb2pJ65XheM01NAzCZe2KiwmE46ii45Rb7\nce05qeYSTpbfdO9xrsP4KkVMnh6GQw89lKeffpr169cD8Nlnn/Gvf/2L/fffnw0bNvDaa68BpJwD\nePz48cxz+9kikQgff/xxj/DQxx57LAsXLuTTTz8F4K233uK9995j3LhxLF26lC1btvDJJ5/wwAMP\n5Dx/WvMfZBLNMRkFbAuFMJ0RBDCA+Hw9VDOx0zhVmIWc4VXxPTo6YOpUnLFjeW52PQ+2Oj3y4+Xd\nm1CmN5NUX2MZKSVMHh6GXXbZhdtvv50f/OAHtLW1AXD11Vez33770djYyHe+8x222WYbjjjiiG6C\n7nHjjTcSDAZZsGABfr+fefPm4TgOhx12GAceeCDHH388119/PS+//DKOm7ftttuOO+64g7Fjx3LG\nGWdw0EEHseuuu/L1r3899xnMJPRnPj6lGNI547DMCXGR/zH/GfMZQ00HPtNGpVk/bX6PQxLDLl9w\nwQCHVvbiQseChsY+fn8sfnTc7n0NSa2UJhrSOXfkPaSzkhkZ9VslMdg/2OrwF99yjoiGeMoX4Ds7\nOjQkHJZoFk3ltZkzHAcef9zamF54AZ5/PtZrHYnABRfY38EgoB24ilJoqPgPImn7rTybyMaNPVQy\nEHC4qtrh2XaHqiq4PtDz3P2J958tYRxCIxxOHBOmdnUA2tutWQrbopSLLoLaWnAc7cBVlAJDTCaz\neeSBuro609LSku9k5BxP42tq4MUX7br//ORyhi+5wdacq6qSjvIqtAFPiQ2U52aH+dKMSWyz4eWu\ngWVGBJk82UZ6CwRsYREqnDwo+eHll19m//33RyTteHSlF4wxvPLKK3z1q1/ttl5EVhlj6no7Xmv+\ng4wnekcdZZ12fkwjezKrq8ZMR4c1lbiC6R3Ql/6uwSgoEs04D7Y67HncAs685Ugq6QAgIn4qFi2y\nefL5cObOhUCwW2hppfwYMmQIra2t1NTUaAHQT4wxtLa2MmTIkH6fQ8U/D3jC+WMauda13nd7BbIw\n2A/WyNhkZpw1axzG+5/grEgTfj9897uw2/2N1j01GsVccCHvycP8hWlcVe30SFuhtW6UgWH48OFs\n2rQJnbMjO4YMGcLw4cP7fbyK/yATDoN/ZZjHzRWM48mu9V01/8suy0r5BqtjNbGPAWDqVGgzDs9V\nOsyZA7vVhuHB2+JiVkQ5ySzlWB5i/NYQoZCTckpI9e0vXSorK9lnn33ynYyyRwd55ZDeQhMvvTxM\n+zeP5LKlh3UJv1fjl2HDbAS0mTOzSkOmg6hyQfy4r/jwzdGoO1m848DcuTaQP7HQ0VW0M9/8mK9u\njt2oZIWWoigDh9b8c0RvNdell4c5ftY4qugEupt5BGyp4bpFZkM+vH4gjSeTl6eLLsK4cSwE+Bpr\nOeD6w2HfeRAM9ji+psbeEjUBKcrAoOKfI1KZW7zwN8PmhziJzpgnjPstIvDzn+dE+D3yMTI2baET\nDEJtLe2nfJ+q9zbFCj4TtWEramtxHKfr+Joaa0JSE5CiDBxq9ukniSaeZOYWrzUQuaWRH5g7iGJF\nv0v4x42Dp5/O2tRTKPSISZSwsfqqK4HYPeiaXaypCWbMwCFMQ4M1GakJSFEGFq3594NUJp7ly2NB\nLsGK1o+2NnILsYh9EeD/djuAYb+5NKe1/aLAnXOA2bPh1VftuspKG9M6ErEl53nnceKYeq6qcnRA\nmKIMIFrz7wfpOicXL4Zbb7WFQ00NTDILgFhnpx8YdsnZEAyW59y1wSCsXQsrVsDVV8PEifZGejfz\nlluovXgca37SqBE9FWUA0Zp/P0jVuZlYKAx5McxY34sQjZl6Ir5KKgIBdW30OibCYVtielOZAXR2\nsu9/T6HhidoyuymKMnhozb8fpAov7hUKPp/97P9uCD/Rrg7OtXIAL897IuWkJ+VEV6sH92ZOnhyL\nQw32xkyfXmbNIkUZRDIJ/ZmPTyGFdE6IsJyW+fONqagwxucz5siqZ0xn9VAT8fnNFt9Qc++02AnK\nOcRxyrzPn28i/koTwWeiYIyIMZWVPcJDK4qSGjIM6aw1/17wzDNXXmm/e6uItrbCIdEw06Iz6OyE\n3x6/nOm+qzia5fzwJqfr+HKeqSpVqydcG+Rb/if4G0fb9pIxNi7QxRdrC0BRcoyKfy/01Tzz/c2N\nPBYdx1X8kmXR8QBcaxp4Our0OD6ta2QJk2oUcigEKyIO05lOhIqufhKi0fKziynKAKPi3wt9CpcQ\nDrPvf0+hkk4qiDJU2jhjt9CghVsoFnrrM3ne7zC1Yg7GX2E7T6qr9cYpSo7ReP4ZkHG0yVNPhaVL\nY8sVFfDkkz3i2CeeT6NZxuh2L4ib/CCTSX8VRck4nr+Kf65obLQeKx4+H8yb12MgV6KL5+zZGsog\nLUl8YsM4XYPpBny6SkUpMnQyl8Gmubn7cl1d0hG8iX0Izc06t21aEm7YG00hjlzg0GHni2HRIjuV\nsN4zRekbavPPFRMmdF+eNCnpbol9CBMmDF4I5qIkECBSUUVE/EQqqvjTu4Eu4YfyHCOhKLlAa/65\nwqvlNzdbRU8RtydZ9MvaWrX5pyKMQ4NZzmGEeNoE2InuN8jn0wJTUfqDin82JPbUBoMZBWtLDLmc\njxDMxYLn/vmEcfBH4PzdYqE1/H64+Wa9d4rSH1T8+0tjox18FI1aV0TtqR0QEuMo1dfDZV9oRO5p\nxpw2gX3LLTKqouQIFf/+EA7bSUg67axctLVpT+0A0cNMtqYRZrleVbOWwb6UX2hsRckBOenwFZHj\nRORVEVkvIlck2X6uiLwvIqvdz49zcd28EQpZ7xMP1/BcliGaB4FuI6ETvaoWLMhLmhSl2Mm65i8i\nfmAucAywCXheRO43xqxN2PVPxpgp2V6vIAgErKmnrc0anufMIYxT3iGaB4sJE2DZstjyiy/a0lZv\ntqL0iVzU/L8BrDfGvG6MaQfuBE7OwXkLim61etcW8UbwapomPUG4Nlj2IZoHjWAQTjkltqxxfxSl\nX+RC/PcE3oxb3uSuS2SCiPxDRO4Wkb1ycN1BIxyGhkCYnX5xIf847EKWXh4mjMNXFzdw3q1O16xd\n6q8/SEybBkOH2ptdUQEbN6qtTVH6yGB1+D4ALDHGtInIZGAx8K3EnUQkCAQBRowYMUhJ6511TWGW\ntR9JJR1goG3WQn7/rxDt7U5XTb+1taf/vjJAxE+YvHChnTdz8eKUtjaNnaQoPcmF+L8FxNfkh7vr\nujDGtMYt3gbMSnYiY0wj0Ag2tk8O0pYTvrN2FpV0dM3IVUkH+70doiphknH11x9EHCfW8R6J2Gkg\nm5p6/AFlP12moqQgF2af54FRIrKPiFQBZwL3x+8gIrvHLZ4EvJyD6w4O4TA1K2LZMUAUH/tOCpTt\nZCwFQyAAFTbuvzGGyIKFPcw/2hejKMnJuuZvjOkUkSnAI4AfWGiMeUlE/gs7ndj9wCUichLQCXwI\nnJvtdQeKHiaCpiZM1M7Da4VfeOasmzkyaNVeRT+POA7vHD+RXZfOx48h2hFhU1OIveP+lMRBYtoX\noyiWnNj8jTEPAQ8lrPtV3O8GoCEX18oVyezASU0ECcfdz8m88rUgRw5yepXk/G23er7HYippp4Mq\nniBAfdz2ZLGUFEUp06ieqeblTWoiqK8nWllFBKGNKm6smqa1xwJiVL3DCVXLmS5XcULVckbVW3XX\nAXeKkp6yDO+QTOQdJ4WJwHHwPxHijaYQTxBgRr2jtccCwnFgRsghFHKYEYjNjOa14Px+Ow98ZydU\nVmoUDkXxKEvxT2UHTmkicBz2dpxu5gSlcEj0soov3OOjcLS3J3UIUpSypCzFP50d2LEz7gIBelr8\nlUIjWd9NfOEO3QsARVEsZSf+8WIRCMRc/xzHboweeRTS0Y6prML3hM4PWMik8uF3HHhudpjW5hCb\nRwc480anW0hoRVHKTPwTbcEi1hbsTaT+lRubGNfRZgdzdbTx7qwmdrtXxb9QSdV3QzhM7VT3j36q\niud/v5wHWx319lGUOEpa/BNNAvFiEY3afYyxwTn/56Iw10de6Hb822/DbknOoxQGKX34E0qF2tYQ\ntQ36xylKPCUr/slMAvFiEe8F4hDmkch4qmgDIAJ0UEXlpHoND1DApOy7STOySwtyRbGUrPgnMwk0\nNMTEoqYGfvITu++RhKiinQqidOLj1eFHE71yOrVBhxkzUpgWlIIgaTylJKVCOGw9fRYtsgV+RQVM\nnGj7APT/VMqRkh3k5VX+EkMse7NCtbZaQTcGniBA1F9FRPxIdTVfu8sKf7rzKAVO3PRfm86+nOHf\n3IszbjmSMW1hIhFr6ps/v/sgP0UpJ8SYggme2Y26ujrT0tKS1TnSNfHDYVhxxOWcHLmH+/yncdzN\np1DbmnxnNRUUMZdfjpkVCyLbgZ8jeYpnXTdevx/OPx9GjND/VykNRGSVMaau1/1KWfzTkiAKMm0a\nzJw5cNdT8sOoUZj167vCcUe7+QmrAAAdUklEQVSBPxxwLT9e30Ak0tPrS/t0lGInU/EvWbNPr/zx\njwBdosA99+QtKcoActppXRFZDYDPz49uCxAK2XDc551nhV9DPivlRsl2+MbTw2wTDhN9+50uUQDY\ndMhpDM9bCpUBw23NyR//CF/6EnLddeA4OMTiAC1erCGflfKj5M0+ia6az80OU9s8nejfHsVnokSB\npxjHM9c+QUNBBZ1WBoQkHTjap6OUEpmafUq+5h/v8jm2Lcz+U8ZDpA0x1q2znWp+XXUdMwL5Tqky\n4KQYtJFu+k0tGJRSpeTFPxCwnXrRKAQkREWkHaJRxOfjk7qjeWDsdA3TXC6kjAdhSRR6HeCnlDIl\nL/5gvTkAnvIFiPqq8Hfat3mn2dOp17e5fIgf+VtRARs3QjhMGIemJli40JYLntD3UlYoSlFT8t4+\noRDUdYS53MwgEoE/TNRZ18sWb+Tv+efb0X233krkqPE0BMLMn99T6HWAn1LKlGTNP775/s2XGpkW\nvRAfUTqjFbwy5kkIas9u2eJF+PNmeom2cxghnjC2IiASE3qd/1cpZUpO/OPttIf5wizvuAAfBgEq\n6KTm+isg+ES+k6nkkzjzj4iPkyNL+UBqWFwV7BHvJ11nsKIUMyVn9om3057Z0dQl/B4VG1/LV9KU\nQsGr0n/3u/g6O/i6WcktZjIbjzybefNU7JXyoOTE36vUHeYLM5FF3Ud3Am8Fzspf4pTCwXHg888B\nO8pbgF2X/QEaG/OaLEUZLEpO/L1K3dVHh6j2dXbV+j/1fYHV357GmEc0fo/iMmFCz3XNzYOfDkXJ\nAyUn/gDOmkYCm5cifh/4/cjQoWy/4q8q/Ep3gkE4K6EluM02GuNZKQtKT/wbG2HyZFi5Ejo64Lvf\nVbdOJTV33GED+3/jG9b3/4EHNMi/UhaUnvg3N3fZ9w1Yu64Kv5KOYBBOOcX6/mt4T6VMKDnxf220\nteOahGVFSYuO6FLKjJLz879rxyAbBE41zdwrExi5YxAd0qWkIjYg0MGJn+DZq/lrq1EpUUpO/AMB\nGD8kyIL2oI3REsh3ipRCpWfgNgcngEZzU/LKYEWSLTnx1yH5SqYkDdxG3MqtW6GpSR8iZdAYzEiy\nJWfzB3uzGhr0nVXSk9TMHwhYrx+wHcALF6rnjzJoJKuQDBQlKf6KkgleK7FbkFfHgYkTMW4ccNMZ\ngVCIxkY49tjYAOBwGGbM0HJByS2D6XeQE7OPiBwH3Aj4gduMMdclbK8GmoCDgVbgDGPMhlxcW1Gy\nIVngtqVfqOfbZjGVtNMRrWLRSwGm/MFuW7YMXnsNbrpJuwWU3JBo4x8ss3XW4i8ifmAucAywCXhe\nRO43xqyN220S8JEx5ssiciYwEzgj22srSrYkm73r+//tcDDLCRCilRr2ezjEocCz2Dfxnnt6n+RF\np39UMiGVjX8wnplc1Py/Aaw3xrwOICJ3AicD8eJ/MjDd/X03MEdExBTq7PFKWZDsxfNC/XtCv5zx\nVH/UzkVUMZ7lPIvDaad1r/knNs11+kclU5LZ+LdbE6a1OUTNhAC1wYF7cHIh/nsCb8YtbwIOSbWP\nMaZTRD4GaoAP4ncSkSAQBBgxYkQOkqYoqUn24gUCUF0NbW0wnhBDTDs+E2GIr53zvxRi4s+drgHB\nqWr2Ov2jkimejb+tDXw+GBVq5KvLLsJHlPZlVazh8QErAAqqw9cY02iMqTPG1O2yyy75To5S4iTr\nXOuKCns1nDEvgG+I3cHnE84btpQgtsc3nUeZDhZWMsVxYPZsK/xf7wxz8rKL8BPBh6GaNjoWNA3Y\ntXNR838L2Ctuebi7Ltk+m0SkAtgB2/GrKHkjVedazObqQO1ymDULli61wQJXrrQ9vjNTR4jVsSZK\nX2httV7F40wIH5Fuk0/tvsfAXTcX4v88MEpE9sGK/JnADxP2uR84BwgD3wMeU3u/Ugj02rkWN+lL\nFzfcYO0+aQ7U6R+VTAkE4HB/mL2jG+k0lfjoACDqr2D3afUDdt2sxd+14U8BHsG6ei40xrwkIv8F\ntBhj7gcWAP8jIuuBD7EFhKIUBxMmWB9PD2PUkK/kDGdNI491XoSYCKaiEjnxFNhtN/zxk0kPADnx\n8zfGPAQ8lLDuV3G/twKn5+JaijLoBIPW1HPDDVb4Kyth40br1qMFgJIN4TBceCG+aNQud3bwwtu7\n0TZt3oA/WgXV4asoBcvMmbBihZ0oSARuvVUnfVGyp6kJPOF3WblycB4tFX9FyRTHgREjMB2dEIlg\n2tp5oymkYR6UnGCACD4WUz8o8wmp+CtKH1hTE2BLtIpOfHREfVx3aw1XXqmNAKWf1NcTrawmitCJ\nnwuZx0qfMyguwir+ipIh4TBc1uwwldlE8eEjwm8jU/l6JKwzPyr9w3G4Y9LjXCnXMI6nWOgLcvTR\ngzMqvOTi+SvKQOCFbGhrg2m04sNQQRRDO9+SEH+vcnQwl5IZCYGfRtU7XLDYob0dqqtg+vTiie2j\nKCWPF7IhGoUnJUCnVOGnHb+/ggljNnLmpDC16vmj9EY4bEW/o8N6jYVCOI6Tl0GBavZRlAyID9nw\n4hCHdfOWI8Hz8Ylh7KpbqZ2qRn8lA2bNsrUIY+x3kw3fkI8JqFT8FSUDEid+qQ1azx8ikcGZdkkp\nfsJheOCBfKeiCzX7KEqG9AjZ4DUHUsV2VpR4QiFb4/fw+6F+4MI39IaKv6L0F43gpvQFN164aWsj\nip8N/zGHffP4zEihxlerq6szLS0t+U6GoihKzljTGObPF4d4LBrghWpnQFw6RWSVMaaut/205q8o\nijJIPNjqcE3UIRoFX1usmygfjUcVf0UZIHQeXyWRmppYKJ9oFDZvzt+Unyr+ipJrwmHeaArRsDDA\nioij8/gqXbS22lm7olH7vXp1/qb8VFdPRckl7lDgveZfyUPt4zX0g9INb45ov99+T5iQvyk/teav\nKLnEHQrsMxGq2cJspnK5fzY1NQ4zZqgJqNxJ5iBWW5sf86B6+yhKLgmH4aijoK0N782KVFRztO9x\nVkQcKipg4kTr3q2FQGlRKH08mXr7qNlHUXKJ41h1B8T9+DrbOawjRCRiA8PNn68hoEsNL/DfX34Z\n5uFxM1jTWPh/roq/ouSa+nprwPWorOLpygAidtEL66L9AKVDUxOcvaWRx6JH8qvOX7L/lMIv3VX8\nFSXXOI5V9gsugAsuwPfE48wIOUyenL/OPWXgCIdh7YIwc7iYSjqoIEpFpK3gS3ft8FWUgSAxEFDY\nxoG76Sbr7ldTE9MGtf0XN6EQ/KCjCT8RBDsdo/j9SUv3QukXABV/RRlwwmH4n3GNnNzZzH0VExg9\nN8gll8QG9jz+eP6FQOk/J9aEGcUifBgMYHx+ZM6cHn+q1y+QjwFdyVCzj6LkmHCYbpO6fzSrkbmd\nk/k2y5jbOZkPrm2krc3a/tvaukK6K0VKbWuIal8nAiCCL3g+BIM99vMmBCqUCOBa81eUHJK0dvd2\nM0CXSWDcB81AT3FQipRAAKm2ob2lqiplmOZCiwCu4q8oOSRZ7c6ZNAGzclmX33/TZxMAELEz+eUx\npLuSDfEG/AxCexdaBHAVf0XJIUlrd04QAV6/vplX18NlXM+XeI3Hj5k5aJN1KzkmWROvoaHXw3pM\nCJRHVPwVJYekrN0Fg1Q9+RrHrZ8FwOXM4uxdYLgzM19JVfpIN0+dpE28AlH1DFHxV5Qck6p2N/y5\ne6wbINb2P/yem6Bx36Sdgx6F5BpYziRW9J+bHaC2kAz4/UDFX1EGi9NOg1mzumz/bNkCkycDEK4N\n9hD5QnMNLGdCIRjbFuasaBOyBV58sZ7aQjLg9wMVf0UZJMKnzGTFb+HCyE1sy5au9R8taGb8mmAP\nkQ+FrCtoNGq/i9CyUDKcWBPmp9GjqKYNgOiChVAfysjOX6ion7+iDBKhEDQwk58yG6CrBRDeY0JS\n/+/EWZ9qagY7xYpH7cOzqKatK1ifv7Mj/476WaI1f0UZJDxPoEXtQSoFrhnbzE6TJrBTbZCqR3qa\njxNnfWptzWfqy5jGRli6FIlfV1lZlHb+eFT8FWWQ6O4JFGQnx3b0OiT3EPJmfSriPsXSYPbs7st7\n7gl//jNhHEJFPEFPVuIvIsOAPwEjgQ3A940xHyXZLwKscRc3GmNOyua6ilKspPIESra+0AYFlSWN\njfDyy93X/epXhHGSdsYXk3dWtjX/K4DlxpjrROQKd/nyJPttMcaMzvJailJ2FNKgoLKkubnrpwE+\nH3kA2waDhGYkj9NTTN5Z2Xb4ngwsdn8vBk7J8nyKUp6Ew7xx4QxmnRrmwgsLfh6Q0seLzjd6tI3U\n6a7++aZLCYdj/TfxczMUWuC23si25v9FY8w77u93gS+m2G+IiLQAncB1xpilyXYSkSBuxKsRI0Zk\nmTRFKRLCYSJHjWfPtnamUMV4lrNwoaOunfkiYYDFI6OnIatXczcTWGSC7BWyHp7JTHLFNO6rV/EX\nkUeB3ZJs+s/4BWOMEZFUs8HvbYx5S0S+BDwmImuMMa8l7mSMaQQawU7g3mvqFaUUCIWQ9nZ3MpCt\n1NPEcx0q/nkjrgpv2tp5es2OXM0jAFTFzdGSaJIrtj6aXsXfGHN0qm0i8m8R2d0Y846I7A68l+Ic\nb7nfr4tICBgD9BB/RSlLAgFMhR/TEcGH4cfcyj98YwgENOxzXoiLztfpq+LxSACwUVjPOy+9qBdT\nH022Nv/7gXPc3+cA9yXuICI7iUi1+3tn4DBgbZbXVZTSwXHwTzoPEASoIMJcLsYhO8N/4qQySga4\n7jqv/WQ2ofFX8ZefLueFagefDyoq4AtfKKF7aozp9weoAZYD64BHgWHu+jrgNvf3N7Funn93vydl\ncu6DDz7YKErZ8MwzxlRWGmMn+DLG5zPm2muzOt3Qocb4/fb7mWdymNZSxb1pUZ/ffMZQc5jvGTN0\nqDHTphlTUWGMSOyvKeR7CrSYDDQ2q5q/MabVGDPeGDPKGHO0MeZDd32LMebH7u9njDG1xpiD3O8F\n2VxTUUoSx4E5c2z10uezo7uy6DEsNs+TgsC9aRKNUEk7R0RDtLfD6tVeiWx3i0ZL457qCF9FKRSC\nQaitzUmPYaFNGVgUuDfNtLXTEa3iKV+AqiqYMAGeeioWZM/nK417KsYUplNNXV2daWlpyXcyFCW/\nZDFktJhGmxYM7k1bUxPgwVanS+Cbmuz3mDE2xlIh31MRWWWMqet1PxV/RSlQvNFEHR02kJj6fg46\nxTinQqbiryGdFaVQaWqyqmOM/faqn0puyMAdqpT7TtTmryhFiJp0etKne5KiSp94jlLuO1HxV5RC\npb4eFi2KKU99PVCcpoiBJv6e+P12MFZ9fZr70tQEW7fGWlWhUMpIncU0arcvqNlHUQoVx4HHH4dr\nrrHfrvKUsimivyTek/nzbWGQ1KITDsPChTHfzYoKCARS3lfHsbF8Skn4QWv+ilLYJIkXUMqmiP7i\n3ROvMh9Xoe8p2qGQVXiwMRsmTgTHIUB53VcVf0UpMkrZFNFfvHvS1GQtZZ2daQQ8sfR0zWnldl/V\n1VNRihHt8U1JRremhO+f+vkrSqmiPb5KGtTPX1FKlUHu8S2J6KAlkYncojZ/RSk20vT45tqaURKN\njJLIRO5R8VeUYiNFz+RAaFyyRkbR6WZJZCL3qPgrSjGSxAW0LxqXaQuhJNxKSyITuUfFX1FKhEw1\nri8thHy6P+bMhFVuPpwZouKvKCWC48Bzs8O0NoeomRCgNoXIJYlsUHDz0mZrwlrTGHcfgk5xTa47\nSKj4K0qpEA5TO3W8nXXkMR8w104Q030XFi2KRTbw+wvTChJvwmprg+nT7ScT/V7TGGbU5AAH0EHH\nskrWELIFgNINdfVUlFIhFIpNN9XZCVOm9HBtDIXsJrCRDc47rzArxJ4Jy+ez2Xn00TSxehKomj2L\natrxY6imnY4FGgo7GSr+ilIqBAJWLT0ikR5jAGpqrOj7fDBkSFdkgwGjv+71npn+6KNjBUBGQxrC\nYUa9+kC3VXvs0bdrlwtq9lGUUsFxYO5cohdNgWgEIz78S5daxQ8GCYdh6lQrpH4/zJ49MLV+r6O2\npsZer792e8expp6nnsrQUaexEa6/Hl/UBm0zAD4/u00b4BKuSFHxV5QSIlwbpMFfy6WRWZwSWYpZ\nuRJZuRKAUGuQ9nYr/iJ2LtqcXz+uo1bEXiu+1p6p+Md7+mTkqNPYCJMnx5ZFEL8f5s4tTLtWAaDi\nryglRCgEKyIOV/A5AOJtaG4mMD044O7u8R21Pp9tYYhkdr10LYaGhl4u3NzcbXHLnvvy5xObGFXr\noNKfHBV/RSkhvI7Se7dO4FizDINbAEyYMCDu7r1Nezh7tm1h9Ha9rFsMEybAsmWANff8x7s/p/FW\nh6rFGs0hFSr+ilJCxAQ+yOubYd/VzVYYXZfPXLq7J/PFBzjnHPuddhrFBLJpMQAxl9bmZh7eZgKN\nDwR7HelcwlGdM0LFX1FKjJjAB91ParIRwMRwEk1NsHhxjzlSMqK/LYZuBIMQDLJTGKoeSW/e0lhv\nKv6KUrZkI4DhMGzcaKe/BXs8pI8tlK6gyaVJKpNzaaw3FX9FKS/iFDgUcvolgPGFht8P558fq+XH\n1/zja9y9FTQZt0Ay3LE381ZNjTUvGVO+sd5U/BWlROmhk54Ct7WB38/3fzqHq6qCffb+ia81A4wY\nERPaVDXudDXtlAVDvPtPa2v2Awfi7svUqbH+hYEa71DoqPgrSgmSVFDjwz9Eo+x7w4WsuQzu2jHY\nJdaZVKzTRQ9NVeNOd4xXMEyMNPK9Lc1se9EubHl3BUPe3QgY663k9QJHIv0bOBCHd72BHO9QDKj4\nK0oJkrSmHQhYAY1G7U5uAdAwD3DsCOCjjooJ9OOPJ9fW/tjnkx4TDkNTExc+u5ZzIuvYnXfszqtj\nxwnWdVO8NPt8Kd2AymqOghyg4q8oJUhSgXMcmDMHLrywWwHAlClQW0tTk0Nbm13d1ma9d9LF+U/c\n1pv4Og44uDutqYFLLoG2NnYEdnD36RJ7d9l4B/t8UF2d0g2oWOYoKCRU/BWlBEkpcJ4/fHwB0NEB\nU6eyzx6z+TFrmEAzzUygNzfReFKKb2Mj3Hij7VkdOxb+9Cd7XZ8v1mlAT7E3ced+/9tnsWvga2mV\nuq/eOxreP0vxF5HTgenAV4FvGGNaUux3HHAj4AduM8Zcl811FUXpnZQC5xUAF10UE+CVK7nMdwSC\nXT6WZbz+BYBg9yo9dO+E3bwZHnyQL79vmL9lLKNYR/WWNoaf3A47dsK6dbHrvvxy7Lcx3QoAT+yj\nwKPybb7+xTeJRIT/m3gp+87svRDqzZRT7gO6kpFtzf+fwGnA/FQ7iIgfmAscA2wCnheR+40xa7O8\ntqIo/SUYhBdfhFtu6Vrli0a6TC4Gd3RwuDZWpa+osKLd2RlrNbj77gycTZy4vw/mfftT6I4BjM+P\n7+a5Ng1r1yIffMAHO+/HQwdMY1S9wzBXoHdJODaViKcz5eiAruRkJf7GmJcBRBL/3m58A1hvjHnd\n3fdO4GRAxV9R8kl9PSxcaFURwO9H3Jq4Fw+omz3FE3wTM8ok2ueTKUG8CSeKEKGCqTKHH9UGcRIa\nFvUZxv9JJuKpWjo6oCs5g2Hz3xN4M255E3BIsh1FpGs8+ogRIwY+ZYpSzjiOVcImd6ar+npYs8ZG\nyPTiAYXDMXtKmpp/st8eUeAN9ubPcgYfsyOPmwDPG4e9QnZ7prXy/oq4evckp1fxF5FHgd2SbPpP\nY8x9uUyMMaYRaASoq6tL9hwpipJLEqvLjtN93t9Eewp0s/mvDm2matmDgGG1byzf/co6fB1tbFzf\nzvvswityAItNPWEcBFt+RKMxEU4m6N4lEs03/RVx9e5JTq/ib4w5OstrvAXsFbc83F2nKEoxkKyA\nwDXDXAVtvpn4fHbelC+45cbHYbizCdauhfCTdp0x8NOfwo47dhfheEGvqUndEshGxNW7pyeDYfZ5\nHhglIvtgRf9M4IeDcF1FUQaQxJGyL75o5+v1auSLF8PWrbH9fT4r/N7ELJ6tP951P34QcltbT9OO\ninjuyNbV81TgJmyn/F9EZLUx5lgR2QPr0nmCMaZTRKYAj2BdPRcaY17KOuWKogw68Z2z8WYYvx8W\nLbLdAVVVNqZ/e3usb1jEjtHyCoZUnbdr1nQff1ZTM/h5LBey9fa5F7g3yfq3gRPilh8CHsrmWoqi\n5Ja++r4nE2zPDLNxI9x6a8x2D90LhvPO6z65S6rO29ZW20LwxoGVa9ydwUBH+CpKGdIf3/dkgt3Q\nEAsIlziRS3196sIlVedtIGBbCOqZM/Co+CtKGdIft8neonkm64zta3wd9cwZPMSYwvSorKurMy0t\nSaNFKIqSJf0d9aphEgofEVlljKnrbT+t+StKGdLfGvZAeNtogZIfVPwVpUzxhDwcjrloDrb4Njba\niNKRiLX1a9ydwUPFX1HKmIEIepZpTT4chosvtu6hkNyvXxk4VPwVpYzJddCzvhQmoVC3EEH4/erd\nM5j48p0ARVHyh+fB4/fnxrUyVayeVNeurrb+/BUVdpIxrfUPHlrzV5QyJteulYnuoDU1qfsT1K0z\nv6irp6IoOcWz+dfUwNSpOonKYJOpq6eafRRFySmOY0f+trZmbgJSBh8Vf0VRssJzFQ2Hu6/PdX+C\nklvU5q8oSr9J592jNv3CRsVfUZR+05urqMbfL1zU7KMoSr9R007xojV/RVH6jZp2ihcVf0VRskJN\nO8WJmn0URVHKEBV/RVGUMkTFX1EUpQxR8VcURSlDVPwVRVHKEBV/RVGUMqRgo3qKyPvAGxnuvjPw\nwQAmZ7AohXxoHgqDUsgDlEY+BjsPextjdultp4IV/74gIi2ZhDAtdEohH5qHwqAU8gClkY9CzYOa\nfRRFUcoQFX9FUZQypFTEvzHfCcgRpZAPzUNhUAp5gNLIR0HmoSRs/oqiKErfKJWav6IoitIHVPwV\nRVHKkKIXfxE5TkReFZH1InJFvtPTV0RkoYi8JyL/zHda+ouI7CUij4vIWhF5SUQuzXea+oOIDBGR\nlSLydzcfv8l3mvqLiPhF5EUReTDfaekPIrJBRNaIyGoRacl3evqLiOwoIneLyCsi8rKIFEzw66K2\n+YuIH/gXcAywCXge+IExZm1eE9YHRGQc8CnQZIw5MN/p6Q8isjuwuzHmBRHZHlgFnFJM/wOAiAiw\nrTHmUxGpBFYAlxpjns1z0vqMiPwMqAO+YIw5Md/p6SsisgGoM8YU9QAvEVkMPGWMuU1EqoBtjDGb\n850uKP6a/zeA9caY140x7cCdwMl5TlOfMMY8CXyY73RkgzHmHWPMC+7vT4CXgT3zm6q+YyyfuouV\n7qfoakciMhz4DnBbvtNSzojIDsA4YAGAMaa9UIQfil/89wTejFveRBGKTikhIiOBMcBz+U1J/3DN\nJauB94C/GWOKMR+zgWlANN8JyQIDLBORVSISzHdi+sk+wPvAItcEd5uIbJvvRHkUu/grBYSIbAc0\nA1ONMf+X7/T0B2NMxBgzGhgOfENEisoUJyInAu8ZY1blOy1ZcrgxZixwPHCxax4tNiqAscA8Y8wY\n4DOgYPoli1383wL2ilse7q5TBhnXRt4M/MEYc0++05MtbvP8ceC4fKeljxwGnOTazO8EviUid+Q3\nSX3HGPOW+/0ecC/WxFtsbAI2xbUe78YWBgVBsYv/88AoEdnH7Uw5E7g/z2kqO9yO0gXAy8aY3+U7\nPf1FRHYRkR3d30OxjgSv5DdVfcMY02CMGW6MGYl9Hx4zxpyd52T1CRHZ1nUcwDWTfBsoOm84Y8y7\nwJsi8hV31XigYJwgKvKdgGwwxnSKyBTgEcAPLDTGvJTnZPUJEVkCBICdRWQT8GtjzIL8pqrPHAb8\nCFjj2ssBfmGMeSiPaeoPuwOLXS8yH3CXMaYoXSWLnC8C99o6BRXAH40xf81vkvrNT4A/uJXT14GJ\neU5PF0Xt6qkoiqL0j2I3+yiKoij9QMVfURSlDFHxVxRFKUNU/BVFUcoQFX9FUZQyRMVfURSlDFHx\nVxRFKUP+P5OxXtvr2werAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3h7IcvuOOS4J", + "colab_type": "text" + }, + "source": [ + "Much better! The evaluation metrics we printed show that the model has a low loss and MAE on the test data, and the predictions line up visually with our data fairly well.\n", + "\n", + "The model isn't perfect; its predictions don't form a smooth sine curve. For instance, the line is almost straight when `x` is between 4.2 and 5.2. If we wanted to go further, we could try further increasing the capacity of the model, perhaps using some techniques to defend from overfitting.\n", + "\n", + "However, an important part of machine learning is knowing when to quit, and this model is good enough for our use case - which is to make some LEDs blink in a pleasing pattern.\n", + "\n", + "## Convert to TensorFlow Lite\n", + "We now have an acceptably accurate model in-memory. However, to use this with TensorFlow Lite for Microcontrollers, we'll need to convert it into the correct format and download it as a file. To do this, we'll use the [TensorFlow Lite Converter](https://www.tensorflow.org/lite/convert). The converter outputs a file in a special, space-efficient format for use on memory-constrained devices.\n", + "\n", + "Since this model is going to be deployed on a microcontroller, we want it to be as tiny as possible! One technique for reducing the size of models is called [quantization](https://www.tensorflow.org/lite/performance/post_training_quantization). It reduces the precision of the model's weights, which saves memory, often without much impact on accuracy. Quantized models also run faster, since the calculations required are simpler.\n", + "\n", + "The TensorFlow Lite Converter can apply quantization while it converts the model. In the following cell, we'll convert the model twice: once with quantization, once without:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "1muAoUm8lSXL", + "colab_type": "code", + "colab": {} + }, + "source": [ + "# Convert the model to the TensorFlow Lite format without quantization\n", + "converter = tf.lite.TFLiteConverter.from_keras_model(model_2)\n", + "tflite_model = converter.convert()\n", + "\n", + "# Save the model to disk\n", + "open(\"sine_model.tflite\", \"wb\").write(tflite_model)\n", + "\n", + "# Convert the model to the TensorFlow Lite format with quantization\n", + "converter = tf.lite.TFLiteConverter.from_keras_model(model_2)\n", + "converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]\n", + "tflite_model = converter.convert()\n", + "\n", + "# Save the model to disk\n", + "open(\"sine_model_quantized.tflite\", \"wb\").write(tflite_model)" + ], + "execution_count": 0, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "L_vE-ZDkHVxe", + "colab_type": "text" + }, + "source": [ + "## Test the converted models\n", + "To prove these models are still accurate after conversion and quantization, we'll use both of them to make predictions and compare these against our test results:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "-J7IKlXiYVPz", + "colab_type": "code", + "outputId": "0c10f56c-dbd7-4cc3-e332-30ad673769e5", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + } + }, + "source": [ + "# Instantiate an interpreter for each model\n", + "sine_model = tf.lite.Interpreter('sine_model.tflite')\n", + "sine_model_quantized = tf.lite.Interpreter('sine_model_quantized.tflite')\n", + "\n", + "# Allocate memory for each model\n", + "sine_model.allocate_tensors()\n", + "sine_model_quantized.allocate_tensors()\n", + "\n", + "# Get the input and output tensors so we can feed in values and get the results\n", + "sine_model_input = sine_model.tensor(sine_model.get_input_details()[0][\"index\"])\n", + "sine_model_output = sine_model.tensor(sine_model.get_output_details()[0][\"index\"])\n", + "sine_model_quantized_input = sine_model_quantized.tensor(sine_model_quantized.get_input_details()[0][\"index\"])\n", + "sine_model_quantized_output = sine_model_quantized.tensor(sine_model_quantized.get_output_details()[0][\"index\"])\n", + "\n", + "# Create arrays to store the results\n", + "sine_model_predictions = np.empty(x_test.size)\n", + "sine_model_quantized_predictions = np.empty(x_test.size)\n", + "\n", + "# Run each model's interpreter for each value and store the results in arrays\n", + "for i in range(x_test.size):\n", + " sine_model_input().fill(x_test[i])\n", + " sine_model.invoke()\n", + " sine_model_predictions[i] = sine_model_output()[0]\n", + "\n", + " sine_model_quantized_input().fill(x_test[i])\n", + " sine_model_quantized.invoke()\n", + " sine_model_quantized_predictions[i] = sine_model_quantized_output()[0]\n", + "\n", + "# See how they line up with the data\n", + "plt.clf()\n", + "plt.title('Comparison of various models against actual values')\n", + "plt.plot(x_test, y_test, 'bo', label='Actual')\n", + "plt.plot(x_test, predictions, 'ro', label='Original predictions')\n", + "plt.plot(x_test, sine_model_predictions, 'bx', label='Lite predictions')\n", + "plt.plot(x_test, sine_model_quantized_predictions, 'gx', label='Lite quantized predictions')\n", + "plt.legend()\n", + "plt.show()\n" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzsnXl4FFXWuN/bnbCELbIMCiHpqKzZ\nISBkYXGZDFECIhFkEWRcUFHHJCAOIo7K/DAkcRkc/XRGXAi7DIQx8+GHbAmRkTWYIMiSTtgUBAIB\nAln6/v6o7k4n6ex7ct/n6ae7q27dulV169Stc849R0gpUSgUCkXLQtfQDVAoFApF/aOEv0KhULRA\nlPBXKBSKFogS/gqFQtECUcJfoVAoWiBK+CsUCkULRAn/BkYIMUUI8W1Dt8OCEKKtEGKTEOKKEGJt\nPewvXQgxsq73Ux8IIQxCCCmEcKhE2RlCiOT6aFdlEEK4CiGuCSH0Dd2W+kAIMVIIcboO6m1U17U8\nmo3wF0JMFkLsNXfgc0KI/wghghq6XRUhpYyXUv6+odthwwSgO9BFShle1zuTUnpIKbfX9X4U5SOl\nzJJStpdSFtakHiHEdiHEk7XVLpt6K/1gVVSOZiH8hRARwHvAX9EElyvwd2BsQ7arIhppR3YDfpZS\nFtTlThrpsSsULQcpZZP+AJ2Aa0B4OWVaoz0czpo/7wGtzetGAqeBucB54BwwDggFfgYuAX+2qesN\nYB2wGsgB9gM+NuvnASfM6w4DD9usmwHsAt4FLgJvm5clm9cL87rzwFXgR8DT5ji/BC4AmcBrgM6m\n3mQgBrgMZACjyzkf/YHtQDaQDoSZl/8FyAPyzef0jyW26wHkAp1tlvkBvwGOwF3AVvOx/QbEA842\nZY3AK8Ah4BbgYF52fyWuk/U82dQngbvNv0PN5zsHOANElXHsttcgGzgJBJiXnzKf++kl+ldZ511v\nPue/met53twmB5tt/4nWp86Yr7e+5PGUd93ttP8J4CfzcZ4Enimxfq55f2eBJ0ucoweBA+Z9nALe\nsNnOUKLt24G3zOcqB/gW6Gpe1wZYbr7O2cAetEHXIqAQuInWf5aWcQxrgV+AK8BOwMNmXVsg1nyu\nr6D167ZAlrl918yfYWj34vJyjqHMc4X5vi+jfR8BMSWWbQQiKnmPJ9trj815fdLm/0xzGy8DmwG3\nqvaJasvO2qysIT7AH4AC2xNsp8ybwG7gd0A3IAV4y6YTFACvowmwp9Bu9BVAB8ADTeC5m8u/gSYc\nJ5jLR6EJW0fz+nA0IakDJgLXgTtsOkYB8AKa4GtborOEAPsAZ/PF72+z7ZfmDtjB3Kl+xiyczXXk\nm9uuB55Fu/mFnXPhCBwH/gy0Au41d+K+Nse3vJxzuRV4yub/EuBj8++7gQfQhHg3tBv7PZuyRuAg\n0Atoa7Ps/kpcJ+t5sqnPVrCdA4LNv28DBpbRfss1eMJ8rt5GEywfmtv9e/P5aF+J8z4LOGI+ns7A\nNooLn38B/wO0Mx/TD5gFUGWvu532P4j2kBXACOCG5VjR7oVf0PqsE5qAtj1HIwEvtL7pDfwKjLMn\nqNCE1AmgD1o/3Q4sNq97Bthk3oceGAR0tNnuSXtttzmGmebzaXnYH7RZ96G5jp7mugPM5Yq1z15f\ntXMM5Z2rkZQt/IejPRyFTX/KBXpU8h6vlPBH00wcN19vB7SBRUpV+0S1ZWddCub6+ABTgF8qKHMC\nCLX5HwIYbTpBLkUjsg7mC3aPTfl9NjfJG8Bum3U6bASPnX0fBMbadIysEuttO8u9aMJlKObRpXm5\nHm1EPsBm2TPAdps6jtusczIfw+122hOMJiBs61+JeRRIxcL/SWCr+bcw3yTDyyg7Djhg898IzCxR\nxkiR8C/vOlnPk816W8GWZT4nHSvoCzOAYzb/vcz1dLdZdhHwrcR53wrMsln3e3NdDmgj4VuYH3Lm\n9Y8B2yp73SvZ/zcAL5l/fwb8P5t1d9ueIzvbvge8a/5toLTwf82m7HPA/5p/z0R7MHvbqXM7FQj/\nEuWdzfvthHYv5WLzJm1Trlj77PVVe2XKOVcjKVv4C3N/Gm7+/xTmPl9G+ZL3eGWF/3+webs2H/8N\nNNVrtftEZT/NQed/EehagQ65B9prpIVM8zJrHbLI0JVr/v7VZn0u0N7m/ynLDymlCU1t1ANACPG4\nEOKgECJbCJENeAJd7W1bEinlVmAp2ujnvBDiEyFER/P2jnaOoafN/19s6rlh/mnbZgs9gFPmdpdV\nV3l8DQwTQtyBNkIyAUkAQojuQohVQogzQoiraCPPriW2L/P4qfg6lccjaKqfTCHEDiHEsHLKlry2\nSCntXe+KznsPih+PbTk387bnbPrC/6C9ARSjnOteCiHEaCHEbiHEJXOdoRSd45LtOVVi23uEENuE\nEBeEEFfQ3lxKXh9bfrH5fYOi/vQVmopilRDirBAiWgjhWE49tm3QCyEWCyFOmPuI0byqq/nTBm0Q\nUGMqOFdlIjVJvArtYQ0wGU2Faam3onu8srgB79vUcwntwdOzKn2iujQH4f892ghrXDllzqKdaAuu\n5mXVpZflhxBCB7gAZ4UQbsCnwGw0bxlnIA3tglqQ5VUspfxASjkIGID2yj0HTaecb+cYzlSj7WeB\nXuZ2V7kuKeVlNP3vRLSbYpX5ZgHN4C4BLyllR2AqxY8dyj/+8q7TdbQ3GgCEELeXaNceKeVYNOG6\nAVhTmeOpgIrO+zls+oJ5nYVTaP2yq5TS2fzpKKX0sLejMq57MYQQrdEevjFobyrOQCJF5/gcWl+0\n0Kt4DawAEoBeUspOwMeUvj4VIqXMl1L+RUo5AE0t8xDwuGV1BZtPRlN33I822jeYlwu0830TTVVT\nard2lhXrE4C1T1TiXFXESmCC+Z6+x1wXlbzHbdtHWW1E6yPP2PQPZyllWyllClSuT9SEJi/8pZRX\n0PT1HwohxgkhnIQQjuanfrS52ErgNSFENyFEV3P55TXY7SAhxHjz28af0G7y3Wi6XYlmM0AI8QTa\nqKBSCCEGm0dnjmgd5yZgMr+VrAEWCSE6mDtgRDWP4b9oo7i55vM0EhiDNtKpLCvQbvYJ5t8WOqAZ\n464IIXpS9c5a3nVKBTyEEL5CiDZor/wACCFamedLdJJS5qMZyEzUkEqc9zXAi0IIFyHEbWiGQMu2\n59AekrFCiI5CCJ0Q4i4hxIiS+ynruttpUis0/fcFoEAIMRpN1WRhDfCEEKK/EMIJWFBi+w7AJSnl\nTSHEEDRBXGWEEKOEEF7mOQFX0R6Qlvb+CtxZzuYd0O6Xi2hC8a+WFea30c+AOCFED/NbwjCzIL9g\n3odt3QeB4eY5Cp2AV23WVXSuykVKeQDtYfQPYLOUMtu8qtL3uJTyAtpAYar5WGZS/MH2MfCqEMLD\nXFcnIUS4+Xdl+0S1afLCH0BKGYt2U76GdlFOoT2ZN5iLvA3sRfMy+RHNQ+ftGuxyI9rI9zIwDRhv\nHg0dRvNU+B7tJvBC85aoLB3RRhWX0VQIF9EMqqAZia+jeS0kowndz6racCllHpqwH43Wuf8OPC6l\nPFKFahKA3mi2llSb5X8BBqJ5aXwDrK9i88q8TlLKn9EMwluAY2jnwJZpgNGsSpiFZguqDco775+i\nqT9SzW0tebyPowmhw2jXdB1wh519lHfdrUgpc4AX0YT8ZTThnWCz/j/AB2iG5+NoAxLQhC1oevs3\nhRA5aA/W6r4d3W4+lqtonio70FRBAO+jjZgvCyE+sLPtl+ZjPIN2XnaXWB+Fdu33oKlB3kHTed9A\n8ybaZVaTDJVS/h+a190hNLvcvy2VVHSuKskKtDcU6wCnGvf4U2iDoItohvgUm7r+ZT6+VeZ+m4Z2\nX0Il+0RNsFizFZVECPEGmgFtakO3RaEoDyFEfzSB0lrW8bwNRdOjWYz8FQqFhhDiYSFEa7Ma6h1g\nkxL8Cnso4a9QNC+eQZsYdAJtwtWzDdscRWNFqX0UCoWiBaJG/gqFQtECabTBtbp27SoNBkNDN0Oh\nUCiaFPv27ftNStmtonKNVvgbDAb27t3b0M1QKBSKJoUQIrPiUkrto1AoFC0SJfwVCoWiBaKEv0Kh\nULRAGq3OX6FoTOTn53P69Glu3rzZ0E1RKABo06YNLi4uODpWKqBqKZTwVygqwenTp+nQoQMGgwEh\nqhwIU6GoVaSUXLx4kdOnT+Pu7l6tOpTap5kQHw8GA+h02nd8fEVbKKrCzZs36dKlixL8ikaBEIIu\nXbrU6E1UjfybAfHx8PTTcMOcwiUzU/sPMKW2YlsqlOBXNCpq2h/VyL8ZMH9+keC3cOOGtlyhUCjs\noYR/MyAry/7yzMziqqDnnlOqoabOhg0bEEJw5Ej56Rc+//xzzp6tfrK67du389BDD1V7e0XjRwn/\nZoCrq/3lQmgPACm1748+Kv7/6afVA6CuqCsbzMqVKwkKCmLlypXllqup8Fc0f5TwbwYsWgROTsWX\nCaEJ+fK4cQOmTgUHB628ehuoHSw2mNp+0F67do3k5GT++c9/smpVUdbNd955By8vL3x8fJg3bx7r\n1q1j7969TJkyBV9fX3JzczEYDPz2228A7N27l5EjRwLwww8/MGzYMPz8/AgICODo0aM1a6SiyaAM\nvs0Ai1F3/nxNBeTqqgmcylJYqH0rQ3HtUJ4NpibndePGjfzhD3+gT58+dOnShX379nH+/Hk2btzI\nf//7X5ycnLh06RKdO3dm6dKlxMTE4O/vX26d/fr1IykpCQcHB7Zs2cKf//xnvv766+o3UtFkUCP/\nJkJFaoQpU8BoBJNJexOoriOAMhTXnLJsMGUtrywrV65k0qRJAEyaNImVK1eyZcsWnnjiCZzMr36d\nO3euUp1XrlwhPDwcT09PXn75ZdLT02vWSEWTQQn/JkBV1Qjz51es8imPkkJKzSGoGmXZYMpaXhku\nXbrE1q1befLJJzEYDCxZsoQ1ayqff93BwQGTyQRQzDd8wYIFjBo1irS0NDZt2qRmMLcglPBvAlTV\nlbOmI0xbIVVX+uvmjD0bjJOTtry6rFu3jmnTppGZmYnRaOTUqVO4u7vTqVMnli1bxg1zB7l06RIA\nHTp0ICcnx7q9wWBg3759AMXUOleuXKFnz56AZiRWtByU8G8CVFWNUJMRZkkhVZUHj3pD0JgyBT75\nBNzcNPWbm5v2vyb6/pUrV/Lwww8XW/bII49w7tw5wsLC8Pf3x9fXl5iYGABmzJjBrFmzrAbfhQsX\n8tJLL+Hv749er7fWMXfuXF599VX8/PwoKFB53lsUUspG+Rk0aJBUaLi5SamNu4t/3Nzsl1++XEoh\n7G9T3sfNTdvWlrLqEUIr6+am/e7SRUpHx+JlnJxK19dUOXz4cEM3QaEohb1+CeyVlZCxtTLyF0J8\nJoQ4L4RIK2O9EEJ8IIQ4LoQ4JIQYWBv7bSlUVY0wZUrVdf5CaPWVHJ2W9RbRuXNxddDFi5CfX7yM\nMh4rFI2X2lL7fA78oZz1o4He5s/TwEe1tN8WQXXUCF26VG0fUtrX5Zf14IHS6iB7ZGUpdZBC0Rip\nFeEvpdwJXCqnyFjgS/NbyW7AWQhxR23su6Vg68ppNFZff6wr54rbG6mX9eC5VN7VtqHkG4IyGCsU\njYP6muTVEzhl8/+0edk520JCiKfR3gxwrYnVUsHFSf1ok9uat7cKIoypPBg4nH0uNzh/1484XunB\nXZk9udjhJl1y2tD7N8FWz4v0+OV3/EJ3QhcNJHH+XGtdU6aUftjMn1/xRDJHR7h8WXtg2VIbE54U\nCkXNaFQzfKWUnwCfAPj7+9fAU71l0e1PoYica3Q5D6CdNkeXAm72OkrU43qij/nzW7fjmG47C1JP\nftcMjjifBod8LhS24og+D/Jbc+x3RjA5cH9u+bNCQVMH2YaRtocQpQW/hZq6oyoUippRX66eZ4Be\nNv9dzMsUtUDAr9240CuZI94/cMR7P0d8/kt+9xNgEiAKOd9nryb4TQ5wqx2dTwwEh3xtvT5Pq0QH\nmByIXeVOxPKlFe7Tog6y8Roshl4PeXllb295sVP2AIWiYagv4Z8APG72+hkKXJFSnqtoo5aCrQDs\n2lX7VEUYblwbT9jmIHDMA/1NTbAD6MwvT0L7BCcHEvyDH5fu3o/uSg/QS+s6HG4R/F8/IoypxYbl\noYuiiXstppiEjnsthtBF0UyZAl98Yd8gbIkXZA9HR+3NQU0gqxqnT59m7Nix9O7dm7vuuouXXnqJ\nvDKesGfPnmXChAkV1hkaGkp2dna12vPGG29Y5xXUJbb7ef3119myZUuZZQ8ePEhiYqL1f0JCAosX\nL67zNjZFasvVcyXwPdBXCHFaCPFHIcQsIcQsc5FE4CRwHPgUeK429tscKCkAL17UPlUShoWFbNyd\nRMcsL9CbigS65SMFSEgK/J6kIQfofHwgpk5noVBbjgQKWpN0zwHiDD7F/Dvvz9URlbeYOOEMUhIn\nnInKW8z9uVrXKcsg7OZWdnM7dtS2K28CWZN/I6jlA5BSMn78eMaNG8exY8f4+eefuXbtGvPt+NIW\nFBTQo0cP1q1bV2G9iYmJODs716ht1aG6E8refPNN7r///jLXlxT+YWFhzJs3r1r7avZUZjJAQ3xa\nyiSvLl0qN/mqXPR6GTY0WLJQSBboJAsp/vmzk2z/hIf2+3W99j3fUft+rZW5TGvJn50k8zrK2PlL\niup2c5OxBh8p5nSRwaNGSDGni4w1+JTZKMvEr4qOqaJJaE5OjWvCWJUmeS1fXusHsGXLFhkcHFxs\n2ZUrV2Tnzp3l9evX5bJly+SYMWPkqFGj5PDhw2VGRob08PCQUkp5/fp1GR4eLvv37y/HjRsnhwwZ\nIvfs2SOllNLNzU1euHBBZmRkyH79+sknn3xSDhgwQD7wwAPyxo0bUkopP/nkE+nv7y+9vb3l+PHj\n5fXr16WUUi5cuFAuWbJElmT69OnymWeekYMGDZK9e/eWmzZtklLKUm2UUsro6Gjp7+8vvby85Ouv\nv26t4+2335a9e/eWgYGBctKkSdb9TJ8+Xa5du1ZKKeUPP/wghw0bJr29veXgwYNldna27NWrl+za\ntav08fGRq1atksuWLZPPP/+8lFLKjIwMOWrUKOnl5SXvvfdemZmZaa3zhRdekMOGDZPu7u7W+s+e\nPSuDg4Olj4+P9PDwkDt37qz29asrGnySl6J6xMdro/yKyHQpW/UCMDZ8CgkhyZDfCgrbQIGjtqEE\nCtqAhGs9jtM+ywMKHHD8zZ1+qffQ7Wd/+h0cQtjmYNqdv4t+aUPod2k0W9raWGmzsogwphK015Ok\nETsI2utZSjVkezyWt5iKKG8Sml7fxNNS1kFezfT0dAYNGlRsWceOHXF1deX48eMA7N+/n3Xr1rFj\nx45i5f7+979z2223cfjwYd566y1rjJ+SHDt2jOeff5709HScnZ2tMYDGjx/Pnj17SE1NpX///vzz\nn/+ssL1Go5EffviBb775hlmzZlkDxtm28dtvv+XYsWP88MMPHDx4kH379rFz50727dvHqlWrrKP4\nPXv2lKo/Ly+PiRMn8v7775OamsqWLVto164db775JhMnTuTgwYNMnDix2DYvvPAC06dP59ChQ0yZ\nMoUXX3zRuu7cuXMkJyfz73//2/qmsGLFCkJCQjh48CCpqan4+vpWeNxNiUbl7dPSqJQsCIzGl3Si\n8r4B4UKEzGTsHa4kyIXE5v4FgJTuF+h2KqiYt4/OBJk9LtHrl84U4sAt5+549CvuwlkpXF2JE84k\n+6cRvGMEyf5p9G8XBDoHnvIMJCI9BYA4jwDevrsXN/wGwq4q7sMGJ6eyPYiajIdQXcV0roAHHnjA\nbkjn5ORkXnrpJQA8PT3x9va2u727u7tVwA0aNAij0QhAWloar732GtnZ2Vy7do2QkJAK2/Loo4+i\n0+no3bs3d955pzXtpG0bv/32W7799lv8/PwALVnNsWPHyMnJ4eGHH7aGqQ4LCytV/9GjR7njjjsY\nPHgwoD0IK+L7779n/fr1AEybNo25c4v66bhx49DpdAwYMIBff/0VgMGDBzNz5kzy8/MZN26cEv6K\n2qMyssD/nI59j3zDmKQBRIUfZunxADK8kwnbHETEuaXwdhQX3kskPt6+770l06ubGyR+VfU2xk2d\nTVTeYmLWuhBh3EFchg+REw+BvoDI/g5w3QeAyDFpINPwW+3PATv1VJRZTAjN1LBoUdlzCJrM1I+y\nsunU4AAGDBhQSod/9epVsrKyuPvuu9m/fz/t2rWrdv0ArVu3tv7W6/Xk5uYCWpC4DRs24OPjw+ef\nf8727dsrrEuUSChh+W/bRiklr776Ks8880yxsu+99151D6Ha2B67NHfU4cOHs3PnTr755htmzJhB\nREQEjz/+eL23ra5Qap96pKQNsKy8G0IUGU+3XVlKzFoXNgUfpsNvPcnwScH90DA27k6yPj0qo26p\n7qBzS1sTMa3mESGzQQgiZDax//ak36GBICByWjqR09JAFBC72p19xjn8xRBIh4cmQaCmlnJzg6++\nKtst1M2t+MzlugiJXK/UwQHcd9993Lhxgy+//BKAwsJCIiMjmTFjhnWEXBaBgYHW2P+HDx/mxx9/\nrNK+c3JyuOOOO8jPzye+kobrtWvXYjKZOHHiBCdPnqRv376lyoSEhPDZZ59x7do1AM6cOcP58+cZ\nPnw4GzZsIDc3l5ycHDZt2lRq2759+3Lu3DmrSignJ4eCgoJSoaxtCQgIsKa/jI+PJzg4uNxjyMzM\npHv37jz11FM8+eST7N+/v1LH3lRQI/96wiKgLSqNzExo1Upze7QGRAuMxv+cjm1XltL+UhbgSmwH\nZ77r2QnDsXZk+KbQMdMb491HiTP4aAIZ+yrmklR30GlVE70dZV0WodMRkS4Zfn0ESSMs+uVb7Li9\nI+DDwolpINLw3N6aw8HRhHrO5emn7bt/2pOJ9tJS2gs612ipgwMQQvCvf/2L5557jrfeeguTyURo\naCh//etfK9z2ueeeY/r06QwYMIB+/frh4eFBp06dKr3vt956i3vuuYdu3bpxzz33lClcbXF1dWXI\nkCFcvXqVjz/+mDZt2pQq8/vf/56ffvqJYcOGAdC+fXuWL1/OwIEDmThxIj4+Pvzud7+zqnZsadWq\nFatXr+aFF14gNzeXtm3bsmXLFkaNGsXixYvx9fXl1VdfLbbN3/72N5544gmWLFlCt27dWLZsWbnH\nsH37dpYsWYKjoyPt27e3PnibDZWxCjfEp7l5+5TlAdOlS1FY5CCPJUXeNGD1svEIDZIsFNJ9XIAU\nc7rIsKHBWjmzV05lPGdq1VPG7AHEvI6SP7eVvNa6yLNogaNkXkfN++iVjrLz48GyzX3v2G2XXt90\nQj435ZDOBQUFMjc3V0op5fHjx6XBYJC3bt2qs/3ZeuQo6hbl7dMEKEvtcrFfNC9OjcHkaiApfQ4x\na12InJhB58d9iQo/zZikAaT77idscxAnN6RYVUAeSQ9avXLKG9XXRiKRksRNnU3kpAyQELuiD7HL\n+4HJUZtToM+ndXZ3Ekakgr6AS3ek0v+E/W5mMjWh0XwT5saNGwQFBeHj48PDDz/M3//+d1q1atXQ\nzVI0MErtU0/YtQFODsU120RU3l7Nk4dMPhvQAVpf4/KdBwneMYJ8vSRmRW8eMWZhQjDemM3ytfP4\n6S4TP5pVMvbi7Dg51b7Qt/DlZRMds0YTmXaKl40pvGvw0dxLRSEIE7duP6aFjshvS+xqd142ziGT\npYw3zOZAT5PVG6jJGHCbOB06dGDv3r31tj+VDrJpoEb+9YQ9G6DnyW5kDf6WAen9iQo/TZepA0kf\nnAyFjla3yvvPXCE8Mxt3jOgx4Y6Ro+ej+McTxaNu2s6qtfjJW2bK1jbZ38zl6r9XsdC4i0GGJURO\nzADpQNi3gVBgHlHqJDjcZMftHRHAeoMzB8MX41mYDoHRCAGhobXfNoVCUTmU8K8nigVCmxyK59Dp\nHNr9JWGbg0gfvAupK+DS3fvB5EhsfD92bttBzFoXosJPs+Sx2RUmcrH1krEYVusqVo6tCutATxMd\n0kczZruPpuqRDubJZjrQmUgIScJxthuRkzI0FVbwN4wu/F/aPziJ/0mLtratyYdzUCiaGErtU48s\nSYnGafR+emZfJy3kfxlHMCN+uUpCfmtoewXy20BhkS42QmZDq3lsGWDCWAlhWN7E0tpU/xRTYe2a\nSw6w/aFJoN8HhQ7Eru4NQOTkY6DPpaBrJhTqSBiZSth2HxJG7gMJfqv9rRPdSnpCPf209lvZBBSK\nukHI8mbeNCD+/v6yPvWUNcUyycqeZ1/oomi+y1rNnafbccQ7FSR0N/bh1342x3fDGZyu4LEnkMMe\nP2m+9TbulZVBp7M/kaq8uPrVoaTbKgCB0XS4bT9RaadYYNRm/Y4bGkzCfT8gbnZEdrgAJrSHW0Eb\nYle7s/32TnzjkUMr4yRufld6VrCbm+b73xj46aef6N+/f0M3Q6Eohr1+KYTYJ6WsMCmHUvvUAhWF\nJu64Q0delyMcGZyMxyFv0BcUCX4B3HBGLsm2qoAGZPgXj69TScoyoNa2YbVkJE9AewMw2wF0SAYZ\nlrAp+DBh3w0BnQlx9Xdab3PM47Yzd7Hj9o5sCtmJqftR7r6ebp0QZktmplIB2dK+fftSyz7++GOr\n//nnn3/O2bNn67tZxVDhl5sOSvjXAmWpW156SYvNv/j/lhK7sjfktdUMug65ReGWczuB0xXGDg1m\n4+4kwo5O49eeuqrH4KF+Z8ba5hS2F775QE8TPikPsin4MGOSBiBb3bSGkL581wESQpIg34mwrYNI\nD/4GvzP2u2JTjPEfHQ3bthVftm2btry2mTVrljXkQF0JfxV+uXmihH81KGmcLCuswsWQUG79fjhf\nG5yJMKYSvHuwJvB1Uou/VugISDz2BJIQkszYSY+zceUXXHgv0X6FFVBWbP261pvbfegcmIv08WDM\nqQdJGKmpumK/8kZ/uZf1wSdutSNhZCoxa13YZ5xDBgb8DDGl3gKaVERPYPBgePTRogfAtm3afzsT\nVWuMZaS9bt069u7dy5QpU/D19SU3N5d9+/YxYsQIBg0aREhICOfOlc6fNGPGDGbNmoW/vz99+vTh\n3//+N6A9SMLCwrj33nu5776VBShvAAAgAElEQVT7AFiyZAmDBw/G29ubhQsXWutYtGgRffr0ISgo\niKNHjxar2xKPaM+ePQQEBODj48OQIUO4cuUKr7/+OqtXr8bX15fVq1fz+eefM3v2bECLCnrvvffi\n7e3NfffdR5bZy2DGjBm8+OKLBAQEcOedd1rrP3fuHMOHD8fX1xdPT0+SkpJq/2Q3M5TwryL2VDwl\nYlhZ8TzZjWt9komafIyxQ4NJCkouSp4CeOy7BwSkex3CI/33pHS/UOP22Y7ILbFy6pqyHjoHP5xL\nvqcH/S6NJma1Oztu70jhbafApAeJZgfQ3+RTzw68a/CxuoP+5czGUg+BJhPRExg1Ctas0QT+669r\n32vWaMvrigkTJuDv7098fDwHDx7EwcGBF154gXXr1rFv3z5mzpxpN/ELqPDLLRXl7VNF7Kl47NrM\nA6OZfuYQSZuDSAhJ0tQcABI89gaR7rOf9MG76LcniF8de+DQcyAX3qt+KOSGZsoU+w8ai/pq7GPT\nSej7pVXVk3DvPnC8AQ55HPHeQ6RvIRS20tRjXNfmBCQ9SFpgNOya2+QmhI0aBc8+C2+9BQsW1K3g\nt8fRo0dJS0vjgQceALRAcHfccYfdsir8cstEjfyrSGVHoH5ndMwNP8WIX66iz3axqjrcfwwgLTGZ\nmJW9af9zEEec23N54yqOfT63Sem1q4qWcyCYsK2D2BR8mNgVvQnbHIzjBXct9aRDAbS6wVK/dkVh\nLcy2gCYV0dPMtm3w0Uea4P/oo9I2gLpGSomHhwcHDx7k4MGD/Pjjj3z77bd2y1Yl/LKlvuPHj/PH\nP/6x7g6gHMoLv9yzZ09mzJjR/IKw1QFK+FeRyo5A1xu1UMyRk49R6Hzaqu7J6H+QOIMPjxizubZy\nJ6zQ9PtNTa9dVS68l8j5f+4k/6GHeP7MPB49lc2G3UksTuwIeW257aQvFLQiwyeFVjmd2RR8mJi1\nLvxgnMfMcTHEG+vAWlpHWHT8a9bAm28WqYDq+gFgG864b9++XLhwge+//x6A/Px80tPT7W6nwi+3\nTJTwr4CSxt3QUPseNV26FF/mSpYW4tjxhjbiTw0gbHMwOOYSOfkY4w2zS+2rKem1q8sUw1w+2xBF\nr0IjgwxLiAo/Texqd17bKbUUlIU6bt1+jDbZ3YgwpvKBwZMPey7GMS3dmraysbNnT3Edv8UGYEcd\nXiVu3LiBi4uL9RMXF1dsvcV46+vrS2FhIevWreOVV17Bx8cHX19fUlJS7NZrCb88evTocsMvT548\nmWHDhuHl5cWECRPIyckpFn559OjRFYZf9vHx4YEHHuDmzZuMGjWKw4cPWw2+tvztb39j2bJleHt7\n89VXX/H++++Xe262b9+Oj48Pfn5+rF692pq5TFE2apJXOdibzOTkBNOnQ2Ji8QldAFM/isbvjI71\nxqW4kUmHPw7geo+fafvrXdzs9Bsxa13YfnsnEj2vUfjTxFLpDhvTpKa6oph3VKB2vsbxL94IP8qY\npAEkjExFd8sJU8dfaHu2HzedLzAmaYD2JlCNiW+1RXOd5DVjxgweeughJkyY0NBNUVSDmkzyUgbf\ncijLfz8xsbSQDl0UjWdhOgfDv2H9WhfAmevdj4FJz9vftgG0OD2+a6fifCyK3Fywrbop6rWrQ7G3\nm11zOQAcCDThmZTOpuBviF3lToQxFaen+pHb8witf+ltVQFFyKXFksooFIrqo9Q+5VCVPNz35+pI\nD7bk2j3N/N/fBMdcwr4bwsvGVMYbs/FdO48DPU1cutQw/viNAbs2k11zSdN74Lt2Hi8a04gz+HDT\n+QKtf+nNrduPYTjelwhjasvQi9Uzn3/+uRr1t1CU8C+HyoZL6PanUHZsTyyWa/dmj6Pocn7Hxt1J\nSATuGDlgjIJdc9HpYNo0bdtZs7TvadNaRiiDsmYhdzkylwPGKIYYFlu9fW51PsNtx/3I8P6esUOD\nQacj7rWYJqP7VygaM0r4l0NlwyUE/NqNhPu3s+P2jhiO9eWq2yGQYGqdQ5zBhyyKPy0KC4smiH30\nUdkxgZojZU0Ie/997dwe6GnCI0kLCxG2dRDZd2RpM6Dv3cfYwQFE5S3m0pc6a5L7rl2b9/lSKOqM\nyuR6bIhPY8nhu3x5UY5dN7cycs7q9VrOWkse24VIXnWy5toN8lgihdBy1paXa9fycXOr32NsLFjO\nNYHvyCCPJbJQp7fmMXYfFyD5s5OMNfjIDNyKna9Wreo+F3BTzuGraL6oHL51SKXCJRQWMuKXqyB1\nIKBjljexKzVD5ZhTD9LhMRMmU+XDKrdU1bblXMvkuSSlRaGTJiKMqQTt9STDN4Xg7wcTYUzFleIn\nKC+vec+RUCjqAiX8awO9nsXDHEFIOmZ6c9X1R3bc3pGYda7ke3pYQxxUdoJYUwtlUFdc6+xKnMGH\nZP80a1pLe2o0aBkPzKYQ0tkedRW6eeTIkfWSm9h2P6GhoWRnZ5dZdsOGDRw+fNj6v6Kw1g1KZV4P\nGuLTWNQ+9hj99jsydv4Sqz4oLHCUZKGQ3R7zlxLMKiAhwyY9Xmy75culdHIqX+Xj5FT3KoymQpDH\nEinmdJGxBh8pwaoC8jMsqXdVWVXUPu8kvyO3ntxabNnWk1vlO8nv1KgN7dq1K3f9iBEj5J49e2q0\nj7pg2bJl8vnnn6/1emtyvPn5+XWyn+nTp8u1a9dWq03VQal96pHQRdE4pqUTlbeYOOEMUvIfXyMU\nOjBvt5Y8d+OeFMKOTisVpdOesfPZZ1umy2dlSHY24bt2HuON2ZgQLApoTa90f7r3/AYTgnwc8Bw6\nHSaHNqo5EoN7DObRdY+yLUOL57AtYxuPrnuUwT1qP6ZzTUM6Z2RkWGftvvbaa9a3i+3bt/PQQw9Z\ny82ePZvPP/8c0OL0Dx48GE9PT55++mlrfJ2RI0fyyiuvMGTIEPr06UNSUhJ5eXnlhm729fW1ftq2\nbcuOHTu4fv06M2fOZMiQIfj5+bFx40YAcnNzmTRpEv379+fhhx8mNzfX7jkxGAzMnTsXLy8vhgwZ\nwvHjx4GiGdD33HMPc+fOrdZ+DAYDv/32GwBffvkl3t7e+Pj4MG3aNFJSUkhISGDOnDn4+vpy4sSJ\nYmGtv/vuO/z8/PDy8mLmzJncunXLWufChQsZOHAgXl5e1sB6O3bssJ4bPz+/MkNhVJvKPCEa4tNY\nR/6x87XRqMWY6/5wgDbKHxrcci21dYSbW/HRvefQx4vOtc0bVsADj1dYV02pqsF368mtsmt0V7lg\n6wLZNbprqTeB6mBv5L9w4UK5ZMkSKWXxEWpeXp4cNmyYPH/+vJRSylWrVsknnnii1PZjxoyRX3zx\nhZRSyqVLl1r3sW3bNvnggw9ayz3//PNy2bJlUkopL168aF0+depUmZCQYN1/RESElFLKb775Rt53\n331SytIjf3tvAgkJCTIoKEjm5eXJV199VX711VdSSikvX74se/fuLa9duyZjY2Otx5Camir1er3d\nEbmbm5t8++23pZRSfvHFF9bjmD59unzwwQdlQUGBlFJWaz9ubm7ywoULMi0tTfbu3VteuHCh2Dkp\nOfK3/M/NzZUuLi7y6NGjUkopp02bJt99911rnR988IGUUsoPP/xQ/vGPf5RSSvnQQw/J5ORkKaWU\nOTk5dt9W1Mi/HolYvrSYP3+GTwruh4axcXcSpswsawwg5X5Yc0q62h7YHU+3nweREJJEpye8SQhJ\nJmxzEI8cO9TofP9HuY/iWf9neWvnWzzr/yyj3Os3prNtSGdfX1/efvttTp8+Xarcrl27eOyxxwAt\ndHJl2LZtG/fccw9eXl5s3bq1WMC48ePHAzBo0CCMlYxVcuzYMebMmcOaNWtwdHTk22+/ZfHixfj6\n+jJy5Ehu3rxJVlYWO3fuZOrUqQB4e3vj7e1dZp2WY3rssceswe0AwsPD0ev1ADXaz9atWwkPD6dr\n164A1tDXZXH06FHc3d3p06cPANOnT2fnzp3W9fbOW2BgIBEREXzwwQdkZ2fj4FC7ARlqpTYhxB+A\n9wE98A8p5eIS62cAS4Az5kVLpZT/qI191ztZWUTITJYeCyDDN4WOmd4Y7z5KnMGH8cZspI2/PigV\nTk2wnLv58zWDrl4WMu/7fCLvduCq2yE6Znoz4perRIWfJia3cY1jtmVs46O9H7Fg+AI+2vsRowyj\n6vUBIKUW0tlW8JVFyZDOAA4ODphs3NMsCV5u3rzJc889x969e+nVqxdvvPGGdR0UhVvW6/WVSv94\n7do1Hn30UT799FNrvgEpJV9//bXd6KKVxfaYbH+XDFNd0/3UFvbO27x583jwwQdJTEwkMDCQzZs3\n069fv1rbZ43vGCGEHvgQGA0MAB4TQgywU3S1lNLX/GlSgt82sudpnStjhwaT4fM97gcDyOl6xhrS\nwTZSZ3MP0Vxf2LraCr1ei5SqK4BCB666HiJy8jEt7s/ypQ3dVCsWHf+aCWt4c9SbrJmwppgNoK6o\nTkjnwMDAYqGTLbi5uXH48GFu3bpFdnY23333HVD0EOjatSvXrl2z6rMr266SzJw5kyeeeKJYyOaQ\nkBD+9re/WW0JBw4cALSY/StWrAAgLS2NQ4cOlblPS5TQ1atXM2zYMLtlarKfe++9l7Vr13Lx4kUA\nLl26VO6x9u3bF6PRaLU/fPXVV4wYMaLM9gOcOHECLy8vXnnlFQYPHmy1BdQWtTFcGgIcl1KelFLm\nAauAsbVQb6PA9/lonls1iZkikEIpWN3LmYT79tDu1AD6XXCwqoA8kh7kQM/ijvwtwf2wPhkbPsWs\n6gnmtkwvLUGO4w3tgZCZ2WhCP+w5u4c1E9ZYR/qj3EexZsIa9pytWUznugjp/P777/Phhx/i5eXF\nmTNnrMt79erFo48+iqenJ48++qg1g5ezszNPPfUUnp6ehISE2A3hXJKyQjdnZmaybt06PvvsM6th\nc+/evSxYsID8/Hy8vb3x8PBgwYIFADz77LNcu3aN/v378/rrrzNo0KAy93n58mW8vb15//33effd\nd+2Wqcl+PDw8mD9/PiNGjMDHx4eIiAgAJk2axJIlS/Dz8+PEiRPW8m3atGHZsmWEh4fj5eWFTqdj\nliW2Sxm89957eHp64u3tjaOjI6NHjy63fJWpjGGgvA8wAU3VY/k/DU2tY1tmBnAOOASsA3pVVG9j\nMfjeFjZR8mcnySsdZazBR/Z7KEgyv41kfmsZY55t6mdYIgl8R83UrWO6vjRahgWOkrEGH8m8jpI/\nt5XMbyPb/XGA1Q00dv6SYttUaoZ2JWgpM3wrcidtCliMsi2BpmDw3QQYpJTewP8BX9grJIR4Wgix\nVwix98KFmiczrymhi6IZev48FDqAvoDIyT9zxO+/4HCTsO+G8Igxu1jANltaSojm+uTCe4mMGBmq\nJYBZ5U6/Q4NA6rjePYPIiRnErHWBDf+yjv4t+RhaUuwkhaKy1IbwPwP0svnvQpFhFwAp5UUp5S3z\n338Adt/XpJSfSCn9pZT+3bp1q4Wm1YyMrP38Z+Q+wnb4ABJa5YI+H4eLrmzcnVQqzIBer/z165ot\nbU1aUhdjKk+l5WC5LreduxOAqNCj3G82/paVj0HZYsrGkqaxKWM0Gq1eOIqyqQ3hvwfoLYRwF0K0\nAiYBCbYFhBB32PwNA36qhf3WOiVTNj6x/RRISLh3HzjkaYUkFHT4rVSYAScn+OKLCmIAKWpM4vy5\nWjYvNzdtgckR8tpy2eWodfT/9Mea8bcq+Rgqg5SNM+udomVS0/5YY+EvpSwAZgOb0YT6GilluhDi\nTSFEmLnYi0KIdCFEKvAimg2gwSgp5OPj7asI5vycoo36HW+ArhBMeshzAgGREzOY1Ge2Guk3EHFT\nZ1vVP8HfD7G+lQE4XdSke2XzMVSGNm3acPHiRfUAUDQKpJRcvHjRbr7lylIrfv5SykQgscSy121+\nvwq8Whv7qikl8/Ja9MBt25ZWEQDsdreZQl7QirBtg0gYkUr7LB86P27CpFQIDcKWtibeiO8LXLcG\nfksacoBPPTsw3ujKSINNrmAbHB2rZ4txcXHh9OnTNAZblEIB2oDExcWl2tu3uBy+ZemB7Qn+twwB\nnL9zP+Q7Efz9YJKGHCBhZCqjtw9iW7s/kLhibumNFPVC4vy5BK/UsSt0sebnb9xBXIYPUeE/MT5t\nHplG+9tVNqx2SRwdHXF3d692exWKxkbjmhZZD1RF3xvj2QsKWhG7ojc7t+0gdrU7SEjp+jv+8YQS\n/A1NzggTg74uCvxmmye5LAoL4aWX6rGRCkUjpcUJ/7L0vY4jo/G/M4YMDBSiIwMDAnA7O5qnc4qE\nS9C/F2BwHaj0+42AOQFzybgShTtG9Jjw72Lf7bYk5kmZCkWLpkWpfeLjwZ4nmxDQpft+9g79D+tX\nuRNhzGS9wZmrnv+hx6XRtP/NCIABSKrPBivKpKTtBiA3V3uIexp1rDcuxZUssnBlvGG29jZQwUNB\noWhJtJiRv0VY2Bv1SQnP/ldz64yclMHwUSOInJQB0uzuqWh0lGW7GXhWx8Hwxaw3OKNDst7gzMHw\nxfidKerqXbrUc2MVikZIsxX+Jd05X3rJvlGXwGj8DDEsMKZoOn1dPkkjdoDDDWJXuzPn55RSbqGK\nhqcs282qn7WQ21Hhpxk+aoQW8XOtC+uNmu9/q1bw/vv12FCFopHSLIW/PZ/9svS8fme0keK7Bh9t\ngdCycSGLTo0KD9D4KNOHnyxr0vekETsI2utpTfrepQt06ADTpqkHuUIhGuukFX9/f1nd5MwGg30f\nb3tkYGC9wVlT8zjcAn0e7oeGkdEnDQT8ZZUnC427im3j5qbN4lU0HPZ0/k5O8GtbA590cCYq/DRB\nez1J9k8jZq0L4VnZuJqMperp0kV7E1AGfEVzQQixT0rpX1G5Zjnyr4o7p2Wk2PbSHeBwC/fUYZz8\nV4rVrTPGs1epbVSo5obHXj7kTz6BT2bNtqp6dg/IxOFaJyInZbDG1RkTgn6hw+FPLhCoBX+7eFG9\nzSlaJs1S+JelEujSxRwS5vl+uIb+ARMCgSTO4ENu9xNwswPG3lpWrheNadzzrwXkXB5Y6foV9Ytt\nohdLPCVr4DeZTZ8MF/J/dxIccvmHZwe8QoM4MjgJ2v1WzAB84wZMnapUQYqWRbNU+5SlErDE33F/\n6A8Y/TfjsSeImYdziJxyFBxuWv9HhZ8mptU8uvePKrceRSNHCDxDg0gfnAxSgJBQ0JrY+H6MN4fj\nLom6voqmTotW+5SlEpgyRYvRPzv9Fzz2aEIhcmIGONyk3SkPfkxMJkJmE9NqHlvamsqtR9E0SEtM\nRnelB+gkCAhOGWo1ANtDhXxWtBSa5cjflvj4ogTgrq4wJjCGD3tq8WDmjLuAyfksmASxX3rzsjEV\n0UjPh6IaVGPkb96s2jGAFIqGprIj/2Y9w7eY+icwms5ndLy7Yh7uBk8ipxzRvHskICSfDejAy8YG\nbrCiVvGcEEK6x2YoaE2/1MFcbneTX/vtJXLKEW6sGISfjLE781fZdBQtgWap9rFgOwvU4s//gcGT\nzwZ00AS/APfUAKsKyBAa0rANVtQqP7saaXfWm9j4fjyVlsN51wy6H/FHd70z8R5af/A/V/wWsE2/\naS/vg0LRXGjWah+dTpucBUX+/FHhp5H6W9D6Gu6pARh7HyVmrQufDujIkTvPI/92pBZar2h0GAzE\nidL+/0/nZOPZ3khmppaGs7BQs+2EhmqZ2ZSxX9HUaNEGXwudOxf9tvjzG473hTbXcD8UwMkNKdZQ\nAO2PhLF8qBL8zZYs7fo7/+pabOZvu4uZ3N46BgKjKTRP7s7MhI8/Vvl/Fc2bZi38r3ppcXu08Mya\nP3/GgH20PdsX491F/vyBifPI/4NJjeiaM66uxBl8uNzjBOS1JemeA8QZfHjX4MMPY4sHfoOiN8aS\nqAl+iuZCszP4Wrx7Ml2iYdgSDgReY318X8CZyMnHQF/ALaer1hE/rRaT9HZUQzdbUcfETZ1NVN5i\nYldp2bgiJ2YQOflnKHQkdrU7441LcafifqCMwYrmQrMS/rbePX5Cx4HWOeBwi8gpR2l7wU1LxA70\n+/kuIuQpMPvzRzRwuxV1z5a2JmKYR4RxDgCv3HCloEsWt53sS4TxIBLwHDqdtDsvwAotHbUQxd8A\nbI3BCkVTp1kZfK0B3QKj+cuZjbTnepFLpwAkeOwJ4sfEZOXP31IxG34jJx/TBgOFjsQuH8CO2zuS\nEJJM5yNDuaR3xmlDItOnQ2Ji0RyRRYuUsVfR+GmRBl+LPtbvjI43wo8C4H54kCb4ARDMPJzTIG1T\nNA7ipmqB32JX9Kb7EX/Q5xM57RAJIUl0PzKIS/12453RjU8+gb//vXTsIIWiudCshL9FH7veqCX0\niJx8jAzvFG0ilwlAEjlFM/QqWibWwG/GVH5ZvRf9ZRfQazN/f+23j7DNQaT+EK8EvaLZ06yE/6XR\noXgPnY4b5mD+DrnaqP9mB2K/9IGC1uBwkwX3K5VPSyVx/lwi3o4CvZ6xQ4MpvO0M5LcGx1voL/dk\n4+4kKCwk7rUYQhdFN3RzFYo6o1kJ/1HZ3TgU8hXjhgbzqWcHbaEE9JoDd0x8P9qe9qag462Ga6Si\nUTA2fAoJIcl0PzIIHPLApKPwttPcPtGfOIMPUXmLuT+3Wd0eCkUxmpXBFwcHxg4OICEkCUwOoCsg\nbHMwI365ag3THKHcOhVAtz+Fov8tm1/v3k3Y5iBG/HKVyKmHQZ8PeU7EruxNhMxWKdsUTY4WafCV\nhYVs3J2EPtsF9AV0zPJm4+4kXjamWsM0KxQAF95LpLCrM2FHp7FxdxIRxlSCdwWAAIdrXYkwpqoZ\nXYpmTbMS/oWY9bjOZ+iY6c1V1x+1/+iJeDuKxPlzK65E0WJ4b3Aiqd9/gRE34gw+JA3bg/vBAArb\nXNecAnQ6pftXNFualfD3G6rpccM2B3Fl2SHCNgeREJKM31DluqEojmVCYGYmjDdo7p9hWwdh7H2U\nMUkDiAo/zdjBAUr3r6hX6jOSbLPq1Yd7X8Bz8zS+3p2CBL7enYLn5mkc7n2hoZumaGTYhvs+0NOE\n79p5fL07hb4/9SdhpBYAMOHefcSsdYEN/1Kjf0WdYzsgkVL7fvrpunsANCvh/2VIIicPfYEjBeiQ\nOFLAyUNf8GVIYkM3TdHIKKbO3zWXA8YodJh4Ki0HdPlk+KQQ/P1gAKJCj6rRv6LOsR2QWKjLSLLN\nqkernLuKymIvQFsW5oUmRy3y57AfiJyYQcxaFx5ZtBQhwMEB7r9fJXlR1D5l+RfUld9BsxL+oAl6\nNSVfURGLFmmB2mwJv9Mc+mGVO8HfD4FWuZrrJ9DLnPC9sBC++67+Xs0VzRtbHb+uDGlcV5Fka0X4\nCyH+IIQ4KoQ4LoSYZ2d9ayHEavP6/wohDLWxX4WiukyZAtOna9m7QPvO8DTxxtq+AFbPHwod+dSz\nAyZ0+Bm0pC8lKflqrtI/KipDSR2/JZmQLXUZSbbGwl8IoQc+BEYDA4DHhBADShT7I3BZSnk38C7w\nTk33q1DUhPh4LU2j5YYrLIRLm+aygYeLef6E7fDhaP+feGRoAAfDSyd9sWB5Na9vo52i6VJKxx9Y\nlHyqEB1GDIwJjCHeWDfOBrUx8h8CHJdSnpRS5gGrgLElyowFvjD/XgfcJ4QQKBQNhD3jmpTFPX9i\n1rqwKfhwMc+f9calduuzvJrXt9FO0XQppssPjKa96785MOkt1huc0SF5cagrq4csIP3I/jrZf20I\n/57AKZv/p83L7JaRUhYAV4AuJSsSQjwthNgrhNh74YJyz1TUHWUa0Ww8fyKMqehvtrN6/kQYU3Ej\nE8+h02FyqHUT21fz+jbaKZoutrp8vzM6rrmlgiggclIGd44LICEkGUw6ntp1quxKakCjMvhKKT+R\nUvpLKf27devW0M1RNGPKMqJZ3kez0HL+FrT/DSQkBewmzuDDuKHBpIV8heGKCQKjS3mUlVWvSv+o\nKMmiRUX9bb1xqZZiVDqA4w0yfFOgwJHYlb2Zn5FSJ/uvDeF/Buhl89/FvMxuGSGEA9AJuFgL+1Yo\nqoU9bx8nJ5g1S3MRtsz6jV3ZG489QVo60MfTSAhJxmNPIJkD9hI7UlfKo6yselX6R0VJpkwBGaDp\n+d3IJMKYivtRT9AXaAWkvk73XxvCfw/QWwjhLoRoBUwCEkqUSQCmm39PALbKxhpOVNEiKGtOiCV7\n1+1PFiV9SUtMpvWvvUFXCDc7cNjjJ2LWuhCxvLT+X801UVSW0EXRdDDr+d81+DB2aHBR8qlCPUhB\n5KQMlvQJqJP910pIZyFEKPAeoAc+k1IuEkK8CeyVUiYIIdoAXwF+wCVgkpTyZHl1Viuks0JR21jD\nhCfDzQ7Q9iq3Hffj0vIDAMTNX8KWtiYVNFBRZfo/M4kjXTdpwl4ADjdBmMDkQNj/DSNhpGYD6Hdx\nDD/9z6pK11uvIZ2llIlSyj5SyruklIvMy16XUiaYf9+UUoZLKe+WUg6pSPArFI0FS9IXjz2B2oJC\nRy7fdQDP0CCV9EVRI57adUrLOyLQsg7qtJDzYf83jA27k3hzlSdup8fg7jqwTvbfvJK5KBS1TLc/\nhdL9jInD7nu1IG9A5JSjoM+DvPbErnJnvDGbkW5GFi1S6h1FFRCCOIMPkVOPgIM5u2B+K2Lj+/Oy\nMRUdEje3qucTapHJXBSK2ubCe4m4+t5LTKt5vGxMZUvPTrinDwSdidvO3kWEMZWvDc5kukSryVyK\nKrPj9o6gNwv+QgcwORA5KYO3DJqevy5dhJXwVygqIHH+XLr3jyILNxwLJRk+3+N+MIDs7lmMHRrM\nnPDT+J3RqclciioR5xFAwv27AbRQIvlOmgpIFBDjqTlQ1qWLsBL+CkUFWEI2PGyYzabgw4RtDsLY\n+6g28zckmTFJA6wzf9VkLkVZhC6KJu61GGvgp0/vagVAt5/9ObkhRfPzNznQ3jiInMsD69xFWAl/\nhaICLCEbDvQ0sWStC0IxldEAACAASURBVBt3J+H8qysZPim4HxpGvl7gRiZ+hhha36uSvijsk5G1\nn8iCt4gTzlosEVMBFDrinNMeE4Lxxmz8Vi3gWtZD6HfPrXMXYSX8FYoKsI7md83lEWM2cQYfLvc4\nAXltyeibxv1nrvCuwYeD4Yt5sru6pRSlCV0UTZ/08yAgcmIGw0eN4IjXftAVMi0tDz0m3DFywBiF\n04G5fPFF3TsPqJ6qUFSArd7VOvN3lTuxK/qAhMjJP1uTvvxtl/3Ab4qWzf25OjYFHCJsuw/o80ka\nsQNa3SDsuyG8ZkxpkEmBDnW/C4WiabNokabzt6h+/NbO42XjHASw4YcRJI3YwW0n+xJhPFgUrEWh\nsCFi+VIQLkROSrUmCKKgFSN+uYqg6u6ctYEa+SsUFVAsZEPKXC7JKK53cSPO4EOyfxrBO0aQ3f0U\ncQYfFcFNYR+L7lB/E/QFdMz0hsI2RE7MIM6jbsI3VIQS/gpFJSiZHvSTWZr6J2atCzu37SBmrQtR\n4aeJmzq7oZuqaIy4urJ4mCM45ON+MICcrmc0FZCA9/16Vbx9HaCEv0JRDba0NQd+k9kgBBEym5hW\n89jS1tTQTVM0QuKmzuaCu+YmfHJDUaIgz+3jyMoY2CApP1V4B4WiDoiP11xEs7I0TZAK/dCyCV0U\nzS//0LHeuBRXssjClfGG2RzoaYJdRUEBnZxqbvCtbHgHJfwViloidFE09+fqePrjpThdLH6DOx2o\ne79tRePGwcF+kvaSVCeejy0qto9CUc/cn6sjKm8xn3TQcrCuNzhbk76r0A+Kygh+qL9Z4kr4KxS1\nRMTypfT9qT+RU45w58MBVoPwZFbQ4aFJZLpE17teV9F4cHOrXLn6chhTwl+hqC2ysngqLQeQZPik\nYDjeF4A5E0+Q4/kf/M7oyMyEqVOha1f1EGhOxMdbQ/aU+YC3l+KzJPWZ8lMJf4WitrAM2QrbQH4r\nMrxTiJzyEwiIXeVuDf4GcPEiKgR0MyE+Hh7/n2g6ixhOSgMnM3UETzPwwpQYQhcVxXqyl+Lz2Wcb\nLuWnEv4KRS0RN7Uo9ENwyjAtPK9jHu5HPIkwpuJKcWWusgM0D574RzQD8tM5GL6Y9QbN3vPCPa4s\nNSwsleWt5HwRS85oy//6dAhQwl+hqCW2tDURk6ipepKGHIC8ttobgMd+4gw+ZFFamatCQDdt4uPB\n06gjPfgbxiQNICr8NHeO03I+h20dpIV1aKQo4a9Q1BKJ8+fCuIeJnJihqXpW9CE2vj8UtCJyUgbj\nDaVn/+p0SvXTlHlyWTTj+Jd10laH33qS4ZtC23N92Lg7qdjTvTJ2gfpECX+FohbZ0tZEv8ujid2k\nqXr+X4AD/X70pW+aN7f33EQ+DngOnQ6TQwHN/W/mzIYXBIrq0f+EjjfCjwJgON6Xq26HoNCB3NvO\nFYv1ZEkIlJmphfLPzGx4m48S/gpFLTLFMJfczauIOrwLg5ukR4YHRwYn0fc3QeKunTwyNIC0kK/w\nPNnNuk1eHrz0UgM2WlFtEk4tJWatC5GTj5HhnQIFrSHPibAdPsViPVkSAtnS0DYfFdJZoaglLKM7\ny02emQn5mfE8IoNICEmmU39vrromE7Y5iK93x+PIF9ZtL15soEYrakTPwizAGfR5ICB411DGZWQT\nFX6YMaceZIuniQjKtu00pM1HjfwVilrC3uhOTyEbdyfRMcuLq26H6JjlxcbdSeip5HRPRaPDNhev\nQPKpZwcobMVtJ/1I9k8DICaxL/meHpodiLInbjVkBHAl/BWKWsLeKK4QPWOHBnPV9f+3d+/RUdVZ\nose/uyoBEgQjEBEIlQqIIIk8BDWGlJGW7jRRiD29aBkjcKdv6+2eca49gWG4g3fZvZS1EEPWONfp\n26O2LjSo04zdEjTd3KZbMYFBQXmYBFAkIYDKQ4iAiUKqfvePUxXyqEpSlZB67c9atULFU6d+p5B9\nTu3fPvu3Dzl3LeccH1GY7QLAUfB9+LtJAAwf3p8jVb3ha+NRKimUOqdyIMvK8z/6rqe1tTf3/qA1\n8IP/G7z684YufzTto1QfcTisVE9b07OLqM5/masaMrngqOGqhkzK86sYemMmFxybcezM5/NEePrp\n8IxZ9UzbLq0NtmdgrBXkU06MBQNr/yOD4vq91p1a3tbexW1e76vfj6hOr8aYiHzMmDHDKBVNysqM\nSU42xqrn8D6K5prZ9yw2HjCZBbmGxzD88yDDY9ZzD5jZ9yw2Ix6Z63d/6enGiFg/y8r6/ZCU6fz3\n6kaMAeOanWf4hfXTgPUXFQGAXaYHMVbTPkr1EX+375fNreAvm6yJ3eqKKvhmKAz4Br4ZSnVFFfdm\nu3h7xsuk7UptV/8diaWB8eonL67B/p2F/NI5Cw+CYCjMdlGZ815rnj8al/DUfv5K9QMjwk0FudTc\n4j0BDDqHnL8WM+SUt/pnO4m0AFYuOCnJfwVQb3u9q+DdnFHC7oWPt6Z3tl43lPL8KriYxNpXJwBY\nHVwHrKD4iWVhHq3281cqooy7O5+aW6rI3JmLedIb+IeeRM6ndqr+aWoKXPrZcU5BXVmpPy/guusq\nWPtaBggsvf9jyr+3DTzC2lcnUFy/N2qX8NTgr1Q/qHfW49iZz0cVVRRmuzBDTiHnrsUMOUlhtgs3\n9h7tx96zzVQfyTmRyh/y32HrdUNxvTcdBjSDzcOwumnWBK8I1NdT/MSydtU90UCDv1L9IP3NAzRU\n/JEp2Yutpl+bc/GUnmT+Zhfl+VVMz+5Z2UdPV4NSvZf68wKor2P+5lzK8yupdFWBAYxwZswhSp1T\nqTeOiOjTEwoN/kr1A1+dd/W4U2RtXsTrO7ZjgN+9t50pmxdRPe5U67bJyYHr/nu6GpTqvZwTqZTn\nV/HpMAPGDjbrzJu5c5aVArrPatbnm4z/27+NrMZt3elV8BeRYSLyJxH5xPvzmgDbuUVkj/dR3pv3\nVCoatVYCbaug5r11XJ/ewitlBrunheUPryN9W0W7BT2efjrybgqKNxs3rGf+Zu8kvbitq35g/Bmh\n5LUMhtTMZfcYK8/f1AS//nV0VWf19iavFcCfjTGrRWSF9/k/+dmu2RgzrZfvpVRUKyrqfFNPwao1\nzGm2UX3hGZJNAw1HHPzVow9T7fTwkyXLqaiIoJuC4oDv76O47Jn2OTaBoUemcOf+qynPr+Lw5kWc\nf3Ndu9d2LJz0NW6L1L+z3qZ9CqG1O9U64N5e7k+puOJrFfDsEGsFqN85U9izYDVZ9Taef94K+OFY\n5SletW3dAPBm9iEABn4xgXOOjwCYt9nVLk3XlUherKdXdf4i0miMSfH+WYCzvucdtmsB9gAtwGpj\nzBsB9vcQ8BCAw+GYcUTr2lSsczq58aaxHMjch+t964ahkg1pXGAwj40pJP3Ycq3r709OJ6WSwrIF\nx7jmcwdnxu8mc2cu1d4qrfL8KrI2L6J6R/urfpHOV/4Qnvsy+qzOX0S2iEi1n0dh2+28txUHOpOk\newdzP/AvIjLe30bGmGeNMTONMTNTU1P9baJUbGlo4MHq8zCgmcq8reTuygLgFwsOkuWu4Ujamm52\noPpUQwPF9XvJ3ZXFmet3M+zT6VRXVGGA13dstwL/uFMMH97+Tu6f/jT65mi6Df7GmDnGmCw/j43A\nCREZBeD9eTLAPo57fx4G3gGm99kRKBXNfC0B3IlgoHLWDpYurGNe5WRqXG+R2xhaZjbSlgyMdK1t\nmm02Sp1TqZpZTcaeHM44DlLqnMoR0kmkxbrif6UCaJ+S+9WvOrf2ePbZyE7V9TbnXw4s8f55CbCx\n4wYico2IDPT+eQQwC6jt5fsqFRNKH3iYZQuOsfaVCWTszYGEbyGxifK8vZRsSOMPXwS/ALj2BQqe\nL9dfeEsOyxYcY17lZOonHGT+X2awbMGxTusvf/ll58+0qMg6EUTLHE1vg/9q4Lsi8gkwx/scEZkp\nIs97t7kR2CUie4G3sXL+GvyVwlrzt6RiIgD1Ew4y9MgUsLcw6Owoiuv3ctWZ4GcMI3HJwEhXXGYt\nx1j+nQ9wHprIJlctJRvSeH3HdqZtWNFa0tlWtH+m2thNqTArfbSEZRdXM69yMptctTgPTaRuyn8x\nf3MuGz9vCHrG0GbzP/koYl2VKj+8H9ods/OozNuKa2se7769FQ+CncAfWiR+ptrYTakosSXJw7yj\nd7debR7+/XZSP55B+V3vU3rVmNbEfemjJRSs6n4COBKXDIx4Dkdrrt+1Na+1TXMDXX9o0fyZavBX\nKswqVi7nUlam1RLYNIIIw5uuAo+d5zKsK9JSSWFpy+PUNXzY7f4iccnASOebeynZkMa7b29tXY6x\nY66/rWj/TDX4KxUBKlYut3rBe2cMHzx0EUwCBzL3ccfsPJbeVwcCD2472u2+/C0qE+mVJ/2p7QLs\nvm9Vz53axcSv5lBsGjEIPzraGDDXD7HxmWrwVyoCFddst3rI2y9RmbcV7JdY+1oGxTXbe1TGGW2V\nJ70VTGlru7t4vd+qDl69hTkXZuKkHrt4yE2rZ3f9MtjWuU2zt4tz1H+mGvyViiIGLePsyF9p6wMP\nwIgRnT+XglVr4I3ft6Z17pidx9KFdUzcfyP/+Ooz7fYh4v/9ojnP35YGf6UiUGlmjpXq8STi2poH\nnkSW3lfHqowcLePswF9pK/ivxZ/TbGNZwUEAcndlWd+qbJd4sPo8aaZ9Wa0xnU8A0Z7nb0uDv1IR\n6LlZY0Fg7WsZvPv21tZlBNdkjvW7fSQ3ELvSujr2jidGXz3/0vvqqLz9fbiYBJ5Eaz9+KnuMid25\nk962dFZKXQEZjpt5sHkmxeYZELGqgOz/m5XN/icgYyUVEQqHo+u1jdudHBoaID0F7JdgQDOurXnc\nW9fIsgXHKNuwAurbvzYcjdn6i175KxWBOlb/+NaJnTgRZo4roQ4nbmzU4WTmuBJS7o7fBnD+Slvb\nandidDh4LmsIuBNb6/kBVm+cyN6x7U+ssZTi8UeDv1JRZPE1Nj744Wp+57zc//+DH65m8TXx+0/Z\nV9rqb+nLjgG89IGHOXjjftb+R0a7ev6EBT/gpf+xPGZTPP7E7/8xSkUhX87aV6niuzGpuCz4BnA9\nES3dQYuK4PRpKCtrn6Of8N/WcGL/5Zr+LTv+zLyjd7Nl/KjWdFrJgBVsSfLEXXksxpiIfMyYMcMo\npToQMQaMa3ae4RfWTwPGiJiyMmPS061N0tONKSvr3VuVlRmTnGzt3vdITu79fvvT2pVPGfnH4Wat\nc6oxYNY6p1rPVz4V7qFdMcAu04MYq43dlIombVaayt2V1bry10PnGxnZXN+u5DE5uXepC6fT/0Rq\nVE2CBvi8ik1jFB1EcLSxm1IxKFAPmrnXPdzj+v+epnIClVBGVVlpm5W5fCulFdfvjbKDuDI0+CsV\nRbYkedo1gPPlrKtS/JeAdoxxwSz0Es7uoH021xCgW2dc18b69CQ3FI6H5vyV6rlBdz1ppjufMnWk\nGzdi6kg3051PmUF3Pdluu+HD2+fwfY/09M77DFfOvzfvO/eJJ83D9z9ljtqtz+HxjBzDPw01k+7J\n1Zx/h4de+SsVA34y0sbu+x/jkWxHawnongWruT2ppnUNgPXrrZYH/vjLgoSrO2iglciWLOn+G0Dt\nwQ95Zuzj/HasVQq7PtMG9haMJHSq7ol3eoevUjHg/2x7hobDMyjPr2LcqBzqrz/oXRnsLUqabwK6\n7v8TKAtSVNT/JY+B0vFut5WiAv9jKli1hsmfnOTIGFh6Xx1vvJ/HgZt2gngo+uhi65Jbxd5HvNMr\nf6ViQUMDG3dUkrHvduqmbmfI6TGtK4P57gHoqgXClbiTNdS8fVfp+K6a2M1ptvFH1z7mvzP1civs\nAU3M//MtPFq/Pdjhxzy98lcqFjgclEoK9dcfhKYUzqXvI2NPDsXeoPedeUvg/lPwSkWnlw4f3ndX\n9+vXW8HZ1xLZV0num1iG7t9r1SprW3+dOqHzN4PUnxeQcyKVja+9BM6pLF24FxK+tf5jywDyvjgX\n+gHFML3yVyoG+EpA51VOBvtFMFA3dTuF2S4Ks128PeNlsg6ndnqdCDz9dN+MoW0lEXReRD6Y1tNJ\nSYH/W8dvBjknUimf+DKF2S7rF4lfg82D/UwauAex9L46nrohp2dvHEc0+CsVA7YkeZi63VoEfu2r\nE5i/2QqE5d/bRnl+FfM357J7R+e8izF9d9UfqK9+W4Hy+b4UkQgsWhR4Ytpfs7WNG9Yzf3Mu5flV\nLP3REbC5STo+Cc/AZisFJPDinf5bYcczDf5KxYCKlcvZQybTNqzgH+r3snFHJUMbplhXwI2j2bij\nEjvuTq9LTw/9PTvm9LuaU/Dxl8/v7huDT8BqI7ebjTsqGXjiekhuxH42jabnDlCyIY1NrlqmvHMv\nAxNuDuLI4oMGf6ViRPqx5eyuX4YbO4XZLs45PmLokSm4Uz6jMNuFG3u77XvTstjfzWLdCfR+PfnG\n0OW6uXbreL8d+QkDv5iA+5rjFGa7+If6vUzbsIJ99kwa32q/Fm+0NKy7onpyM0A4HnqTl1LB8d0c\nlZW92PCYmPnZLmPAzM92GR4TM/uexe0av/3sZ6E3gktP93+zWKBHV/v39qrr9vWBzF/o/3izshe3\nvl6k8+cUzQ3rukIPb/IKe5AP9NDgr1TwysqMsS2aa7KyF5tL2I0HjLHbzfyFi82IR+a22y7UAFhW\n1vOgP3z45dcEOtF0dyLpblwjHplr5i9cbIzdOt5L2K3Af/9cvyePQO/X1QkmmmjwV0q1mvvEk1ZL\nA28EPmq32j8w68mgAqC/k0ZXD5HuTzQ/+1nnq3/fc9+JouP4TXq6WbvyKTP3iSe7HV/Hk0dXY40F\nPQ3+mvNXKgZ1zGmPr7Wx1DxG4SgHGMNvx1rtH7LcNTDr8hKQ3TW77El+vi2HI3C7hpUrrXGuW9dh\nknfWGn40p4S5d83lf8o0ipYkkPjmm63jL8hxWW2aL65mTnP7ENZdS4r1663fBxprPNF+/krFGN9k\nbNuAe0Sc/P1tDsrzKxn26c2cHXXE2/6hlszKu6m2Z8K25djtVhcEh8OanO04wWqzBa7GSUyES5cu\nP/etJ7Bokf/XiHgXX59VQNbhVL4YV8tX13zJhLox1E77kMTGkVxKOQEXk2DQOTI/vI2aW7aRse92\n6q8/GFJf/kBVSSLw8suxsXqX9vNXKk75u9JOM1b7h2Gf3syZ6z/E9m0ym1y1zKucTI3rrdZvAG73\n5eqdH/+4cxVMoKvj9HR48UX/V9ydXjNrDUPuWUjSf8+iKecWJjeepzr/Zc6M+IxLI+qovaUKEpu4\nNOw4JDbB4C9JOjGe2sz9re0rOvbl7+0aBaYP73eIFhr8lYox/gJcA1Zf+7OjjmA/Mxb3sKPYvk1q\ndwKYfrx9OLh4ER55pP1+Vq2yrujbSk6GggLrpNPQ0Plbw6pVYHOtYbqzhDqclBx/hfNZf6Dpuo85\ndcMuaqftBHcCnms+a79j+0UQsJ9No3nMAZyfTKT++oOd+vL3xRoFvbnfIVpp2kepGOMvtTHdWcKe\nBatbUz1GWiD5K+TctWB3k3xmJOmfD8Ntg3FnbZRv28b07CKqx53CrG/fD8jXv8cX6AsKrLx9u28b\nRQUkpHxCwQejuWSHBLdh0+wPsDVdzbBToxnZmETNLVXgsYOtzc1nAlxMhgHenV0YAYO/ZNin0zkz\nfjfzN+eycUclpc6p1opmA1bwr2XLerzcpL+UWG+Xu4w0PU37aGM3pWKMv8Zoe8d6uHP/3WxyvcW8\nysmU37kXPGCGniTp+CScnw+zgnHLQB5aP4kfZudQnf8Sjp354HRSkOZkTuMlnhs/ADwtfD3rG+QH\nn9F0YjQvXWri0t80Y0v4Fk9yI7bmq0n+ahgXRhyiPP8QfD0ckhrB5sYzoImvm66mZsKHVkC/fnf7\nwTdfDYO+At816eDTjDwwkxPja8ncOYtNrlpKv5hq5fq9ffm7W26y48lqyRKoqPD/LSWe9OrKX0QW\nAL8AbgRuNcb4vVQXke8DTwN24HljzOru9q1X/kqFrmPAW7UK1tevIbG6hk1j25wAEr+2rrzdCWDs\nkHCRoQ03cc7xEZk7Z1GbuZ+SDWmA1SMfewu4ExhZfwMnJnn/fboTwd5mptcjYDOXf7ZhP5uGe9gx\nrjk0nbNpn0JCk7VPPxJPjuPStYcBmLwzlzFNV9F86i6qUjykH1veGrS7Wmje34kw1q70O+rplX9v\ng/+NgAf4d2CZv+AvInbgY+C7wDFgJ/DXxpjarvatwV+pvlew6vIJoGRDGm9kpFh974GMvTl8mXKB\nc+n7GHpkCl+9uK81vZK7K4vK23aDgYyDWdRN/S+SPptI8+gDHd5BuHzZ3uZXgFwYgbnqNAO/mMC3\nIw9ZJxz7JeskYRJAWqyThTuRyR/exsmUZm4/PJg/zDhOkvsG3C9V+A3iEDjA+9pLd+QvJRQr+iXt\nY4zZ732zrja7FThkjDns3fY1oBDoMvgrpfpexcrlFKxaQ8mrh4Cvqbx1t1VKKW7qst4Hm5uhR6Zw\nzvERhdkuNu6o5I1deVTmbcW1NQ+Ayryt1jbp+7A1jsaT4p2o/WYIDDrf+U295wIz+DRJxyfRnNrA\n4KOT+XrMAQYfzeSXf7HC0PNZQzg95FsaB7tJdM7j1L9d7sfjdMKRAPcK+IK4vwnnRYv8fw7d3c8Q\nD/oj5z8GONrm+THgNn8bishDwEMAjni740KpflKxcjmlzTaWtjwOAmtfuYEXJg9pzfk/9raw9Tqr\nRXLWsFxqM6txbc27fOW/J8e68j8+ybry913oDzwPxs+VvzsRxAM2N82jD5C5M5fazP1k/amIansm\ny+q9Qb7e+mG3wz4PON+6HMS7y+sHWm7S4fB/5a/hpQelniKyRUSq/TwK+3owxphnjTEzjTEzU1M7\nLzyhlOobW5I8TDo7l7WbrHr5EynNZO5yMWnvLWwZczUbd24ns+Z71Nz8HiUb0ri3rtGK6fYW6iZW\nM/LAjMspH3dimz17c/1tiQH3QCbvzCXxdAafp1xk2oYVrTeWddT2XgNfyWagYN1dEA9Umnollq2M\nNt1e+Rtj5vTyPY4DbVdSSPP+TikVJhUr2wfdU362caxaw4+b51BsnqEgzcnaN7Naq31OD/0G21ej\nGX5iNE2Dm7g4sBl3m2qfSR+P58ioMyQ3JfP14GbsX2RR+9XN8G/LOQOcgdYr/a74UjuBJm67C+K+\nbwOB7kGIZ31S5y8i7xB4wjcBa8L3LqygvxO43xhT09U+dcJXqcjStoKot2Gj7fq+PWGM/womDeKd\n9Ut7BxH5gYgcA24H3hKRzd7fjxaRCgBjTAvwMLAZ2A/8trvAr5S6soJdzKTjXbS9YbcHtw+7dw2a\noiJrctfj6WJhF9VjeoevUnEmlLtce7pMY3eSk4PrCuoToWEqImljN6WUX121WA6kL0ojhw+/3G7Z\nH7vd/+/jse9Of9Dgr1Sc6a5s0p+uGqKVlXWuqBkwwAr2vg6fZWVw+rT1zSJQBc5DD2llTn/S4K9U\nnAmlbLKrkkl/C6i88IIV7P3l59tuD9YVf1OT1W9nyZLAC7GovqXBX6k44y+QA1y4EHjit7sVsoKd\njC0qsrqBilh1/WDNKaxbZ41PJ3WvPA3+SsUZXyAfPrz977/8MnAffN/rugvwPa0iWr8efv3rzhO5\n3c09qL6j1T5KxamuumGG0vQsmCqirqqHRKwTjAqNVvsopboUysRvV4KpIgplcln1LQ3+SsWpUPvl\nBBIooB850jkFFOg9RLS6p79o8FcqTvV107OuThod19X1994i8NOf6iRvf9Hgr1Sc6q6CJ1iBqoh8\n2qaA/L33yy/Dr34V2nur4OmEr1IqZP4Wc6+o0MnccNIJX6XUFdWx2VvbOv1ALRl0MjdyaPBXSoWk\nq+oeXUQl8mnwV0qFpKtS0b6eT1B9rz/W8FVKxaDu1scNtK6uigx65a+UCommdqKbBn+lVEg0tRPd\nNO2jlAqZpnail175K6VUHNLgr5RScUiDv1JKxSEN/kopFYc0+CulVByK2MZuInIKCNAeqpMRwOkr\nOJz+EgvHoccQGWLhGCA2jqO/jyHdGJPa3UYRG/yDISK7etLFLtLFwnHoMUSGWDgGiI3jiNRj0LSP\nUkrFIQ3+SikVh2Il+D8b7gH0kVg4Dj2GyBALxwCxcRwReQwxkfNXSikVnFi58ldKKRUEDf5KKRWH\noj74i8j3ReSgiBwSkRXhHk+wROQFETkpItXhHkuoRGSsiLwtIrUiUiMij4R7TKEQkUEi8r6I7PUe\nxy/DPaZQiYhdRHaLyJvhHksoRKReRD4SkT0isivc4wmViKSIyH+KyAER2S8it4d7TD5RnfMXETvw\nMfBd4BiwE/hrY0xtWAcWBBG5A7gAvGSMyQr3eEIhIqOAUcaYD0VkCPABcG80/T0AiIgAg40xF0Qk\nEagCHjHG7Ajz0IImIsXATGCoMeaecI8nWCJSD8w0xkT1DV4isg6oNMY8LyIDgGRjTGO4xwXRf+V/\nK3DIGHPYGHMReA0oDPOYgmKMeRc4E+5x9IYx5nNjzIfeP58H9gNjwjuq4BnLBe/TRO8j6q6ORCQN\nuBt4PtxjiWcicjVwB/AbAGPMxUgJ/BD9wX8McLTN82NEYdCJJSLiBKYD74V3JKHxpkv2ACeBPxlj\novE4/gVYDnjCPZBeMMD/E5EPROShcA8mRBnAKeBFbwrueREZHO5B+UR78FcRRESuAl4Hfm6MORfu\n8YTCGOM2xkwD0oBbRSSqUnEicg9w0hjzQbjH0ku5xpibgbnA33nTo9EmAbgZ+L/GmOnA10DEzEtG\ne/A/Doxt8zzN+zvVz7w58teB9caY34V7PL3l/Xr+NvD9cI8lSLOA+d6c+WvAd0SkLLxDCp4x5rj3\n50ng91gp3mhzDDjW5tvjf2KdDCJCtAf/ncAEEcnwTqYsBMrDPKa4450o/Q2w3xhTGu7xhEpEUkUk\nxfvnJKxCggPhaVF4kAAAANlJREFUHVVwjDH/yxiTZoxxYv17+Isx5oEwDysoIjLYWziAN03yPSDq\nquGMMV8AR0VkovdXdwERUwQR1Qu4G2NaRORhYDNgB14wxtSEeVhBEZFXgTuBESJyDHjMGPOb8I4q\naLOARcBH3nw5wD8bYyrCOKZQjALWeavIbMBvjTFRWSoZ5UYCv7euKUgAXjHG/DG8QwrZ3wPrvRen\nh4G/CfN4WkV1qadSSqnQRHvaRymlVAg0+CulVBzS4K+UUnFIg79SSsUhDf5KKRWHNPgrpVQc0uCv\nlFJx6P8D4Obclx42P3sAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jWxvLGexKv0D", + "colab_type": "text" + }, + "source": [ + "We can see from the graph that the predictions for the original model, the converted model, and the quantized model are all close enough to be indistinguishable. This means that our quantized model is ready to use!\n", + "\n", + "We can print the difference in file size:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "6r42iBnULP4X", + "colab_type": "code", + "outputId": "afe526c9-498d-498e-d768-1edfbf21e870", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 68 + } + }, + "source": [ + "import os\n", + "basic_model_size = os.path.getsize(\"sine_model.tflite\")\n", + "print(\"Basic model is %d bytes\" % basic_model_size)\n", + "quantized_model_size = os.path.getsize(\"sine_model_quantized.tflite\")\n", + "print(\"Quantized model is %d bytes\" % quantized_model_size)\n", + "difference = basic_model_size - quantized_model_size\n", + "print(\"Difference is %d bytes\" % difference)" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Basic model is 2656 bytes\n", + "Quantized model is 2640 bytes\n", + "Difference is 16 bytes\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "C2vpZE9ZshVH", + "colab_type": "text" + }, + "source": [ + "Our quantized model is only 16 bytes smaller than the original version, which only a tiny reduction in size! At around 2.6 kilobytes, this model is already so small that the weights make up only a small fraction of the overall size, meaning quantization has little effect.\n", + "\n", + "More complex models have many more weights, meaning the space saving from quantization will be much higher, approaching 4x for most sophisticated models.\n", + "\n", + "Regardless, our quantized model will take less time to execute than the original version, which is important on a tiny microcontroller!\n", + "\n", + "## Write to a C file\n", + "The final step in preparing our model for use with TensorFlow Lite for Microcontrollers is to convert it into a C source file. You can see an example of this format in [`hello_world/sine_model_data.cc`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.cc).\n", + "\n", + "To do so, we can use a command line utility named [`xxd`](https://linux.die.net/man/1/xxd). The following cell runs `xxd` on our quantized model and prints the output:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "l4-WhtGpvb-E", + "colab_type": "code", + "outputId": "f975721f-bdd1-440a-93af-55f13c4c8690", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 3808 + } + }, + "source": [ + "# Install xxd if it is not available\n", + "!apt-get -qq install xxd\n", + "# Save the file as a C source file\n", + "!xxd -i sine_model_quantized.tflite > sine_model_quantized.cc\n", + "# Print the source file\n", + "!cat sine_model_quantized.cc" + ], + "execution_count": 0, + "outputs": [ + { + "output_type": "stream", + "text": [ + "unsigned char sine_model_quantized_tflite[] = {\n", + " 0x18, 0x00, 0x00, 0x00, 0x54, 0x46, 0x4c, 0x33, 0x00, 0x00, 0x0e, 0x00,\n", + " 0x18, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x14, 0x00,\n", + " 0x0e, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x10, 0x0a, 0x00, 0x00,\n", + " 0xb8, 0x05, 0x00, 0x00, 0xa0, 0x05, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,\n", + " 0x0b, 0x00, 0x00, 0x00, 0x90, 0x05, 0x00, 0x00, 0x7c, 0x05, 0x00, 0x00,\n", + " 0x24, 0x05, 0x00, 0x00, 0xd4, 0x04, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00,\n", + " 0x74, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00,\n", + " 0x14, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,\n", + " 0x54, 0xf6, 0xff, 0xff, 0x58, 0xf6, 0xff, 0xff, 0x5c, 0xf6, 0xff, 0xff,\n", + " 0x60, 0xf6, 0xff, 0xff, 0xc2, 0xfa, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00,\n", + " 0x40, 0x00, 0x00, 0x00, 0x7c, 0x19, 0xa7, 0x3e, 0x99, 0x81, 0xb9, 0x3e,\n", + " 0x56, 0x8b, 0x9f, 0x3e, 0x88, 0xd8, 0x12, 0xbf, 0x74, 0x10, 0x56, 0x3e,\n", + " 0xfe, 0xc6, 0xdf, 0xbe, 0xf2, 0x10, 0x5a, 0xbe, 0xf0, 0xe2, 0x0a, 0xbe,\n", + " 0x10, 0x5a, 0x98, 0xbe, 0xb9, 0x36, 0xce, 0x3d, 0x8f, 0x7f, 0x87, 0x3e,\n", + " 0x2c, 0xb1, 0xfd, 0xbd, 0xe6, 0xa6, 0x8a, 0xbe, 0xa5, 0x3e, 0xda, 0x3e,\n", + " 0x50, 0x34, 0xed, 0xbd, 0x90, 0x91, 0x69, 0xbe, 0x0e, 0xfb, 0xff, 0xff,\n", + " 0x04, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x67, 0x41, 0x48, 0xbf,\n", + " 0x24, 0xcd, 0xa0, 0xbe, 0xb7, 0x92, 0x0c, 0xbf, 0x00, 0x00, 0x00, 0x00,\n", + " 0x98, 0xfe, 0x3c, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n", + " 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x17, 0x9a, 0xbe,\n", + " 0x41, 0xcb, 0xb6, 0xbe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n", + " 0x13, 0xd6, 0x1e, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\n", + " 0x5a, 0xfb, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00,\n", + " 0x4b, 0x98, 0xdd, 0xbd, 0x40, 0x6b, 0xcb, 0xbe, 0x36, 0x0c, 0xd4, 0x3c,\n", + " 0xbd, 0x44, 0xb5, 0x3e, 0x95, 0x70, 0xe3, 0x3e, 0xe7, 0xac, 0x86, 0x3e,\n", + " 0x00, 0xc4, 0x4e, 0x3d, 0x7e, 0xa6, 0x1d, 0x3e, 0xbd, 0x87, 0xbb, 0x3e,\n", + " 0xb4, 0xb8, 0x09, 0xbf, 0xa1, 0x1f, 0xf8, 0xbe, 0x8d, 0x90, 0xdd, 0x3e,\n", + " 0xde, 0xfa, 0x6f, 0xbe, 0xb2, 0x75, 0xe4, 0x3d, 0x6e, 0xfe, 0x36, 0x3e,\n", + " 0x20, 0x18, 0xc2, 0xbe, 0x39, 0xc7, 0xfb, 0xbe, 0xfe, 0xa4, 0x30, 0xbe,\n", + " 0xf7, 0x91, 0xde, 0xbe, 0xde, 0xab, 0x24, 0x3e, 0xfb, 0xbb, 0xce, 0x3e,\n", + " 0xeb, 0x23, 0x80, 0xbe, 0x7b, 0x58, 0x73, 0xbe, 0x9a, 0x2e, 0x03, 0x3e,\n", + " 0x10, 0x42, 0xa9, 0xbc, 0x10, 0x12, 0x64, 0xbd, 0xe3, 0x8d, 0x0c, 0x3d,\n", + " 0x9e, 0x48, 0x97, 0xbe, 0x34, 0x51, 0xd4, 0xbe, 0x02, 0x3b, 0x0d, 0x3e,\n", + " 0x62, 0x67, 0x89, 0xbe, 0x74, 0xdf, 0xa2, 0x3d, 0xf3, 0x25, 0xb3, 0xbe,\n", + " 0xef, 0x34, 0x7b, 0x3d, 0x61, 0x70, 0xe3, 0x3d, 0xba, 0x76, 0xc0, 0xbe,\n", + " 0x7d, 0xe9, 0xa7, 0x3e, 0xc3, 0xab, 0xd0, 0xbe, 0xcf, 0x7c, 0xdb, 0xbe,\n", + " 0x70, 0x27, 0x9a, 0xbe, 0x98, 0xf5, 0x3c, 0xbd, 0xff, 0x4b, 0x4b, 0x3e,\n", + " 0x7e, 0xa0, 0xf8, 0xbd, 0xd4, 0x6e, 0x86, 0x3d, 0x00, 0x4a, 0x07, 0x3a,\n", + " 0x4c, 0x24, 0x61, 0xbe, 0x54, 0x68, 0xf7, 0xbd, 0x02, 0x3f, 0x77, 0xbe,\n", + " 0x23, 0x79, 0xb3, 0x3e, 0x1c, 0x83, 0xad, 0xbd, 0xc8, 0x92, 0x8d, 0x3e,\n", + " 0xa8, 0xf3, 0x15, 0xbd, 0xe6, 0x4d, 0x6c, 0x3d, 0xac, 0xe7, 0x98, 0xbe,\n", + " 0x81, 0xec, 0xbd, 0x3e, 0xe2, 0x55, 0x73, 0x3e, 0xc1, 0x77, 0xc7, 0x3e,\n", + " 0x6e, 0x1b, 0x5e, 0x3d, 0x27, 0x78, 0x02, 0x3f, 0xd4, 0x21, 0x90, 0x3d,\n", + " 0x52, 0xdc, 0x1f, 0x3e, 0xbf, 0xda, 0x88, 0x3e, 0x80, 0x79, 0xe3, 0xbd,\n", + " 0x40, 0x6f, 0x10, 0xbe, 0x20, 0x43, 0x2e, 0xbd, 0xf0, 0x76, 0xc5, 0xbd,\n", + " 0xcc, 0xa0, 0x04, 0xbe, 0xf0, 0x69, 0xd7, 0xbe, 0xb1, 0xfe, 0x64, 0xbe,\n", + " 0x20, 0x41, 0x84, 0xbe, 0xb2, 0xc3, 0x26, 0xbe, 0xd8, 0xf4, 0x09, 0xbe,\n", + " 0x64, 0x44, 0xd1, 0x3d, 0xd5, 0xe1, 0xc8, 0xbe, 0x35, 0xbc, 0x3f, 0xbe,\n", + " 0xc0, 0x94, 0x82, 0x3d, 0xdc, 0x2b, 0xb1, 0xbd, 0x02, 0xdb, 0xbf, 0xbe,\n", + " 0xa5, 0x7f, 0x8a, 0x3e, 0x21, 0xb4, 0xa2, 0x3e, 0xcd, 0x86, 0x56, 0xbf,\n", + " 0x9c, 0x3b, 0x76, 0xbc, 0x85, 0x6d, 0x60, 0xbf, 0x86, 0x00, 0x3c, 0xbe,\n", + " 0xc1, 0x23, 0x7e, 0x3e, 0x96, 0xcd, 0x3f, 0x3e, 0x86, 0x91, 0x2d, 0x3e,\n", + " 0x55, 0xef, 0x87, 0x3e, 0x7e, 0x97, 0x03, 0xbe, 0x2a, 0xcd, 0x01, 0x3e,\n", + " 0x32, 0xc9, 0x8e, 0xbe, 0x72, 0x77, 0x3b, 0xbe, 0xe0, 0xa1, 0xbc, 0xbe,\n", + " 0x8d, 0xb7, 0xa7, 0x3e, 0x1c, 0x05, 0x95, 0xbe, 0xf7, 0x1f, 0xbb, 0x3e,\n", + " 0xc9, 0x3e, 0xd6, 0x3e, 0x80, 0x42, 0xe9, 0xbd, 0x27, 0x0c, 0xd2, 0xbe,\n", + " 0x5c, 0x32, 0x34, 0xbe, 0x14, 0xcb, 0xca, 0xbd, 0xdd, 0x3a, 0x67, 0xbe,\n", + " 0x1c, 0xbb, 0x8d, 0xbe, 0x91, 0xac, 0x5c, 0xbe, 0x52, 0x40, 0x6f, 0xbe,\n", + " 0xd7, 0x71, 0x94, 0x3e, 0x18, 0x71, 0x09, 0xbe, 0x9b, 0x29, 0xd9, 0xbe,\n", + " 0x7d, 0x66, 0xd2, 0xbe, 0x98, 0xd6, 0xb2, 0xbe, 0x00, 0xc9, 0x84, 0x3a,\n", + " 0xbc, 0xda, 0xc2, 0xbd, 0x1d, 0xc2, 0x1b, 0xbf, 0xd4, 0xdd, 0x92, 0x3e,\n", + " 0x07, 0x87, 0x6c, 0xbe, 0x40, 0xc2, 0x3b, 0xbe, 0xbd, 0xe2, 0x9c, 0x3e,\n", + " 0x0a, 0xb5, 0xa0, 0xbe, 0xe2, 0xd5, 0x9c, 0xbe, 0x3e, 0xbb, 0x7c, 0x3e,\n", + " 0x17, 0xb4, 0xcf, 0x3e, 0xd5, 0x8e, 0xc8, 0xbe, 0x7c, 0xf9, 0x5c, 0x3e,\n", + " 0x80, 0xfc, 0x0d, 0x3d, 0xc5, 0xd5, 0x8b, 0x3e, 0xf5, 0x17, 0xa2, 0x3e,\n", + " 0xc7, 0x60, 0x89, 0xbe, 0xec, 0x95, 0x87, 0x3d, 0x7a, 0xc2, 0x5d, 0xbf,\n", + " 0x77, 0x94, 0x98, 0x3e, 0x77, 0x39, 0x07, 0xbc, 0x42, 0x29, 0x00, 0x3e,\n", + " 0xaf, 0xd0, 0xa9, 0x3e, 0x31, 0x23, 0xc4, 0xbe, 0x95, 0x36, 0x5b, 0xbe,\n", + " 0xc7, 0xdc, 0x83, 0xbe, 0x1e, 0x6b, 0x47, 0x3e, 0x5b, 0x24, 0x99, 0x3e,\n", + " 0x99, 0x27, 0x54, 0x3e, 0xc8, 0x20, 0xdd, 0xbd, 0x5a, 0x86, 0x2f, 0x3e,\n", + " 0x80, 0xf0, 0x69, 0xbe, 0x44, 0xfc, 0x84, 0xbd, 0x82, 0xa0, 0x2a, 0xbe,\n", + " 0x87, 0xe6, 0x2a, 0x3e, 0xd8, 0x34, 0xae, 0x3d, 0x50, 0xbd, 0xb5, 0x3e,\n", + " 0xc4, 0x8c, 0x88, 0xbe, 0xe3, 0xbc, 0xa5, 0x3e, 0xa9, 0xda, 0x9e, 0x3e,\n", + " 0x3e, 0xb8, 0x23, 0xbe, 0x80, 0x90, 0x15, 0x3d, 0x97, 0x3f, 0xc3, 0x3e,\n", + " 0xca, 0x5c, 0x9d, 0x3e, 0x21, 0xe8, 0xe1, 0x3e, 0xc0, 0x49, 0x01, 0xbc,\n", + " 0x00, 0x0b, 0x88, 0xbd, 0x3f, 0xf7, 0xca, 0x3c, 0xfb, 0x5a, 0xb1, 0x3e,\n", + " 0x60, 0xd2, 0x0d, 0x3c, 0xce, 0x23, 0x78, 0xbf, 0x8f, 0x4f, 0xb9, 0xbe,\n", + " 0x69, 0x6a, 0x34, 0xbf, 0x4b, 0x5e, 0xa9, 0x3e, 0x64, 0x8c, 0xd9, 0x3e,\n", + " 0x52, 0x77, 0x36, 0x3e, 0xeb, 0xaf, 0xbe, 0x3e, 0x40, 0xbe, 0x36, 0x3c,\n", + " 0x08, 0x65, 0x3b, 0xbd, 0x55, 0xe0, 0x66, 0xbd, 0xd2, 0xe8, 0x9b, 0xbe,\n", + " 0x86, 0xe3, 0x09, 0xbe, 0x93, 0x3d, 0xdd, 0x3e, 0x0f, 0x66, 0x18, 0x3f,\n", + " 0x18, 0x05, 0x33, 0xbd, 0xde, 0x15, 0xd7, 0xbe, 0xaa, 0xcf, 0x49, 0xbe,\n", + " 0xa2, 0xa5, 0x64, 0x3e, 0xe6, 0x9c, 0x42, 0xbe, 0x54, 0x42, 0xcc, 0x3d,\n", + " 0xa0, 0xbd, 0x9d, 0xbe, 0xc2, 0x69, 0x48, 0x3e, 0x5b, 0x8b, 0xa2, 0xbe,\n", + " 0xc0, 0x13, 0x87, 0x3d, 0x36, 0xfd, 0x69, 0x3e, 0x05, 0x86, 0x40, 0xbe,\n", + " 0x1e, 0x7a, 0xce, 0xbe, 0x46, 0x13, 0xa7, 0xbe, 0x68, 0x52, 0x86, 0xbe,\n", + " 0x04, 0x9e, 0x86, 0xbd, 0x8c, 0x54, 0xc1, 0x3d, 0xe0, 0x3b, 0xad, 0x3c,\n", + " 0x42, 0x67, 0x85, 0xbd, 0xea, 0x97, 0x42, 0x3e, 0x6e, 0x13, 0x3b, 0xbf,\n", + " 0x56, 0x5b, 0x16, 0x3e, 0xaa, 0xab, 0xdf, 0x3e, 0xc8, 0x41, 0x36, 0x3d,\n", + " 0x24, 0x2d, 0x47, 0xbe, 0x77, 0xa5, 0xae, 0x3e, 0xc0, 0xc2, 0x5b, 0x3c,\n", + " 0xac, 0xac, 0x4e, 0x3e, 0x99, 0xec, 0x13, 0xbe, 0xf2, 0xab, 0x73, 0x3e,\n", + " 0xaa, 0xa1, 0x48, 0xbe, 0xe8, 0xd3, 0x01, 0xbe, 0x60, 0xb7, 0xc7, 0xbd,\n", + " 0x64, 0x72, 0xd3, 0x3d, 0x83, 0xd3, 0x99, 0x3e, 0x0c, 0x76, 0x34, 0xbe,\n", + " 0x42, 0xda, 0x0d, 0x3e, 0xfb, 0x47, 0x9a, 0x3e, 0x8b, 0xdc, 0x92, 0xbe,\n", + " 0x56, 0x7f, 0x6b, 0x3e, 0x04, 0xd4, 0x88, 0xbd, 0x11, 0x9e, 0x80, 0x3e,\n", + " 0x3c, 0x89, 0xff, 0x3d, 0xb3, 0x3e, 0x88, 0x3e, 0xf7, 0xf0, 0x88, 0x3e,\n", + " 0x28, 0xfb, 0xc9, 0xbe, 0x53, 0x3e, 0xcf, 0x3e, 0xac, 0x75, 0xdc, 0xbe,\n", + " 0xdd, 0xca, 0xd7, 0x3e, 0x01, 0x58, 0xa7, 0x3e, 0x29, 0xb8, 0x13, 0xbf,\n", + " 0x76, 0x81, 0x12, 0xbc, 0x28, 0x8b, 0x16, 0xbf, 0x0e, 0xec, 0x0e, 0x3e,\n", + " 0x40, 0x0a, 0xdb, 0xbd, 0x98, 0xec, 0xbf, 0xbd, 0x32, 0x55, 0x0c, 0xbe,\n", + " 0xfb, 0xf9, 0xc9, 0x3e, 0x83, 0x4a, 0x6d, 0xbe, 0x76, 0x59, 0xe2, 0xbe,\n", + " 0x54, 0x7d, 0x9f, 0xbb, 0x9d, 0xe8, 0x95, 0x3e, 0x5c, 0xd3, 0xd0, 0x3d,\n", + " 0x19, 0x8a, 0xb0, 0x3e, 0xde, 0x6f, 0x2e, 0xbe, 0xd0, 0x16, 0x83, 0x3d,\n", + " 0x9c, 0x7d, 0x11, 0xbf, 0x2b, 0xcc, 0x25, 0x3c, 0x2a, 0xa5, 0x27, 0xbe,\n", + " 0x22, 0x14, 0xc7, 0xbe, 0x5e, 0x7a, 0xac, 0x3e, 0x4e, 0x41, 0x94, 0xbe,\n", + " 0x5a, 0x68, 0x7b, 0x3e, 0x86, 0xfd, 0x4e, 0x3e, 0xa2, 0x56, 0x6a, 0xbe,\n", + " 0xca, 0xfe, 0x81, 0xbe, 0x43, 0xc3, 0xb1, 0xbd, 0xc5, 0xb8, 0xa7, 0x3e,\n", + " 0x55, 0x23, 0xcd, 0x3e, 0xaf, 0x2e, 0x76, 0x3e, 0x69, 0xa8, 0x90, 0xbe,\n", + " 0x0d, 0xba, 0xb9, 0x3e, 0x66, 0xff, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00,\n", + " 0x40, 0x00, 0x00, 0x00, 0x53, 0xd6, 0xe2, 0x3d, 0x66, 0xb6, 0xcc, 0x3e,\n", + " 0x03, 0xe7, 0xf6, 0x3e, 0xe0, 0x28, 0x10, 0xbf, 0x00, 0x00, 0x00, 0x00,\n", + " 0x3e, 0x3d, 0xb0, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x62, 0xf0, 0x77, 0x3e,\n", + " 0xa6, 0x9d, 0xa4, 0x3e, 0x3a, 0x4b, 0xf3, 0xbe, 0x71, 0x9e, 0xa7, 0x3e,\n", + " 0x00, 0x00, 0x00, 0x00, 0x34, 0x39, 0xa2, 0x3e, 0x00, 0x00, 0x00, 0x00,\n", + " 0xcc, 0x9c, 0x4a, 0x3e, 0xab, 0x40, 0xa3, 0x3e, 0xb2, 0xff, 0xff, 0xff,\n", + " 0x04, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xb3, 0x71, 0x67, 0x3f,\n", + " 0x9a, 0x7a, 0x95, 0xbf, 0xe1, 0x48, 0xe8, 0xbe, 0x8a, 0x72, 0x96, 0x3e,\n", + " 0x00, 0xd2, 0xd3, 0xbb, 0x1a, 0xc5, 0xd7, 0x3f, 0xac, 0x7e, 0xc8, 0xbe,\n", + " 0x90, 0xa7, 0x95, 0xbe, 0x3b, 0xd7, 0xdc, 0xbe, 0x41, 0xa8, 0x16, 0x3f,\n", + " 0x50, 0x5b, 0xcb, 0x3f, 0x52, 0xb9, 0xed, 0xbe, 0x2e, 0xa7, 0xc6, 0xbe,\n", + " 0xaf, 0x0f, 0x14, 0xbf, 0xb3, 0xda, 0x59, 0x3f, 0x02, 0xec, 0xd7, 0xbe,\n", + " 0x00, 0x00, 0x06, 0x00, 0x08, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x66, 0x11, 0x1f, 0xbf,\n", + " 0xb8, 0xfb, 0xff, 0xff, 0x0f, 0x00, 0x00, 0x00, 0x54, 0x4f, 0x43, 0x4f,\n", + " 0x20, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x2e, 0x00,\n", + " 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x14, 0x00,\n", + " 0x04, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x0c, 0x00, 0x00, 0x00,\n", + " 0xf0, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0xd8, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00,\n", + " 0x48, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xce, 0xff, 0xff, 0xff,\n", + " 0x00, 0x00, 0x00, 0x08, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0x1c, 0xfc, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00,\n", + " 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00,\n", + " 0x08, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00,\n", + " 0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x07, 0x00, 0x10, 0x00,\n", + " 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x1c, 0x00, 0x00, 0x00,\n", + " 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xba, 0xff, 0xff, 0xff,\n", + " 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00,\n", + " 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00,\n", + " 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x16, 0x00, 0x00, 0x00,\n", + " 0x08, 0x00, 0x0c, 0x00, 0x07, 0x00, 0x10, 0x00, 0x0e, 0x00, 0x00, 0x00,\n", + " 0x00, 0x00, 0x00, 0x08, 0x24, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00,\n", + " 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x08, 0x00, 0x07, 0x00,\n", + " 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,\n", + " 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,\n", + " 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,\n", + " 0x0a, 0x00, 0x00, 0x00, 0x10, 0x03, 0x00, 0x00, 0xa4, 0x02, 0x00, 0x00,\n", + " 0x40, 0x02, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, 0xac, 0x01, 0x00, 0x00,\n", + " 0x48, 0x01, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00,\n", + " 0x50, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x26, 0xfd, 0xff, 0xff,\n", + " 0x3c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0x18, 0xfd, 0xff, 0xff, 0x20, 0x00, 0x00, 0x00,\n", + " 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31,\n", + " 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x34, 0x2f, 0x4d, 0x61, 0x74,\n", + " 0x4d, 0x75, 0x6c, 0x5f, 0x62, 0x69, 0x61, 0x73, 0x00, 0x00, 0x00, 0x00,\n", + " 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x6e, 0xfd, 0xff, 0xff,\n", + " 0x50, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0x60, 0xfd, 0xff, 0xff, 0x34, 0x00, 0x00, 0x00,\n", + " 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31,\n", + " 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x34, 0x2f, 0x4d, 0x61, 0x74,\n", + " 0x4d, 0x75, 0x6c, 0x2f, 0x52, 0x65, 0x61, 0x64, 0x56, 0x61, 0x72, 0x69,\n", + " 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73,\n", + " 0x70, 0x6f, 0x73, 0x65, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,\n", + " 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xce, 0xfd, 0xff, 0xff,\n", + " 0x34, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0xc0, 0xfd, 0xff, 0xff, 0x19, 0x00, 0x00, 0x00,\n", + " 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31,\n", + " 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x33, 0x2f, 0x52, 0x65, 0x6c,\n", + " 0x75, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,\n", + " 0x10, 0x00, 0x00, 0x00, 0x12, 0xfe, 0xff, 0xff, 0x3c, 0x00, 0x00, 0x00,\n", + " 0x03, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,\n", + " 0x04, 0xfe, 0xff, 0xff, 0x20, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75,\n", + " 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e,\n", + " 0x73, 0x65, 0x5f, 0x33, 0x2f, 0x4d, 0x61, 0x74, 0x4d, 0x75, 0x6c, 0x5f,\n", + " 0x62, 0x69, 0x61, 0x73, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,\n", + " 0x10, 0x00, 0x00, 0x00, 0x5a, 0xfe, 0xff, 0xff, 0x50, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,\n", + " 0x4c, 0xfe, 0xff, 0xff, 0x34, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75,\n", + " 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e,\n", + " 0x73, 0x65, 0x5f, 0x33, 0x2f, 0x4d, 0x61, 0x74, 0x4d, 0x75, 0x6c, 0x2f,\n", + " 0x52, 0x65, 0x61, 0x64, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65,\n", + " 0x4f, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65,\n", + " 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\n", + " 0x10, 0x00, 0x00, 0x00, 0xba, 0xfe, 0xff, 0xff, 0x34, 0x00, 0x00, 0x00,\n", + " 0x0a, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,\n", + " 0xac, 0xfe, 0xff, 0xff, 0x19, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75,\n", + " 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e,\n", + " 0x73, 0x65, 0x5f, 0x32, 0x2f, 0x52, 0x65, 0x6c, 0x75, 0x00, 0x00, 0x00,\n", + " 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\n", + " 0xfe, 0xfe, 0xff, 0xff, 0x3c, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00,\n", + " 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xf0, 0xfe, 0xff, 0xff,\n", + " 0x20, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69,\n", + " 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x32,\n", + " 0x2f, 0x4d, 0x61, 0x74, 0x4d, 0x75, 0x6c, 0x5f, 0x62, 0x69, 0x61, 0x73,\n", + " 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\n", + " 0x46, 0xff, 0xff, 0xff, 0x50, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00,\n", + " 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x38, 0xff, 0xff, 0xff,\n", + " 0x34, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69,\n", + " 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x32,\n", + " 0x2f, 0x4d, 0x61, 0x74, 0x4d, 0x75, 0x6c, 0x2f, 0x52, 0x65, 0x61, 0x64,\n", + " 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x70, 0x2f, 0x74,\n", + " 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x00, 0x00, 0x00, 0x00,\n", + " 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,\n", + " 0xa6, 0xff, 0xff, 0xff, 0x48, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00,\n", + " 0x2c, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00,\n", + " 0x04, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\n", + " 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x43,\n", + " 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00,\n", + " 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x32, 0x5f, 0x69, 0x6e, 0x70, 0x75,\n", + " 0x74, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,\n", + " 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x14, 0x00, 0x04, 0x00,\n", + " 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x0e, 0x00, 0x00, 0x00,\n", + " 0x28, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\n", + " 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00,\n", + " 0x08, 0x00, 0x00, 0x00, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79,\n", + " 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,\n", + " 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\n", + " 0x00, 0x00, 0x0a, 0x00, 0x0c, 0x00, 0x07, 0x00, 0x00, 0x00, 0x08, 0x00,\n", + " 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x03, 0x00, 0x00, 0x00\n", + "};\n", + "unsigned int sine_model_quantized_tflite_len = 2640;\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1sqrhBLXwILt", + "colab_type": "text" + }, + "source": [ + "We can either copy and paste this output into our project's source code, or download the file using the collapsible menu on the left hand side of this Colab.\n", + "\n" + ] + } + ] +} \ No newline at end of file diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/Makefile.inc b/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/Makefile.inc new file mode 100644 index 00000000000..879042f65d6 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/Makefile.inc @@ -0,0 +1,6 @@ +# Settings for the Discovery STM32F746NG board. +ifneq ($(filter disco_f746ng,$(ALL_TAGS)),) + MBED_PROJECT_FILES += \ + BSP_DISCO_F746NG.lib \ + LCD_DISCO_F746NG.lib +endif diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/constants.cc b/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/constants.cc new file mode 100644 index 00000000000..09d464bbfdd --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/constants.cc @@ -0,0 +1,19 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/constants.h" + +// A larger number than the default to make the animation smoother +const int kInferencesPerCycle = 70; diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/output_handler.cc b/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/output_handler.cc new file mode 100644 index 00000000000..cbfe75a7ab6 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/disco_f746ng/output_handler.cc @@ -0,0 +1,80 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h" + +#include "LCD_DISCO_F746NG.h" +#include "tensorflow/lite/experimental/micro/examples/hello_world/constants.h" + +// The LCD driver +LCD_DISCO_F746NG lcd; + +// The colors we'll draw +const uint32_t background_color = 0xFFF4B400; // Yellow +const uint32_t foreground_color = 0xFFDB4437; // Red +// The size of the dot we'll draw +const int dot_radius = 10; +// Track whether the function has run at least once +bool initialized = false; +// Size of the drawable area +int width; +int height; +// Midpoint of the y axis +int midpoint; +// Pixels per unit of x_value +int x_increment; + +// Animates a dot across the screen to represent the current x and y values +void HandleOutput(tflite::ErrorReporter* error_reporter, float x_value, + float y_value) { + // Do this only once + if (!initialized) { + // Set the background and foreground colors + lcd.Clear(background_color); + lcd.SetTextColor(foreground_color); + // Calculate the drawable area to avoid drawing off the edges + width = lcd.GetXSize() - (dot_radius * 2); + height = lcd.GetYSize() - (dot_radius * 2); + // Calculate the y axis midpoint + midpoint = height / 2; + // Calculate fractional pixels per unit of x_value + x_increment = static_cast(width) / kXrange; + initialized = true; + } + + // Log the current X and Y values + error_reporter->Report("x_value: %f, y_value: %f\n", x_value, y_value); + + // Clear the previous drawing + lcd.Clear(background_color); + + // Calculate x position, ensuring the dot is not partially offscreen, + // which causes artifacts and crashes + int x_pos = dot_radius + static_cast(x_value * x_increment); + + // Calculate y position, ensuring the dot is not partially offscreen + int y_pos; + if (y_value >= 0) { + // Since the display's y runs from the top down, invert y_value + y_pos = dot_radius + static_cast(midpoint * (1.f - y_value)); + } else { + // For any negative y_value, start drawing from the midpoint + y_pos = + dot_radius + midpoint + static_cast(midpoint * (0.f - y_value)); + } + + // Draw the dot + lcd.FillCircle(x_pos, y_pos, dot_radius); +} diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/hello_world_test.cc b/tensorflow/lite/experimental/micro/examples/hello_world/hello_world_test.cc new file mode 100644 index 00000000000..6ca3e88b6ca --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/hello_world_test.cc @@ -0,0 +1,112 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h" +#include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +#include "tensorflow/lite/experimental/micro/micro_interpreter.h" +#include "tensorflow/lite/experimental/micro/testing/micro_test.h" +#include "tensorflow/lite/schema/schema_generated.h" +#include "tensorflow/lite/version.h" + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(LoadModelAndPerformInference) { + // Set up logging + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + // Map the model into a usable data structure. This doesn't involve any + // copying or parsing, it's a very lightweight operation. + const tflite::Model* model = ::tflite::GetModel(g_sine_model_data); + if (model->version() != TFLITE_SCHEMA_VERSION) { + error_reporter->Report( + "Model provided is schema version %d not equal " + "to supported version %d.\n", + model->version(), TFLITE_SCHEMA_VERSION); + } + + // This pulls in all the operation implementations we need + tflite::ops::micro::AllOpsResolver resolver; + + // Create an area of memory to use for input, output, and intermediate arrays. + // Finding the minimum value for your model may require some trial and error. + const int tensor_arena_size = 2 * 1024; + uint8_t tensor_arena[tensor_arena_size]; + tflite::SimpleTensorAllocator tensor_allocator(tensor_arena, + tensor_arena_size); + + // Build an interpreter to run the model with + tflite::MicroInterpreter interpreter(model, resolver, &tensor_allocator, + error_reporter); + + // Obtain a pointer to the model's input tensor + TfLiteTensor* input = interpreter.input(0); + + // Make sure the input has the properties we expect + TF_LITE_MICRO_EXPECT_NE(nullptr, input); + // The property "dims" tells us the tensor's shape. It has one element for + // each dimension. Our input is a 2D tensor containing 1 element, so "dims" + // should have size 2. + TF_LITE_MICRO_EXPECT_EQ(2, input->dims->size); + // The value of each element gives the length of the corresponding tensor. + // We should expect two single element tensors (one is contained within the + // other). + TF_LITE_MICRO_EXPECT_EQ(1, input->dims->data[0]); + TF_LITE_MICRO_EXPECT_EQ(1, input->dims->data[1]); + // The input is a 32 bit floating point value + TF_LITE_MICRO_EXPECT_EQ(kTfLiteFloat32, input->type); + + // Provide an input value + input->data.f[0] = 0.; + + // Run the model on this input and check that it succeeds + TfLiteStatus invoke_status = interpreter.Invoke(); + if (invoke_status != kTfLiteOk) { + error_reporter->Report("Invoke failed\n"); + } + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, invoke_status); + + // Obtain a pointer to the output tensor and make sure it has the + // properties we expect. It should be the same as the input tensor. + TfLiteTensor* output = interpreter.output(0); + TF_LITE_MICRO_EXPECT_EQ(2, output->dims->size); + TF_LITE_MICRO_EXPECT_EQ(1, input->dims->data[0]); + TF_LITE_MICRO_EXPECT_EQ(1, input->dims->data[1]); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteFloat32, output->type); + + // Obtain the output value from the tensor + float value = output->data.f[0]; + // Check that the output value is within 0.000001 of the expected value + TF_LITE_MICRO_EXPECT_NEAR(0.0486171, value, 0.000001); + + // Run inference on several more values and confirm the expected outputs + input->data.f[0] = 1.; + interpreter.Invoke(); + value = output->data.f[0]; + TF_LITE_MICRO_EXPECT_NEAR(0.8071436, value, 0.000001); + + input->data.f[0] = 3.; + interpreter.Invoke(); + value = output->data.f[0]; + TF_LITE_MICRO_EXPECT_NEAR(0.0964818, value, 0.000001); + + input->data.f[0] = 5.; + interpreter.Invoke(); + value = output->data.f[0]; + TF_LITE_MICRO_EXPECT_NEAR(-0.9352637, value, 0.000001); +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/images/STM32F746.gif b/tensorflow/lite/experimental/micro/examples/hello_world/images/STM32F746.gif new file mode 100644 index 0000000000000000000000000000000000000000..e427bc867a7296f9916e1f2950d5d1286d637071 GIT binary patch literal 299034 zcmV)MK)An0Nk%w1VL$;u0r&p^00002ClmrH83Y;)1RxazG$I5$B?bfm1}Gl}FCzy9 z1P41Q2RM9w8eZE+!x`F(Dx*BS1MMK|Ul$KqWjhCQd{uBpWI_DJvKXD<&o@ zDlsfjNG>cQE-Wc7Ff}hdFfdR_GaCmpDI+s0Dl;!BGch$YJQ_1ZMm0V^HZ?sqIXE~d zA~-TCI5aglKsh-b20BhlI#o|QIy5{wKt3=jKq3f0R#re{T|q!VK|n@9Tv$XzOh!O9 zNGcRdMoLUV4o*!>PEA!$E*MWdBTzpqP+nhBQe0Cn8dG3mQ)+2cT~k<9UszWXT0}Hj zUT9l&bY5j~Urzy2ad(7rPBn61BXmzObxbdHPd0T@Ja%qSc~Lcbc7uCTIDAhtd{#SrRzG}NK74zO zeM&HWOfi2}Ie%C@fKfGoR5*ZFIe=I?fLl6&SvrAYErLfagI7C)SUiMUJ%vy>gh9`<)MT&TLjemBHiJOmROOR|*k#b#^mupX#a8Z|WRhM&DmvmZ~W=fcKUYL`lnSOSfbXS{pTbp-YoOxcIOf;QUKc0SE zp-nuYc`2rgZl{Vdr<$Rsn<}TFE2pI=s31zHi!!K$_tCTydn=q`8Lae7#t(QTrn=h`S zF0QU1uR2e!lr^uQGq0mNucRrjq%*IjL9eMZud6(-tthXrL9mlLu$nZnlS8tTPO_Og zvYAw~mQk~&L9?tov#&z5nnSdtI<}l^x0hGCo>sh|alEUYzNUM_sffg?f5fYZ#jc6P zu8hjEkO2PxA^!_bMO0HmK~P09E-(WD0000X{uxhjbZKpAdSzrFb#rNMXCP&IXK7|G zV{dH$A^8La6afDKEC2ui06+mi0RRa90AtA#NU)SCRHH(9LRj#is)qv&Hk?Rnmc@%0 zGe)eak=4eJ9zk*x8FFMtk|-fkT&a>_s+TNV!b~VLrp=iy9oEEoN~h1BKY<1vno}Xs zqCAl%Rm!yK&ZRe_rZhRVs@0NLpGw`D^{CdbT%igbOH(W(`@6%Z?2@we8!u8?$a*n|ANnpMmq1efP2D+{kzTUe3I^@aNE- z8&6KXy7cM8uWOI)TetS&;46#&AMeV#^5EIAr(ci0yZiTL1LdG+NtH9 ze6s1~nse&;=AC91O5!ap(g~=bjM_+Oo{A>w=c0c`I%%VZJ}RlDeFnNIr#zgwI{3;eaEaIM{y=EqLC87cRNtlk>fJ3fHM`gfj#KKkn=tByMCv#)Nu=Ca?OyYBjq-Fob_2QPf>!xO(d z@5f)B{PO?)UcB+oM-Tn<%p+X^hfg~A*$`9N_wnDKzy9zyE$$xps~`UcSg!FA@PGzPVE+*KD(4MQfDBZi`Xorf2vV+q z6||rR*`u!fbP#d?oI}Y(h_n!rP=qHmAqrQ>LNj+o3N9p$LWJ5HdJnGEG7MM=t1LQ;=QWMV2+=|xq} z!3?k50}-J}%UVWp4`LK03L44FUG@@X%8nXh&OlB_$*+(plv6)g&wkdCp8yT0KL<+Ce7@132rZ~R6UxwrIy9lWn8rLnF@+qA6P@2gCp{s+(Re0+ zlOXM=Na0D+dwSEF*-WW6QK|%%ZnK#gHDgTOiP3c0^qd;?rA~Ls&XShXrzQ1h8_NjP zXY#V3yHuz?mCDqO3bm^vz< z*UHScy7jGWovB>2x>cU0wWmOxz!;y(SC7sBtzeD7GsV!-z5+I?Wku{#`?^@gHdd+* zg+(Q(3JQrnp_p{dDq|BHR?dpmuXj~zTt`dVxteyZnXPMR*NNBGZZ@8L9jrSa`&!5U z+P1d0g)BoSOW93ORRuCltyaq#)X$pLv!LaIUCCM6<~rB87^SIZB}rJo^0cKZt?o~Y zn%C~S;IQ5$E_ibrRqSGyw&+c-dLhdN$?~?h78R~J1AElt+V{Sp6)jx%3*FO(Gp9oh zrgoP*MmJEiy43V;FvAPl1$P&@E?6&l391Aa(BZtu)vbgd8qN2*7ZbX$!bMeE*N)cL zxFp4Ica^J7*}fEzF6FO`XKdQw-jtI&wJwfP;7lIJ)U_wxERO~IW1>EGyc3pcVGEn# zBrCbY9;PfAHXz^a(zm!x7AXRb8`=Y}l#ql)uzqRm-@mdLz(C$HYZ=T-BtKaHrz$OH zb&tGZCuh0Nc#bTTqs)mtP@u|IzVdxP+F3$l`OM4w@>=6e<}&~Hrd$Q!854(5Pv?eQx@0@GcS~tJU}PWr-Hv6ok2AVYRzthg(jEh>t8MKmXZz6H#xkZAoMUqHn#NwH z>$t(`Yhat3;kC7Qt>5itCtKIN`6g^UD}HbEVnN@$?L==maRp01eBuxP-u9$be%7KI z{NM==_abFJgj3i2*9ZUWs@?rbBMTbB9S3^PMH_Tz?>WkB=)f9J9%+{2ddpCcy3>X2 zawoie=H<@%rMJ%YcdDD%1OGYChh5Lka=g%J_h-eWZqKtXblDEYE@eGD1Em*y(TbL_ zO~nmk^Xl90a{qf8s7?67$KmjUM||QHpZG8cKD&<3dm$%pb*=9p29VDD0D$UOZfATeVdPYuavHT-c#(JF6}QK|?sq>3+lT!3eF(nrhfjRO z_ulZx_kHen&-~>#U-{6FzVxX|eBM_7`iwX5;*GC;>}S9GYydz148@=O4L0>}e6fO!O1fGvQ43V47F*nlvG ze*K4h{1<`!2Z4iUfe`qB|EGbJw?`h>fgp&1AGm=5Sb|1yf)3b#2DpMO2m=Y&f-Eos zA25S7Sc5jG0XLX~H%NLe_<}Il13(yrHa3JyP=tx}NTGCh>Sa~1@Jd@&S72s-+&6(R zHh>^#f<}miT4;qDIEC1Ed|w!b6F7zxSb-vVhUUkCYPg0W*oJP%gzzNF6P=N$T<|c)V_;`T-6o48_=c!>g{laLayW;s_=;KBin2I@epHLLc#E`Hgdora zgK&t5xOA!2h};K;ig${}XpF+Rc)&MV(+7sjD2AQLi5O^seUOHxc#X%HirTo10=SB- z_>Hp|j=6}7AjkrR=ytxwh|c$X%qW56cYdXqjE8r8&sUH3c#qQ8jQaR{`}mCh2#o-_ zZvk0+(&&W)>5K@OkpHNV)aFbwct{3zPz}X&-K37ZM~q?M1ek}3duWN2_>m3Zk&=js zPJn*&h>s}Qj|<6=(CCTrh>bAmjo^rdKNyJ|Ig&d6n1??Ii`__y)A)`W`2k!clo6?9 z6e(NE$O{*Fd}B~ZuvmvWnTaU~i<@|o`M8cM8GS6dl>_OLKlzd|nT=LRhcI}PIBAxb zn3jWxmJf*rOwg7@IZsA;loqLb2zh~H`IJyuhdc>@Nx721mw14Qkp~%#UTKYCnV8&| zmwOqNZ26dw8JRcfm@k--l^K^2r+)<2h)c2&ZX~u1S#KX_&MBNt@#+leYPaKL7;jxt{D<1MPVO@9CcK zIiK=bpZ1BH_?e%(sgsYW0fi)-7nYe6S(nS%drXKm&tMG*s-O(oporq252_9j+78B2 zp%Il2)8G#pdJh}Qp&t678F~*^6`~+Yq8=(y7&@XVx}uWRqAq%%FZx0C17b8PK)!RM zIJ%$^3KJ9BqduCTFR=?e3Z$73q(NGwxo{RpN(x1)2~FChvg8O+I;Bpyg8v7c07{$> z1%*mTpw8%__D~OEI;LbA4`phmXxa{Gx~6AJ4{kc9`VglbI;V6xrzTpbc*+lYx(|CQ zr+kW`f9j`#I;i*%riKcqhnlF0x~TL2(58v{rjFXC=`g93TB(-Wpy!aOn!2f+x(?LP z4W1gRohqv6P^xUYrt*-gs+y|vPzEe`rT-auO?H&8_Lsi52^{*Ur@E`Ws;0jBtH5fg zc{;2cTB!YSti%ea%DSw~YOK!stc=>IkSd|mDy@imt<{>X)~c=9O0Agct(Q8g;+ml3 zTCU>?s_1&I=jy6f>IE@?rCt!A#c7=Mx@w%cl({OYy}GaZnyrIMtQT6W$eOHyN~q9! zum~Hi;0meR+OQ7Gt^EqI;X1JtYp&{Qu@{@Mq8eha8l3(~1z9SfsD_-$X?z8W3Hcha z{JOH1+Nl3Zuo^lKFx#vHOS3lrE369Jtq}XLJS(w2Td_c^vC<&4L`$lo%B}|3pRwu& zqqe17s(<%dpctyMEL*kLIo#Evts+R3#+q1d$vWJu4=osMBAzz z3z{LjUbGsuUmB-WYqeDCvRgZ|6BV#sTd+6lwYpli4Z5>rd$?wcwu+m!M$5KBi?j(C zt3x?>S{ipx3$;?Xd|(KucZ;_x`?A9-vwd2#e@nW8`?Z7%wurmB!P>ct%eamkyN%np z9Gi(Cdx8}ut4}syBa6A2i?4R;x}LkM!pgUPJG!PDxXOFF)2h18%ev5;x~~hn)H}Ph z3%R(Py>ZK77xua|CR!by>Fxtg#l{zz@8>?AyMBOTij!vGt3=HJrj3yuq@|!KtbS z9_*VS%(Mb5W+ZE2O=+v)d#F~+xD1TEE$qTh48svT!&Kb1Hk`URT*dgS!#v!lig zSKv!{D7waM9IPz={Kjy6u!y|HcpS+T8?1F)$vb?>*6YJoT8H{(Zw=LuW=zPBH+Tpv zzK@KzZp_G09LGv5$$ZSjc5KOetjCZ{%Uz7gUtqf_mxvtmz}I-zx>MyH&>%f%F7928VV2n zddthquc_?J(2Tm*tj)nN&$4{an9R=VkjtAqgH|WTP3v()OwNus!UpP$hpeyq+|HAH z4aEzvI19l!Ys~aq(fEwb7HzcrT$yiKfI-~Ljn%ybjm8MA59++m`?}Bz>#Ppl()@7J zEiJq^jKLZIUDFqx&C`t27_HL#{KI?H0)UKM(-z9Ui^PU3zN=cbJk8KC-P9HQ)YeSX zIX%@jZO=MQ)mL598y&ep`O`%SVZnUHtofPxmd1@IQ3{Q!OU=}5ebZKr)%6V5ajgvb zEZ1%Q);k>y+AIY={n3Fe1|!ULYPgz_M+Ulp(rUfdRgKq+ZP&vf40H|IdF|MDoz<3I z*QR>Cx;z3OJZEGqV1ry_!Red5mx;IF4vDSUja}J{P1#)h*swj>uI<))z1bcO)GQ{# zL(Oz_SlWFE1+IYFsy*D7-P*H#+!#C8vK`yYjohqV+?bu&jmy`Yy|f^`SR$Qj*92(| z(16_k8IpP!+{6vtYl#mobKwdPR*MR=$Jn1tzO&Jt>TDIUy9C)X1v=l&e`C7c&VNZtj_0pj_f183$`xk z%}(pie(T4M;%&)Aqpna+UXd=&OQ&v$NI(jz9`43o4a&~!#gOc^j_$Gk>d?;a(oX52 z9^XzGkv>OZ+9zFW-c2IV?GDg~o*?d%KJEb@?F3)&>hA8E-tN$#@Xwy^?|#Yg4(Bfx zTiZTe{O<1npYi8T?&WUq50CI2zwiw|@*e;3l^*e&?SY@E1D{4vhG)Wzj#+R2?gXhv z<^3L#f}rsOzw;t*@;<-pARqM4?cb%k2h=2|Ca7 zCI9n6zwRMV@?O97LqFd1XXq(Ud59)nEI-IyrFs2M^HL9pjBxc>pY?QK_jV8VUGMN; zfA?W;=|-RHdlby#6`%zi>~3%K7(e%lul0J*^$d^qj=%SOKlzMrs$);~<9OZK&gS+# z^L+^ERG;&UFYp{+`li3}qfhye|M-!g_mbc9tY7(E{P+EAg4USmp>1wZ|CZei`iY#{c@Vuj&?6``9_gqGodxUQLt5~gW<=R!N*I;13DjPOd zY+18t)k0g_HWO2~agEMpdp2uRyyCFs-P^aI-@k#O@+6q>V#9|M6<5@OCF2AL8#6Yc zoaCg*8JQhhUdi$%lb9`L*1Vb1+u)$6hpwjU^zLieX=BTkUE6N|-P>ey5#^ z0rM3+y!f}{I}IaFEYaZ>=OjK(4*>$DlF24FGwvMYbLi1FU78`BS~Y9eaDfYF-=25x z;I?(Yr!T+$RQ&PR>vwP5e{$r?^JzKeAX;pQEVvkBh>ap!qPieR!bn2RQp(H(&l-S9 zrZ`B$sSfkbLvKX&_UjKm`%rA_92M_7@xHfAl##y|0W|188wre|Ip-3@h=Rs8h%A8} z2zX!$&JZNx!V9;c5JMimYe2L?JTy%}5lJjD%Pnhkam5sAWU)mu#hmd>FmY7VMmB}g z(V>NUWNeJbhU@`61)p3}$?Q6ll1eK<vSVA)U_(H~yD~&Aqdo1_Ru@gw+BA8kwpLkl-F4D$D|Lr3Uk%)JJL+W8 zEJ@5zV98YPmi=H#FRFA_k5#3uc3*6-oweJ4y%m^SMQ2ckFjr0yICap- zD8}d{)OMM4*H3vZc*$OmS0GPeV@h7;WRwZ!ci@0q_Sa>WV-7drgKI`ep@vyt&PN4_ zNVixEGu9YNk3pli!+W<)d1aNGmKkQKnRfc=nGv4<)?lk^*0G|O4m=FX28%2jg(gT9 z8s1YoZZ>2&l)hT(sbAJNYP$QSI<~Cy)|<_(A&N-O&C>qp0R_qqX=ubbHu~chk|ufS zx#PauZMiA0d-58wuKLn)y_T4xE<_y!k zFyDemAy0moGr;Yj zBT5|OMp6hqG#-+0iChOalHtfk(ql~Lb7L(pS;IP7@|L;8V-NGU$LL&-4VK|uAdPs+ z-=$KGh_r??VCl+6=As^v%;PRA$;(;)b`qPo%%m?lNj++AF-|bXg<@CvXrD0ZP`t0ZZn(O^kzOa`ArQ1w3A85` zIn;hCte~Jw>85}LzzHlBSxRK<(3+sos7BNYHFXw6ubK>MMRn;ooRA^DYqm_Y>$K`F43Dj7Ps z;H-p1VPom#PF7lUo_ft}Z9Qt+R(001`y|K|xR?c$Vvm%Gbwg(y$~|SyH41f=rafMn zQ`+K|v(lYzb$wya!0NW2pXKdXe-+%flCXO!Sy#I@P&#ET1SiU6?oOOh+30H2x?Y_x zeSHep>}r>mSmLcfe9OPGI;=6!*&cCA;F#kgH;6?`!*Z*71Zk#b8}X%XeeHW;3+vaH z-0kjYzk5(6dB(s5W^Lkzn_Rj+IKmXZu6!?BVHjuF#WQ~KZiPC%2y@sN|NU@){iwYm zf$G2sUZG6KvmgaL_*N|crg4*FjN$!0Im$M^F@JNMO&*sP2#^)wLr08Wn1)xOua)RK zb75IDI~l)HCMlFvZzyS zYGMz2)!1e=Ia*Cv(gxFWXa$l;2BDp{GIqG`!gUcCOoeGn014IhwY7mAYy@!{=tW3A;e?%d*{(D_>-f2y<k-^^LM*^*E2u*v6p@IYcFuF`*_*BHznDW z@cV;vdi26Cz1jtWeC4Nq9O_5L`ZfOe*Z14;@t6Me`F-{4#~Sw!$pgSO)8NdU0xwSh$#VbJs zY`LEJt%3$aO48b`&gVv_$O-x?*yHve6yQt%OgG>A%$ppD#thE5Bu?-APUS34ryEVvyi4;8 z&*juh=Hxin%uE0h0I^9qGKfjtiAd~x%kAV&;RH|d>`(tB&*Vf;C|pkhWzB17&+H?( z%{wBu1jIkYMnT#_Lt+6Wv4P&ifZqg8{RGPK1W*qx&(vJaEZWO=G0J#EFg{$+9<QkNU0>i4As!?tkDhS&=39308LI2HP9(U&;_Sq~z`;B#2)MurRg4Rj z0)xm;8okjDt#ssZXn)hdlaynrNa(#4d~beK{bJyZeZ&qTG-^jy9DY||vns4b0C z_8g%)MT3i&H6FCi9rVUZ;|M`LQ$m$eQe{&}Ra7_4(nj^sNo7@UleAmRR3Pg_UPJ-H zRG{eGolt$!LN!!GHC0qyRjXlCV%5?|T~uT>)>d6s64gI1KuZ1tzP6fHCWVkbK`|rX z)l%KoU+valHP$6dI$_1pVpUaTRZdEk(>a9IXT?K?git;WKiaf( za!plpWmbR{N_OosC%HgcExAm~#&o0CX`8oueOP>5Q-1Z=e?8a#fW6o@yoQ05)vp^o z+{8z;TY`FpSd!J(eBD=5oY-+aRhB(jSH;+urNfOqFp{8EEfl;XP1t$0K^PU;hJA)& zEZGg^*Kjq5mZeyWUDO_h*_h4Pja^uqJT{ z+L@Kdo0VF$eaeXuz-1Y}3yl%2ja!rbR%D2{t_{e$eL$5xTD}e2vHe?f;Dj%zv@gxo zsKw2Rlz<*Enh+9?18`WmjabR`+PPKKy_KTN?c2&-S)=V-&;4A`1>9LI+o(0zv^`zV z+_I2$T*!^w$)(%Pg}|iC+!v%>xYJy}4PDV?R!$&YculAO??GMRMa;!r-M7tJ*Nt7+ zh23hKUE9T7=ta(@onCOEUa|FE-aE13-QL|(T*ghiBx~K|eci21S<3C#q%y??d4wXW!pSNI^-Q+`%NZ=>1g-39PRwyX{wT(_@(LCa_i9L{FIlkc;CbcS# z-W$f@EZ$=+-cl|GVFtCe5k}h}FymHGhyU`-L3H9W6vq3VXmnoZ zNJQv_{$GZEXH0%*mgeN&f|ClR=-(Bj_&rP#n1nO72H)yvaz=QTHoM|Gd3^KL?e%1tG=72g$+$k_bkPhgP*5B@wW#>gtg-&UtUg=w2 zYNHLzuL0tiW>KgY&crvL-05iv{q|}7N<)s=D7aC zYE7r^BVII0=8*ns%H~|lMroBc?6kfHb3klqG~2}W=Q|ehck>3+vAA=dSMSUhd~+Zt^Z~^JeZG_yUDcZ-q#2hIw!Jj&Jvt z?;Idz_!fft&TswRZzO1M|Nd|K2Jrcog7qeF_CE0QPH^#Fa0XxQ6W-@CfN%-#0uzpK z3cqkO!0;)^a1MWh3bnl-ZtLfSX_@|QLeA(UR-u_ruO=443svS9AAlI2aT%|18ozNH z&v6~E@g3iB9RKln19BmU@fUYoBQNsaJn|u5aw2c?8kZ@F{+bf~VCS?{DZg?oA1Al= ztu82W6DPUgPH`3QK%gV@(+PwB6i{*|S93ORb2oo;IFIu+5Ar!jay!3sJg;*)Z)85N zLMuGUEC=*I7xZLnSI{QyU={=`=zxw5RJCyqIe%kGpY%PibUlysJWujVFGEZ3^iBVC zPY-o88gxPr^inr<9K6M|h0r10Js=B!^8%#YG;`ltWZ=SdT+ek~-*sN^bzlE=QLpq7 zz_U|Fby7F>=ok%rUSoD6Wc&7KT5nPxQH;E^K_IwxY|phD(Do1Pb`ThL z_n$U*bWeASRCjUzb|_B)c#n5^2lsZLcY6jXiW|6V4hC@O0eozD|3ryAq%K=t7QYE>T+)Uc#zLCj}Lj0U-*&dvV>nL zl}D1dCh>w-HlrFhgGV-kXL*~ic@U5Jo4dMVIW>@kGmdEPGxH4C&E%r;fvb0%r*?1e_MgZ4d@tMwQTMJlDt_ncw7>eZFZ(`E z0id_`eUx-Rw|2Yd*-F3r*WCNJPt0tu`)%+0y)X8`@3~-#h|J`p&%XGlU(zq;yUHddy$@GJOHguflOp_tES8&kuOH&wM^7z!x=r&0qV{|N7NW`?B}^c@Oxr z1pL>(&?W})7dUxPB zA@lIz6D2bLIIxL90_4bzCo7C~t-vu?fmwGG&^ zRnv}Ld$w(-xpj9_ed}>1^1lX_Cr9jDrB23-6B!lV4x{Jbdr+qp$#e@IDWW`SJ4~Ui@xeeF>->a|JdRQw2JBfSGm; zI{09A5JvdePT+a=UVA->*PeMDviBi}?!^b<5hIdl;(qq==aXWf)_YnO7WX(74xtI2xFvOgq}t8jw9T_LFrKPWYpdml65rkU#=3XrF`@+NYBn zD)r=vi8{*Yh;Naxk(QHMN+ug-W;!M&nf9@%muPb8X_}>qO6sYlve_n_&$&9-WJtkq zClHTnnB8uG`nhYMhx+;}u)zw8Xoa^bdLg65iU#Fwl3w~OrfWotDz%+j8|Jmunz}8w zr4I7xst6``)JZv*aqbl?!Q^V371DPDtr=4ILS#QZq*JfF-lQ+T!}|NLt^iACtg^>S zX&b@eX#@te4LgkL9NS8pEyUAWT(P&6fNLYi9hk8PC%Wk3K@K2`ax%*5>R9X&14ozt zflx2I>~dv%2E1>-IOi+y&OQ6gv%m#EN~M*SGJG`BNh|%Zv`u>&anlYnZSf`=Z!B)P zCf^dW8myoU3fN(ru`bylt%P$Hqbw-EjQgjPgjG{R#dUK-hD&iw-#Uo z&Nm!OD|Ix~5hv~J)KI&jaTZu-&5735x&q7Pm}maU6CIS@uFiGCE&AvH8+v-&q^G{R z>IZyq(cpXw0mbY{(0;oXe-nO%;YTE`__K@uPUfYWnku>Hl~=vI=FMxqM-HD;FvCUi z$(yju*=wIC_uYG6`uCxWKlk|Mlkel*dE34|?(OFexbMC1p10v!Ha=~D9Y10JIrErn zJk9@{rw8egURu!tql!4P1y z0}CRNiA{Xs5~CQwCerU&Q1p)z_tTRrW>AAm`5+iaC_*uk(1B!Ri{Pa2z7(=ig>Ky4 z3ft%dIo@t~mr8;`jza*F@vnb8)S(Xt&?*}`XODVpKxGJt$VD=;0U}T&Bnf#Y1rhQK zA}d1~@R5m3-r{V_5EDH=*@?~^|?rHN2^JykMmNuXqiXc|eH6=bjfjkTizE_12N zUEZ>nbd=p2-N!UM5)&(q?4uw3*9mUC!g-yO-XICUfsV*>ny;j$0*>=YQQFW6qS|IR z%V2;npr8XeC@0xqdCeoLbClEbh#>_j&v`nsSmC?p_P7TWe0tALvQ#A6s5D1f&WwG! z)DuDzN>E-h)R!jwJ5$P#YSOc(KBecZSQ>&BXlR=(Wuz39*u;98uR2Lp5EK>D z)6Fb&UcPK-RkM24tzxySbUYy)3Oto7U_bX$X@-EWLp`Y=WiZdDLKQ~S8!Y!y0?Hvwl%m+|sX}RKP_GVj zmz^Cb3fV^?A!OF73(cjulF|f8hV^k{tpor;>d{Jslnv4`DQ-!U)>m;sk7v>?UZ=|1 zYX-KsI1O%Mp9dcJ0!j8Pl)0|?4j(Hl<=%zZ>C*ieXL@EnB7yWrd<$v zQ#+RET9>9>cF+u1qx_oe0-W84p4F;?KRR8&4w4rygo=b2y!%6^T zzP7Ar*w=(}L|_bKxCt>;AOa~oVNarzvK;2{f$v(#;6_oNQtWL5nqXNMld{7(W*3YV zEZ9A|n477s0vXih12epDzA}#IlrL;t6i8OWYG%Qw;yl)}YV5XnebaM6>B>_ zNlXL%WBVefT{5VlU60u20Yg|;TK*hToIFIGszAv^WG-F@!(%lsdCQzu&>U>Xu%V_P z)PGACVB^YGO`ln=dZwO~_3V<}j+YHJZsz~aKp`IMm@raSLP}!YR$(XL&xuwkum{8H z`r;AO0RAvWgHWSRGufEVh7MajcWFbeHq~w>?4!f~?P(#qDz)asc9>bi>2<$Z!0di8 z5Y7PKWfJwpYVGx}xe#n+6EoQ{zBhmg*aZ|X7~a}W;(`^NQhDxV$poj*V!s_HD<>u- z;=Z`TNxhOOu&#fURP{fr`Mdwb2m8Nvu^jiXF2G1S9;S~BIdyhzVK4f z8=Xd(6vLsb@s5|s9LTT-LBNpmmcRTQy6%bp$#eb_UvoJsxXz7OI79THFFop0uX<;2 zWA&`BM6=$tnN4~95w);A?sHFwHWUsIYT*6vgRiu{O=`Vq5j*meuYBQ|Pm6wC?!7u(ic)$GL|Gpi~e~0t4pZ%G~{3YNY|M}Cu{-(FZ z=<)CR)4M(y{{KJg0buFDf{5wHq(nzDJb1)d=Et)B|6pa`-cAGjd-`9b@Q;0xBE48|ZJF#gU>cU84aT7yCPEv|AsrgR9U?*?(i>pm6=tFqZekd!VJE&J8-k)8h9W77 z;wY9PDyHHos-h|Gp)0;3EDB*B^5HBBqAU&~Ar@jGCZZzx-y5t?L5#*TX#m6-$?Vh~ zCMIJRe%&WFqbIH+D?(#5{(&@BBQ;KAD`KNH#v&eeBQ4gVAO4{&h9fTKq7mYvE$pHp z@?QY9qdWTIFRo8MWsfmx0Rt-kqyK5*GdklwYU4HfBS3DWKxQL14x~5kU_pMPI4&eZ zj-x|1q&cReL{8-Xu_G@!VK2?&z!*w*-6K9C<1+H2NCqTHmZU+RWJ03kK_X;ILgYi1 z<4eZm5vn6aI^YuCl16?{6?kMODq~3YBT$;;Kd$6Z7Nt?DBugTtQW7Oog5xd1Bvi_z zR7NCC*5pNEANXJZ6!4@zc4bh0r5i4#STS-K=SqUBV+Wkd?1Rn8?{ zdZg<;LIGx_R&u3R{v=q2C18%FLZ&5Qx@BRmC0q8OSsLbHCMH})B_XyYRMuow)}>w6 z0$OFIJ%VIk)*c*q9Wea=LIEI%osNhIOD0g>vK(_#}pgr-t??X%=XL3h03% zXo!yJhc>8*KB#_1sDw^Pg;FSpW$1nuf@j94PU@XNm4;j`Apj65?%wBo^W) zi?%37z9@{ICymbkD2EcKj)EwGifE5Y>6H4YmCmP4rYMAJ=>ukFgc#|OcIT3g=~=3w za}Fn!#w3(JX_|T{b6)9*!YPn$DVOf&?cFC)2@04xhM1BmlLmr}N}!(#=b1(+o3`np zCTf++X`D8yqb6xSP6Cncsf+UFpr$Bd*65qMsgtg$qI&A5E-IrwDyhOD&8iWcdYRw|~ts*(mOtRCyD&ML0b>a8wo zuF~eHP9(2R0-jDNr5*{fW-GGB>ZV3(vns2zhU&A5>$pxUPpayw3ahpn>$)Z@r+zEA zk}JG2tGAZ_>$sw8u(Il2wyV4L>y6TDy#8yr4lKd`sI=beujcEX_Gz&;tiL`izy@r@ z3T(kv>}>iV#vZK38tks(tHP?Q!%D1;TCBWI?7W^Vw5BY_)@z-5?6tOP$nxvRk}S%W zEX&et#;&Z7y6neVXv~W2%yz5I;w;eC?9HmIy$)+{@@&uI3cJcI#1gI1PVLcFtTRB0D~0y0;woz^elF;W?&!WQ+#c?9aqZgPuG^xi>r$=kj%w)Q zEbXFB;_4^vifHb7F7ocK^71I`j_u`UF5i|e?{e$%>MA2t@8e=G=>~6@+HUdAAoYIl z^p0=#N^b9-Z|v$S_i8Tr9&hq$FZ`}<^M38~LNCakEck|Rc&FR<=@FbMx{2e&WYD(3*> zi~V*l{vL4uwy@W7FbpfO3wv+^%WwJ;uO*yt0gJE(@9zwUung1i4>vFoSL_5+aCU&G zf#wx41_yBu53v&iF%L8G6i2XIPKjPNF$qVp6vr?Ze=!tqaShLCARciF7YXlKuogdY z7kjaqx-k`paT|B)9FMRX3#f>lv7spO9VagfgYn>pFICSP(US8^4LF^8h|5i|Z(x z@+p(DAj|R?2Vp<}03rDV1rz}P04x9i002M%Kmh;<{{UmH5}2wKs)M6Eg+i#x6vKwH zA`+Y^u_CQm7&8vkXptkUjUYRIbmeiR$crUUQcStBrAvneW4=5k^C85UH$&#E$#ZAV zg)nLU9BNZ2P@+YVCS9uVC{vtHmHJdk)nrwxSXo}(3RS1nr(BIr9ozM5P_bywnoY|V ztXj8l6Gkn&Hmu#OcuDHjn)j|>vUC3eu1okWskVj-D-ImD730Q_=}P81d9P)`l`${g zJQy)&&xs?84n4XwrNf_HS^)YqeEA2{5kdO%d`Kkj@0>f@4kD({}LYlyLs%^(Ic13KDYYzrn#plPrtnM z>F@F9Umst)fA#v=SKol=$v2;Y{JEFlehR|pAc7AbNML~z3Me6j6lTcaf(`!1;fJGv z7vP8$Y6#+mCvKSHiYj7wpNJ${7$b@^wiuy@{n3~lj3vs*V~r;2$fAun5-B8*KKAG$ zl13IeWQR*G_+yeUQpseLS2n37lt*T%q?bW268FimD(a=0a@whN=MuQNgS2ywTp`ExYTA+pM|V(rc){Z=%c9y#lK%r>X|GOYXM(Iwr5f|L$4v#Nq0j zFTMb0+^@#GVVrQq3j4`1!wW-8@y94zjPSl2mke>ltETL6$TYj$aLY5xyt2(a<9u_@ zK!aSf&m@}~^u;OOAC2_WNiWTG$t*ivv?x$ZJ@uwm3w`j;Sl{>a)>jj~l)WdX z4NBH)d;N9TNtUfO+0CLosm^eN?e^Px=l|XJ+;mIa_1;nc4fx?B3$FI!Z2Rpv-;YO* zxZ#vTF8Ae@C$9M5m1pjG=aPqxIq0L8uDR)_kIp&aqPLDZ>7T!Dw(K#l?mFza-=6#K zv* zZT|V?dtYbw-&Zfb`_$Wxe*5&RUqAf#Z%DrO^V`q9|Njf100Ef40=|zk`%9k!3CO?! zI`DxIjGzQ3*ueNLuzwhATm~!1K?-*8g9Zd4ZFJ!s5;})q*1`k^_s}*Ly6|f+TpJr+ z=)xJ=5EL_1;SG1l!yeAzhd>M>4*!Kn#3BZ94Mt3&5^K0bCc>+UdLV-trT~U0T!D&H zB!d(GMZ+$7P=w4uq8P_W#xhFrjA-lv7KgCLCj{Y*aFl@@=SasB*ii*{#A6=!sK+=` zQH_B7BNYNk$U>e$kVHTv8x?8AMmk}UOMs*!Bk9K-P*Mk#v}7jZxJONTa+7roWhXa4 z%2JxLkEj%-DpyHIH@fnbS#)JA(`ZI0dJ$|nctHpAfCX-Pvzy>FWFp6TO>&y^oQvcpHP>lOcDl2g z@N8#1r8!SuDuJHS%x6CD$^T4!M&O?SEkGg-5KsmfbO8mO074haP=Y$tln;%lL~~hy zic(;r5v8aeF?vuuc9eu&$fY&kiB2}Yv!4m=r6jmeid;C8F)tD?z=}Rx_QnfCXpD~4}Pn$s3y3WS0y`HuUb~CmDq#|^TCO-7PGHTO>0|II$F^h_Oo&=sZRCE z+Sc-wsOkjmUj;kNwf~}(1Bfl`X|Ku!7ksm;#k?$KhfCVZ8uz%D%`7K0JJuYaR<==f zYiZXy-IgYxx-y-oYF(Sx?!NZBLN%o~pIgnGlC!tDO{{L;>d$#fH=)FZEMv)u-1^#A zqFkuNG?a^p=0Z2L>&32V*(=Xr#l_y>@u0wd!8|<3kughI~K!<{xM=#UF9o-70a=X zwze59V*!WSua@4mxLGUfU%&a>aYizzKa6a{a@*PM_A5e@O>HV)GYxKFZi-#0?P$yS z*73&mI>$}$^Wu1@zApD{+e}PTKYG~$r8m4A3fEEJ8P5@5cfE~!Vl1n$kC=XDVBZb! zei!`Y2meQScT?U|UEA2w5MSq_-K}SmhxnjsrZbHVD)Eh9HkKCV@UzY;z#To{wyN8@4FCWDhCbS! zPu#K{&oU;a;DH+7IkSZh{VPV_3DSq24k#Xk>Q~SDKD7Szu#dgyO;7vU+rIR(uRZ8@ ze|p}_-u1u_e$a;>2O9#t`1wY@@`=Cv<|qH+Jl!c8q)&alV{^49AL%Yz@JXvHe$`5V zHUAj`phvXVox=EbKKhYw{Xx3_{f}gV5$4~0`rAMLZO8xn)31Kv_y7O=7l8cNe$#hI z1ZaQ&@Ha{F7l9Huf%jK|J%9rjICvNc1Kihv8>oTX7lIx*f*_cG^+$pF z2Z2)%ZraC5?NkP+Wn9Oy2SMOj1*UTXsDAh+cMq6?K8S)p_=9uzf4}E{MyP`Yn1ceC zgiP3k3fO~C7=%(dg+G{qZTEp#Xagj;g(JWMUHF9`5QZ8chGaN~W_X4ckcMceh8eJi zAmD~@7y|jlPnGp?XUAd-h<5^*1yo>vTnL6`7>9reh$d)#O1ORin261HfQQ(KPXG9b zPZ)`kIEj2%iBU*}Dwv6y=t!5yNS>&Ppk#%WK!?U9VcM2cj#zi@XMz^kimnKR@yCY& zD15$mgo?O_keG{4*atk=iLvO5m4x(2BbJyCTDPahkLiz zdqv26wn&7mcYD0Rjo$c;xfhP&IF8m>i{^NasYi>~7<;yuj_rtg?+B0aIFH$ggvHl~ zA@Bh!I09%uVR;6P(I{fm*p0NedUS_>e;9~th=zlBhFxd`dbf?>IFaQTeD1i8?0Ac~ zsEZtFiNKhG$ykDaD3Zx2gn7t>+L(@EAbdTF3{#{bBH9XN<1 znT!z$dK5{HLWzzQhS16f# z`In}7nq=sOstJaH36U&mnEr^Ei7A$Gz?ck(mRiV#K(L!dz?+v;fj^0n{b-QEiH@Ba zmvRZ3bs3t&=!dx(nX8!sZ-|7McVp8EM}{Mnyn_n-O}pqh1_1bUu5G(#5XMD2n{Nsnve+(a~Kk8p%jXt8%m*!;GrOD2T>%VBs!u~qy{INqGeD;DC$Zb zm`yFvorh^zi8-EYxsCpiqxw(}_Q0b)S`Y95q(VBRLmH$-dZg|^q&}*nOxmPQ+7C|} zrBVu|`9P&sdZk#pqgA@4V%4Kw`la&_reHdzWLlCKnx}|rr)j#VJ^wnTj@qS<8mU|gsg!!A zVKt_VnyF=~shrxWo|>uZ5UQd&s@6cKrh2O9@C>TDs;OG4sLHCa+N!d;4zpUT-H@oa z@C762r~Jry-f3C}AgBXLkf)~)I?ATFnyi{?tdgn={2-~&I;qlnsnc4iU`nc=nyuQp zt&57S-ukMx8m{6h4dZ&N=6bHQny%`)uJWJ@x(b4Q8U=sam4eoAM`ojPN0!EVsLI-} zjLNLh8nDuurPW%n)~cr6`mG4NungO<4GXFX3$6+)u@oDt<%+Hs%dWM01tl1-^D3M6 z3T}ExjjXq?$m*}S+NjSeu(v>|0=u#X8?%@iv93z04*$ERH_NRP`>hpQvl*+gKMS-% z>#iMpNIhV*_!f3vC9GnpuO@r4D66Pq3a|ldFJeWoFAK9WJF{)-v)-DsIvciK+p}cr zwH6z+Xe+es%CQ<41%B$9AB&h#CbG0isD-MfP#d*UtFl>(wS9ZAfV;3f>$PHAxHLPq zWox!+ySQqrwz*og^XI2Y3ttbFaN+rtBWt(%^|X1bw}{IQeA}#3>$jwu5RTmeQ4L6r0d$&8Ra9 zyv`fH@|(H{d%fCAz1EAr`is5yTf5uazC`=I;LE#~SFa;xzR!iet+&4Hd%W)pzYuJ) zs%yXfOTp8-zn`nP8tlIn?7wXLqF5Qe&RD>e>yOc>z$Kfb4eY?6`@j)gy7OD9Eu6s} zT*DT8!8gpm9K6Hp>b<>6jKxR7B^;0_OuWUr!VV0nFAT%ZJHZ=l!#ZrgIUL12{J&Pr zy*|8(g*XO6Oi-nUz<5~1u7?l)z^{o4z|(uROU%Tk>%>nC#c;gAI6TEvOs>>m$5))k zKbytjyTx5>02Ow^b*s4yjKpY+4Wk;mYyZr~2MfA%9LH}A$#opbcwEJ3i@O3`m@u$d z)>T!%+gD=T2NG<^hitQoyvQ-!$m~GLlH9++(8`l+$$GrFeEhvQFuoR*%N3Ob6b8zn zOvYw>#*90|sEf+aTff)}%d)J@th~&#%)OXw%O=>nzWZCJ_{(K{#H8H5h}_M@T+C5w zsp1T_u*=NaTgm3!%+CDG?3}K(tjW~ORp@0`=Zkm4JI2|(&F$>X{A{M$e64H>y>mRn z>g>wvoX!V*$I-CQ+#Jo^*T;srl7BW{-!;hTTYLa5%nZ%X{cNPy@X;V0(vFjQmxihJ=N+eQpm{3rPu@qj7|*~17^tsVNhtSnATMt*Mu$DaQ)V6?bK|o*o@uS zi;dWa9ogt`)rBX`5lw4ab#GuCZ6okV52<>rY1m|4*kxVXqAl5S?bfP|+6ztERxJfJ zoz)!o$$15AZPx&Pt$MiqkWW3@zWv*$&Dx2*+K=tn#eLk!E!?d=*g#9yon4nefXmK;M^vO*-|x!n@x2ZaNC`2;rfl?DbC?2 z9^4>);y!NTKpx~fp4#im;%^Jz5Kd=sS78`e-!$$7Hs0Mhp5r<`OV<$2!anttUA?%pqI%Y#mF zy^OgM9$1GS<%;g;s{ZJ#zUr6`4X-}woKESRUhA-K>#;uTVgBibDSx5fQGq;@NziEy zz#a){O3HxbA6)@QwTIDi7@o zZ}ATQ@(*9}GXL-Lea`3^-vSWbfcGZEAusY^Jn+W;+;Omwj*z4+zw#HK^fOQIOyBfN zAM27%@lyZvg^iOy9Zg>#dT*UXX>4O>}&1-2Y<%kaP-A6^-90?R1fhn|MqR4 z_HsY>bkFv0{^w%;&~AHofXZ`;=2F)L_F+Hc_(t@Lj`nq*_=>;yj4$yt5BGK-_b^}c zHt*@-F2HOs1a}ly33cu`2v?f_0E2JxDS+|`fA)ya@@n7stUvjZ@AR<0`nVqTuV3|F zp72;dczwO`XXa1yX1Nd+_+X#zgx?2-kNT;P^vj_7tzY|)ANy9{{IWm&&QJ43fAhPq zNU`NsbpK0iq7Uw*U-Ze({K`-M(jW2YU;XJH{p#QTkigN)925PL4i{HNYv=dqu7$tWZKl}Q>Y2|q> zSTLCqB}$lX%H#Euyw9*khpKm(67ar_Rp6VKpb zhltZfeJIr9+8M0&0o|PIN&Div})1=)^wTksC*RXCy%q5c6 zh5ziha^cQMLKpW~y?izN9V1u)Vb$OgFHF|Fd9cROLs$H4nX~niny=^7-km#k(c!0$ z2VdI!`P9izvu^E#3T&@k>dv)$)@&0ZLzcAN-Cyij07Y;>IO5V64!PwTY7Q6%pG%Os z@*b>^d!(uGl>>C0E4?>WOE#7KmOSik~ z^UDdpS_tgG0Si;^!CV?N(!pLhj0nQ&GE}L<3%_%6N-CjjvNQEU9C1q%V?YtU*M5V5 zf&_y2pn=%<%a1o40}PJG0)ZST%OROt@XiLSWb#T6HPll|Km+~LN{ zeXyxHs7leQ_`Cp1hZf1~VYn%7R5Q2V0NnAw;^ZXg)=?W=uvF=mt;#iCV|x@WNvtFJiq6-Y>b4T?74WVm%%;e&~U z`Q>hJwG2_56&1HpT`lglE@1Q0SfFCz{TO5v@ZywYPxabM1C&$l=FSCKmj9YyYf(+O zW`wz(*=w!CwmD+5%OYfK6@F1k&m>;2$fU(eikg{U2(x@u|97Q5lG zY5w}~!3lR5;+)NP{OmP)hK(bgVse zr*Xy~hg@>7D8JZE2$mQOKh0|lS>(GtcN%o5r~Mnr(z9mW@YM@%e(Te9mR@4ocV#TQoS_i2@Z%teBH#(#e;SIEPx)f zk+C*sH@wAxg?szLVv;vQXQ;1+HuT{Gb0|R_cF~I$wBHYjXb=sm?L_hEfd_8`K)RLi zggIIk(daP60}64Adjum7zxc;K0y2lpl~S0dJmdz|B^y_^G6F_}Fh%)EU;lbCn+_Z%F)>LlL~120 zid5wFwh*jVHdBuMvCA`gU`szzvzoue<~FstxJ`Dkn?vNMH~UErZ=_LemCGF~mD$Kh zLh(&fY@rlx`Arzglb*e_XGO1>xP4|6qG&WG5NN3%5(w*{3rOg0NU#GHF7%xbkft6b zI?;Q=G^R4Ws6TBQjx$c7qaR%hI@jdCk}7m$cWmJs3i;A`(p0GyWolBFs?nov6ftH{ zLB%FV&X1yS1Fq~{P(#SHk2dt7QbnpJd-&9|&eW+j%?MQg`Bt(Pai?rM7FXSXDX)%i zimwrCS>tNWrq(l*Xf>-g0Q;DmCiXB-m}}ekGpTr~iT{a{<=s|!@YTICl%*DM=wJQH z*25Ckv!V^HVof{M#>N1%ki9Eq*_op`s&;^UA!;-^Ym?9V!mzv5ZD~m>*tG`s7``nm zamxr-(}L7zN>r|0N!ZN$tW&m%B%o%a+1cFk*0{grE@*@M-R+)MyzhWVF&K+VaYn(T zU@1=9SUW;>Mz@3)m@W%xOEQ*9Ar<8iTz13T-GQkUy#F0=fd}W-@tVWDsqL6$&9asf ztSErCd2M~Gi{IP&SGyY~uzus*aAnca-!gs#zG?@eR zt6`jSxWhXZZjVF!V-P3UvnNI`Ry-6z`?3Z&3tl%*_UAzK;Ca~?5}w;a{DnkmIz4%0aeY9BIJ zSkO*(LK@(-H{2qIzino;o9X;$NJm=Ildf});~dv(EV#(GJAs%34PE-qu?b`$bQ}(? zfg3AY)vI#RqI^FGdcf4nf?03tW-W#>Id-&W1 zu5{YLWa_CHHWx?#>DAx|K)AXWzHXB5E{LseJpZ;^s z`^T<=r&$6LkV3*!e)NduyV%^`YoiyPSsC*a=#k{Mw^lbXVC z3px|VaPXlITj?h+b~BK%_Nh0$;ZTn{H|KqI7;IXJWTMp93lNt%juhxs5eQ(iK=QO3 z{V;H6yVK)7cey*h?vH=@AjD7uzCUcnpl3F$`AYIOCR!;cfR`FPxIXKx7OFk{wf8}&f$L_ z{Do&5@)d9X^J_iy>#f-%zydr#+-tw*V?PF5K_>(p}z`Bgfw_mv&0|36j%fCrU znszb1{u{d06F?GVJ_kg=6HLJe%suPFIr|ecgz`ZB6FU{sosvns63jspOu+QxK@@zy z27EIG8^RTgKqC~Q39Kp?L^7{y!dEgO4!l3^u>l)AKEx9O)_ac?LO=Y&1}v{4 zEJQ=3w6Emlv*}lIMyErTW5c)X@*b4ref;yyuz5&1hbiF*>K`<1=Km^25w7Nq) z#WF-K`m;Dyx)c}`wKp^~-lM}S+(bP5M7PsJPz*&}G(cVy#rQMAGN3@L_`KObCec#@ zvD-jMbizsuHd?%{EbK&T>_t+%Mj!OWY_vvf1V(S%LkJ8?2+5MtUs8R1`;3Jd9z~IIOC_8hJuDTE{k&La#7RbxJ^vg3~17~SS zqvXufL`>i`P5;si#&K*-=IS>AkS*G@O@v&U7!k@8fJd?HP2UX8m>kZP{LSnnPWWR@ z7Ys9x451#eOy-=z%>2vfq@vBNPT|B((JW2vbWhKm&-iRi`y?;<^Uj-e&5&dy@eGnR zC{L_(m*S(P^khi(Y|rhi&+CNG2BpshbxOumO$x0^<2 zPz80+_l!^mO;HF%(G?}a)U42Ylgm^x6P-&Y17fVLyiJo-uV)0zE3|_3BvB!)NELn1 z5|vNxq&Z`%${p}Ck90-XVxnYRlO3%=-TYDN%+e!0Q7+w5Fa1(7gwboL9yF{SClx)9 zx2nvm?)Z^hEZ6IXII*K+;Wenr=HW!F^nQ)@+2W|Xgi)7CQsSJea3U=`M51=wPx z*#C4D*oz%lf~C1VfYvmmOHOq}C_Eu$OebnR)c6S3h?Uri_1B3FSd7isjcwMMdspIv z#Wg+1pF*(==+%eCh?Qm8mvz})Y)_4)*`t+7cO}Qykk&tQ#5gQj{X^LhJPs%TTA|Ha zi&fRFZPgr1TCm;NGyDov;z(ejRj7?xSZq$JwLu(QIJni?q5Z;^o!fJb*_l;ZgVnyY zEzgF%M8qRH8;V=Pm0P=|TbPyEuFc!9t|N~jU15A&)io*OJ>IRF!^tVP z>2+S)rQYgIU-c!+?A2a^J>84b)pQaX$)#Mj72mJ$KhAw#++AP)RXWBEVE9d4osCUp z8d-y^Qq0v~^F80)bo_KdzX)_+%0m3FxDPwhUO<|1zTW^d%os{!81qP-sEegLJ%& zgtlfNj%ZVL>ZdMNe~#Xb-f6aO;`Y62q14o%mO^{p0HNtV9UbeYK5Lu4X*8BCs2O2c32+k;{(Tc`zA_~Pg~Nb&Uw7!g4s$N?!hZ}k2G z^JAZ}ZM?dCu<)-0u$XZ)py20UvMzFK`1t@D+%W|3+{I zr-223a0owe|DJIDu5b&t@GGcgA}9ll>+oUJaNantSr+jTFL81_aTCYX;0=S2!fq45 z;i684PzZ#u{=moXg1{`d8VlSV=W!qZaUc(JAs=!gFLEP4awJc3C13JP+$^3f=oFXH zD4%jEuX1v_^3Cpn*PLq?;c88A=nQR_*%I#_XL2(y^ZzqXb2Qg+Hm}1re{(gDb2xW% zI?u%YymCCxb1T<#$kcP|)-|mz>Bp?+gJ#p9w74~&^FUVgL}zq4fAl(sbV+w~N{@8C zGHgsIp-jgMH1u;0;%vk|SOHJ!ZT+e_w4F)U7gb+%R&RAze|1=oby=VFRHt=Yzja)< z^-J$`Pw(|U|MlCs9xf+75$9Xhm4QhrC%QU=acXvF&o?5gxFP@oYCocCzjou(3QNp( zZD#>*pT%vrD&FJvZodI4Ga#*e|L9ZcX$VPdarkO|MqYn0emmXH`#Z8 zPp|g~_#+eeTq}5eJ9vao`1WG2EZ_1_U)>jJX#Y|0ZkJ|L97uL0>TrY4c%IVu-1PW? z2l;0od6F;r3|>T&kM<{U_LYZrmydUqk9nDg`I+Z+ct`n{ulY^CID-H9gtu;>|5%&f zc_WbGhllaz*kO2uTx6duvNp+-%5#5*`f!)`c+b_S*ZIAXlZMrLsDFB$&)g;tdrP-^ znzw;P7eTT|^KY;Dr_aQhKbNv6dUfx4DDL*)%{t-Wv!bVD7w1Y1TIom_>WUYZQ~&z6 z@5z6kday@#$ESD1*W7exd?#14#s7AbWOJ*({K&Uvd&m4bzxotV{2b@;&8PgprTWJo z@6Koa6p;4Pze2_5v_w*OarZo6r%FR(`v0S^DlJfiDmPmR7yaTp_tm%k%ZL8SZ+n*~_@U4HlrMj0Z*1%yNo6E) zQkP8(OZ?Dx{j8t<<{x+c|NQL-h$2e{3VhO|Adnvj7cxxf@F78n4HZr_SW%+IiyAj_ z%xGj|l#oq{xygQBGm$q{YdY5oOXmm@>^v2s(Ezh){rM&z}Z{PB2lF!v+CL z6D(yKq=5qqQYBEW8r3S+tXM&C?V94l*RWu3U@S`!2U!+s#j^FesVx|N1v@-VDdOg&%)7e&+i3JLlhDv2_+0 z@d0+zZ3dHO2ws+5WZSvaofqDPN1k}&tybO!I=F_O1{~hzo^4hw5h92slBm@IDV_)b ziS>0zp9e4IcZ6Aqxp?D!$^GYFavsfOQFRVN=*3897-`Z?XehZPlWc_1B$QA}I3bl5 zmezUhl1ifs9~&2>%3&CYre-nrW)ICYx=_w!({X!pR(OUE*k8kN-JH#*stz zd8Ctp3M%L(eWW4fpozw@=%I=-YA6*|Qb6IPlwug?WiUh~(47v!nBqd|srezPqnhgG zsi~q`qKB>KI3q=P!uYCTeA*fsuD7o7r=yM<`Y5Bl0xN7El}vddX*mS3s|U*NA;`04 zbTF-@1ZWwOrs#dCSePPascNa=g1hRtXO28I)g)V8_0?EseeKIocwNQUR)npE*k#L12Fz!Z@kyX8&%E=^bI6^qz&}qL zG|_a!g7n_`A}wvxfe)0i)>|8Xc$tZpj5y<2KY?}|WFs;8*OY@T`Q?|NO(WT0_>i{T zHNU+l-2GNoX;YvWoOj=qv|gy{Pb}?p5=drt9qtNRw!7}Ux8OVQp9&8a)Wt__{K?2G zzdZBBH~;+b&UdJC+HfC36B>P>fXbijp98jy4j6kw8w1R4qYOMHK-gUjA< zy$xp1Y-u~f8y*+Jn2C;i;Cq|*j1;IIAfS9dU>zsi*S;C{Z4WsJoTX4Ff=ZR3hdcZs z5Q9h=5$GUyMtl;3(8984g|2`*Na7QRXvDEZ@qh`uVgM27#0EmJiwyLl7P}^e%yCc% zWjtdCwTDJCvToc71$OvHKMrz3aUvulDe$B* zBw+-848k?8(SVWo(-J0~>2k)8zPWIk!gL}F?`v$BsSF&4d3zN?kN6Xhs5 z=YwsD3I9ykYaI`$)<``mmaY z>|r4ZNd-ELftzuZ0a5~hf*X=ka8V3FHHoNB4GD1o*?cGT2=^vf{;rRgryK(b z&kJyq3yqv6Mwv!}@)2O6O(>!|6WU3U!n3BMbms#oAjRDs52rZYfkHiM8KK6Y0YfmP z;${<6h7xsBK4qmpXIapxQuU0s{3k%G8U%^}lBIuqLLGbN%t}zSP%)BNMkjzFqabCR zH2;0%m6pnUtpj)urGJ!Bgj`SHm`vs7F0&VrjEZ z#=3wA3w>*3w<1-xP>z?I&E;l8drO20w6s|T=o0E>S-5&N1w$Ka`zY$ZGf)MvXie*F zuQY=(JhGc&D1knSVq2hm)271>>{Byzf^#AkuP|+)HP_kFul}*8U1hBf@w%e0faWb? ztL+Og075O;GDvulrQBZ<3qz>Khfqyz#2A?3g4_@hY{i{giTGzO7#o>m)8-gjYwYe%l0{`R) z^F2;27oA-hfKjOF)Z}KCz0E!AG%pb24>z-!_6-;rYy(&;sRjkPJ#vu;L0|zhQp16J z=}DnGKOz`)uhiUXVvk!`ZCci(Ft#jY<+$Z98~43}^YDk$3}PC$c4c0h0(Yg=xxX5i zesr!Wd|SHOBp4a57;;08#~OvGAUTKqZEFJh>JuF=`p``g12-cLX-I$6z#El8lv|7f z1RI&17bY>A)vVGX>Q~bkKJ1O#E8$Rs63Gxgu7lUCg5}D&F>8Q=%ji6VM7CJQHr=tF zp^RbdqVx=bChL#iK~*P?v}>;CGuP?d*Y$4dH)>82vjVF)dxLj zkQk@!xIwG+YJa`!8|DewFRW;`K!q*1Aq(B#-Wh54J@A7s{NX2~_}nSLr3^|PO| zr?pQm!7y*si?06XH$MjRZ~y(vOaE~HKOpdd01hA{#GV00!U0}_0>U08ET9Aa9s~*> z&k-L58lMIF9xTif=d=WU*ag2#Li0tR2vQ#jl3?FmANt`R3EdwHwjlPoApP+l49?&k z#GMWLpV-Kt+Pq*4@*ocOU=99Y5Dp;$5@8X_AOIfWA1q-aApe3PD52{ef)qZWBvfGo zQXvEOUIa#91$N;D7T+5vi3XxgMrcdpY#>q2$moUz9AgOp&ZVk5b7Nb)}ayR zp&c?I6#Cx{;vpdJAs_x>9~Pn^24E6Cp&};YA~b>`IAIe;A|qa*BU&LPULqE9;qQ4N zC#qpC@eA%$2L_x0jR4dev|$>$p(@fLE4HE_-k~1GqAb4RARgi@-Xbp2qAgA$FG9j1 z_M$H$;1mWUCR$=KW}zk~V<$Sp7bb%+2@@;zlZwCz^ra#yuHxQdqYt`bF6v?}cH=h! z<1UUPIF=(hBBD6{qA;$bF|uPjX5#F@Bk?ICGjifH>i?2bdC8hk<27od_4y+=ZsR$I z<3JXqL3(34668TjB1698Bf6tQMx;a{qZZQRGBzVLnvwLh$~2y0HCiJWMjt>5BuSQ} zIVvP1E~HAHWJ02(I=19XPNYm`B1F#Q7ItCpEhFWO06?MFK5Bv6_2WqX<4^|TN**Ol zq9al!rArPYQ`Y1$+9Xs`q(xTbMFyV+`Q)e|!AFvy^A%+_2Bb+YC0V9qOrB*~qUBRI zpHog{R7xdHK4e^0rBzmAPD0;2dX1A+FtFu4P)vrCK&( zV?L%^wk2c|qhwZQWWFV4&ZTBv;!V<}UE(EZTK|Gn6^?ibrci?ASo)!2HY8(CrfagM zQd*{Cw&rZYb5191N@sRPr+c=ic3!7^awmOK zrG0)Ua7G^~{*HLEoq7HzdIl&Xo(&ue3Gb&-sgm}Xxmw+aK`8vgwSm8rH%UKhmvQG zf@o7RsDhfPkFF??q9~EJsF4b(gdQo3V*e<4f+v3#r;hF@VzT9w3TTL)D3%6kk*4UD zB5477sf#M9m~v*6%BXN==#z%b3UH{DI;V(gDV%a?m(D4ej%kwGscnwnXXY&DEp&sg@BC4X+DHeh% zo;vDWMyjdKXm~=YPy%YEYU-wT>ZsDGr^c#mifWjWYED9`U{30}pctmMDwVz}ta9qG z605BqE3SU(t#0R;(&(T5DzMfnvd*fpQtPxPE4Ef-eb&OQrD@1qYJoy)v|g*V7Av`) z>#}C+arWM$c58>Is<_5!Vvei1%Kz${sw=wEYPzngw(9DB?xm{uYI#=cz3OYd60E(J z>%r#hzBVko?y9#|YF~nDxFRgQGHk-WXT};V#!_Sg_A8@GYQ$2Fz*b+yZtTS>ti>)Y z%XTcs;;X;5ou67w$)0Sy&TGoHtjeP7$|mZ|{w&NwENnj} z2Cci2rVv!=(IRcqDs9hRt<5$q(?adjvMR5Z%ha+eeB!Lt3Mh0n3ZQGJF7JwN;)ZJNLLjbg?uM>S?z-ykLN4<5?&SI|^OmmPP9z)LF6SO^^D3`F zK5rB*FWPEv<}NNKs4lp)Zt3do^>(lKey{1Cuh-6O@meqUVz2qK@0ZeV{Elw<=I`&m z?Z}P``SPv&2Co1g>HMlM?H+KC#$)t4RN!&Ca(lfFyYp&{nqdHS}+FR?*?md z?BXv7>u(4HFBE$1ClK$qQm_iIZ~$lU1;4Nhb8rmL@CUCi0^6|jvSau<@aj_R4Yx21 zkFX36u@DDw4U2I4x|bcjrE19FU6kT55EF3`8?h8u@f2V25&y8uqOjPquop+M6;m+* zhcOva{vH$$h}N_L#5}Per?DB+ zu^k_>9wRdPCUF*)P#g0l%!JM$)2SR+@**oTCMR+xUos=Z;|Yren$Qsmcdu%eG8hx` zCa1C{uku<31OOrV1O*fT{{Soi0000$0YCu&2>$?82~1^b6e@!U3C3Dja3Dj6vK~fc zHL>EXixyo`+(-*!Mvfmfb{r`(q{ovcC8kulGND42FkPO4Nb_Y)n>ZWJytz}S&z?YK z{uCODXi=j>lOjFpsj1VPOqWW98ndNUlvlB4-O9D zZQ;r#Tau*Kt|0H`)l1jxs=s{$uk|Z8?#mIxix3doj1#dJ=(PG+qg^X-hDbXYu~wD&!$aWcve}4f8M-%bmrW{|NjQxy?XKU+R;m2Z{0ij_wZMXAHQ9B`upay zM2v*3Sf(=qAVT2sc$6$sUUWj3c z4}!SihaGx|Vu>WSD58rd!nmS}G|I=KcovSRql^W{*kg)71}UVEJnr~okw*?W7|uTUioE~WOg~Gm}sV1rkPx}x#pB$s@aN-0-jll zociUd=az838Rws2LZ*qJIoj#xo_w;YsG)!|87QHL7JBC=llHl&n~dVAX`PPJrHiDX zR%$7tiHdq@rl>ahXsV=+S}LZlvj4hin31-6swl6*dTXq_s_JX4w(c72u)y{@>#?#D zo9nX7ChKgc$i@iEEz^eD?6t~1YwWM9LR&1h+kP8vv~+fB?YY#VD{i*hk_#=Q=0aJm zuIdU)@49IUI4GyI;>&Km_xf8dw*$w^FKPfM46eNby9@8ZB|`jgcnvR%tHtAHJZHfM zM?5jd6$i|)#st4xvSa&xta8ZzVob8bFn9d2$QQdj^T;vhd^629<1F#cCxfgr%RZZ0 z^w2?@j5N_b11)sYP#dW5(JfQ$bks{z%yheyUhOs3k}A8~x(S zO}Bma$zs<{cFl3St@m|5%m1CVeRcy|-UthiU(Mxar^xucC{P^ORzdiTnr_X!$ z;0y1*`tWBTKl$g&KfnEcd*A;1_xImE{r3V_E?8o-*^gl0Cc$<1zhQ=6RpqGa;HVY&CrnWafu=ImsV9Y{MO~oFioVjSsm!WYx5`zFQuM1{&1F$d8q%^x zm8>;IsS>FA)PZVMr%&Z+PH!q#y4v-wc+G2Fzqtuc=;9unK&M#83R0xTb*WH2tWnR( zRH@va^|Yo%?OsIoOOcmb9K#>sn`-Qxv3D zwR@%Qby>^Z1st>!uy`%-+#p!m5*D$!?X7MR@LZ4n*0}JEEOK*M*^R37l_o9dLia0% zp%;35zA)(?G?=;L{N%Pj0qq|)&deyt$#Tu5aC7tXl z<4fZC+84Jan65y}no2f2^q`m3Y=A*5;|(MBz}bDUf*agjyau4fVu^*g%U1TVG5tH|l3G&FT!;V)c5LHDn|aGzPA{6} z3Sm0DHoWsHW}N>l;5sLOpB&yap07IPJ{LL8oz!JE2Q5xPyBg4>jq!hv?9UlI`oR7D z@?#c;)4+y>m@P|VPaC_#0^YP_OU-BDo|?IlW;Lv*9bmXR`ql`}_LzBXU|$d7 z(r?bAvdvpgU{i|<0iMmUF~R^2YRYAZ|uZ(&U1}FTjA9HxQ+Q8@-(AX!_h`Lwztaw`9KE0 z=7Zxg=Gx79dPi{Xn@fGM(2ndEsnBz-JK^V9hqky&Np^cL)9Pq1_kdM;_H?`b*q$ac zO&jolx?}UdOW!UmD-G-}c%ZBeb%Wq7P58pcf#oYtL*f_D_{JX|@sE$Z;V1uXH^0{M zm(RTBH@|txdtUO6AA^&_;9@k8p4497y6O?S`qPgd^|0?c*qbZkT?U%&4M=<2Z>(l+ z796%~5P%-*8RWua{`i;&J>(V7{Lp*h`L2(C^Ce&T>LZ`?*w20+w*PJJd;j~&pFaQS zOK6?Z_V^$17-Cou!RYJS7DTOdWYwH z&G&rt#{@>efDYJzql5zzIDr%xffjgy7YKdgmwp`Bf#O$w><4}C7k?x;f6@1VUI&2~ zxPmdjf-cyCFxUbyNOd1jgEn}B8i0d2xPv@s0X_JGKp2ERD1`kOK*4VXEhW37CQl_<%3ig&FvD&Lh=y+XiE{{tp?HV!w}+;9iYl0jsL?#n0S`=3dz@fYIuIcScYQQ3BZ_)$f%6W*o@0~eb6|D+Shr)XnDhU zjo6rt+Ng%0D2j6EjSR>FT!)KpfOj(mbri>V&ZvyvCVJmkg0hH;%x8V;SdY(mkL);& z*%yt}c!vJCjok>2@Mnv4ID+ffiO0B)$QK4nKy~Aoiy()N0VREw#tXnWhKl!flBk1} zhl`kh=48WlD4>%F)5QX`ITT9 zmXYX{M!=9hNsdVXat1(@j(3zuxpiC_i*m_|n&^E|xp`67iJs=U zp67W5?fIVTX$9^npYvIt^mzpLiJ$t(hcWnkGDw?2Ihi^KaqKpcmFbrEkPiwv4-85V z4q6Wn3ZW7@q3l4R7J3a9+Mo=op#IRI9{Qmm8luy{p(2`~CR(B-I-)7sp(?tf%*mV@ z`YAH13o%NgHhQBtTBAA|p*tF*?eL=&st!U5q(hnxMtY=3TBJ$3q)ZwQMB1cEiVaaJ zrPffT)(}+vnFaqD1+^)l=-8G=>7n*;p=9c$X8NROnx<&_p!z_fA^N5$3a4@EqIAlk zE$UNkil<|`r#{M~eEO#o+NXZ1q=U+%QwpWjfT-u7sM7zisNL|Wj@qb^I;oUesq1j5 zm|CG&3WNUnm|fahUrKtqd70JMq4l5-WvZsCx~8l;sBSu^_E4vD3ahfJr+NCPfGVi2 zx~qlStG;TejB2I9nyAk}ti@`n$eOH~x~$8(sayK28L5i}8fO$alGSLcd%CLHx~<$Q zp|L8bB6_E<3a+(!t95Ft=&GxP`m3p`uI$>X!>X+EIZN`rY6p6e zrP`+L+O3LOrr%1ga{5>MfUx7bu;pr}xf-z%Dy&7SuDSZIz?!j#8m}CCtoB;3%o?)y z%Bc#71OBijV4y&^}Ypy<b~ENL(73BKJ)L>aYKrd;(jvF{`x)o3moNp*S11Vmr1E`?F>n zv>UswM!U8_nzj_Hv~SzAax1c2nx&c8kWy=#RQpl=I-p>RwOO00TRXU2TeAvFwutMs zK)bl^8n=$iwvgMlZVR__Te)$`r>nYW z+q!65x{n*VNSm~=Ypj%OxgaaDmut5?5JfF;cshVxCfBzGT9i)-u-zxRT1&cWTeyn* zygj?RjC--LTf5DBy|gR2)qA@h+r7A}yCeTgvUmHtoQqmv_PLcA2ce6qqszPm>%7p* zy73FNNb9}!d%r}xz50v4+{?eY`@aATzR&Qq4B3F2I|QnwVCZ|b*Q&fR`?0MGzw%aS}zZQJK7;L*89Kawfz-rn~ynDb2OuSZ@oAZdghlj1p>%OWg!4w?5@@uoU zy1^fu!x@~tJKVn^{JnK6fwb9zT1vjbE1;fhwMGfJF5A8^e7$)(!%}?1tjfVUe8u_8 z!`5rXSp36Yyt^a|t>rtE=c~j^+`vsN4;CuIPYkwGT*J~Ez0}*qTujHvs>OAj#~%E> zU+lCu5XK{5fxw%<2^DkIy1q{Q#8m&f$fg^|Zw$xuJFz~@#~h2tdc4PyO37^Oq(R&Q zUKhymM`uQtX2bhl)0)04o4gJ@xN_Ubj||B_yT+HC$#z`JJjoEpZ1 zY`zL?#w*Ok>?_OMy2-fg$}>E@&J44&yvzO@474oG)_l#p?8kt7x8-=mdNs_%ymE+~ z$jQ9S<}9Ud?7E`cqwdw{F(L z$BfL%e9y6)t@r>B82Zo*=WGCk8dZPPc6(=}byJ3ZA{oz*<;)cGv84OzkrIMm;k0PLk~y_E z6;VeWX6H>x>FvoS?vW_&)<~}6HE!b)4&xLq*1KIeD-=8#?tP!0{2KIi4E(!p!o zgN~Mlp5TbS=xM&_4oVD>{_0f@=aW9`m0sshZrAhe(IDODtM}=1wO>E}tAi_v@Y$`F6n*Ulcs&#SkwcK2Ixd!-nb;_xkm+Kj_S{y z=*+(E$pG!v?(WlG?Xf=Z*M95vj^~@a=^-6|Bi&NMe#*T>ckF%c8E^%->kN;u>Ix4G zjsED^F7Ndo@e+^koBfBs?9Wu-VpbO7IE7FmPVQMi@J~?9&wl16pAGej@CvW&tp4y2 zAMY^#?i2t2@+@xghK=!4kdgfkPQ_PCJx|yt?t|t|1qwjsQwLE|`M!(5Ouk`e=>SbLaG&*ie(@%lg}G(Q{}%AX9>^kp>uR6!jlSy2UiDK??{m-ibZ_^N-}n)(_2anr z@0D*|9_&C5@`GRUN{{lN-wW!X=#BpLk{|h}pZa%?`l_$wppSt??0^HQSO9ltqCWOQ z-}#}x`%0e+YVPc2+WK)X{KLQcuFm?o{??Sw&rW1eamBYEf7s=&`@CQM$Z!0MPyEGy z{MY}V{NLdHB@c+P&-HNxW1%$AU=R4(Zt7=m{qEoW*suNa|M=Wr|Mq|X@DKiQ4NMph z5NL*YX;R|C!3Yo%CM3eeO%jMiW;Apd(Vz|;7A)}K(O`s=K2#Jq@K|J5Jaj2lu3Ooy zWj2^Gi9w4-vu3fIIbZ6uIgDpdmq3I59I8xcQJ>P1B4yh2oKvMxiB3H_6X(^XR<%~; zN)_8zuwlQ6WjWR?*|KTntz~;HQ;!%N<5G#j#L1Mp0$X6AQ;?w&7lBELp(6&cM2dtB zW5lSTBuOKENtVpT7M$gsmt}6=>eVyW&rw5*?rPL@R;f=>lTO{b^y<#BU(ZHcTkZeZ z+n9Ab3L*o?T)KIMXYpHMVZnhxEDA=voN>vHALC@1jNwS-%$l`t@3pMjciFT@k7o_r zwR!a6&x>cT-hKPr@omfB{W~sjBX);#8A8H1Ll83PAcVpeYfdr#80+sb4M3u9vg_R8 zF1z*81C2Zig~1TM3QsHV!w%mQal{SjGZ8m$a|bwUL+{E1ZVVbqb?@u5hET= zc+LTiN*d%r%7&cqP4D6)vdJXrbW$}unVi$kC`m*SB|xVPbWkc+v@c7%h}-{Rpd=9T zG6TF2)Uk;K74-2p$fgStJ2&AhGQ~ba<@3%xK`eDtR6i9|970*GveUG#Y|+I;!x)Hy z=VlZz(gJlXu+qkU)D%KaJvH`7Wc4I9)n!j*_SruFT-8-+VNKD{S!X3KqAnBofCPd*59&_tFF6j)gi|Jyp_Q*amOw9 z%(&D{m)Lral^9fbE52~vdsW5Q;*2|%lG=RrH4$6jK1!HO6be2e+%-BWlUy-%96{KJ z+oW$|cR8k4=bd@(nO%=_&JbjxTQ$1qDoKtkKOAV?F-E~2oq~;uV3z-xVIwm2LS1Wd z_WA3Le+GK&dQnCC>~)W>Rz9U|HCfC4CQ{iklN~ZpM-*hHnQN~#_8aW7{}y}ji-iul z@U;_1TXD?VHsWn21Y)!}g&GY|gB)Yl{Ax6_Mwf504L>||!3h_cbi@N!dvV6QX3KHN zB~Q-3|0aL>ZoKtw!E+wcs96FQitW2y;wer&Y-IsAU3KM4PnLD)qlXptexul;i|ZSj z!BGmjM*vq#H|Ko!2R?^*_~M%v9re-KU%vhIrC(isYp*E-u79V9ov!S+&zn-Yxj<4@ z%iF?cMm`&*AxMsJo%@cbz6T=jeGrUa(9%Y~)?ILV_}d*__DBDM*+F1nk{Y1+#>WJf zjRABGtY8Z%$UFze@P!e4PX;k)!?xV-J{)mE2lGQU5iCk&3nQWluXaG*`K|%gv4aLr zg}xcW(0NllT@ydFGgX9tJ&}gPLmLv?nOF3zWwh<>qQs%r-KSh0ko} zE6I6IbY4iB(bQxvUj&3*x}i)%{G|~ONz77?L4o2_W-Fcf&k3gUlI&dPkZ{RGrtPvZ zCs5fu?di=geMFJ_T;d8GInI8X6QHsTs6c61P=Y#97YDUwLiO1{U;gDGo-ANDX}33i zX4Ib>rKv|XO2?3b^mXohXTtbH$dv|F3H%uO>Y&lc#XoA@|DE7o@s0qDt)@MMrW`nHuAxRb{J9*Q!=oVl`y!9O*?$$|V=QG$9U! z*+GUW0+?15tx%omTcg@m#3mLrS1l+>6?M&REfW7~wc~48jq_LNEY++|O>1G{df4tL z*0AMt>uBf4*mqc_u3H6|z~aVGm5#8lmxU-k>2}JIAX7f&AgnLK@mbJ{7MgR^X>f-- z(Bhs}u5P+U4yyTBt_HD2vE@r(EL+6l*cMD^aLfZp@&%M^qqpPmtZuT*SvCap_E+=Si@>hUuHJH zGfQQ7`FmarXZXLr4RD9mS77$)rn#GhY-CT`Q2JKQz6#nv5G9<2G*B3L;Jq+#%S+V_ zmnyj){@RW~oLV3U`5m`#Y=Y0Qr3F*h!OZ`m?GaxbV*q%!1~lgFcsYy=^4{3I8V0bB zd2C}ZH&DnT9x-csVB$bBxCAFA*BDz`E`&T;#+X{4NPR|-nEl7 z(g846%B|}@Sd4Ox@{H4AXGljn%Xijuj`!GKKmWLX&NcIq(IC$>$F|9DE+QgZ%xF|n zS<eq$C+*~rx69;EmZT=#~DPg6#YL^?>Y)dgmS9!)apIUTG>`+Hm&>Y z>@i0>wR!F?Av6g8`IBLF1welyb#y=y|}*i zYXwee)4ZGGz_+s@$;fcV5+Zk%whvDg~txW@+`a)gVVab0yKfcy~zNHPrTpOd4_mhS94}So##5Ide5tFb!ktN+CVRHy{EfPCck_E6Nq>W zVlEvVG&%$1mb%S%e)Frx{q0jftbyU-<2{`GF4bGw-giBAzyrSJVlTR`mH-2pM~CL< zYkSm>LHD`48|#vXyxeI=cikOKK@KptiF|Bf%g%K_XPaBP_xd zbU`!Bx-&$>X|uqE_uPXzb;_I*776|q_`AFIaa(vD@>$1^gU^$MqIo`Y~)2? zY{y$fzHKDM8wn2BLPZ4#$2C|3Wn4yPj77abM@pQ$got(C5%byS&$MSg1L#x8{tTA$;q7*NUj`9q2$A_ z1k1b}N=qzDzjR5a>?xUq2vd9%x5R{(gTu>0qPdL5OCZR)M996oOUjf?%lu2d%*@H8 zz%dKVCe(|kY0G?kg2$msmZQmMj6u4L%*)(N*K{?$jLp|%$t3JdTl>s<@ZOBUJo5@KLn596Cb`P8->p)r30l6wmRbP4dLc^PIHw z#7uJ_&|kDo+0(qHk)1chx8+bq?^>gzvd{Eus{P_m4BgNEG|y_(zyC~55ba9?ZO*KF zEmKlZB$G}7Qn8OES*gsJm^hxrngpoF<)7IE_;| zrBXUAPaO@=EzMIstyC7t7BOv%Kz+aKtQ>q~0sXnS1Ms~&XjD^mR8&pXN!`*MZPh!K z)K#t20;N<@1XClM$07y0LG4uAJ5*4;tushcIy+TjeNfRFvBi z`C2(DjJT@APKgp#Qf1U)HP&PG&sT-j5M5SsZPpQmR@_|6ir7?kRZJ%BoHyN8ZtYfs zRMl|3*GSD*ef?H(Rn~Gf*Fb!W8}L)Yq(OEyP4<(~cvZl8rB`p&Qhx1Me`VHx1=#-} zL)Tj^Olmz?oGeEW2vtyBSPUrDhILqceb|WI*NLrImZjLUXxUH{Sm@-pjqTW~3_OMX z01@O4leJf$O;(jn+0i=Lm0d}fJzAJGP?^0oAYiI8{Y@pXS*dMB3RRSywTzzKP@fE1 zt`*v%MOusHL|vP`rUfHijhd;Q+MK=Gto>P{McJ-}*tmt&q^(=K9nqzY!G$weK}A@# zt+>Qv+qNCtdVO0R-P*Y&8o5o}u)SNyJ=f)Pt#0I7w5{2{P0hU%T*BSl!Zln^L|m_B z+^>CHux-}7B?w|H)5FkBE~r8j02UeG)r19Hw9{P9b=#2y-MQ7>#TDHEE#3ddeMq-h zDCeU+GGdO*aa|Lz0#Tp@M%c#7wYU!m4%@w5+yz>aUed-MyP! zdMdjiSQ8OIQ%CTGTCj#WfXSQ1LV!%W+_C~3wchJx)ji|h?hW1VHQMkUVDTka@_nZ< z(4X@ap~A=!p5iqw)InP&h(WGqLq1t;-dQ$EG)gN)^`}JRVj%OdO;;75!Wj(@qw&(v*&gWq^2YlG)A$FU8 zPDbU;NJJC^DpW*jV zL21TO@#Q#~=4FRI>SLfprA}(MSZb$MYN*DEOo-~Ku4<}=tEZl-SN~-A32t<8A-D=xvOZYs+OzB-Vsjrr@5Q zPZ@niJH%z@e(nK)?&y~82%T=~uI}q@?(CNCM(b|xhVJdwZt)Ip>?UvX9`Ex;Z=PIl zlJ;%)elGZa@Az)M6BTY`)p7+we{m>}aw(s3 zDz9?zwsI`baxK4dDc68i>~SBTZ!pgv$u-}9-X8ydJUI(?V&}si+&;V_hjTf<@PR#) z9kPKh&+}CTr*BCCKL2w-2PZur^kO1(j|74`Pjp59fkQ6>M}PE2U-U?ybV;{!Ll1OJ z&-6mybR@F2_(C&j1@%yuEK)CZQX=ikg9B= z`3TSXm*07%SNh$?^FDWSdiVB^xO1u}Mf9_9B3|_?!~;sW0zY5#3GendzxH|uc!DQ; zj5qs-cX_pMd$-r^N_YFXXM1_4_qnfor)PJhr}Vtnd$FhZjMt^H2X{`&dadUGU1g88aATBkZN@*)~r>vZsqFr$k(u9b@CwF!U+qsOn$a#OT{ePfogxQeJl5FmbYHe z?8TDsVT6i-B?`7^m|~2LC13wCZoHD`0hW>-Po^CCGUk^?H5}#axpU{ILcx+2iuCkp zs-jh2?b>=Q*C(=P)4p39FO@U7ckAxUm$&cV!ub+sGdzv)k2^-#8~SBoWsySIM<3J9Qb z1sd1~g2yqpAaV>I*r0>XU6jBA)dg^&g@>6SgGy0_w?ux|g?N<*ExizwdMTbqo{BBH z_~MHr!8ap(`q6jeecR=T-&J@G_~RM`28jkJ4;Gl9gGMTOU?qL{^&ABk=0L_Adyvt= zlxA3IMh+cdSXf3KEiwPl2PiSc5mrW>xnobBt;iyr?ZGJLoOIfG=SlM6v=e`Qj(7x+ zfeJdLC&g8nT#%S3iV2}Z3i+sl$u;SugcC+IB`2BcGL4o+uHxw&U4qG>m^qmkA9rqw z1OuLR#@Q;YuF5)Vj5ON%9&7@Fu`92=`ugjjKq4yW6v6f;X_E+1xa_i5aI%M{i;4ou zEwE5~Eva8Z6hs55zB*oK-@Z96t>uz?uDR*v8K4zfQ~?DN?#c@I*5M2vYjc08d;i?V{EyEUl;&fe0?T5vx0Kcokb5E5#XWoL*YI*8A?q@q#=u$iI>X zh7VW#8;Gv{`V0TCaR?F-0<&IXim<>7>pZQ^+9pvfTrmjU*3e}cO*GOnDV_AvZ0h+m zs@`@BwbWEwz2ntcW3BboW0KK^y(EkLal0z}D(@3-nB8lTEw2o7qsK0%gQXM}jPoXW zS29f}uuQ5mw})-J1s8;eaX1!vDZV)4jRW_1L6MUsIb3yFemUisYo57nUU%*}=q`)? zALOO~So(0msV?`UYPcTu*kqTDHtn^84ae8F)4q1T!X`&I0#OFv0p5S_&9~oxdntI$ z6;$dR-3b*feF;ZbZ$0)qT)(*WTB^L|mYma_{rJ{vpgsDPdp$Y(>Te$U=<&I1pYvqEz&Iq28?P{B^8Chz5`htnGgAUWbif7S z4R8MgVYJR5K8VOK@^XUP10^4WDNH}QP?6Ki7Iv_cK2QEEXb|HLGk+OPF`nv&kEE9) zFR9IKLXv-#p0wO1*TYVBUxk7my&0D5#m>#E=B%jDZ7b zeCG!0`NC%EAbP%}&*#sYoq(8E;NV={wU*4fXHZda)P6d_ zgGSw_1U(s1ifZ+s5KX95|GHU@y0M@U+$?JU>DZZlktCp1>{L(1Ovl#247l593T(S8 z+&<5%os_6TFUB^;)Iu#MaW!b1prOvjrKbs{2 z_Q~4Se)hGa9c@g(+k_#EmxA0vpnGk}UD%qJzF;lyiR$1)-0ES!zJ2OM9eP!;%C}$~ zZEs^I$cA(lmYq9QY=)5Q&k0{(oXvxh0Fq$T4RhFoj2+cLsJbB*+A^>pK%oC$1}xRC zYVW-B95Gt|+XEQDmx9UyDpKuR-yNs+prJCGUpw$MAm7)*9dOtjoE|Yo035Ey(NPE%eQTN2`RV{2~8eN)2Fp(-m z@dnKt#vALn&#&cbXd93+r_6OGa6U6sv#e-@{`Ao9S;B&wds8aFgwa^e>^#%EAu4K` ziejEFh=Y4(I-}3gz3iiD@eANoi&#T^_A`$`3+OMs(yuCrWsE_D;5@&$*C8xl zx&gV@-rAbf>2^1~Pvj@jtti&zN#=iI=zGm@_Kr60ZNOjo+KlGX*F+{9p13Is0l zT??&0ViH{^1la#j+kTTB8({u6h5>L;dNVPsZnyJ`F0H z%aa)*$*7^df@hj;P_!+B#fZ=ncvXen+h@==W!nUxnI(WOdp_~=_x=8 ztkO*s(9*>o0QTU&4ImH_paM$Z5Eda39$^F;ArdB`5;Fgx2xj2-L7^0u;0aQp30mP5 zqTmYtoHelE3&J1_#?DFM$Enc38{}XA>7Nhc&Kd@x5H{fwI$<2jAsyDC9bRA(V&N2$ zUlsNtAO4{hrr-{q-xq@6`)N{MRhq4=)ETOwBer21K;jQh;v3H49cthuJ|P@hA|7Vq z9+Ds^`k^R}VkiQlDc&3*wjd!EBGai2Nm_>@z`@bgqV-iH8(O0+;^HQ5qc3V=CwgNyexf*n zV=$7V9v0&Fhc{-Z#0 zU^*V;DH>!#wj)E*qC@T=Bdy5Am`5aX9S`0k?EwQmhMhigBR>`-L4IUE1|%e$BSNa< zNiHNxo*+9mq)R59JISL;%wj!aBvxLdR&wGu0;N~>R((ormV=KMcO50wx(rbB4&am zXqx40)@EDU=4hg#OOobla@}cGnPWaCWa_3puAXeVW=C>mX5!{x(k64}CUiz;XewiG zRwp6KC2-P;YF1-$R%hfP=W7OKY>wx1Hs@#FCL^F{c}k}!Qm1=XCu$1kRHnci%z^(+ zCVuLqc%El^IwybrCxDvgda@^h7N}ajr&IoBaLQ*$G-7vpXZ7W$axUj_2B?JwrhpEp zfo>=Yisd6xqk^8tV+Ij%K4^wcp?>bAe^MxUwy29!(&GF%sjP%)l&Wcpmg$Pxsh+ZFlJe=BYTcLW zC`UkPoswyV;^_kV>7HV#qBbg*HtB*g=ut%IkV@&DBI=qlYNcu_r#fn%mgsbjCU*v^ zN{lFv9;%O`DyDYorY`D{W-6!+%qmLhAeqT;ef>#sg5w@T}@e(S6fYc0*RmLbJv?8r_mUn1Du0H-g@ocel5)|F4z(&)Q+MM1T4fFDy!mc z;Sw$8Ca&T#?n(yffClPriVW`MrtNnw?ZKXI<{B>Qif!t)F2=g9=Z@{bf^JmqEYESS z>YA?RZZ7QdF5&_$>qaN-P9+#TE~i>I6dJE@A|@IPZ!BWn)c$Vs z>MrzdukRkN_kQp5zAo26r}bLQ6-Y1lhA;VQFZ;eP{JL-Vu5X*JLF96Wq+;*r(l7sl zum8#~00*%6jxXO%?Zv#Kz zR^Dp`i*N)}a0$C<)^_kGEZ7$?82`q&w)Tn|6Q^i_{kfFnevLZs9MX@5Qi4rYlyvT9ms*fHuiX7Ro zq)Ch@Q<`e1aG=VTEMK~uNfYL&l?QLuv>B79PMgpsdt5m0E#j15@*Qj5;iUs?Vtl6?>(<()abS>1JPTjUu8QJ+?Ac{1zH zq&>rq4O({S(YJA9-u*gt>)yVB(;iNIw(aA!lPgDVTDa?~(49Z$&Ahnr>erX&*3F%| zcj?oAgZ~eXy}bGC-L;e2p58I}_T%G+7cU>Z_4D@Et7l(&KmYHm)iE@MZqUqqAT-F)qn^WRRrJi@LX(ye30@^2^gzgDv zp@aJArJ!my*CwKnCMsu|iZ&|gqnKW*seY6O>glHtW-98Wn^xK>qh>;OD5-|FIx4HM zp8v`zsHwi{sH>~iT57DOTDohly5?!?ud@1z>aoDV0&Jwq66>^F~cFByz$C$ zPW&&*DP!C&%OjHu^36Es42sS)%RF<$5Ub3y&oB23^v`I*ymQepzd5VJH6wkr$V`iA zb<^%;O*GUdOFgyFRbL&p)>s?uHG^P}UG{WbZ~e5{xtU9G+Cz(N_uXW>4fo!1&;Ly| z-fQdq$>4&Y{k36zC$2c%gaaOU-i|}wxa5>qZu#YHXRf*7i!=UsT2x zJo3|1Z~gVmW8b{!-n|vQ@x_NZR2uz>?7id2L60m^$GhhT6NJ0NK(194d(f1}O!46jNgBAo~ z2t`Q33rdfJ9@HKtO31<%jxc~OOqnh=NWML+5EN`^+#2REI3DKEhd{KU5dVYtLn0cn zh)6sl8kD%hCOYwnP>iA!r%1&rTJefxFhdlcz{MqWF^fSIL>P}q#v$s^j0>z!7uE>I zHoEbRZ~UShgCK)C+VPGjP=N^axJN$v@sD{7q#)_I108sA2veX!64=PdMk?}=iCm;4 zBZ){NP_mMk%%mn~KuAM&(gdF@q$u%7%25t-1Ashb2vUj4R#JchL|~;WXZgxil9HCP z%w;Ke*-1HSF^+JYgB#0e##DyVlP=(-GLH$$W|Dvd&%|W~rb$g{;<5zLv}6+|sf2BM zlM7e?r#Qz+&TwwwoaP+oH?_%5ZnpECn_Q+moteyfqTrtR%x5&|$^TD%Li3*h4JZec zAkYaE)Sw7OB|#aG&p^QiNjr#F+z)P4H1 zpHMw$RHaJQ3Z%5E8_lX#LrTk!TJxb@1*=%c%2kHKVjBIx#7UdV)~j9st2V`HT;)p9 z5-b#(c-`bW_li`$`n9jhTxmSp`qZf!Hk);=Di;#_)5T`>u3L?)WG743%9i!AUgg9l zFoD(_^i!}&eJNb&T2s*qG^+13=U-O~+Sa=Es)RKxN(*~f#sAi|u`YnAZYj%K%O)1H zz{PAhE1_A=a>ACeh3#8^Yg*Dacew-oY%C4Q+Ui>OwS%2)Jr$~hoi0|oJuRwHp({|x z!j-n8HScbFyH#zDm!iH6ZhQ-{0dW$yxVq?q3Rs#_{?2ovEiEl`*Lz#=rdG8GzAj*~ z+uFiy)ThPuG z!#9=bm0L_(p^8_=fF)2;Qi zd;^eYC=Wu)#a(5f{R`>m7MjqdMZj}c4Qd-V+R@+5t|ueS-C5@K*FQcoRfVl#V%NEw zVnwa7M{QYCpSsT^{{UVFNhjoj|Ghnf1|fG79hmb`Ryt9|ctBN?*Z?eLo! z>Tq{+)Xi>)x0LVL@jtsh-w|)T0|WF>+>Q@j54u#dg$XYYE|+y3;o&pqyUzxv*{{`Y+VKJEP;csB4o_{b-I@|Dm0=1+Ku z#*aSXr%(O5%eeY+&%X6ddPwYslH5$bc>gy`9QtSC+?>uoe(}AqMYp^E{YHpC4-}6B z`rE($^UuHj_rLx1t6%T`Cw+V;fCR{X&}V=Mn1I`7cnsKnZg+r7^nP^*fAUv>`*(pY zkb(CH0~)x28aM(U0D>Vnf+Sdi8eoDbD1s`8c`UGk9@qmg7=tp{eKa_Ovh-6TrYypw zZ=+{@e+Pi-w}1^uflN?@MtFooD1rU=f5W$YPWXgS=zjqyg;a=r3V4M=h=p31g+thd zNce?dh=D+We`II_A9#i>n1(61hHQ9(AP{!8#0eBOU);A}5axq`=YLt?erULYZrA`8 z5Qu^}h;1l>`G<$a7ll(;h3mJ45dZjykm!h#=!KM6i4K^AVVH@UxQR(Cm06xw~Ta1Yk@U)$Hxn|M~Zuve}3qP4lsxh@P`EXk8a2VRB(61 z<&EFyjqSLO(|C@$SdG4zj9>VIYFLN{xsiSNf?&vi2*`*RNRRPlkN1d=a;JI`NrYn9 zk!RSFNGOX8*^u4Hjt@DG68|}o6j_m8Xn|$uk{qd#1v!*NDU>THcN!>Os~C8DxQQ_NlRjyIK3I$3xRCsKleUX?%|$&IoY0v9(%GCDIi1#dohOBz z8Td#bD3?sRWgdl>dH}f9Q(4OxJpX=E-GjyK$a1Z#IpZ=hq z`Dvg2$)Eihp#SNg1bUulb)bEQpbDy>?t`8*)H&%npYl1O&!7wfFdAgym+N#|!tFcZzP6t<+krpGvB!imj-+t=yWduKKOu8m_iVtK&+pR9qs;^mkt)S|y-FmPHYp@C%r{`*?4%@I0 zYp%E&sIKXm6c}OP`L6Jak91I@#cHqky0DV!ulxG1`)~{WYO*Q|umWqX1S_%$8?(wf zvoI^GHvfCG5i7AdJF$G4t{kX%zo~@k<&?l0tb;kMn}DD73bG*!sVQ5sRysp1OR`j3 zwa+@RE}ONYTC+0SwKU7M;`*>VJGMMqwt~8@?3#fXd#E?pgDL3+9gDO{`>{;hw2xY? z`}(w2ySG!zw_5A61sk^U>9t)7wuURXbPB5sySQYVvp$=YY0CtL%2hixqa5q8OIx>g z+oxLEw@Nw>C>yXW>$xxcwSr5yrklElyQgB?xUO5aJX?mS=>%R7my;V(Z&tMR6t9@8 zxtr^)oeQ=8s<)$iwZ{9mL@KhUtFVQeysP`XuN%D(`?x?GxocZ?Z40b|x4pfaxxb6E zLH|m;!h5_cOTOkirp{}yiYu$k-Vxao_y9}2(j`@pOVzOAdX^Enp(y?7|TI!Y$0W6->YUM})V_bCvsVfEU0d+_0)kJL+)4 zDXhXqjJyjh!%DowF)Y7MJi|0h!P0xdW8ii`%K{rbcJ1ZCv?i>pM+clW!arQX@0kqT z01Rhr3(#7`N360d{Ju2|!*V>wOgzDHY{NG|f1TN#@8|?>z`>jMQCwWKAWWKatF&Vr zvaSD+#*BPcY<$4yd$roCs=>g(b_~UsOveKp!B5=Do-D;=OvSspoj5QB*Be6Yqu+>3#bb$?8^09(7C`6HEq>s zoWxme)?*FU{v6hTEewK9*oHmQW!={#-Lpom$NIb0m8H^6#fMq2j2fr~ut3*~P1N(O zq0~&%GhGWBO4fdz*rrX|qdnNEjo78l+GJ}7c;uod{dGybSgiJ6`*&)92!cm|vYY+e zYI@hgUCp8G2#Sr`$oG^gjDK11*!=5nDR*cMklS>y+av#g*_sX9 zL@nGEY76B3+^vn?%dOnw&D@3U-s~OU>Yd*59NlOuv|)u&&AiTS&2-!C1o23U-t7ad z5Z>Z#*9|Pzhke`~z1|M4;LtD(?+xD*{@@Ny-xz-2_Pw2w$wuM#P~)|`&zHR-@ZbNf z+q(?~1%BY@(B8Q~;qxuuuHD=)F5?z1(lHLZ8;-w+XuGt{)*+6H-~?KIY*-G64MGm7#p5jXGKJ>52~Naz5#)-s-IW z>Q>I?O}^u^+s9o_bPNFIQ|ADo-rWKD-Dhs<#D40se(cB|>&ouxtIq7OzUe8O<`(^My#pEC0}-g};&hUg z)V83$s~SM&hFxk~|(oLvQpn!e8QJqc%Ds70nop*%z z3E}?l2;cDuf9@c^<_0hFl0NbuFYpZS*v4#pl{9hL-pp0!>-zuB@8Hhy0ACF1F7O_| z^CAD}Jg@Uup7I>dtCA~^4lQhbXSB#H=tdrhG%tdC@TfOW4B0UCKM(YGe)T55^*&GX z;*Ry=E%ZjP1#|g;bVqdTTyXRx^A|q|{ciPekMmkT_a%SuJ8TE0$_>52YdVlv>U-yqc`CkwAc#O7yfA4}{QPY)nzsB~55A3_G z2^-(|a)0`cFZns2`jk)gtl#wnpXQ9+@Bm)QL)dN+F!9XH#UNZ-~Hr|{KjwpBn4nwKyqS}*SJYZnBURd3F5K6#Vb7jR`)mJg+_~4vw%vQT?!Uo>3no6;_-TZP z7gBV>xZ;kBkU37y>{v6&%_2{J77<$JXqzx$nmL{Nr+L@uc?+wn4ESyMzT^LX&$d4O z_3hE;zd!u=|Gkh)2(iQpbPl8GJbK9k5X3lXr0qJBkh|_a5N(X`a9W{`^E&i!KM>X9 zPrniS!%w~w?L)D|6;1T7xEIR25KQoa7ufi5LhQ1OP=^XH z9ApFqG}H;oCa+wo7ZIi0l1umIBecas5gq?DP(`~8(49vkWzsJzdZ_VCOqp==A_p(n zQ34p&AOp@gIcSO;1psV6W9S+H;EKGPO>}RH_vI+p1W-l>2{lGs41`8 zs=OP9d2e)Srb6P!hzOic2N-wuaE@1H90N5TN4jJ{C5PN{*(-Pba@@zt{OzN6k{M#Z zr+|*K>2wI&aMV+^K=rg=cix!VX_wyd+pC{^_LlG7U3=c2JgFy{$5PCC|Aq;JJ zzxr_yH6H|F503*1DFAMO5*e0{5(tvxv1ElyEP(_c_`dj6@Oc)bVg{{v!w_xHhv*?p z057Gric~;N$10z^74(?@ico7p%H4d5@z{v0UiNlAe*?poDi~&Ws#yI6FEmz zrqYy=lw>IZXujnDFImEij?Ty^BwPkfCN7L3B2~FcVE(d^IXopK@7T(BgyS8Q{GzZn z;4V$x@{BIoCE7k$$j1Mba+ox%VK;^O&24V*nBC)Exx5%lVPR5q(Ns__nTRI`hSCV5 z)TS?s8BA~D6PyPt=QxoW!e(abZ;>fpCOwHb^MNuV5Ts{4?Rig#hI5}2y<9*2X;CQx zad7BNX9&o*BNZN$h5TCRJmJT>TsZWh`6Q-9T}siHvht!bVkY>SlbAPojR-_CT-e&Vo7Ieq>PSn11=NzPIy{#s1i(RS&xxcr84!c zYh~JEu1GpN?g%L?gRe zy2f@Vd$4O5ItRMytg|DbvyNtvs4-{>)d@->ZE2HwTGXmlxpGXc{UH0=+13`X>eA~+ zcN?UT{ua11!GSzimxIC@H?hcFu4-GWOvREnpUykRfAXGc3}a#A@yW6y<57`W+K1{POpJItXzyP z7?0A8*u+kSzzP5Ko%T9;SWdo$vK8@d;dq`2iGKL5l(TDO?YH5A2DAH5$=DPw2c@am%ueH zuau!oTZ0b{^TEUEV^yaWkEZi0;K@z5Lg@bFM zA~IRSqbBr}eVynwlUmgRF14^>#_Colds1_FH4$k!MowPlGBZe1oD90@gq&=`+e?Jr<>UU zb$0)Q#aywCm>{>BKB8pu%;|gUI@J3j)KlOP)z~nwWh_Yj-)@<^J}zXZ-MsPdwr~uI{^gjh#d9I|HTz_=qn( z?Swb{ugC59xraOJ=6F0gdR_Wtn?VIUxAQ>rj_{I4)9-=xSLV;YdCnKUTDt!QGNAwd z`_N0B>WybS=uv-sx(bir^JU4%3!wMd2mR>V4*W?+i}u{}e)pX>e)o(2eY)2j`LR`g z&(|gEpm$yDn?ymySTIQI4_@u_&%XA3fB)WtpZLNbzWyCA{_%T)$dkO0i$1Yq0n9st z_`|;XYe4QJHt$<6|C>P1Gd;x{zvR2S4KSY9GZyZ!yvlRGD5xd{Tpb2VM9 zlfVdk!3(s&7|c2WT)s__muD(KcfvHvOTf~QKbBiT`V&HCct9AOLExjn7K}RpM7<^0 zx{aE>1en5&5D5@mzbmYm5wy1-Ji+ZFLNGMKBOF5|L_#J>Hd$-Ei_k2}JF))_gs?1( z!{?(yA3OjFxWg|fh!hM$+zZ1BTtYH5LqHTn3{1nDinKKfomuNG-I7BCc*HrJL!num zI|RZ#^uzr7Lrn}sPAo$VltE=H7B$Q)^&>@#DGf)2L{*$a7^uTa%)?8>L|NR#POL>w z1Vvo@#8wM})pIZ$IGZ3dIy9Pzb`zddj6_zvwX?B9Jk&!zoW&u8#w3J5YVWv<3<10JH(s184eUj z2_neKD9FD90fe*}1YF2;h((8#N0w~KiCnGVd&qi}NEM_>inPen)L102b)?5=g}CBbY4&+JUagvi6R#kXWk417!n zyBPw+uu(Y!%S5OibfSx~0kQ&3+k_FqRLjx)zS1;#H`_C-1E-^wKFVhfrQO}RcfOtwslOikzUD5lbQ0f#z4ed=` z0=z;I4My2)F%>mS7OhYreNh+HPY1=&28~g;Tv8>aQLHkafCL&6wI%e7 zi32dA%QI0R0F}Jl%%lubFl|vW9n#&DQ6)t^C(SG8qeB1X)KQ!O0EH+}3-i(E1kx~d zK{6fFJVnwZJ=6aDPFO<3XIsN4ZPVQXGwpB}0vLcdJkL4pFXu!8y0BA9jZZ$s(+QPO zO)XL*^;1tpQz0@Cb$K^KmAwf2D~Eg3_IyS9tkgT*R6UK=OPy6F{nP*y#S!{3&Dygy zZ6^|KzGInERo%i$w1!uu)gzTvWwp;_tyMF1R$JA&_|jE?#KK@b#^kgV{^C-JiPTng z)nnb$Wfj*>MOJ7%S9Cp7I|x-TxUC||q*5)_HsmZwtP`Y5Id2WuW`)&pC0Bnf*W&2Y zXzeHDOVa@L$#``{1H_5&`O9LZD{zfYiJjPet=Rv5Hf=uZPIT*QS4Py%(I-0+MQtnhwT?q2wJRN*-S0kp#|8aB}Asx$U#j%FWbwh4UKN~ zSsDP^tkqh#C0efSTBDWO?4jEX4clDJ(L*gqsWnKx49k2?*|s%TW&ql_JzVRw+oN4v z#g#$4bywM|QjzuBuSB_TTpddsT+lRJm*iZzMcnOdT*mEKSjt-iil#&biAOBC_M=<` zP=HXOzODJRwY5#ngBZ1pi(u=l-uZ=J z`nA9Nty}!eBO<=%f;FWw|3!cvyrr{Z`U_QNJ8&19L6Q&4DuL0*~!m*xloX2!osO;y(^! zOEqN06lEDkgFE-40YG{mhh%V$q2IPy@Xc^@MC$}eDOwj8x$m_oTYry^j!5(bFF6=8XY$;e?#9r*cX6(j(?81g@ zI{@s--fPRQY|NhQJ;;Qw?rhKgY|pObAE{)~E^RXy?bA-}lMuj+c!EsZDAha@B@3K{ zrGSqnfpSh}W}X0J9_s(I##{!e#LWzD;=W_!F7D$_ZslHX=5B7{-ZtkP0O*cx>8@_; zo^I^U?$x1g?(XjG{);V$p%K5yYh@8IrdUgSElHaJH`a0~}; z4%cw=CaU)n)$kPY=_+v(KXDYVS&&wifi{Cv2y260VZi(C5ASdr_wWnP@d@8?9^Y^r z*YODNaUdV^AMfyLn${FgawYfX6UT3XLf93a0#2w`kBae!yV)6US0N{IF5hx5|MD&e zb1@(DBKMb!6x;t1PxB>T^Ae*a{SL|?XaXuf?i{S#9!-ofQ^z0N^Z2XrGXHZx4|G2t zbV4t5L%(rEKlDVmzMoTbG=FsFh;&J}8A{)pN{{V-=4k|#0XXNVhjdfXvbw{7|@F8(m&vjcbMgnr}UDx$r5B6QBbzr}BV;A-( zw{&Hf7-mm(ReyF?hvZ3@;(KvgLxloC5b$Q6QyU=mRd0152=`-m_ESH0Z|8MzKXXeF zcH?Y!cMtZRL_m3effV=wbNBTc1c5xScapsKBFJ}r4|sm}_kaI)M%VL#7xyz4czz%F zcYpYOr#=6LuXQ%Z6y>N(2l}E&Cs1qmBLGi^S^x#r19bu~>!C#S5bySGk9bjU`QXlX zdq;SeX91e8d1DXvK4&tBkNKj`?s>Nmp8o+ocX>N(_>_-#K4%=GAHkr{d6=Mgn~!;% zH}6}H_lY<7@xFR|F9M>EaCJ|20&!=gSXz88EmM|_1>{Jv!TxU@6h_K_xyYR{MoFwo_>JX zP#}PSfdvni(4fEq!wd}{N{EO7qQr_0EMjzM5d@1HMLs?ODRLyqjwU&NNRe`-%9bZR z!iXso=FA^JY|^x$b0^Q9Ie!B6DYQn=p+t`&t!dQ9Qk+h2LX8@w1rr=qjbL?RH3}83 zSgE$?n3c!XvR<`zJ$sgo2?`Jl8h9&rZiKpV@8X4vP{fQ63l(b2NO7>>i-ivphA98G zB;&C&S&ne|QnJ#?O^ZIgoD{R>&Yq80oop7YW3ORDpDrDvv|6-jvviF^!tDaRbZg_z zYkRj2zljk8R|$ML@!}3EOHDq~GG*q@H-C;Uy>sK9Q(w2v?gcyd?cULR2OmBTc{goy z-%hW4_rZdF4Iei?vG5Z4_Tl5tzi>Z){r>+###v_31^AqS0)D356l5^ipo0&J$3}z^ z_EFx1;8~cVh7zXH;f8L=MjLN-u~(vAL4fc8Ln8fU6G|+s*xY;)o%EZE|KXUUjyv|( z;{iSbNn?T`4R~OIMj{8Ak`OX^hJ#K*xdw+Ha=1x_RC+k2mROb-qKO@Zpu+zUIe=NF z3T1xz-ilMj5D{ueeni6VoY;w5`I%ps{Rqh1l}9;BQq1)992xaL$X!Wd(s!U<_7kGBSz zD*%4-ifgaDzKCRz%01{H7{?ads2))U5+x{^GKy)Fj&{oFZ0wOC>Mh!eq6#a|cDo0e ztL}B#|Esg!Nwx1DZ-z?2=5c$?}ejZpkOdo3H;G{vr{x%rnaj z^S%IIVepcmB|OI(3A?OdrB6Cs;RzxlP^QJAPRg;QUf5b%(B)fbG>!e zUhn9j%`}reHos@%oHozPvJJG``AVyBrR0f#G?*AO%}Iq$KWzrN1yGH}7O)Y{ooj}Z zmADS7Dc<;Mi9a5BYHvb5x#gE*o;l^4E7!T_jx!$m5~Hzpxan0PzOdN>uYNPyIj8*w z>~QqlHtlcUo_pL5hhg^sPv|Y_-oa19Y4OHyEbiZKH9_M*Z(vl&{vyn6rHtn=T0>}>#0fBR#~ zz}{B4qY)r!NuyZan#MQeX-o#oOOFC8ShuTLFoTiV-~~A-qPpo|3?i&R2Pq{5Qh`Dj zT?i8bK{x`f6sQE1+u+qS_qpkC?uMvQk`6^DxqGpQfI?gW4TNyBvKcXnMN}dao2Wzq z`tOJWlUbv%CqWP3&4J|M*zue;CaHn(OXW)7t~8i5_>d5TWLzTxvXl@7iXjta$iX#& z@P$2;;Y3H~=c1WwhOoVJitQRGh7)eWF(uqgR zUlgZUv$MUZdD`>d7pbMi26E#Sd_z$f#V9em4Dxs6lVSgFE|S4kx`z(B976@vxSm$J zpoMOkB`wV;$YVlMjUw~lA|LQW9yDMy(M*|O#MhfYk`RO;?DC?73y2$W*8xL$FU-j#8V-Y^O6>NX#TWuA>*F<_X5M1LvW11_ym38!PF}mZVdX z1a#sMYuZF|x>TG>?0^|a`9`Y2GpJ{r!UOY$PgO#2g8ZBSEHiik4!qNt5*u6vbs5#A zDv*|1y(LE}>eVn^w4^76=tw!5(6LhBn#CkS2Fw2_G6bCUpIb#*ER*S`#jwc(L?L3o z#F-M9;Wa9Sv@1??s@KDYB&7n8>L7`CN~CTKpRibiiu!5U5nS*U0vDA|BJjPrsTcxI^{8W^&V61DN0?Wb+-fjhalmqQ(Ai<1k zfT0tO4`E{&H4j`in3i<_Q5)E_PHYUj8-r11yDQZ1T6UL&`)p?`NKxoHmzDypz)CH+ zgA1}&V!vZyX+_D_5IQfc;v?jHZYk7%+IG3|m9KyYuvpZlfM5)AOM>r5Fq9D3y!b7! zMBOV^sDjtL6+}buv89>(#I>^aTqtE|EY&<^Gm7%Ntg*mM2=` zv?^A&61J|!AQ(YgdKWHF%`tja+LyJ;b~jv@GkIGKQYP#12`kL%X=%&gAmsPO5w`8< zT6Lxm9+in{q+)G|J!KHoVLthc$Xj-jF#jWtwUYgtI0_(ynUPz*Vp%vU9 zgY1%cSc`)^zl$W=dYO#W%K6Td|1p(4UrD!bDs~* zXF_M&(1)%Gyglj@tynrYJYfr_Lp|zGpE}iRfs3mr!|GbsdLMA0=&J18dkPL9*{x9a z*}V>wI!8F$hs8}YtuyX&ug-Cc#A&qSz2|u+QX@a!T)#6H@Pd~z!xl!|!`Gpwwp;xA zP}d+qEFSWbpFHJ_UU^1f{t*9^xCG~qzWL63KJ=*2IO&~H_0yX^^{UT0>s!x_RARe) zlc5yqSN{ms-+mIhCj{+y-}~SHJZQrIIq(y2{5&B4_{uN*-6Owz#5?~Ee1KXF7M%9$ z?>+lKPyX|z&wc2BUk}rdezVFi{_&$f{eDPg8V#_TLZU9t}Lo z&YZ<@*iRA3jsYs60_OkT@4bQVE#L!6pcGDE30C0-lAsl0;TCG)`E}tHP9Ybv;1{mo z7?$A-!eAPrAq}o!3+A8>zTy3yo;9qV4+5d%9RIoFV}M z5v{CCZw2BK8Xyz?UKEDmBG%$1;$kkgVI^)NB=RC6^kOIOVlaN9CmLfhB4Zqe;wbW9 zDY{-Vq~bR%60P9LE5f2I9wIGjBP0UQEoPxEQsOU$<1i8H` zW@nluU?%_OXNKlzD&{=yAY(S>Tt+2iGT}c`re$(tNs1~=4{?3ZGvWQmS%7I zCTgapY94`9!V6W-C|v#^yKLCU1HsZtf;@lICw(r)gp*DB|CC9w&FgrB1LW z)nTJ@dLeT*=T^qyb4ur6t|xW6XLZ8ob?V^hb*FdIr|zw$VeHC&_6ZS@=YQVi?p>xy zVy1dV=Yi^Gd&cL2%4dUW=Y7`aHC`QA5a+uXCx8ZMat7mh{$*KCr-8mFdnzb{I%sy9 z=z>CMgsSL@<|oQrXij2ihH5B!BB+Rh=!oKIh=wJJ@@R>ssDxT2g@$L0c4vW>=ldaL zg699|lHMqj>S&MpXq2w#lrCY5%FBxqsd{oKk|rrU5@?e;X_)TllaeWvo+y=~sgT-a zk!GokekqOGXq<}ahk|LDnyHkYX=vK#)%l~EPGy3zX`8y~n{ug#)@X;$X`R|>qT*?u z@@R@$X>r0qpbF}sI%lC8Dx=b=oGR*_E~=+G>W`*rT)HR}R;s05s*!T)p{nYtYU-wb zYN$5qkNWAAPU@+)DyFiktG?=(c51BlDyWKQiPkEA8f%aMDz4@#tOhHuIxC_wYqJ9D ztjeT=e#MYNtFkUDs6K1A?&`F1>!L1Qqh9O0V5zolYq)-EwEil%N^AVdDw+mrxgP&( zz1HiwW-FzptCGGeyZWoQ0&KrhtGa%qCeW*`>TAO8E5EL*z&7l^I_$vCYM^pMvD&M> z`lrI?XsRwO!#=F9My$I^Y`ki$#vbg&Vr;UatNFocmlABIf^5u&tjH4Vu$Jsc{vp?_ zz^&danEE2kLTtOr?92i!ypn9s3g-x5tI%?6%MxwCK5fxTEz));RsaUdqO8g?Z3uSk z$40H!e(lIoZPk)0!lEkFHm%W4E!wVa(01+1QtioFD9@hl+p2BbhArOa>n6f&d*;=e z2Ey2~?BLpM-rlX=vTfojZr{o+*7Bp)ZY|fk?BNHW&X)gb;P$KJ z`sUDTCh3+g%z|y|My}9~YwO}^<2o*U(5>L=YT=sh?OLwt;x6Xy?dq~Fv6ii*j&AJ| zujo!L;-c>ED(~_7Zjbh^(;cl*dal{hu1gwk?k+F$ChzHDZ}xhx=GNbvLa)Sv-O@Ji zkrprEcCYSk@AtCr<+iW(ifs7;ZswwI+RE?y!f*QWZ~pqP`vPzP3-EQ)@8?!;|KcwK z|L+2GZ~QWF0}rtN?(f&aZtP+J0EMmsYj6Z_a0fr|2ivX$hid`5OXwZ22j4FUpRWU} zun4a(3lFcHG6mPv!T2U>0;4btw{Q;cFb_j;2pgvB#;^|Fgdy$85v(wQ4IifuJ8cg$ z@e^~f3pa6?3h{eZOc6K4u~KCcoA9sVFcgDv7z6S8lCcweWbD3YYf zlMNXjRFx8BON6Ibx|B(?=F6BWSKh>_lc&#}J%0ktNi=BDqn=FTY@H>_W@faeN! zJGgM$!-nG`P8`_o-oAMqN5=U0ujR&a8572=d2!~>ia$f{JeqRl(w9$9PQ95m>DI4f z%MLA2v~0<^SL;^Io3m%wzFh|g{=2yG+rpEZ?oD-fbLPX9M<+vtwAOF7p`SR5#AbhZfCHjtl6?hMcp!onUdUjB96D&7hZ}Z?p@IpP7~+X2LI`4nA3hjhh9r&{f^VT&vhx#NsI`uJm#M>ZK`jz$(4rISc9xnzw`VyR`8Pzu>4 zl~r1KC75A)>7tZVe#zyQN{;E~nr5nr=4MWjc_y20ijwD^Wy%@noV3{qC!d50>gS(B zrWvTAeHJQaDT79OsG@)-DruvRV#+9@l-k#drT*#isHT}3M(U=W-kB<>r?%SWtEHm< z%BiZPOj_%wbHb?WsIK-p>#vo{3Tv#q3VSKAtR|ZxEX5M*tFg!iyDF{EI%{pU$4aX# zh`Tn6C#2ba3vIO9PFpUv=z5#3tt-}f?xfu^tM0tw4!dr+?0)91w)fWSFSr2Pd+opE zzU!&M2!|!_E%Ej{Zo!M1J6pmI(~B>`I!4SdwfSoN8^aZcd@;umcPuf*4O=`g#sZi8 zaKSIHJhI6&2V66#ZnTZK}vmm%TODT6c{$r#ZJxwc2mT4fot6*Ddx=AUAzB-D~&%&9~hr zW9RnZS?`_p;eP`jc8-Lb?YQB2Ev~ralT)s@%9um`N#%ZT=P~D?hc3Eijc2a8+?#)% zy5*y@p7^_Y$4>U@ougj6?X>59yY8*;el^ymKkg{+xcfem>cl7Cd*F9B&z9_$7caf? z(;JWc^3=PI{r2Go4~p>MXMer;J zU%&t3`=9#kcfZ2*kADYDU;qKgzy=P`fdxdI0w)N;3R-Z17tA0AGZ;Mza`1y21fd8= zNJ9B|PhglpU<&uMLKV6&e{4`;3=ud7P{go?IJ6-Rb+`xJ&9H|#45APJbNIp{T5t}5 zSc4h#pu{8+v4l|cPYZX*Lm*lahfT~P8M?5=E_!haV0=Ot!7#=$iqVWwNaGjJz{NIh zL5Xa1qa4>5#yQ#%jd;wX8u!RYCM;nGef(n({3ysmp7D@_M5H1UsYn&bz>$!AWDyog z$x51G2$#&{9oLA(PI~f+MhsnaC4}Ad(?~Bn4)UVp7@mKJv*@fOMd#ZpXCgwK)IQN zbY|0_l5AxeP!H{ZE{pr*5^NKI-!gUVETHub4c zU8+(UAl0f`^`7{c2OE5F0*E@a1P}eHO@Db&q4t!h8cnN4d#Tn0oE4$yrBTg)afRwpr^?vIUe&QtjqGD5o6k+aA|JX)=wJCN zShJqBtZn_QTM4^ahsrdsbVaRd_xe%J2KKe2?Py|STiDwFmh_m66(&{D3fbQ7Rx@Ro8X3!lJll2y)7M-P+`&M)#c_hc3fI-Lw80M!GEEbl%yDY>$LURSX~#KL5}Vk>*+pl5r%dIga$yYp zeQ_ri=7to2ccNu5e~Z582X`4mJY9?Ah7E^~ao! zvyd13>php6)X(mzp!v++UT=2H@q9I`cPobRdb_>Tu631NJcx2Lde_zUwT%re>`J3L z&DK=5yyv}aPcIqNuDrLXO?_%$1NX+;jq4_ zlrbK2WJlBI7ALQpO)Kq3oz$vdr7=r=(;uH zBmg^^-EHh&20h4U@43CdtnQBU6y!&*`L;t1a6GHrs%ox!qd~!NwvXN2fVcDCV<7lZ z8{Y6~PkeLl%=p9e;PBDD@Zcp+OUf7C-y_E8Mg!mO&6nA|pltvE93b`yqo26;{M}IcxfZHd6M&N;u zw}U{?gFlD^JphD0ScF8#0|GdG2q=L}2!T!*fcgi8Cpdx}7=<~ggGXqDS-6E zQW!_2n1QG`g{o+RtVn{ccuHCrgpcTmVn~ZGrgTcxci$Foc{hRCM@VHDjCpv6ZYYek zc#KGRftq-UoH&Zkc#6=NifOQliim~Q7>n3=jgW|pvq+2E_>I|UjnpWOUa*53hKuzy zU^`cXxMzHwaD4EHe4EIO(pQg5=#KH2kG!Cd{Me88_m9d4kk8i%1Xym9NRLg3kP3N_ z3E7a#_>gSSiKO_976^vrhy%fgj#TAbc{hOmXps9?jf>cgB4% z8I0W6gDI(!EP0hJsg(8?l=mo;HEEO4h?6`?f=Bq2W*L=eS%+$wmS#DXA!r2GN0LQp zlt;;qS_zWVSdrlfj(fR*54e>s$(4ecfV|h0H2IYi8J1%iNQ^j@ZTXmx>4%XimE1T2 zdKm*@AcS-2f^}(^`&fKSo*9p!DVnmloHo#$&iS0s$(%&VeF)fm)YqC`V2Oh% zlX%&THz}L6`HCa|8J)Zdosqc$!>F5*sh;J@mg(u37Z{uvIgNC=i+z>^H)w9h=X`O* zME;3HOXNiX`k#V8papuM2s#I7;0d~?lGE^@5c-+N_n^wPcM%Gq4vL}jVxbkc61E~iuy@= ze+s6TN~WIrs_CE(j!LL@TC28ttNVbby1J{pdZ=A$sIwZZi^{6PTCB!;tg@P{vHGdY z+N{oc4$%4x(K@ZtK&{p)t=F2V*}ART+O66u1%lV47g?(H>1z>~rC7SAkP55ps;t7A ztGF7k^4br*TCc16tN2Q+j{2;~+OLi3ukISK0?V)5TCfItun3!}q3VU=Dx6VzU$<7A zREnxtnx$L1uI-w!W7?(j>ZKhEulx|QAZxEAo3Hx1ugog2C%du$i?WjXvIHBm*gCTb zTeCL*>#es?Pz;-y>vskY+fi3_l=zvFS1OO{im@8YuCzL*)4&b)Fb!{-RUbRDB`dX7 zd$on?vMKAVE!(yIIMAN8ORrO#uT<-+imJ6+%e7vcxL|v@VM`5TOSX`Uw%w|>J8MR6D^hz191=-}}AU%e?~J zy|??nj~l+Z+q=I>g_k>Ze${R9LJ0N z#%|omj2y{LObaR;mv@}UG2EH!3!0ug!GH{_gSx9=%(sV3#viQ6a4gBHjJ<8F#Hl>T zPK>jaOa)BZ0#1O*Cnm#0$H!KT#h(1hpzOs@OUfHOy{n7LkQ@xFoXpJJ%+(6Zu`J8A z9D_&*yvIdnRZPPM*~t_P%#SL}V_eE;{Kw8*&dc1&Nqo-gEV-gE%kmioxU5g|TmViN zw-H>!zZ|;J+^nNp%pP3EXf z${f;;%h5jV(M?>(By9u?jblT%Zq3xk`+1#NJf;+#z_iN^8;#EC+|fXN)mXjLR^8J- z&C|O((hF?Mv^)jzOkAsm)I?{2Oqm8P`GB5~RcZXxa_!2GOx5V@)pD#1!jRQkt=E3d z*IP~3czv)6z0fCJ*28twHkP?qhJ?wf2Z1UNIGnL_&DDY3)gay1fc@9Iu-Tsd*_h4K z?rhTL3$ZH=X_)B*Cfy0Z`FL{ZP~nC+qTWw&%N8To!inq-Oioe&0W|-P1;odfY?g)&~O`3^68|< zJ%-9H+0DJ)+TGjO-QK!=-NPW?)lJ>d?cQkn3A3!6Mi;z8E8f@C02_$hVK4|;fZSl1 z-sr&I3=ZG;{oeLX-_fww^DW^K?%)`n;q|@YWvk!Q9K~oxWx8xy{FQ0}UfcwJ-U&_* z4bI^UuHgnv;SdhXoo(S2KH)Zg<1((}(2(OWKI1R0-jeO%OnTBD*xyrz%_P2IP&aG? zZkYyNhiag#L0;u8{^MAlU z;0Ue-EN3EYpu*LNa_mg zv+n7Ip6j}f>-XL3wf@9Ke#ddNavUjoenjA|PU)~N>$l$Qea`8i9_`Z3>tasrSI+D& zp6F{U)`xBCB7W2*e&Rbwdf!b2lMaT6K3KI4|-kuk!G2@F&mk zI}h|c->obU@!W3779Uw`K7U0v^N&aF$*ut%|M5AG^3`DVIzRM=?(cNmUkrB7_;+9Tk8kUEANh@+_j1kmzx($g`EO_)Ogy%R8jsQ(@9~Pi_>O=2 zSD*T*kM)xu_CjCztxx%SfA6A7%YVP>O0VSB!@qqc4IZWCIdhsQ6jNR`v|NE5xP7327{NSGr!+-q7-~7(c{OEuF@xJ`e zuk`}I1}%@0<^ECFw%B|{_--%t-ar224-mzm4HQ_=U^Il(q$O0?aF{TLyBZ|=!i z?#?blaVQ<3G)z;}C8kceP!+}6Bw4dwr5$^${}Z=oORsI)0Lra$cZJv8Tm0|w%Yrd4 zkNi0KbLh^cPlpURy5;NHO=jla*|Yc0JlBXe%_PcErbW|;G2xo)*srp+eq}Fq><6?) z^2Z6&8*cF7h%;_L>k3qkx$3l|PQe8kMDRff9W)HQ?!KeYyU<2c$_OXSTPilFqH?bY z8A!bCy|9Kj5jP8n+ONO1{u3}j8VN*>FzgykFh>b{^zla^ufwTA3yY*nsG>IX&^*+@ z;$XEMawE|VuTtbyRl#~p)2b4@l4RQof~2U0kLj8svy^3eOZeDP5LJ(Dz3OXJ*>(_xDx)>B~} z6lcy+n~l_0JC!_7)%T!`byh;FyfroZD9CTu7$N-?SaK^h*GywSO*UO*j|I|MW(jlW zS-DJ=wpR2)Xu?kfV10qmZ0idP%S5%*H7)*l9hcN{)m`^ociDY-VszsTm0q8sU2@6O z&};Qo5oEZQP!JWBZ$ZJnn7xr1VuzcnncUZygQyorQCr|2bH!caB(Rp1amMx{KlU+2whKCW;4qK^)^_kOw70 zTc=O%@Y)u6SkHo~tF}7ko4@w@Z@;q+yvLuzE`01~?bQ=)4m&Pb#E@5RI&OnimKtv% z1fd#dz4`WBaGC+J*`hbSK^nFf&>Pukz#CE{&ybpdH zLYw?l)F=A^s$Z|NkN!^P|GnIV4@`R)pf*f+!mEYD7%OBU1N+ts1IBQH4OAcdn)kqY z`AmXDGlKLQcfmwapb1#19+d>?e*hF;2~h~c84?hIG=v}(3+TcqBFlzbbl~b( zH$e*KOG_0XLVT<;IV5(gZs7A>S!h5)CuXsQPfXzy&v!*T=241rJQo+Mc*8G(u?Kj1 zo^4ul$S3hHk=omy5=+BIyp1r9bmZe4u_(hAV$zCT^dk;kw!<)n@n|7q&1yg<6*D$c z1ZzyoX(EZ1I!yACc+BJ;ZFx&vda{$d{A0aZC&(;VFi;sZC41nO#uyx|1+I*d9>$?U zS>|$<($ph0r&-Ax|M60rIpn4;!*h*dETKg4iyaY9Sxj4Pk8T=RW*gsPhF(E4nv}#Q zEv;G2d*aiZoZO}t;So;T-3tfUTIDcJdC*4Q$5oHyQW~}-N#M=1j@O)MKJ{r)idw3l z{M;uv#p$}DajaUzgi=D)$k28^^qujPC_O7`&w65XlP{GiO*cxTn2&zH*;i{;}R47u* zqEx0j^%(Musa0+2)|tMws@cS9R<#Pyu8LBAkDCgWbdXA;K24b&Yo=P)8r8$HwXGM0 zt6}9jSG(5r|E@gAgirG-(vW$@pe`U79SByNreSagW@RT~)B0GbJ~g$gZEP(ayI9o1 zwxW{dXaobg%*Fk%1se=2Si<00bM9|{$waAXVfk8LuvWFpg=}-3+eg|)*S43PmKW&9 zHmS@2Hkq~6SmVad@0B(VDqR6Q8Jpb1qF24lh3Rw8D_i(RwwuzO?`JCe%Ykm}r_|c4 zScB``;Yz8YyKRekk&6uFDz>!?c5iIsE8qIk7eGO{t#x?_1VWCstYm}@aJ$Qaqp}jM z$EDYS+3R2wr?|oCRWO5>E8)p*?!F+su!Zl()v63)3xD-+_W+#4q3R~7%q*~p6Rctv zC;7oA|7I|ZAuN}V3bYG^G$aRgyaOW!_Gu>|?T9e|h7}`O%CWW$dYj7D{pzyl$JFW9E#^JBbKGMzICpx&1+6$yM~7LwUZN# zTSf2bx=B>EtKj3~`?b2*U3NAP01!zlG~+YVrZSMBu{h;&vYzwG>Mcmt;1Rh#&xtBr4r>ssR{9`(R6)`PR%i?BH6pjSLQ2s$8< zZ9rx=fj{(ayv3X1V9^-=NHa+XXZjMootL|aN(Jdb^v}IoN zd*@v0I?s011>J36e>~&{*X@Q88flgchJv0SrK)KyH7#+#Mh0#|PZ*lP9=_O?`q@72WUq z8~g&GG?lXo>g-O>1>;Q*aGcNm>T$PvvvAiM_{C#-+au0|F9qZ z#47QnKYim>zxqGF{`mXGI_$-xcgBUgx_rld?iYaWQX*7?tbBRe+di)Ev)%a2hkNUl z?|A3Oo@zmtzHh5+eFGd+w7l>A!$}XI;^*J|=T|f2$^U=0QopPdz`PqL{i-1T@;mDb zyX>pKgXzAQBfjt(zdc(%{{z7Nvp{V7CvHnW0i>D&9JWS-oa)2A%>%s$5GUZfznPN@ z{rd&ft3VAbzYCnf0QA5Q6u1zqvFS^<+k?MTA~(VVLN2pE#2Y@fn-Tm|!XnQ{bgA*x1MDju+B)fI0AZSAZ2^5eCY(haSLo;MUL`*|9gqJ9cK{w<;NPI*`ltcnF zz}%|Cj>$v!&>q1HI}{MD&|AS6$%G^{L<&?yR7^xwY(6XWr8Z*%>fxUs3__*z#0FeC z-xI|nBt<49#9pj5dxODKq`?e4L}OeH*T zv@ZV$#8Ui4Uo^!=EXGtsMsj4uWQ@HpND?00ucs(FBB~y1+`~|8kwOU|QVU0Y6vtyE z#x=aTa~#KjG>3G&!4F)=XB+}R@wy@m#aqNih}^z_60ajf!hO6*|9+G~f#kPx{6}*X zvt=wukvz!R3#D2?stl?daq_8o%tn`#$YwhOe9TCBqsfk}!H-n2H)KX<+&hwM8wDr= zfEkaLd`Xy$xqGZgUK~V>w8@;@NT`fTo!rS*L`V?~s_khw(OJWTB5P3@dS#RSgb{7&`L1;?aA8#_+0lFT)L&7-79zpT#9 zyiWLx&+gpL-y}@?Y)jH)M)IJ`9ec-=49cOTKXq%h*qlv!#LUFg%=dIp29?k3d{FwV zPy7r{@U+75G)btSpZ^R!^$dX!;7d1JDg|v&5_Qn~j8GIkQ3?krM*JktCexLbHp{jwnHi_E^`y&N^Rzhk(v5La|7z?YqWVNQ?URoofB_f)087r4 zxibvN(k=B-KOIv*CDOt{QVK;=NK{kI$~@ZxExz)?Sb4YIVF4b^%pCetFSXS94Ajru z)G{ShLlwy-)w@x`Eb8luaa*@i>XY0MFiMk^JVgUZ-P7`_R7|~8TeZ}s)KpGYQC`)* zLjBYaL{wC93MV6`qHOO%b|Djk-Jw-e%?NW~ISav;zfdyECoz{a* z*muo@h8-lenNhopSk4pBlX@PEg;ki_SC75fat+zuJd-aKS)cvck_}pg_0(%+*!#J> zGMZS3mC}fO64i_~N%b-*=!t%X))FN}ot;6fMbeS|SfTw|lf72C-z6+u#3T6)#XbsB-_i4mEoTB|)=6V2MLHABUv zTgScILB-p=?Ld?TIph3W%ymxZw1BlWnii>ypVGwx2;I2tUDQ?G|4_?S(yd!weO=Z)-nfii%d@c_{M#Cx58I7i zR`p!aRS4g`-k|N-;ayzfMOfoSUhow_<>lKGEL-V)&fFE;wxu23#a_e}-NoJB<&t0R z1zztB-^mrY@x5I6JJ$0x%2-L?JgHegbl>+q;NPWI`sLo#{aF0XU+&!B4{5(?JSPHu zH6p;q+TF`#jkPmCyQOMh14dx@)wBg}U=&{6?|tAE4o(SXUft?n4ludEqXW+qm?w|{ z^Iat{Tq+MC;Aa?N5+36EEn@q%!4o#(Bfei2o?OL&(R=~IDXo$qBn3^_15H4IuhhoX z>;WH6$`B4>AtqvRJlzBiUasxl{}yiIHBQGDroEOGt+iQ;H|>I4sD)N&h0xgISLt91 zBH)l6V=_)$)HUN2X5ceMy*6g!2ObX%v#>RFfjCYi-04CysD)&(g;o$5+ZC`@`W|Ql z4nZE|M7~Z#K4e5buti>FR_3-jz%M8cCVrvPauX31$>UhiV-@IRKL+D>3*=GmSyC?L zojqkWX5v<6wp0007tvsu>540`{hIX=5Su;U{q(XHE2{; zVRwG#NQ`Gxxkn=GR?m}U|9$RBKIUPN+vb0MBY_@hQbuTk9%mKI+dm;~g_>FAvG=!8z?7IedtMrq^SAxK{7S=P}>wuMK)752!lJ;qn4(f&8YOQEy1Z-KMGA9Gb;Y+|{S>T>3y+u03u>8Q2wpc1n zC}Exs=@T_)tiJ24KE17GWR-Sivo*E~?&{X`&;n380M?KLed}43YsN0vtH$dkeqRU% zU!sO%u*SV0D!go*My1_G18oaOXzYGfT)URR(w?K!?rE*w>s96|=}T1F?Uu1-4FMR6 zRNX=z?PjW$>lI$@|F!CEcJ^)71}EaC(Bk&g%6@1&yzIYz5+tA%H4vif6T-CCS2$AU zeO%?bHV3Tj?o9A*ko|7)9&hq4?=lc?J3#O6K5zD}<}zq+_;&C3j_*up@Aba#YF6*` z-f#UbZ~y-9^4^0?xCIg_@M|dW0Y`8wPH+LoVh4ZlGT4JIsOC9{a0|a?GdM}FUK>p` z)>Ip+n6`vi5Q^#6;#*@+h}*S;cZKcXBQ_a#)^2*S18I zF+ef5TtO=H|4e)x4rlHKB$Ss{YNi$^YrM=?LW_T1aXim+J>PRa?{h!@^E(H0K_7HN zFLXmcbVR>$aD`5jZ1e+kbVy&`NVn|f*1ahJ1wpvRAZPPYu|Qwd&Nip_F(t*ulDQMHiJD#_H69&noe zr`V~k|COA0im!MexOfvWbBy13j_-Jn&v=jz`H!C#i!b?##}|{Ac$H6il4p65Z+VwD zO_UcY>#F%eAv%P!d7a;Ro~JT`_xXVL-%8g4nU?eFKKdyLJ)?PeY+?FYQF*72dZ}Of zsh={akL0V5`ILWoWiR=ix~+inda!2!7#RDPPk~1&`=}N9k2ia_w}Fi(d&Jg|jnDdj zzeINATVbQFG)RIdy=cE%>ZMj*W<~lCJ$w-O0JN`ov|sxnhx-Zx`Fczlt~ZfVKXs@* zdn~Q<#s7NCvivRge8vy`$j|u1*Ye8$Jw6|K(BE?kYJ1YB___=^9s>BJ5cj?39U60k z|D^wWznyJPuLX5zY5}Oy!(Lm&uT8mMe&&x_<$rs~-+Vk%`{rN%!w+HVuYU5v^RHia z?MHhn@BX)!a`t_J=+FMLFS|S!|L@20@pu04XaBK(@*!V&aM%6Y*Zaqe_+kYJ2nGZa zToAz^!h{qy&;X+cOa~PQ1yC$Nu|x?6AC_?J8090#DIrICB>5yrM<7L1u4HMX2oL0pFxKbby@T%Qk+6bQpv)!smP~Mp-N?9^(s~;RHqcQsIcqA zfl5@kcp>8g7q4my=+N=N0^GQAC1jYZp)TGOcUjy>O4BTizgdcC-e3TKQ zPhUmNe!Y9`@l#iY>W0Z~*|TZquh`q|Z^0QioPYxwsL_DcCAZvy(K+{=brDK9-FDeo zcwU7VW~d>D=)E^zdo8FZB8f_Y*cwy!>9=24{PpJ_fO#EQ;EXjQXyXA2=(r<~GWvL& zd=FAMp@Kwicw~7C>Lz2g&hV_v#)t`ZKIp~&!8p=i=US@fymyII&sH2cd z>gYmax_GIjC&ZY+b2<)I7kPA1w;*0Q_O<4mZlIvAbd!px> zpuG~h2C%_SI%%TB8XM`cj#i@LiUCA~lC%DG5bc;UG=N}Areelmku8DBf~P>iI_|5i zmU}L$>8e}Yy6yJJT#&TlY3sb!G3D!^!Qwl|qx>d&1}2g+%7(B46T55@mG-u93RCRC zi7pZEkwY0~?6HR#PfQzT5ZB(-DXH7m`z>*1nR_z2?xwu511hup|MIs=#tUArfjV?E zzWWxeFBt~&4Ctai@4U&f%raalC=e?xu_&vg!V1(AZ=9dCzX>)csb{K*GPzvC{B_r0 zhaEPND*$1$6!8RCbyB#+iINNQvzXbcKv!IY7yfDLVB&~G7t_%*#EQlv= z2DOc6WaijjM=rV8lT&Uv=96DZHxf{E{`uRXi!L|mchBvZLqhio21%R$TJRuoxTP@C z4;M}}?uzRUiqtM%1cS$rXFfdfl>=Zr^2saj>E>;K?m6_&Ll1Z9oLg_YRIKwo^cZCV zo6qz|3ClzkNm&?0*Jo`<%&=D z`_4`J6h9Nu$0le~Tl63(z3QEgZf`ir1vB^%U}X$@HQS);xYxb!jv~my z7`_O{CtS=7U!a8_c^8%X0r50;98p{L*MJc8KvRWmBq8lYtIsfyk~~agA?H;qcnP73 zpX($jLn+GVjgpF6JSFJ(=Dr~~;VdC|K+gh3zFD4e|97vHmZfkRyj}9rYP=C6FoSu= zJZ2$`4d}u(IPrxtTw|Haq}>~1u*m*pQkcEmC53>LwQXvxo80{7H4mVt708N|G$3a= z(W%aLma?4^oFXbysWHK@kCjcBp|xTuOWb*BeB5{i8WAUv#^q9%R(oWnS~h^{nBbQL z1>+d#Aki_zhFUoo2(@5iM@%Min!ef8LPILjk^ZPa;0#cZ_{W10OjCFVsviPiS}79n z(lYBD&gXD?)19WXojx@uJArymx7`w$(EL~}`=GlN{&R%^QzNDd3d|Xh;GS(6%Sd%& z&?HE7EMR#kL1`*X6X>9Q9W|yk)7m(adVsCd|7;~Bd#M7E?%DJ(I#gVJD zClNOAxV@&+2Puf&Pu)3GPm;2yjxD86+!sa^ptT}DC?h|K#?*kWa1B|-rBq>r8?tVo zvXnI_1-LM~MbaT*mBnaSt$Eei25+*#1nXKqdt2E26*jjefcF+!*M%UIxUO{pVc*nI z!S0c?Z2YaPJat&3P-n5BvM#(%q50@({1HcB>wy(8-;F8)2r>?iNx|Ql} zK>I!OcEvw2oa01uBVXLwR=;jMFLe1U;FuOSe*qTb4Cs5*hDOUb>QXC!rA5+o*{=&A z0YS$y?2rgqj5BwYCWym%P?L&xWA7y}|7vA`*%PL)d`(5)UEsUl`O?@Tr_IkGxT+Om zw)U&JWuRz~e@QRTgl8ubVrBm=I% zY=t#+7s~W?vTGrH;*4UXyvBU)RyjdS71H^|L<7QGW}GPopSi|97DMd1G*KoZH%tRS ziD{->Pt~G1p@)WnZ}kk;C1BX2KwPD9gFIFIL|M%Umh_cFz2r%^t&<|b>l{q2-PekB zvNLFcru(eRRxi3N1ab0=VTlRO&a%Dk9dK0F2uC*BI+_W%@mdBQ(PigRt20)Oq8(X* zVt@LWJ-~vQK^^5t50sJdrEy$;|94`nYP;O#M)q2NqSqokq{I-0v!~(x+gblN!sner zQdh_gY04VDDEPM$tV&i`13ZR|_OC?=&vL6P7v9{jCpYTr{$E-* z#^IiE;|z5}=hI6r@_zf-+*Y1z+_`mNBv-Aadw)7v&Z@?VW4+>OS3Ba3RN}=?m+_ih zN(DU_*$?te5Oc6R<}a^_|4(qf^PbrJBj7NJ4b~wKcsTtXQjhx36QcF5e?9Bl5c^{O zm&?WkvKCRNdkZ-pSH3^H#(yvT#t*+q5;;Ed#mn^yW@l~8cWK%8I;ysMpayjD;Fq(n z{Xml5V%=X#_er1m@sq#&=BLCZ(k}`0gTDQsmjyTA-wJ2Yzy9{WKmL=EfBnZF7V42IwtuAUn1pd0338nPh<)}b8IVIjofA1ojo_Mso*AP@fFAG+Wm4q*`*Vh|=G z5;9>E#=`wIVH8H955NL^hz4A6(9XSqCUzkwa$zU(VHs-B8;;@{&Y&KiVjG&GDy||c zrs5o;A}PXRA-3Wy)?y+E;vU|jBJ83rLINT#LN6ZSFDl|97UL2=VkAzX6n@T@kj4?v z#Tjf3IlVzA@*fypV;G8JENJX&N=4kcI$B~cP%Q644!g;H{yNsm+kOGafuP9;?)Bv)=FB4DLX(&b&^rC#o3 zJm#fWhGkHWrB4QCJ{G1e}x}-tQCT+^4WqRgqZYFMm=0)!2V2WlV zgl1`uCUF*FQJN-fBB%YO=2EWat3YOK$|hyjCPen8bz)|2@+Nln1Pz>gIRer*;Nsf95BDlBa-{r+OAB zY?6p`x~57dXMD~leb#4Z?x%N3D23uDfCearX6S%!sCgo%NPeV&S^_XgfD2rBRc3P;0UTB5JD1p*whjJ)-8s;K~D04cdiJGX{v}J=jXp2JVi%uwv&M1=F z|7eXeX_Mk8aq_4osOYwo=#Qe5iV|sEYH5)gX_6|bjE1S0Iw_Qv>63D#hbkvj5+s6x zO^Qw_oO0=wMyQwmr+$Jdnc``lnkk?5sg9;;oCd0(vS*uq4ScSsoF=M^+UK3lseg{? zo-S#mLh70N>6s$uphD${+JKKZG13s-|kCTI!{`WhWLY%=o~l zk}9p1DxFHIqkbu+?kcaU>Z-EptcEJAvJ9x!>X$O0t=_7x`YN+FE3p1*u>R?R;y|Z* z>W#=MAR4Q-Dl4=utG9NmvxcjHN-Mbr>!wz#LS1RGZY#I8Yp=d*qlzoI-srr7|D!ZT!w+bx5)~mtNE4~(HzryLhTI-IiE4u=$smiOw4s5|v?6X2^ z!#eEsEo{VAr!6u9cj79>lB~&|Y{_Qq!Di~lq6@b2;m1~N%3iFU(yYqf?8*je%R21K z#;nZREYOms#pdkH;w;N@YUG?K#O7+Y9&4QjY|uvS&=xJx8m+(XY`OF-(*~D}V3zfSGm5^mt4?Bn`v+~(`y0%_thuHHtj^$ z9xw6=ukr>a<|6Fs60g!;?&(f1_1-Zvw^q#N%qHp@bulv3)_l7TiG9s|fuSV4G?d~uB=CA(7uL39V zUIOs53h?MI?)^?M0voXU_U{5SFa`rB1k(T#@-GF~ZU$Fyyl$@sTkr-8E7HD# z`)?E5Fa&d64?8goV{r{%a1;-*6w7WACv8(+u?UxO7n`vbr|}oBu@1Yi6=Dt<$8i>` z@fp+c7ISeOqjCFwrUMsnw0Z(2+#s-!D4KX#uq3g?;-MnjF(Wr}9zXIF<8d7a1OOrV z1O*fT{{Soi0000$0YCu&2>$?82{bh-6e@xSV<|j{aNxp+5Ld~XMX{nqRueO7yoj-5 z#E%?lg&gUzB*={pQJzf2kY!4hE@QTo`Ld?XhBI-V+DVgVLY+Q)0?kSEXVIZXk0$lR zRH@UYNN>WNYLetiiC3$Nq?(m$*QZ;*3LPtzY*4dj)1n2-Hfr0ZaG}0+>$PrJyLhc) zy_lQw|cx~FqaVtMg{4(#~&x1ENKAkyr z>(`Sn$6gq?@95scJO9_Ly*hbP+_RrfU%vfc#^1q@pAWsheD%`J*`knOeu*W4TSD0-nOu&krj=p>*`}Lmo+&4qX3|+^oME!5=bIFId1sw- z{<-Frduj-$V~DnbD4s703aF!y`dMhAit@>5mV;uNX`qsJYAK-zHrnTaDhN3ag>9>PhRbkM0^QuEO@Z>ZixndLXg6DywR;g~|sD zu+5HIEwj2h`>d?ca!9SV*AAQQtH$!mEwbH;IPJH^VmofT#QRpTdeWM8S@J;$P(vk zvbFkB>~Y8=f6Oq;4Yn+^y(-sivnV*Dd~nAIi<~geH|s3)$~3#|v(YIhy>rk+H+?kH zjseYd&Qwo5^wOVPt@P7ecP;hSr9$oWxg_`e_1Ict4Yt{A7hN^lc9Ly1++NrHU&(5} zJ@?vlxBqRoi4o(Sx88lvS9aQd%l)_DXA|x?;*dXn_~MFBPI=>lORjm=kr!V1<(7lV zc;=gzZhGga&rD_LFt`4>*g1}l`RSa~p8D;tv!3{evhPhg?!ecsy70m44*Ky)_3r!f z#M@r{@Xbdrz45v44!H8aXa79(+gDG0_uemyz4o9uZ$9_aho8Rs>!W}E_u|j~e*5&- zZ@>HO=YK!^u`*x&|NEO?00&6G0wT|VuCpHb1js-HI`DxIy5W z?vtMYy{AC$$pse_^q{ol!WP7U1%^6wp$d)YEFUUSgw}G37tLrBN}$V*_A-`ow5Ldi z$;^?)^Q0tAsX(EL&w);$r7n%>OJ~Z`X;$;5IQ6DZXUbDfGN7k9-DyyTN>riR0~C8S zsZvo2fqyQusZRB$RHxe0Tjn#E^|WO|x2jdHdi1Ly1?y7Bs?xHWRix zkP5c3m_6%a37XfRHWJ^|>Zcf^vbH+Ulluw$!z*X}8N=Ybulz z;047r-Wc58n%A?rO|N=CyHfKCH?r`RZhVVNU&eBD2y~DyZ&NB=H$;@8plvU7(;DEk zzO}&Gt!{r0nqAd)x2O@V#Ja%i$;)DQy-~$*b4d!;_s-X{^$qcUPe9`1>bI;52Czam z%;Ev-7Pk+Mae@JwV$#ak!8?txge5FqBE;6hh0U;rVXWbKw)e5Jtmub_E8_Bwbd)1@ zE?LovU;>*m#WPs3E=_CW8K1VSw!N{Kxk&_v(xDR*z6BdP0A$&|82`Nq81kF#yIlWD z6qYJpGKlBfWaCB|rX|qleFq(As8ZR*i>)X_Q2+sna=F1{7HW$ljpGIAc*o)0!zSBY zXagfQ&Y!00oY#s~CF}XreHQ6PpIl;W2HDSqX0eZ3%u=ha+SMlwvzK$M-(EYm%amR% zL(dEgOj9_sQq$e}wyxgwT&ql7U$+upeZuu?G|;%FQ6;~+-?G#T1ie#<+%ME^*5jHOFsiyHi=0I#$+ z%j{oux0Vy~tgojj-s&Qgy43f+ILDn#bc-9^1x0svzvIpHfR~)c$iC^p5iav=vT_VI z4=(~X?QVZ|TGU$?wYJ9|>p=^>-pl>vpVz*Fo(HN~%A&0}5Bm5;qEXTRsO+n&!($T!>f?e?();P)cHJ6s2^a+-@FyqFlm z0E{{CCNzG>&>y|@RnL0PTmSk)%)a)wFMHQtzx3dbzW?~gPyX_UpZe%ezxuJC`R%LT z{pg23_p{jj`qTdY_@{#Y_rFH|xAOm95P)3-QTZo;0O)^7XGx7EW~}r=G7tcIWO0i} zf9*$kxyJ-XfPrwdfk40m9r%Gb&;lVif*|OD8EAqih=H@0d8^lg?Dv8dNPz=5g9uoI z7kEY&m_-_RgD04RG4O*xNPF@we}q_wGH8H^n20vGh>VDXa{qXTb{L64D29|+hGck&V>k$6Fm*Au zhRC*p>{oxeSAkYgh>&=TxG0IbNQ5@Pi@x}a zJrImVAdHyAfyQV@{f9<17z6pmiQNTpAUA=N7k<+he}E{6f!K$k_X^zDjo$c;;5dEa z$cN-uj(W&>oq&z6c#Z0~j`Mej?`VkdIE%E{i1qk`UJ#6`=ZxG1jTskvj~9;J2!5G2 zi?-N<2Z@K&D3A=<2@N@p<`D2mO$8;SgDnT`Cwh?m7b@V z=NEZ)8JduZkghbDZW(%5M43f-j(%vAoY|S5NtR4GiKpp>eYu>>*_@;~d#kwvC)tUw zsd=(Vkes(gX+%fcc?I72o!uFpg8$S+Ls@Lo0FV1UjGwdZ0e=P6@i84BCq~@Su~pf6+Oeu34R58IWNK ze)o_M_%IJ1N)I5)4k0R{BwC{CV4^764k}s?E6SoH%Ax(>4;w0@Gg_lITBFlIqdK~y zH;SV<`lCJyq(3^O_CTb%FbzkVq+6k+OxmO`stX_rrBXVgRQjYTdZN~VrCFK|TFRwf z8V~1y4Pg4EVoIiEx}|5DrqrOO&w!?F`lfK23c{cv&rm*^1f4xllG53l&p2Bi2bSzWNQ0Z_;s6TqDirT2H>ZmO`t&rNOmx`%K$_+w#t()4d-nyyIs;a3Pt{+OS;!3U4 zDy`AFuDlAX?YgV*+N<%psE#m6EkLXyNTHHuq0`8${t&Lu3atXGuGDI+*J_^HIt>Z? ztql9A-&&*&Yp$$1t`vK&7K^Uz`mP!q3>(|A^7^hrn2&*3topjIDLDpN5S#uQsR28% zF8i_|>Z#nSusO;N3;(OF4V$wM8?gXOuor8p0sFH;>$4u4u|@l_NSn0udZ(&MvV1yj z$?9>-%8dkR5C0mpFl)3v3$Zhst@mKFH@l%gIZp*uxySbh_uO3^kciXh1D@+jPr=J+OR11E(aJ8-Lx*7YT25YGb zJG(8Ju#OA4x&Pa+ra#b4())vsp(1?=x`;gQ%bOjC!4Zh8NC|H zy4IV%u*Kv8%$nYryDBybK(}3oOG9%(=$`b_&F zK|HmKIlUkp!k#1($WT!dM0uY7x2TIP?gcg9DY z#+s|dOaJ`FPE5eo>abEwxpX|qbzI4KY{!(W$2gp{T`ZVRu!TD?k}11Uf~>(&OHU}P zvL4(AMvTUYJh6&Q!f`CeJWHkHo5{Fr$(X#$TAat4+{wl2#ehu8&3nqSWxp7@%B}3m zoBX<5E6a`?zRcXqx$MA~e9gYB$%)Fx_=thROa=aRSj0DY1x0VjoXq%p$ot#OYue84 ze7O7o&Cx8WQGCFXT+R5L$=H0$*{sb<3(Vde$}j-Vvo_Ae$G7yB%7LoPSE~>3FwN|& z#w(h=(agpL47B!~&*_j0aO=;!{L$3h&;QJ{p!~-KO?4S;&P1Hf5FODHEv6Qoy`(zR zH2+=GEv?ZV9nv}t(mXBFNDI)ZInaT;a?^&omuJZC;G!=5&P$Ea*TA}Ld&T?A)Huzt zR=v|$-P2Va4L&Web|9eL9Mq#sbqGCH3T?*G8<6?C&K?cHRQ=Ld8rLAwrE@*ky3Eh_ zeAQd6*TT@(ex22Leb<2f)@|L)K>f`_U7_@ae`!Fqk=F|9EU-&W*l=Chdu`d5o!MOd z*P5->UH#Kw%>(Xh(uqx1&1lZNg|ebudWfhAkv-Y4P1&72+qB)ewH?@>?bf>O)jwc_ zqlnny_KZNhaa%Z&%d3i$V5q+;*|ELbn=RXzecRH_(bR3*oNe9H4c*NR!C#%ce*Zko z@tae_P23}3+&s|OJYWGuSO`kJ+}mB)+8y87J>OY94EBB9&yC;uP2cie*oVD?rOjP~ z45+oVTn$j(CEW>t*@f)=-VV;+5YE@wz26jm-}znPf*sxc?c5yR-F^(6q#fM-h?vvn zgrB^HoVVaxhzBgn;V%B-7arpF7{>^ci1_eEQ za#i3#+`|t=Xa|1c_p0DE>4bIwqBH*BLH^_KY~wjT*gF2@yTIdOKIUYufSWXPkQ09E@BZpAq2A@Bt_+`U>WqHsi=OI@zTJ>+ zi)_G;r90fM*jP`V;C8M7Ret5X?&!T<>a8BPt3K@YP3#&T>}me$=H2EZK2&AB>oB{J!xVpW6nn?H&K|2e0vStL(*{k5<&TNbY>~L{U*T@$;VRU;kj`*^cocujTvR z?}mQp0Wb18@ALj{*p>LlC!e=ibmY!{YeNiT_ciS!An)|kKg9O9k^${{b^!^;SRXUSIcJzx4ut_x}9#i=Ojz z|My`Jtlv)dYcAG*2~lbv0jE7r$Jh8yuHH{i@lPlPCkhT%Z~2%n_jV8XoR9aN@A-TW z@P5Dfr2qNS?doN3_k$6HSC0FobtlrQ#|fBTwm`noUbps)9#|NFon__jaN zzJ21Wh_b{yS{_AqBj)zfE`?K{`M5v*xxf3?9}T15`*{!c!T;~)*Dn%VSa7dG8W4csnQ>amUNi9~jnpJ~Yhi%;@jH_0!VKYViDzq zW81ddTexwRNprhau3chz_3j-DmhWG{SpAZP*pP5V#PSkb%&0L76^~9piY(cLi;I*j zGn=R^QwPq^IEnJ~2~?;B)JCZ~ed=xX>#MN^&&FH(+5ceNx$n~5J=b^dy1;D*CSKgQ z?!$|f$6K!0x#BN88XMu*BID%8HI<;$%)xR@@Fua032$U{&kqzO#f!Yu}m<-v~kTe6Wp<|9C>ue zN9m}GFi4R|8mR(4BOs#4&XOoELqR><2|Xu5wEyxwEgR+XOEw`TQ_V9gwKP&pFO73k zI8XHSO?K$C4oE!jRPx9VG*A^oLWf|r$S89<6hskqx-v^gwb@cvUU&6X(oD~q)XX+D z4ffc+j&*a{P7xC|B%YuxwXzBmIN=1M(CC7u@R%%5Rnn-eRVPMs4d>Thy;K)icH1S^ z*kYA+H&RaJY<4jzcKFfGJB>7w!fPeCVT~bSEpH|iK%3IEMQ_9P;VvUycVbE_eivSh z=T%YO7eTeRxo02bH&27B4fq9u4G!-+4No%y(JE~%_tQsxv03AIFV=bIj5!YWUZ8^( z8ee@K;}^Rj>OA%2lx^A&;g-RLnTm$Zh5tC+jVZ=?>zuppnP;Eb6{lX0ho*T~2CuDC z0h(H!5QHe`b-wR=QLMI}RhEuFz0dK*))j`M$ z(W9NyV5dQr0WnaUF^1-QVjE?MF8EQgik+ID4)=tG%^+z5dlb(M`uM-> z1uzUmED9Qrh(r{=agA?%2zsbFR+{?EhBkNTmL@iib2StwEUI5K7Fx+9@L&F2U$c@s#2NHWM@0wDNSm| z)0(x+rZy>ANjrwqLMq`F5J?~_j1=>jj(Ep9QQ1sI!c(E|oMu8LIZudU^OpJ22(<87 znZ{Y@Rh|50M^j}Tfs!(r24!eT5o*$f%JZZXi>O5F>8U-;&zpC|&^}e-M}LA-kONI8 zNmuI7lm=C(Lv7_lU0Tm=CeBW++Y`XPsF|F~K&LJf2~T}`(4PX8s9RO1O0#OzmLAoL zGMy<*nq<=panzF{DCZ~%iG)4$^s1?3;#a-ORlMf4t9uP=U&lwYJ=`;wGu5MJyogh^ zdb9_z1E~g>YSp=7wg0Ym)hkLXd)Jrxb(yIXtR+Ws)`T7DC5#)UVoyfZG(dK#OZ_{~FlldEsS*6+v}PHiXfd zCRQi`WOyB0RVxXiyvv1dfh{{=10UGEtG%raDQewmaf^qeg01( zRa%dt=9A8Ya(m1`<#FIR%X8*2maWWXJTo}X4qmK^N7m#co0$&6om`XGTw@N07|uB^ zaigEBXgSl_&I`U}Zow=hA}^VC_&sw}H+<+e>*URPUi73TO=?s}+R<9hbE{MBWg%0? zek7~!9;-DTL3J84OYUe)01awY*=p3KCibcCtZGv;ddrY*Fkmem1#W(pw=N#Urj=Yd zY-jt|PbThok9{+6EBo8yMs~78yzD(UTiv)-Tdbjtr*;QgPe^i-ruVz;|Au-6!}f5o zjlF8v&@m2N(jx`wj0Lx-O!DL9N^|=cFku_=_)FG#U@W$Q;%G2U3+KDORM;y(%|wh zz&t+Nh`G#ZZd;p2{pspDxYSQHq>cnM_w?4DwJmim2o#dn%_vmC&K->w0N;Yo+~++H`m~2$ z_oTPFsS*D^`gPY`hc7PdQ(JOvw9Z~VnAbetI)8iHE4vfDLtU(Vnumu!Bk~2Z3!XM+?zt>|yA~-%5L<0@XJPc4j z863bRLLvfXBab^mxi9*qFRVlH^OKb*LL7iWGBm^XgOoK4#5Bag>?=e=JVZns!6#4wER3}d zff~uL%dnKKI}v%JizG##Xp2XK@>$n)WIrz4mTXWyF)?8*?@_& zLs>L{SJXp3%VMsZ9vZc?fxKpOnYG+ye0PGUi5jK&8zNAVJvzR*T{ z+(vE$$58x5ehfx`{Ks)DMsoDN$S^YMX%A<7MuiN&3<$$%%!O=>$a};`fUL-Fyh#5W zv`CG-!C|bza(u&pF-S0c0y1DncYH@|iN_+CM{1nN7<9>v+{bWqznMJ6fV|0OTgAcZ z2<*zBS_CD6nS+x=NtIMHZ85uggh{2u$bPIzr_4c|yg{9Ox2+QtefqSZOh}a6BZt(y zq?E{|q{v^0NwU;Ovt-J$Ov`esIz?nekc>AljK{3hN}?=+B{0e00TAv0}L=t0k}iQ%%%T_`2y3MZ>(bx*1J2*aJ40#oN5i;5>sUP_5!LPG%5Gq_#KrN+b<7A5C^C&0%F43=rM{!PY>PC0S!;({7));&ADq(kSsI5^Cz$q zzex`)LzPy*$+;~dfER8Xj7PPs$TECdN4=#X5)PPjSJ6>Y)d!yWBBuE^Zb z8jaHA6j1@)Q5&^V9$io@WzgANxJL|0SOl*@`Jf>rfDIb}i1W4o+s^-;I7$tbQaIJo z5Z%2jty9A6(!?QBo1hR)dXZ(x<%AM%_|7 zt<+`%(lE%=p35{!B)`8)fYG=_`@~BpeYr(F)k#fNNqtmEZB-Jj(p7y`5Hm(vtyNp? zQ7ygGijpr!6w*EY!4_Odgp5YzJA+b%QdD(RR+Up&jZ{jt)Ld;aJHXRk{l4qer|aPY zl|V>Wu`kEOp@u|4WK~v1jnx~)O!iq;SzXs>rN~=_)@Z#3Ox?L9_|jJdRw1Og>6x)M zHC8b+)npCWa#dA=MORln*mh;tgoRgzCUoh8|vJz0}w z*mPJ}z*jH)(|1(b#|*XyG2636+gMf2wJqGX zb=J7WTE#tCy7ercz}vmm+jr#GpD;qe4P3$fSpkH2ZQ$7=-1wDW1byHLUS5xS4EFRnGCLK7xi3=4 z1?r4h;OyWJ2H_AE;jtuPA7H0NTv1`+x)sSINfMeEv3ODwj> zA4cFj4r0Vr*D?NMxHZNu4HEsu-CykjGoayE@Pwr0;3c>LLX|H%mV&Xk<19{KE^gF4 z=3_4w;UNa(bt_{+reN2rfbFU_uVusxW}e6myaKjj*6m9!p5-s*WG(jOPX=YEl&bkM z<3ffiB?hgE?SKyG!qbH0$IMhZwqr}afirB1>h%ehDX3>G>&8aA?JT4A|VK9ftJC9 zUguyWTxn)!T&`t?Zs=W(uZY&>mW@S|aO1E&W{*A%TM_Az?n9AA>2)>eV-RYUUTKAP z;FeBaP9VruzOOPpi7S``$IFaK9fY*`Y z-^{ia<=aKvp2zst70~Lf_E5S`X`L^>lSf1>@u58QxSipAazuxRA5(9`X>{`p)9|VI-km~9(W__XDxQ^_>t?ZvB z(R!@pv6yYzrfnq{i!zw)+}`a>*zNx^_-)`0Zs8v8C-{QsFmB|Qg5);t99VAWes1U< zZs~UJ=??DR=56ebsq5D6nCfoo{BH0L?=lE)MDBv`HiPnxPxW@*M~*g_uG{VjZQPYA zJj7>BKm{rB-^>_Qqa&rr^Z)=4aK%Q`0WWX^KX4jAa0Nf`03h%Ne{cwoa0wp(3a@Yr zzwkk|@PEy44ksir^l%WT;}93|5iju%H}P=Aa1~$i0IxNDI761uWx5bVYA|#S^k7Vn0%N7wQuXy;*L43L zc2o~^QcrPV4|Y3O_GBma4tI8FZ+2;q_Bt>0n7(#w@0YgG_ATmmZTBWwKkMv4Z!0k6 zR<_<1df0uWX?00|{_<;}je<%2Z z$9I7*bA=}?Z`T%x->v_LzodzG9eV$EGZ%LsCj>~iZ*KZcTz6~&PI#obbi3U5^e%Xm zPx+ERd6y3Ym7mQDb?;S=Q5K-k7no3k-}$SaP@hi$pttu$e0c#+aS#ytA29i(|9P6d z0hf1rs9$=cZ}EzXdKhr}ssE^|clBPb`DmGrhut~)!FD$n?Tyc9TQEtGhuLv1aXKIR zBfwCgzxk&9^}e-wV0XbAUo2*)` zH~dQnfx*A}zYqK$i2M_m`>2;u&QJVZZTYR&BCj8Nts`@=x1i-K`<6xfH(vX-KP}8B z{IRut`tf|-*ZTj`ulm|ARlGm^->>%Hul?c&cH<9z8(_XJSAM*=`{aNAWsiR8A9mb7 z{;a2d-T!rxPyO8gept_TjCY;8ZR#Q4x2djZ*>~%NZ~uiS`n`X7%0GCX_jmc{di$sU z{NMlk?|*=JQe+^(f(8X5G;(lZ!iEeLK1?`;%0!A5FJjE7(FwQt&r zuij{iLn~JxUA<-vD|Redt7XsHNDD(P+O}WanuRNu?Od=+M8=5H0wlz~c>VGK%r^?* z!91|ADN+AnC1aK;9dC%d_$FnXI(Y&OS~PRd%}6~b1+Af~Q-)?ipiZrNHP*Om>%x8= zdp1+LwPo9OoqM<4-fW%h%`04R;lM49kNCS)^x9{W6uYW)N{zv))#{^c!4Tqdw$stH!Z_Y7T z+!%38cU^T6NT^+9m38->c;M}V{|_P*<+A#5b1_u79v?8c1b3XA!kd4NTO#r03f1=R$>|D0b6p}rI%lV z>81aQDU$gji|&~zAC0@=SP>aJ3Wsz!0H!D^s*!uqPL zu?AX6Vuf~4(iAG>+N-aJdIwZica3lWnvVXr;+C%wn`x)cJ{#?)(@I+{wVontpSCrc zn(d0HstPVfsVm?F%$)EmO3nP{@NZ2R27DE$qn$S%3ltN5Y0(fbob=I3OFOC4D-g!97EoA0brn}z zeT5bqZ*7IgSETwx*yxfScAF@p%*G~tti0WJU4XL3%rIjDiz{=-UB=EAp0Gm$8wM?9 z;DTSdwBbx6et6FyX{EhPP^{3r`pEUlLJrr@4{nVIoQVQK}PbLd;WRIuJ+-D=m?b5 zf!w91UJ2^k&yll*L?Gs4j>?g5V1Y5%W~2J%Z;PD8BP57E`}424zWspFU!(tmHO+fs6{lA^;ULP%rFdaD&_nA+V_DMG88qiXnhs7{?ePFfJxJ57Gglo=~?k zlmTBaTb)cqz$^OM(Tr@&Q1$+n$MXS_TlHH``tB!4{q?|*^7A8lPP8H>E)kNFqyZ%> zImt^-Qi)B}B=G9+ITOBQI@ZIY$~ve;)bZ_57b3usV8_T;if4*qoF)IDu+YacWa5Ky z%t0AGXqg7+agk;OCNPKd#Q{_imalvi(4N-JXT}bifE%1NtLa5$KB*wg{0t_!=}n|q zGMt#)WD&`UEKoKfoh7+m_SB|9?QyV!Y-pV-Q%RYg>`@2#jM6bz>BR>LDhbD&02jHq zlVKu2pNfGYPx{!(Stb*i618Sg#@I7g3M-oom0xhsrvblVl$w28C`Warm{@W}DJ$S6 z6T$h-mBO@=Mj0nfYbws*RWex+?WYwfB@Rzmc4uSCHY^J!Ps;dX}_JTB_R+AO_Ux zfDY>2tYf@8-9MrPwx#uM6@B~Lh_d!|;+>usK`JKsiZ`O23MLUc5QG7+E(H%3Q1Som4YSztm6}>b` z<{vj4UfVKqq@ZoUT3!2M`F1wI01hu~Da@-7Dwt~){TS8EU{T=G6SX!5GEH7WnoXAKb)ihHr8Ew zWI7F^;hEn~63bpPLd|y8siCIOsW%QRf5*FZe8eeB`@q&%B zN7N$mibwq68Si*YNDrYrV7qx6Vz~q++HwDC-+YP~zG%)*81&FZxahy~_S2*O@?gsu z;V|s0ks$) z`yT(aPyPdkzyJU9U;hDM?*SksIT`U}UIMCK>YZKy^1%ZVpaV+a1TNq2;ob#eAOu$6 z(nVhfHs1$IUkG{v^<7^HZlC+1U<#Js3X)%9*u@!u#elFM;{_i4$=?mO0sYzE4)UP> z^DgArV?&5hnj35<1}%9$^%6;08)zA{c@qVBsJ5!4_H}7hYi& zdLRgjq4SZT398^3qG96GGu1`e7gf zViWdZ6%OJddZ8gEV&g4h7b1cpI^rTkA|!y}7>1z;Vxk#dpEaamC$1nQxnQzXoEvJu z8Z_4&)}bA$;vDi}D;A<4!lD*JqAbQDBF>^M(&8=VqAoTf@_k`19)d4M0wqqOBvzs& z8Y41hVi{_pGjJjuo}edw;u}B`84(Oj0fB}H0V<~ADn8I7ugrDq}ug;xdY03Pz(emLKpmSH(CJb7?_0s-g}q zP&qo}E3RWZ#-ln;WJOx!Mb@J}Ho`{o<39Q$G4^8_N?$XQWC;pnGF*~1QdC@V;|<0i zBQj(|%H%_mBSqRIM&jf==A=&Uq%dkEM~0+F0_8_`BvF>+QA%UeRTGCKBqJunLPF(C z(j+1Fq(nYqRbnMPW@SbSLpn6WnYG+Se|6yS!0&y7H71iLb7FI#^hqgWnJc_W8URn$|GKaq+}MQ zWrn3+BIEY80%rmyG%o*MGfj-`Fjxt?h}rlw*tW?R1FZC>SU%BFHQCuP2-Zb~O^=B7wmXLVkuCI%!~ z66attO=+s-OB$zCjwfrjCiG1wb3UixQD=H$=X=6uc4psp65eM%!g0t>T8d|R9;a&h zXL7EmdU~aM1}K3VXnZ0lf&wOcc4rHO2hr%KakA%K{-=3HW`R2AfnunJa%h6Kr+kW~ z;SnbzdgswZXn0O2e^#i4E~kKQXotS&fPUzV(x`{V=Yo!CXga8f6eekcr-Y`cic;ly z+GdOzX_3NchT8wAeAcLfGH8yH=#GL&kCNsQ&>@ft>4o~FmQJROwrF%Bsgi=}k&0=O z?xvY8DN@!acZw(@cAJz^X^;Nseg>&kt|owvX`X(mi}ER%hAEmhX_^Y^lhSG9#UFn1 zXv9=Oo&G78;;EkcX_-c9qXKHBo+*Zz)# zsRpW(YHEW%sHc914=`%2I_j-LiQ4L_lIoV?YM_p4vSur{E-SP~Yq*Z(u)-;|B5SuQtG6<1v-WDcf~&Z~ zE4jw1tU~_^t)A<>qARxM>bI&ZyY}n9@@u@l>flXl(Lh19=If*G>%y+9zXI&T2JE{E ztl<@Gr^@QV;%mbuEP6sJ#Coj6eyqEK?8Mq?#nNkt*lWgWEH6fE%dV`*x@^3f?8##6 zwzliFZfwlH?9Ps?&W`J#&a9#)Y0fgN&+@Fp!felyq0**k(0b~z9&NfNEz(YH)OIY% zGHu~GZJa`_%35vF8g125ZOCSA**0yt8m!Gu;m0a$*naKUx-IoBt=VoZm3FP#zAf0s zE!^^L--Wny@8t&D4sowr=iJq*n;;nSzY~lW`;wCQQQZClkYT)*0 z&3gZ>zGm*_R<77qYUFBekd7ed7UtkG?%IN`>rQU#itgNUtmR@Z>Dn%IjxKGg?x(&k z%*L+l&MxiZZt?o9&h~DV5-t@AZ|&yp^EU79O7HSMW^ZO{^0MTT{x0)wZ}d*D^LFpz zim&lfZ$H{8tNH-iLa5x5Z|IJ%_rh=dLhtt;@3&&F!`vhJz?Z}`eD_yVx*{_g+_ z@chQi$2d@lYF!9hw#}LnmHLn(|}Qs3o6P-8ph=%$#G(p8Xnh z>Cd8TueR%&bz|SXb;A};yg2UTxs%^kKK!?H-n)TEH%^^)_3O-SXTSa&`te;=$E9t4@zlX1We0}@p!=_)Jf4^q;00ubUfbHkh7;AYK@vcL`=FABP>H=--1Vj##0AB$_DWj3=h3qK!Cu zc$|wO#z^ChHRdQJhD5HYV~IcZ_@a_QB3a~8Zr}+H0Y!CL3(7xH5a}v&KR@=(5g2D{QsaQoAX#+(xMD zx2ZNOuC~ff`)j!9W=pQR(VmMYEVqE0?zQu_>+ZJR)>|yL^y!OB~$6{9f$x%OamF zbHg5k4D+xzhaB^|H1pgtC^_#u^2|irTXWDoCtY;LKL3og(6}zG-_%qqt?+cHm4`z4XyQCocHqj$0o2=8$u)9paj2j=AWaXWO{vrKc`=)1Zew zI_a;&E<5Umx7)hwvFBd*?z+$J`|qk3p7-sz^KN|az#pHy^2;|L{OZoT{r2?4Bj0cI z%xACt_Si$;z3{BtEWY*Mb8r6n<rXQS&$7ktYHp6xI-NJa1K9QLlFOkNW>x<@rX!F;tZ9@ z#3eTIiBQyo6UhLEC_LecSTq9_r%1ytdhv@Vyjd5hIE68q@r-3mqZ-ku#vr&+25^j{ z3gmc#Io44Gc)Vi_?AU@3+yQm9X?A9kWPFEvnI$xZGkC!T3cA zY*LmpxMVP0Kulm}@|UelW(Ah1%1&}%lf#^W5{S@*MNSh7Ti}8=vB}LbWOJL~{AM_* zX-#sP^PK3MWHeE*&PcNJoyCkNCFgm}dLr|gN@(UY_sRdy3y>h6{q(0mo4J4qOn?jf zET}>wuuz8j(xDK2XhS7xN&{GQq6NsPMlXudi{^l%APp%+4X}ofrW2a&OsOR?Y61u_ z^QA6LfJ|lD(wUBQp#IdR39k9bZu)eb;=CqMn;?dArt_#sRi{er$<(Gg6`lb#XjG@V zPjgZ=p&iYtRz*rs26WY{73C^e$12u~+5;b5$bmeex>R{a6sK>6sY`1rSGeLcp!7^? zHTTNbz8Y1pfDLS0*ZS1LK2@qz1t?ynsnfVBcB^qMD`X{0S+Z7Eq(IBZ96F%X!M@Y6 zkqxbAaZ1>=3YD9FO|5GGD%f`J)w70;ZDLV*)!P5|^Ry@!?PParS>E>cx0fv|7dWwr zJxn95NiA(WVTwuJ#+0tlm91-6i(1xFceSle?l70hSf1ANuR+D;Y!Pc)gCexNp!KbK zIf`570{6X?B|=e&3$JNd;jXZCD|4YsUib2Myn(H*b+4P<&Z_i});#Zb7fACyeAKbJ@vw4yk;-8D$f5VvwFC zXN!;N;yYWI$eBFtP!~PsGEWz&Yo_EMxtr$0%9g?aee+H~TmepFxz2j_E?}_=YT@#E z5K^WuZL3^bL%Z6hn}oHit4!uv;~2Y+F7kdCTWL#EGs$g6?5aQATRNB8yohGCo$dVQ z2lsi(cFgZ`cl~Ob!g$TXj&w8|eQR#xTH3wV_K(Fq?n6_!s>Ckzv2&a4Y3i5Vd1kh= zpS=m5PIYC5-0CU^vop?p&H_v&_qx$op$9(8V-v`p0KJ=PTz3My9`qj7o z^{f}Y>}Oy4+S|U*4UU5wbkBPq?*8|{55Dk+Py955x$bvQzVf{g-n=*8`HUe0P2V=zC~je&jcO2$+EJw}9{0fPI7m4)}m9Fai@;ffo1y6qtb+c!3-! zfe`qC?iYa!XhkH*b_z&HB`8eB6n5-&Vku@?aL0f4w|@a>f&zGgIGBSrs7U*FdOkRV zKsbXkIE4I1e?_=_NLYhPIDkxegDLogI>>+^NPQkyg&KH;8>j(UsD&25g$>|^UHAZD z2!>?%g=CmY5vE};=yA%Yf8eKg5r~Cyc!pajhYvV?dPju!r-ytO_+#K7==67h&uo1NRU{9k?4er$cP}YdoG}cftH7EIDT;GfFsC=ju?P<_k*Sw zgq=8ugs6&u7zY5Th?F>ru_%k97>c%7eYc2I#(g=IhSdFnqd&#Jbr>Kpn*p1@%jouiJ;s}HVh<+o$e84DY@HLEH z)_c$Rd1*ie?PrBsIEHjMhZ)EOh@^YgIE~fFjoi45fyj!5Sb&EJi*0m*Q@D`y*pLhv zf(s~#21tZp;!*oGbnk9H@NQ7DqUh=cpck1n~5 zQyGxq2$KXklUj+8j%brNIg&XUmSQ=ULCKI|uzu%=heat|N``Rh6qVkzdVAP=@|cSy z>68k{e75(G9VvVCSCv25jLImGSy`A{$(5Aolwe7gjM0zm>5{Gql0FBRFG-jKd6p6kh;=m`s*0H5*sp72?peg-ew$wQwIL_9=)`WXk= zgF`8#ADKWk1sWCyN(u?8phQui4hk^^>Ie)fp%fZ0mr$XCfT0<>p;}}IxnxT)K!tWk zfznx-h?RZvrCdr6UFxGf8V_PRres>CX1We& znht5Ire`V-Zu+Kh+NN?kr`8~u)Q65Num#h(mMChSDA}S<8m4qQsAt*^Udp9Wnxu-l zs8av>qmG)Wj~c0xs-=hOrRs30Uuvb5x~ZJnsh;|&n98Q2da0xO4CipFsA{UJnyReY zs@>qKu)3zQI;&|~Nt21EC8~QmpouC9gUY9!@mYFP3Z{fws>NEVkGiP3@S}{{td07t zlPam6Y7L>fte>i>pqj0Qx~-vVtlk={;QFfKDh=aWuHkB~v|6TidaLqyt9i<&Tt#c` z_@|*atb%&4$I7jT3aQS@ul)+8&>F1*`=z8xtp*z}+M2KmtFPVqtpJQcmuqzv|11qhZda$Fqur~k8 zunn8D4!g6fO0GUzu|PYqvbqEzx<@s*vG6)k@|s{-aGfGMvLsuwJWHoE3Z%-avMieq z2)m>&3$rmxu$FqVG~2LLN~|}FvpPGrQ`@s_8?+QFv~Vj(7>j$r@&yopP@w>!ONYO73Hx4Q~oa)!5g+k+zOq{9E}wv1c6 z$UCI&nWU;4w$UrS#@f8G3%%=0z4AM;a(kyc7z2Hvxm+b@y^6QK`@LBkzHnQt!-2#`(L4knN3Ttp!d1MYqFzj zz>Zp<=$XJr>cC?`RcFb z>%`*u#O3L=Tx!KT%*9)r$9R0lJRA>|TeR-lM>cp~TV=*<$fBB8!U6xB#C<%zZY;R) z>B4f%3s6kQuxrPd%*T7I#d@sCaSOxe2kL} zvpCwxn|#YOoXej4#a|r8B#MGqP;?tL%mtuehm6Se`l5`?%GhA6OnkMF3Z!r>$8rq5 zs;tYqjLV$t#kQQyw5p4NyaS`GzZ~{j$BfF!9K6fS%%b|t=4-fJ$_?^7&-5&w+N#aC z91Qr(#@yV@{@l&cz{>*d$3$zA9P9=x2v&ydvHzRMpLfm2I?(M)tUr1U!h5LnEYFnu z#P+bw`>fC29K!$|(&)g<;0&GOjD8<%lp>7Ig8+UIU91xQ&ouwd&r2)|z(CJAEu+|+ z&(jOi$^g`n`_Usk)H5B>jr_+Zd4i-&R*lutW5Aj87t=D`&qYnt>VOOKd2r=8eMkaStq*4mW> z`RLkdV1^Tz*}Xl$vd!C@?cBRP+SV=F(w*Jf9o-?wt~>vj+Hd`Rxu;)j_k|y+H*_|gu*-0JP#3a;Ps z?b-N^+x7k65-TFAj|Gn4~_h6CM0BBj@%Z=caUD*r%-V`3+4IbeR{?o(2 z;`d$Q`OV-H%iq79mzJqj#?5L3?gaUJ;96(|3C`Ye+TJLB;w#?bGLFIY9pg)$0RL<1nKThIAZsc8#6>Z&fmrEcnp?&UVV))@}zIbPfW-e5^k zPj}vhm_Fir&gq>F4WBORqu%Pq{_4W+>dLE-Od43_JiGz9s0;6{+? zzdr1t{_V+5?8n~Z#Xj!M&g>PA?&@CdQ2yn1@a#r=lC|DZ9zIOtv}Tsh1lq3WTFB?Y zUJc@I?%|H?e!lJofA9m3@M=!*?j1^}4ee}TRHUX*)~;QduG^$j!@O!4(J$bx-tpANd0R_mp4xelO5~pZN^m@Mk~k`>}uf z&TsqBAN|iS`@OCEqU`9spN7B&X}$lp_`c5Z$-n%qKmL<1{pNq$=b!#5PTJOQg7Ge1 zLb+@GD|Z9#{Tu%QsW1K?PyW;o5W}DeBv{bk!Ds^&B3#&w;X{WFrA34|(PBl688s&4 z*pb>tY#~98j3;s($&)Enu9U};r5zqPV#cJ>!sZkwUc%%|qJzVQ2o*K}g+OA4lAbPz z936uNDV7pZeLApd!9t5Pajb52R1zJyui3zk6-&14Sz^_SrDf}J?OV5S!nPUf=jd_`Qeczc0AIeuk10 zaKPr+Q|vj&YN}4Uot)~4GtN8^A&AVB`izdIPBTF&)yg|h0j}z)ZAAX)GqFVb0NgJ{ z6<2ieKNtxtaJlvloGCg5DYFhU6d*hy0}#4k1HG&6;r&f*;KML8H8%J%^+NL4J9;W3PU$h z6)(wU#|v>vS!q>t)<|#l7T0dW{Z?FV73H? z1wpaRmRw!69T!|J2L|}QbkS9qP=?vP$9U2^PieN>23D6Z982(FHHQdQ z&QS+(_G%8b?pXt`n?CzvwZ~2yWUvQE8)d`YX4>V)U%F8`WT$KDI;lD1E`$$0;EwMg zdQ8zV~2fq#%qtK%ZLTP#sZePwhQ4HZj%O|4T6&Zh!3loQgBQ%;^f>s# z9tQDj?;GLrU^cnnajGdP6du-K2t%FGX*>?h0P1kqK_F%ki(2Gj4~y78|G^D_Nd%uz zHZ;DXq)|@`j7kHu5CMs<4Gu~QNHl@~S!K60qHzKY zXw&D|=*Bd}@qu-uqz?VKM?PlqlH39$|FBm=G|8+@Vu+z48~H?xL6QeGw7?wwcSk&G zLzc1JWG(Zz#XxG(l7BQ}CtsPVLCS<~#abeqN@+?|VojC%a%ClL`Ai??QkS%ZCNHho zza0NuMwnz|Lr#9=BQ&y6nNmca9IY`+XYR6(w9g5Ca%JZ6n%O>}V)dL|h!6)D>L8$ z#v2d9z!&IiOgfYX+)6e|f*$9ddu6!%Z3=OFN?kjYnmRV7w|=0zDR7 zm%|KZB;Q%jV}|QQl}t$Sh9EWNwW42#2Xp) zt}eYD&hT3HS>hANb;Spz?-|3ZeElBZ%|t!EQVx91=#Foz6;A6~r+nThe>kjRO>vmx zThkavGiBPS$j+qCl%nP=c?y2;gu}aOLO=P+XU%em7dhq@mwB!`Ewc_Yk>hKA^Uf2U z>OL7<0$Zm+(2MT!q$9oMMrXz`z^?M8H+|+$kAlqhiSq?i9qw0O0>!k>^>u&!>|`Ii zyu;4tY_q-4PQKd$KrVL%@bf1f7P)zTzAvv6%H4YJd*%ObdB7vN3xfBQx2JA+#LpdH zy2+% zrl%d>X5jqgIBa<6LqGIzA9>cf?z7Vq-|xp~z3d;Z{N(?v_O^#QazC$o-bX+BA^-i> zvEqf`_iOl*pMTl4zxSiB9`@V^fB11d{uc12ivphOXs_P;IQ2`v&S^i=dy?#{zx%_# z?eo5gLOU3{z;yAy@MAu0s;UzzwCCeEI8(rW+r7pMK=?Z;2pkmpi$DqFKMK@88EmQk zt3jXvz%&6g0s^)J+Ckb{FD2kPYWcl0h(FqB!4({bB#gm^o51~>!P&z=8`Qv|!@&%5 zjBdIs155xeRT2g9FZkdJr{gI7<@uDR73ydi^5GKgXY7W%-Mia8v;ZM z!_ON+=~%WhFvBxcLO~qFK-?iG%s;h*!ZwsbDr`C{yr>0{x&b=@85l1FOuz!jL`~#F zHfSF;KtfOq#X|grG+aYP{39lW!#bOSIgC2-=r<=}0a;8xJ=8=I;Ka_sfRO{lQ6xlP zL_=V7!$UO1v75q16u(w9CRnsPTEs+KB(Eq~K*>@;UhG9{9L7}4Mq#8jW1O_`vm1*_ zkPPubE_5%STZ3i?8v8A zG$z<60)SkN9r6)pW{uJL@43x&f+9aQ#@5m5ZzNg)l)IW z%4VYa?(|`yONtz zWtCNPz0_xAR!x=GX{A;&>s1rg*1}7`ZB0obY`hUuH!>^NbM04JUDkBHRdt0{b|qMI z;FW_-*mkWrc)ivvl-DOsR1Tm|QPV8{b_`d?i-O){HDu*ie+Afo71)5qRgg84Fa^VpA_*^m|4ktJD^J=v2@*#%`;i4DsKn2zimR~(gDqpjIyz1e|X z+JY@vKvmd<x;;(*WPRM$70=f#-q>9`%>5o@LecgV3Ng?<@i->7{aj%x zr{0u5-!SN&bi%Pl{<;?|07 z0x4TxE4YMYsD)Ti16X`t78a}*j^OWY;u#EH`;Fn@y;5o#^5SGP%Q3Q5&Q|g>EeG2NHDO1=EU6u7PK=?!Hsoc zHD+QbR$VBTVHyt4H;&ynzNgFB;`I8iQ`KA~X2~=jWbe$oLvCY4?&KO?WGgn$M*cK5 zAypK9lRVyIJ|4^uxMWOTW3OdnPA*tbu4PS|D>>#?9<+ifumxK<;8k8>O9m7{eq};l z+)Y-)WcK7)rsYt!<-Ltwn#f`Ug1bYz0zIgOWDtch4(3$_X=15$2&1#m|&maOYd;&AjW>)}%>FZ-*j*M0&=8za?axP@GgknlXXIWlnXJ+Si zRzh7!#;QFE-Lr%LzVu@k=;BKDCRY|{Sf1f@PGpa6=yn}t|4q@KmFTW~Q9)ZIQ+3;1 z)My@?6d0~sO@_jQhP8D=Xo3dm;S=d0)rIr{uUuB!ZOwrWP{w#}$9^utjpkRVMOUBx z=#b{f9c|}p5Yem7YOUUCuHNcfxazO|>OHsyHy~+rd$N-Ty$-kpP3TMW+t$&$(3sxo zsg_`0U@*MC>rB{dzV2(k&V(}9>%azV!Y+frHf+RBY{mWp#%^rKe(W5Gn?aI-$&PHx zzHG{dY|U=$%>(K*@J_&RU&PVe$oZ}v8C_Xd>rZg2TsZ~Ct9MRjfsCR5(tZ~c~= z9Q5!14sablYoCpigggS`-eyXWEu_ohhVik}^BF%JOzZrdJcpfvGK z`Azg9pK?2=b4O=$mqhe6pL9yEbo*9xNVoJ%*YrfU^7X76Y8Fb#7o@tI^Z*UnTcI9ophT+ek~-*sItY1;N*N0v@t*K__B_Fyk|U;p)FANFNe zc4lvOXE%0ki}q)ic50{1YQOerFZIZ{b)E3-P|sp6KXzyDCRT5C(8~c|{siOZ9IGvK ztPNdH!KYpq_hf%g% zZ%)q1)4|frnv-(wjQO)q&_$b%g>~;f9P#hs z#a@Q@8FErG^2*95>i{A8f#;*qr&F(P{iolwskK}EU|J)K*51E^FVEIC?b+wiS8cyt zAz zFS7V1oN>-2-;Flbxdxs^>bc30dg|!so|qWen1db2K*k<8I=Luim5Hf=LSktN)d;O! ziDhO~mAR>PYnu5fsGf2v>XdS>#~!9FhGZwJt>($&tFH<=E0DGR`6r*15E|tkoapka zE@lX_2P1Qs5kd!yHu~rfRDL!VP^V(2)w58I8tS##l8WuN+E7uotgedmE~^9fX;K18p47n_v-+C~zyT+!tYyef3j|Zpfw}Of2!A`~!x2k7 zv9=XkEb+qTrmIC297EBu6(4)N@fB^9%r44$7&{=!biynDtFA8So3965fYR%~pupk^ zCbQfu${sloEQ1IPD{S<|Nq4F=#Y`jZw9-$^sD{T#Sba6fSZB?#$#5J>1BY5uCfL z(kp8kCd@p5)V}_k>&|iad~Bl+%Fw{kdF#D*(|v;r0N{ZOKKS2-`wcbI8dt5g$c%5z zHQ4AvJ~rhozYMQ|C#?N8+;V#k%4KyAkpz8;#g|(TgZ1W{>aDLXF6^nkK6~uz0p>dH zxu1^u?uhwLqsy*REk)zSKOQpjk$*jTx|B1oxaG0>M<59}?W-l{oaoxgG_X|i`5Ia= zD$^F_=SMpF>7T#;yzMJiy6N%v9-n;j|4#P(_y-UF{Qms+=f1oEcqc3f@NDfup5qc| zyag@}at$mV4YU?P9|+-O0fdNi2%y0!xUF*@{GR88RyPMBsDve`6beP4!WFWxg~)Ot z3jvgc6VgyvpesYL%%F>)5lwS3#2^e`*MK0hj$B66-H48e#M>#cFr_&N1-s;>DRvBN zLbxIpC#c0OX0eM_djs(lc)T!b?OgS00*-J5A)EQDPdx*n_d+qVchkmdsSDo~Fr78m%;>xg;nR zmb!vSQ94KQ;uKA(%2b|`i>MSMEFDLfIzpuX1f#npve@{+ICL+L+<1lD=r%$+j&J~? zv?UD*iO01uiVkjYW)k$s%my{dIFUpEZU_lUg8{Oep-foD9H}=I9Bq=cljb)?Lqs2* zK$*sT0E<=`#a6O1g7m!SEA#2fdg@aJS74zx?f8Taeh?PFj3XSMFaTmQG>|SRA1={3 z6F)X`40KRJ5@^^Gf-OpOe?%r0Kzd0^2DGH_G$u*kS<6YD6s471sSaS8(b2K3rGf)f zJ5@-8o$dgQXk4L9R~Z`e>@$i+<>FD3YDJ+kwWv7&!xn;oKI@@?qID!_K@aM^gT7#) zb0pyfMu^M{#FV5olUXqdPy#Lhb*eP~<7hPjNdl2-)1+{{YfHIGGvi7Z{P3x|7Fo=t8^sbD}mquU9O}z5J|mnV{S{~yWA$7R0Y=*Zg$c2 z+~EmRw;Eh)6M`Gu#$HdVCK+u1e9tR`^m4Do-aCU&-22`Xdzik?yfHKZWZvblExITb zEk^$xJ$L{pd&AWT)`Mp(cTwlZa&n&E$1_)s(`nRf*$*Cx|a z${*(DPJSrsl=9b#+M1}tQ;kXq0czRFItTpQHQb+t?b-j54vF1 z2}4)1(u84RGw5SPOE|BiyAI*=X>Q=QyZ{P zHI6ld@M-2kTi1Dpc1dUdpx5927SloX$`zuJ+HKC*)hv zxNo9o#>&BKSj5rPde_g6cCn|u zYvE%zrU<>fqSKv}BpN#2<%0CTc9|@B|2q@`-}k|9S>%R4JmM8oIm3E3fhb%szZT7RKH8xleYI=8?N%^j%cRya%X7c` zljl6smHz$eXCC~Xk6_{JuKvVtpIv0ngZbZY_W8qq^Y%}D{ZU`~)8D`U{^vgc9)jvg z0s$7F0bWAXA)wVQVD8z%?ls`5zR(fSu_9ZPU zqF4rI1eT@Jpk-RB$W$X$6 zrf4FgB@km_lBQy&rfM=KJW6I{zNTKnre??}W@maPZvy6MiX~wlB^aP) zC$1)II;LyNCT+qcWZI^3>LzsJ=5y|5boM574y11);%{!9QEm$1sATh%p>dAqVjkyH zPN#G>XL>p(dakE?UMGCUr*&?pC8%Y0_RT^bW_|7_d6s8u{wHR>r+`*wb;{>}7N~*p zCVyt*mYG`fBXp+|dD2+1d zeA;D?^5}@Fs3!jCcV6jGq^OjxsF4=wT{bD0hN+W|>5bYbVxH-gPAM2zDVBa`kgleX zcIlTcr6S(*VzOzQDkM_&X`ITbp&qH8CaR+5>7F*KpAsrVu4$!s zr<-0uc*5y|>ffPyDWfW>ry}Z|b}FBaDyiz|qn;>`T56$k>ZZ=AsDkREzN(|nYDYdQ zq*?+SplYCgrxs+Ydai1Vw(6(GDy$BxtQzaFma2R@Zq=M43W~r~5Yq!%tjUrr#HK8}N@>VOsKugeutKBC?kmi4tj(V6%CaoQwyBp)-Nj~R z#?I`<;_S^nY%b1d%f_pq_Uh1XY{C+4%|b2B3N4=|ZK{rJ)0V8%POZWgt=Cem#IEJe zVyVGKt=5w2&zddQZY{ooE!K`LmR_IM2Cdt+?bos`-two_Dy_U?t)j*(s}5k^s_WfG zXV!jfg6=KfmhIsNF5o)u;6`rYHZJ1&D%^g^n@;Z8)-7{NF12d^ZfI6*9=YrS-tFUh zZs&e3=u+Mj_VEUFcR+xa0>S@_?B!y001HR1O*fT z{{Soi0000$0YCu&2>$?6tr9q@V5ozrOvO?~u%WDn3>Qj-NO4w0i>xkYq=ixAMvntK zg8b-8B*~K~N2+wFkmbr#31dc_Nz-M`mN*gSglTgp&!0eL3Jp56s8OOveR^7|v#HZo zPkSPLNwsRot5#E*%sR2F*PBwoik&*PY*?~q($am~ zFUh>S_vzqERsRn^K0D*$=aG}w&K-LD<<8&BkI#NPeEa3mt2eKIfBp6JxhG(N|D^|D zf%)O*AAbyrryzj{Lg?Uk5mxx%g$?$1-+=>4*rA8>QMloVBxa}~h9;iK;fK$)$YO{f zmWZN-Gpbl)f;74qql_=|xZ;mM-e{wcIf~fhkv80nWmX&!nx+0WVX2=ooC7!r<-r$d1sh;PFW_O zg7!(MpL8Zzrm8k*^(q!M+@rk6UJ>Z6~w zTI#E)s{cCbs;U*j8hfm$$ts)cuvNiw zDz(lwo9(jD*14#n)8dNlv&@#ut+xt-t1eXRhN~Zi){eWax#vdv?Y-@GTHv+B_Dk=) z>#eKrb^^QG7P$!jTkpWq5^V6k@!V}wzE@GNIES$ffP^@jm3tK!|#u|Sdvd7(C zEb_{Da*T4qEE`B&%qi3QvdN-=Trs^s9)~+>-iow`s|IDZaeF^x2}8WyYr47?SI#PyXwRj-#hHa=Z^fL0avh zgP>r;=`b+|{E1L{B|KdURkuO{p0I;0#2^M!7>GI4FovBI;0KLI!VvDThbR1?4gZHo z#5EZ4h-65j5~pB>B{K1eP>iA!r-;NVTJef!z+x7u=tM4dp^IP)V-&ndMlpWTjAmRy z5M)q4kZc^}2~@CwMmjPC zkEB2(CrQalk|2>YIAjuQz0S=f`lIXm^IfGfwU*b{;@ZC{m5|W|43hzq^r%Qd%FdCN z^rRs@rXy9FPL{g#r7(qQK6KHA6f88M357sAQL59Pnjir^Z7EMRDo==lvz|yLWhtq! zRDurFsRqUAO{a>;i$3(KxFo7oHA+!nYIUIqpf}Qnq3O7BhvU zR1eD4vj+99c*W>Cse06;Hub50?IctOOW4BB6|0CfszoJQ*TwR*o)z^dN5e{4wx;#6 zm~E?NH*3~->4FnE;Ok*akk_BuwX~>JtYQ!9SHaRUu=g~qY)6~Pa{r39m$xOTYIob% zuIhE7@mytPhnrc=7B{VtH7;kBYgr|5;<<0#LkOap)1CJAw5bK|Em;fO?gDnUQ7vk9 z1^G|LZnp&eyeB|;%K`IV7o?MY?@8m!+~s06xbCg5eK||dN_ch}wxq3f)D#p zmcs)!ZH%)^5aajMrygsemn()=BRbJPIPYO;JZ4&#+03#gtd1|uYfOt5r*+mer#rn{ z7+)9AjSlmwLoDOo(l^S`Mlmtd;Oayx`o^=KF|8jh-(Qng*S!w+D%bpGAm`T8=*G3W zHC)?dlX}@ORkaCF%}LYBJKFfJwzb*dWkzc|wFt2Frzb7$f;*MmugqtqG0jldik6-p z{xg^Z-Ciq8na>cQ^r1KWYspe+(&HZY=c}Pxa+9CD)-A27wo6cCn9H2eau#;5c}ex7TOHI_uXV@a z&Caa{{Ocw4I?(l-WiEqO00%v^8$!NzYP)?5aF4s(=T7%H*!}KpM|9rtK6L;3y%T>Q zMBoKa_`&ym?|esm-RaI$FYLPUPLI5RWl8y}r<iw?w*5e%wi+_6XE_u=-qFZ}9v&wIoB{`a{LzVNl2_~O5v4s0;K;*~G^ zLTd;rp`~Uy(*Lcrod&9?k2Izj#mjw!leGCYB4(J3A2!RmDeE|r6 z_t$?In1TGKfg9L?Bk+M5@Btw>f+ARgBzS@-Xo4!Zf*{y}FyMhP5Q9B%fwmM&i4;wY z6itaQ7lZ^Tg;dyp2zZ59n1xn2 zgb_GJMYx1&q=ZQ*gD?n#ESQ342m)y6Wow6VCYE_k2XcKteT+AR8@PseXaRbNhb}0C zOxS$HhkOG#h=a(5hIoh+xP=P{dtLa2M*rxDkQj+zIEeu`hCpD6m?(yn_=u8NMxH1| zFIWj&Kyh<5N{QEdghz#{IE79Kc(3q^uo#QIH;bx=im3>Nw+Mx4;9rE;i*%@kzzB>E zsEEYKiH+Ea$e4^|G=E-@f5_*C%+++Pw1|SJWxODZtM`1xc#O#id%A~>;24gxD2~8a zi{*HZPne6ixQ^?%itXoo@Ysv&IFIO9k8?N$ei(z#_=eJmV{;dax!8*ND2@y{j1sJ1^LJXZ5U|% zXo_x!jj2d&keG=wnS>HZfDd_-3;$`4^ca!uNRbt3kr*k4WC)TUxse{(kw{sTBLI6F zD3B>hb}LzTuo#flCzNG4lNVT%yO@(W$%}TUlY_vD61j>bIekBQh!?4hR!Njf*_Lkk zmPI*~erS>(zyo100w_6(D#>#{NP8`LjZ|rYGD(yi2!IRtm0-zw-k6v?IhMxflV*9A zLHULMCzmX6nfwQrN4c45*qNRQmzK$Yv&REnD{{`I1`%2Cz9zR}h=B zX$7-+o484vu~|*LsZERNchdlz!U+pr>6Id+lW!$*%Gr~JxP8v~oYr@dX(XMGsGGHk zn@3=q+L@i)Sq0tsov{X4A1=1WtLE{1}xs*_wiP z57S@|{7DbD;GgOMpaR+s1R9_Q$_@xRpaMD%3d*1Pa1Z_vp%MzA6gr_4x}O({4;5;m z8~UIa`k^2iq9VGVpGBf2dZPc8qAI$gm4TuzdZPRipr2)+F?uc5V52vhqXLnmJldnr z@Si?9GrKUPL@J~LkqJgh3P?&aj<5$!%A{lvo}wv`J20OV6>n3fnx`js`MG;=r4RjC z4`N!NV_K$W8V_iirfN#2Y-*-p%As%?ryA;^a{8urN~d_5r~7cH5$d3K>ZgFppn*E5 zVOpqydZrDEsEL{m2mjir*x;kml^Us+im8vfrt+Yvlj;tj8mgU&rl3%s zb9sTNSzjx8eq4&5`3a_B%BiwCtF-E<{t2jcnyVPv54yUidOD}S`lp0?sEfL&hDxl( z+NQ~BtIYbT&ibt95UtWWtv_0=(m<`)+6~#dt)3dK(nW^wSDL43ntPe0<~nYy`HF?t zr;4hmwfe4&imSQGuEE-?^?I+vdaMc>ugFRd`r59`8nDd@t=w9$2CJ$)ys^{95t*Wl8$F2Zdrrk=g@G7Q$IrXqlx;j{wlCA zJFu2&tqi-cHvfCGHQTZAum@1kf$?dj^%;lhdW#zSu}1r`AuF;E+Nb{TpM1)zB>S`{ z3#=;Jua+9HE*rDTDzh`|u{ry-IBT#DtB;0=mupw96-#h_xni#Rr5k&+a0{|hYp=S% zpiq0aBg+qSOSLImwJQs$S{t~5>$PB8xM8caVLJz=I;HsMV#tTKX^XDE*tQS)wsC8; zcB{1Xy0>_%x17tjpqsT^OS6Jox?4NCglo8}i@4p&v*22`RB*N?)@ElmwCftH?#jBC zo1jUnv~}CNog1}Niw}NlwWRB{rklK}+qJ9Pyt<32V~YbnYqnyiQuYL1lgoQ?;I@{# zyQiDEn*Y1GOB=kNJG`N5wF%0+1^d0qtF`KTyv}>Ahx@#qs<=K2yAU_K5Er?UYkQOn ztKDn3-^;n-TfXEgya;^0#{0hO%e4;7zNfpq5*)wI>$;~}rCC6|fp)z@d$HNby}9eX zNL#=OOu|#Uzzht($Q!{A+`=tP!7wbV(L0(LtiiKupMGhLA6&U146h7ptH5i*2W-A` z`mZT`#1p*25S+wF9K-Zmzl$rqPSBJ#d|&%3TeeGhJlw+~JirDl#9++BqT9qye6=sk z#AU3)72LBI9EMro21$TjqKCtOiLu+e#a;Zw{7S%KOt(f%#(-S7XFSMhEWh)cv-aD@ zCjYsBJmpgKBvo>VcmK=9KD@hp+{a&>r(+z*!C(!!fW`t$$e>)vtDDGE`hgV)$BkTK z1;BC^E6H|z#{hh$mE6TBi^+Vfz<;a8yiBv5+{?fm%%YsBqpZiqjJT#OpYtc0EWpT_ z)rL|9h?7jq$6U>Fo5_Bxz{K0Po6O1HEX>0U&Yt`XhFs31th;Lv#q`O{UZBeQ3&~k5 z%am-*N*u&_o3-fsvJ0BNPP@zE{LjBk&H!D^1bxovaLCYWo_0CK?Cj1^3D4P^dUrQd z^Gwg1deIo&uCsg%0nE?MDiP|?3g0Zy1|7-;UD61xvwEP;8Qjoud|J@l!EYDKY5(8} z6iv+*ozl>nzIiOOJ6f&mVxR*}(nBrG0bSH6UDE0Nx>QVia_nm`$+lq$lYy|GwVcx# zoz*AZ#Lm#w$p8$%@CZU})XLz?WNp@Hebi-5)Jfge^t;shCzu(`h}gA{bG6V5-3e*X zly6YCql(qljMSWL3nA^*VSU&m&DLpc&Wx?tYfa2oZl5N`K{K;Q^*sC4Z0jk!G?bv4>*|e?G)eP6C?9y-8YMFLps7#u6 zsn-nvf_*L82Tj_gZQHe-3nI-8j{w$hz}m?W-IEI2r)}NXo!zmW+``b^*Z&RK;Qiey zUD^7$l$U*3v@2p#rvVHd+*#lNBd7&qjojhQ-S#csXPpbq&DyL@-P)bs0KVP-{n+@u z)ymzvFYvlQTU0lEaDKL30XN0N?FGU;+#?_b^G)CNecv12;T0SVh8^99jo1TT-~@i+ zC{Co_t>P=*;<9bx2Y%d#8>RPK!!1C=r{!=-;A$)vf)c)z6&`}}t>GTd;YEJr`90mO z4c1@n-zEO!0>0cZj^r_prZld|&CK9dj9}(A;Vt0fVF}+tp5a7pV{^E0<;!v*URG#N)KH68#xJ9VtWcA>^ElBK5n$#=iAOBDVhxmbpGj}UhJLj>!@w&MBe8yAlIv|1H0|&92HKo zu7|N(>tlZE$iD5QKI|l|$$-tOE!<>P)1lwISM2$LH; zTGf8&w!Y}b9q+{8=>Goh6D1F(|f&+ZN1?eMU^rn2kkH|?Si=8 z@(177j_8Wc=mEd*yDsnp5Ai0C@Cq;T55MvV&GICl?AqR~VE>TdC#lRmQ1K`yQbQo^ z!6j@Cu<`ny?IAz(M1S%xZ}cvo^eM0O4$t&R5A#Wlk3M_9_I^pMY*I1JO3%1SnwIZ* zsQ`O;>mU#CWIysMAL&K^^d*1vXTS7q@APjE_iLZn8b0q-%-O@p>dba-UH|j59_AK~ z1hD`PXu0a|rfAfB1$E`i`&o zq(AzL|M;d~+n$J0fNu_qW|yv!T3O! zJjL`ZA)l>(_K+x3*iG_W$kKfOY%j9oRR?lPFd0czM|6n4var zE=OVcGSDzTZEohVlt&z=4LVCL-Tf+R@Tqab9`BHR?((+PuTSrLcklOt-BTlM*rnp* zDU%GT&_8nXCTenSD9SQBN;{=I1C2ZHz6(#a2irSOLiW^>?LrGH)R4jrF^tbc`Qiyt zMBsR6!MGEPgNZQ`B)h;rp$4R^K+O=WPJ`>R(~iLr8;tNr4ub?SL-d9;Qb;7vgYPeQ zMD&Kq`9@sg2m6jw(WNAal1vFM_v_L!J!9Cg(wp30{9ktFP zk2RK9QBQTY$N5$jl(;CWT~V_WW<{XI2ZDepIT^>q41r!11WnChi(NLxEWI z8LA|dMKUQ68HA+1{nDpf4NUk%UolN{&4;y-*%+>7wz=jfBleWtiXle!XNfNsT4Q-L zcGBK@@GVJS{rHVehnNs0`3aMEh5wE~t5>cxM_^<28t0*T78__c$u2wSu+uKu=(UmV zF$gM6y6@vDRt!_>4+MtVQ0WvMm(tF-ZrI_ked&efoe@u5ZLrUF{PD3NXFPJWFKqkr zqlwnsPia${wrQ0C78q-Tvo4B1h0pA|@S9uD`E{Kum)-2eYoEMl*Dz-r_RZgpI|jr) zkIu2j@TQum=lX6@>%dhXoORw6mp*ptZMU9w$+5qFd+l%ces|q_=llkysb~Q|FSghh z(V{ZAVc@2QDBW)zMo9hF=QoD_aQ+*w{(s74&u82tVD}1mJMbm&H{@fQaELR!g82t^ z3b25`l4rW*QQ&?S$e#eiG5^5t%?^YfJfH|mD8BHaP=&rL-}wq9ztB~HZ=_Qm{qnYw z=8b`aJF)}e_IE-c0x*C>G$Fh)r@$4u2W~ACR2ZBXCRwqMCKjBZ4RL5G{PFOIuH)he zLs&$6kr0fGOJNz4s6Tipjfsa@So+l0C<#ijidMX03!JCLElw|bLG+&xzi2=)dXbNV zG$Rtx_$O8^ripI!SAVETuozviIm@fx20NHX!}XDnn+&8NIqAti8nQ`eRO0XE_Cjnq zaz-bcBmFFiM@;rlmR1*1j>E^>;8`j09hg+agBuZ~Wf6fAuh zOHJA`l)4-vE`fMNUjJrul)k(MFh2(}_7#(cHKe2nUimWlK$Du*tR*|$S==(Ku1tmPa>gI%L|kOm+;ZGoMM) zgq}2|Llr7fS=!Ph+QFj0Q{)R{2(Ss@G^aWhs4|%;QlJ*qs9WXgP>=dmiYhfYENN;R ziAPKsQieHFo$91ElfHs#=b%4TX;8J=RlL>}sbKw}OYd1$*xE_NS_y zfh$(CYFEgX^#87THK|_rT2GgvQLHkZDHIW2BIhIcumKwYTM-a#dV{1o=j3e=8!NTLsQ<86NV<$^jve%<6 zmD@bsar6~hR!$^|9pxzbI#{<@NLedgpsgb35C8@(^ES*JWg4eB(NlKwbm@HC66-n0 zLoR1PqY&nvJ_X5#mhzdO{OJyRSka$ubZy6a${skCeaR6sD^(mR(QSz%xGX=Bif{;hI6;6 zduq?&^Q;X%0fAcNY7apxAcf zpm|1!HoOZx@rz^Jmllutzike1jSHOPSvL279*p9ScBztkqp$-%z4#=Sl41+zqb03|Y1Be-H0jV$F75NH~h@bIJyM(Ip{dfTy0^IbQ&>T|bx z-36ob;GnwITd(O&!4B0}L)+*rhe9UQe)y$Fob76MJKU#^bGmz23S1ZWrNunVj|1N1 z3un%(h;Z@^1bXmLiTBzUPjRTv{PaqWeCzGbLhBL+bWa8S0#eNIeKx%8OCY@AP5+8WWnJhLaD1j{ENYB$Uvsjz#%9( z0o1`A1Onc(lqm4Q6I8(=Ji_@iLL~e`FeC>UR6=i?!8+qP3fK?q3c=knfd2w4062_8 zIqZ)EBtf#v!W7)XE_A{B^FmTf!hoYex&a;t0>Lf0FE0zO5yX+;0gWEYLp`KGKa52G z8bdOKz2w_E+Jk_Z2rdHryEg=>!-|nLtYd? z8>m9TDZyYo#gG_AVNAwVd`4JgMq)fhV_Zi*JV$pd!u;C`0R&Nv#~ppcKAe$V#IOOR+>s zvfRJPBRH;uDheWlHDCj%4zyi1Q1%W@P+p*%{x1k00zM>2d#kKwua zs;Q+o1DkwHe)Pwk)JUw1%)FdTuB=SUEXuzOHkhVB+b(dPXE@tOxVQCvQ$arEX&zM zONb1C#Dodn?9I7sLf{lm>r_qbCSB7fUDP<$!yLU)!3@wVO;3U$QV!^n0#U2> zynrxu&I81*#7u$bJTgR8R8xIZD1|vleN@b$Q%Rl0LLpHBWJ8zv)Il9R40y3q-Bm_K zAvm2W zIxSXmL)I?+59P^CBrVh>rPgZ2)?eLLcx_Z}?N(N;S8u)77$k#Y&8S&r!;iE;O#fU25cn*Bblcb-kgI_uRfUDoQe@cT zeAmE?*lz_-eMQTl=(-Th#x@L1(>eo{BRd6{){sruZiUm4C0Ss7(uYM^p_SPCq*z#e z6(VTLm(@p%z0?)ksp4?IoYh(5oY8r8&7loittDEP%$H2eMH~#<=o~v-Jpi0NSgK9i zscl%FW!sYF+O36K9rarO>w;Gi(tcG?&8xLbTT6naAS5u@wY|=sZCjHi*|^PGxg}4! zoko|z10m(*L{_i`@yA4-WZcz%(Y$T_0?=` z-mBH!+^ydH>)jgUOwa||?y+G`B+;B=1 z&y>FBi`-5{K*WGu^JQMOUElPrUD~}}+AP)4vk;@e>bz1eb)W&RGzZGA= zCExz_UjPnZ0Uls%E#Lz#O9N@6#Q3oijiQ5zBT^6qGnfP6mCmC}z-Sc$4A$WP-Czz@ z-yHVf8g1VY{^1dZD}L!fPTZ}%xfST6npUWVRPY1|Cf*7@-WtweD!$?K-C-WSRsz;y z5dPu!4PuOX;ou4iB<6q?u7yXih5uH78=RCw9*ow93zjOLV;kP$f~?^V#$tN)VK2T! zZ}K=60laI3up~AEHNJ&4RsjVCUkcJXDW+pcw&E*h+bpb?EY@T2?Bg!J(LV+(qs_q+ z?X!f_1WI7xn4REMi{h4Zgh*~>Nv>Aki)A~u&xHW@g4=OtxicUb{`6W{3r2UM399GsF`n<_`7&=<1H@5 zp2g#lL}xC3W@n~mcP>vDTFS&o&-uLoE+}T3Py4y_%oAzj-ROg;nXp#QTpU&!=3+m$N-ju%Q0vtbL{^jD4&JuKLh$H90J!q5MX}3;j zp7v?29?ubT8m}g52aC&YHsYS>&6qmtQG{b%jp~8EYQwJT!_L0B&SiYC(`$y@u(dUQ zuCzGpn3?`fLSt3aTHUTxMsZLlb9{t<22&ZOBE zZH%~uYrt*8!|hjwTdeERSV?J6*4Pa| zgi^L>DP-!5HmEQ7?(fcoFA#6?9&ht5?=nDdJ6P}ZZg2H21Ne?_`37(Lu5T&0Z~V@0 zVaD(M?(bp#ZvYQ)8W3<%C2$Woa0E~ATwQPlXK)8^@D+G)36F3Hr*H$mZ~*sjX`~-= z%8XT6DiCLB5m!oyiEKXgP-bU0shH*a(`SM(ugbVr|bNr!Yx|A3gn^Agr{PXCy%RdMdk@LL2lv$n%W zr}R@#byZ(=R&RAze|1<_b2;C1TJLmQA7ZCET~OEZKF{!&-U0R#;u4`kVjuB~bfG{w z_H;$kI+_|~a`qdFc4=?+XP@?E|7&KKb8H{0WH%*a_x8>y_HQ5Y4;OcH4|hU2_uo&dB1mi$M;q-;iUy-C~$5;SejoC)uwJsdOvu@On8O2 zn}y#%=x+Fjp924hx7&$t8j7!YjL&$DpLjr-5?rqqkoWkIFL#SVcak@GaCi52Uw4&f zAeVpnl^1tUuG`9^@-wV?JI8ok-}T7oaweu&atU;dCc29(k&y=uYTw4A7x`>oc8*{A z0zc5HTzWvMN(DUmB?1i{H~OPD)I|UAqtAdG$9gIpd#*QjvnTsSm&*b5T4Wa{sP8eH z_jDX`!oKrf+Dzl0p8`P_`i9ebI9GOR7kk2wbHQ(Vt0&ZLw{}IB7R1lUF^4I{$My#| z+p=$Z1SOfqpL}Kq{K-uL&VSH1XX(&Se8unf&bN9@r+T_qjJIbQxu^NLcdEMu*k9Mi zKUiYh^!xu4P<+u-PA z2M8(xofyGEupmMv2NgzH*pT5vG+0hRh)|&7MT{C3STtb5#f*?cJ_ZnA^5jJl79^}} z$?~NGm@;F|)R3Zv4V*by?1bU-Cy*jSf7}3S^e9gdHsPR1osK0N_N>$+Y12MT=+Gdu6 z_Uznv@9H&@W(!5Yg#!;SZWuA+zxNze72lF>>68QtlIR{;I1P&PBfd(D`z=F>;m;q=W zMCciI)m670R2XUq-gx3&mKlg~P0S|pG14HQQwO!#t5TCGukL4e#TAk zmyU)Vrh{TZIR>GDOFXDxf=MFj;E_!_NhFj}MhTW?RbDt@h8uQBo{1%T`K5|0-Z=l} zCNkD&CYougY2P5g<+o!=A*j$ne{t4n=YK#V2qFuRX~rE{l%=JCl7$+|B%)6$x@e+} zE{cJ2OIbLnmX%t%(}3G-rCQ^B z@O3+GjFp({CI!Wj0qZU7UXn^Fq_n%o6?hJ(>q2p&5y!Oa?6@RA5W z8?c8y9AT{$*J=UA#1UU1akUp?OzpLvcFHlx*`7GZx2lSeK)NZB(ok2NvBnQiUF|0?Lfmo4 zHwbc|; z>@e1WY~8ij+~SBbG; z?4Zj&JMFbkiz(HC>;AVDyc2Ht)n>q<2k{#>B(mdT+9&7OELU#1xtG_hE*E2@czt~0 zW>3&wyou=jZsAJ>K5gmQ_PvPbqfa9H?XR!>_VA-PKmFo@TWa!T$YKBS;J)|&|HXnQ zJpKXjX=*7P*TNJ60x%E(5PDwpB6vBq3BqQO)4@kDn86KVFbNtYANoEhLhg012Mn>; z#Xc6o5$>RKEqtL1H2}jIiVrq63{iVR^uZKQ#UtDyf!vhi#aZv-D-;$`rB|2?) zPBh>XeW$zOc`8#u0FVqI2tBn$@Ks~;QREyb#s-efR}DDh7y-t-G?K9aNvO}**d>K) zTq6f`)STNu=LDUNQ3SX_BOCb$8u4g|bcKA~Arq-c+qE%uH+0xiASFpBE|CT+xa1@) zsmV=pl8L)hLlmJ1II3Z03vXE&9;Tc`ybe3Bf>GNKuG-lQAD12})CXPmPwYo$-9B zJLRdgbaql$@l0tFQZ|Er+A^Q8;6^8a3e?O==%4=NfC=6B@){OE}IjjH7B@1^-P>njLEKC7hS1f77 z^Ar)1QhJa%+38YDMog7NI4n=YxQ4T#YKyOY>?UAAhJhOOs0&Rjj}CJmoeHz5m9?r? zbt%z=Cf2JAB|!;BO95iSwU+igEg9#8OUi6jWu>)dlc}Ub;GEL{&2ZKL|JSzR1*=@g=^47(@~b34j(mM2 zfh;=qu|p*)dhayd>vH%w!lV)Q(A$I?jAWySRRaGP+KYlAFx5!b?dmic7* z!?`+4e#GnF>xw~`W*!8Z%be2=GgB%ic5ii28(ThkIZpL8A$->==MCiO&Xq9B8WQ99I&wm z*?ek{j2hLc{&Y!pJ7)vy7^OeP^gy;rYaJ_~gTxgzH4zKzTNm22%Dfey({;m0Q~A<; zmhX*4o7!owcFa}ofUhKyDt-DOXByscvR(h3OdB`z#M?!{1mfJ`F#G3bs)X-tDHQEO z&$~0Q&M}UKUCcor=i9};b#-BU#epl6#V4KgDo-jp#m8V)VYrHva#@HVlH ziqt*7w&NSWpsI(Ragp0fDI zdGWTztvv{W=$bJ48w!F~WJ8ka++n&6nX+s1-rVN*uDaDxPD`wFz2+tF`pxI`bf=R& zC$2ruY%zkAX1F7&+{{pfzD#NPuicu%wf72H5YD-K_H zTP&XO&alPdA20dIQ@-**)j93LrMUk`OoH>BCj{t2k2EThe)Oa~{UcM)_7c6l^%^S! z++)A`*)LA+w%6YQj63$-?|iR#=L7KXAbdO!{}09A-|+MB`_1c*TgqR4@S_h1L`+}$ zdcVFTHvg{ebD!|uZ#?2HA9?YUKN;pf|K*7|q78|P47h;&`)hH2&&!|wpZEOm@NatA z-(C%`_dfszAo7*o0LC5k)j{$VU>hi)@jV{`K42bLO#cxe145t!TA&;pU;-YW^>v^h zXy6BWU|RA|sArA}*pMJ|ZMaA{VM*C0b$`W+EG+;U;z;46Y&gNnR+%p&UNm5BiM` zk_E-=VPN^q9|B?^5~3^~A}wOzBvN7|HX<(OqAv2H7w#e^rr{+DBQbWOCK@9$dg3R- z;V70O9oC@|A<_{n1`@j3}OHNs*xb|W|TqdAUaA^74y4kS7XWI-0=FuJ1~DkMXCA}Lm$ zJj!7uAyRVW<33)bKk}nD0wh6pq(>&?L8c%=isT}UU@(B zSR_UYqDJ1NEpntmn&eIf<45wOPm*Lv3MEk%&}2=D z9!^^2RsJLy0;L#krB{|DSBB+Ku47P&C0TyuQ7Yv!s^n5WWlJ_CL_S5)_yASvqfOrB zAZB4Wj%81xWjgw$SpsH53g%f7W?>#?TC!zguB96?qg%oyWX7LdB8+6vr6t&9RbpjU zYGqe;reXf2XCnXRXM*KumZoT`C2FRoVlpOcuH{2MW@Od{(O70(;-zM8re1QUX%1#t zs^((;CUCB1YYrznI_6u#rX>J_Yz7T&+Gb|rCUov*V)kWk2B&qNrf+5^ac<{vBBu~8 z=fPZ3He%*&MyGCGr*vXxdZMRxs;6+iCwF@1Wk%*C3=4A#B7Qn&dG4oqQfFYUXM3`z zfWjw%254#;XZIoKdn2#^M8kcy{-;$@K@ zX^OTeU%vk+lY%CIJ}H!P;*3Tqjeh8jzGIH=sFpgYkalU8@*p-MshB3|UM?w>PN|Ga zDVmDF6 zm*Od&5^02zX`dS9f39hu4yvKL>7j0_mB#6u)+Y~`Xn3-~coyoUD(R_`rln%4sj}*- zwrZwoDyJ%{CH&EyI;yWEDW9e)uf8g*s;H~_YOKoYl`@_tcq*+Tsm0O7Z0f4B_G+c7 zs<8U!uvROv&g!9FW1}*Qo<1wIMys?=E3mR@u!`%oo-48zXq`q^vnH$dv8qmztGUYS zug?D~x?(H7x~rYyYNM=xx58^jo@>2^Yq`>^wdO1K6|1_ED7)TjqxgU}j_bgFtCQNR z#mZ{IUM#}i>zLZ=sFv!)Qmn>?EWsLV#*QqWWG%$zEv6p6t)k?9c`+&K|1AI_#IGfanRW$Qo_Y679`Gt;!~A(i)7vuI#RY zB+yoD)Y`1ner?!Nt=8)7&UPupYVE&vt-y*c*v{D6*;*3K=pqAJ%WEZMp( z+`etx9xdL|ZNr}J(l)Kr>h06^E#r=D;x?{InyuaT?Bgo#-coAdLT=`2uCdOlZtC8y?SiWeV(aP}ZtK45?EWt7#%}5I?ewB9^3pCj3hth2Li3*O^JXvfl5X)1 zFZr4;_NK4;mM_A7FZgDm_)72iqHpxlukzL}{<3fU9xwN~FTaNG`kt@-UM~UPZ}RG| z@dmIlHeUA@sQ>JbQ@1`DtSTki-XunA-E2BUEG zHn7rBCQDFm2v2Yc=dTQ(@C?UrF4Ay2&TI!iDjN)L4YREZQ|}NLu@U2N_4YXo=niQh zd;<3&aS%r_3E!|3Q?UqJu?=r$NMb^X>=Lqkz{P+0PG;^NX3G*dFo;`yC9s1K}(W6L{ zD)q!vXVa%pg-VS{wPi`GDyxdTx>YMzuT#N>#W|L!(Xyn@nnkOYt=qLxCvYpt6nX-a_7ulKU;q7IyLOHq*1rFotrmn$iRQ^9!~pq?c&Ch6JO4}w{zXlgA1Sj zT=r$;%CT#2ew=&t@8F|{{~u31c6RgD(a*lVKD~GG@X5>fO<#KX@$S{X*PnlX{QvU> zXy1U70m$BV{3VEBccU~|pMMYvNT7lDQF!2m3NF~5hUYmbA%hl%2;zet_E#5%CU%(N zh8#wyqKhPoSmJ~+!f0cPD6)v-jXcWu;g2-}xuT9O68YkYJ|dZ7@mBnyIFUMrtal zdxi=ss->P<=BT2w|GK89pZW(&pRT^zs;s=GI^iv7;=1dwu=Z+ft+NKJ>!!I98|$&i z`f6#j%Tjyit#E!0h;6mmhFdJO(nf0_xZesZuDaTio2@FHIbYrN*g~TXAH`&seKsi8g$=ga(GmUj z*K_ABHrq(r|84Z$fFB(=-Ftiex8Q>#Zua7GEAF`7Hb3e2;gsY3Roahd-uUK(N6Yu* zmTN9L=cJDwdFiL8UiasQS8h7vu)qGv?6lV|`|X%>jeFssyB>P%y9b}0>bw`U#E`{PVusu08hK=idGI;DjOyLPvn7I3G&`6kgp$ttJ!y1Bu4eFy|?Qpm~9o}$; zEDWN-|4g_BBC1e`AqaL5o#x<644pxk!AO|VP4iXKIh)m=X z6uHPoI&udj*y9F5Fv&_r|vhE=SAoxni{x>l32^|NjTt!PJk*Sfa#l`)(KCw`h($!7Mjf;FscllofE|33Ad zRBfzocgtJIYIU+>1+Hr?d)ZqecA>EiEon__SEQDfuFqxe2DlJVPFQyudthyAt6JO3 z&KA0dZ31N#+gtLM*S9lp>TiE5f?EbOxcd}?3EWUngI3nIusyDK>5AXvF88!b;B0gU zEZvk+cMM{^2yz5Wc5`cYII$CU(S&J!pk399dYZce}#uuuzRkVGeh- zzr{sC1XwI&&z{xADpoOaVf$hjr&q=_w(eazP-G;RAjdpTb1=sn;`jP9|DhPh@EGEp z;aZYXqDt0kPy_qqGe^>}`pq#|6Y5j}`!~y^MKg>S9bhn5;?ZITshNRFQ6?W+%}ed^ zMPCWzED)K_psw?hah&8m`}xm&?yja^E7BuVxB{fUG@?6e)`e2J)>oc2j1L@UNH4gX zH&%3})x7B;e>&8ncD0i~Z0b{!Rn;7Ta+DQ)(nTj)3D3=_XRXcX1ov9ZpunW4g-u&a zKX|S_#x1gsU12u|o4xRMw!EiJ(n`~t+PW6)KBs(ZA#<7A&*k;EIbmZYbC=u_o_C#~L|5iqV}OcR^qM{NKBSe{Ri6TP_tS8x#cs&10I;qEk^v5XNM zhQ9Cp?|uio;O(CG!pGs;knY4F7SH&^H~#U4k9^(-&j!l--O~!iyel>Dgos}~>!9Cy zpCPG7W;a`(&=i3J0Puj+vpS#BeEXNHYuzCX0Ov%&`{Nb=Xx_iV_mV%n@Y6v2;v4_? z$RB(kmcM)8JKp)wf4=k~&;06J|N6tve&w~Vd+vKb`P&CS|KBf9{_>MC^PC4G`qyuM z_MdjCk*(*yHa*}n#>Cuv+1i1AKlz>(e?~BXOi+LZ$O8xn1URsO4A_7P=ztGM zfbxe1;1_%rsCVLLeDAk`n8$$=2uC0&e*!py6G(v~NP-d=fhrgSEdYTo$O0`GgD@y{ zBk%zlV1qb#gF3i_7SMw}_=7wcggH2bG+2a2D1#yZ14^h#H^5EX#8=id0pHYLIOR?! zH)$Q1eZhBrDAcZOz|hRc_M|JR1x$AKo;esUOxbXbRQc!wuQ zMw1Yc)wSS#(0d#=zGz(jA+=5*hhSd$B7v@htz0?)rg1Kn2p+4i+iX>WXO%A z7y`IBbZ@nbsP%|+kc_}Le4vPmvq*-_M~vz?kIG1o%s74bh>y;=k80SD!We#aIE~gQ zjat}``go9z=LG{e0$UITEHI8DXO8H&c)aj?kT+)tXo*7T0F|hbM%a1KD3A6ylKJ?H z26=%5d5xh6j_#O>EGUT{Nr^H^h&Tw7FWCbs|7d`vXhw9nkP7*bZcuGM_gE8Yj2Eep zBM6JF=!&rjk|9}=P+5{E>5sukfzr5*E6I{cd6QgelQCI?UrCc#%krv5#!iSVhxs-D$hM95Rfhl~p;A)`*o_*^OJdlwo<8f{2)w7>Ftu z1{sN#K}n7_C6Ps0kw+PTb;+2TIhPYSjJ=1CXpoRkxtB-Dm)5t5fC-pcIhd%}l}nhI zU^$j4Ae*$Qn6SB-j@g2_nU=a}Q7Knnqj{O#2nNJioW^+t$C;eQ*+)1OkM$URhsS$; zxe3!poxbLm)d!H;sewG%o!%*!J6W8=|GAudsGP`Y1?QQb=((Qi$)4`{p741D@;RUM zS)V>|pZIA5HL#!jnRNQ;f^xZ=kr|P@*p^1=meZgQxe_WZHEC882S9Fw#jt9DY9Gao^(54p}4{#c%ayqAUTBi=Gp&**4dWxre+NUI{ z5BuP!fI6szTBvL~4|jT~in^$7|N5ql`lyZ?sZW{?R(cLtYN?ot4V0RxoVpI3+6|!k zsiGgj%UY?_aH*pjta5U8tpO{r1Z%B56{b{h zmILUmVOMK8hmHsqtAL8G>RPc(>ZZgBuk<>v8e6X(d#`V*ul_o+B%7}QTd*j5t)ZH- z*^00%hz1KgXAPTkM&@Z4|0uBvO0gEZvq1{1?m7=1+p!-DvN~(BC4030da_2Vw9?SD zESs_`%cvgwBd3`?L`XrOP&coSQ+JX^84$`3-TsA5~TW;?bZ%dbg$u}IstYx}lO z3$-bWs@s~BR|~GMT6{R`wO^aAVSBc`+OB|$wnMA7O6#_Wd$>z0w~Sk`Qc!i=YPB+p zwQD(kdb_o1nzMY{w~;Eiyh@}QOSl`Gw#sUuip#c%OQEW}x~zM;jr+PPd#$?gxOEFj zWGa~`^_#jod%cLcvZ}e9%ekKWv!ctfi(0I=>bh16w{M%e$_u;D>#4L$wRY>6xa&+7 z_nSXfnV5U7n;X1K|JtTLTf9Nr52I_iOiQrLyS}a4zU;fW>Fc!7tFkRyy9ao;JEgsf zg{HpSwaE*&m*JE3^b`yd_M!?i;_k5W)=%zYOfcFxpb1*)5*&e6+{*NIy(?8`7#RkP=ZO-ZEetw6)21xbAN|uX9lHtbvhIupD9r%+Yi1OTiG$G0 z83_l3jL%0+)Er&YTW!=l?bBZk)MD+`LEX_J9mV0M)F#c;YI$(9q_Ah%39TH1L!H${ zeb;4u#$dhIc^%e%y$gR0*kp~@eXZ38t!5!nr+y&9on7E*Pp%H zW8K@8ZPw|G){PvFl-p>syk@c80*}qoP5|2)AltN!+q|vYT`kzuJ=mcA+a`P4*?rdQ z|7^chJ)olnRrcj4fRPf*N8{7g;;0SKr9q!=@p5P3w z-6PJjIiS`!X>p__ao+Y`r?%hyec`Zu1q=S+HZJ0ujpI6gEnS+IJn zZRJe9=1yMXS`OuEe&rx8bYAChPTx_z&;ur>NI#A^NZRTkH+$+rHsGjPIzUr*r z>Pjr>u0H8@F6*=2;d|`k_08uS*wD~S-Y^H_q@Lt7KHE(m>&E`(jNa(9zU;{k>&GtC z$WGf1o~lfs>uC^5E-ud;2JBQ&=ooP5ZK~?xo(<$)49{Nci(cgn?(EDS<;9NcF;LPa z=#;gj?M8-yS|#cn=>*;m?!|uY(GKvroe8;c>gmqz2%qo)&)z5glJZXPNmy>Q7Un*t z@4;^9%|7lLukak7@oj$W1t0Pv@9rHh@an$smfi5VUf&$}trH(~XJ+wC|IqISzwtI- z4Ih8=#IW<`UhZM)vsBib0|M{M;_pER7 zLZ1uC@CK%D{Ja12tH1ogFYSphf3fd(ktkT4=Iid}XFX`-{C@dK|DOE45Bk0E`{1wq z%^&j4VEX8P{D=#c0EzZRDuYO_gNr%#|ih2F$g)F`!YN8>3~SkUQHs89Fgz`-NN)htt-Y~32BYZ49-#G+7wrAZgF zNz~3@t9DCS9VO)I(9poDg%LdBcyZ8ILXBMVnkJlB7-Gc36cr=RIdpMj$B~1ctQE87 z<(N?=^MpK@<>bwvLo+3fw6tcysWs1`YSk(gtgyYzUR%L7|AYubD14|pf(zWXX=~55 zYXq;}Fn<3AHcYrVH@=HyhW{=u^xT#fR zzn1OwEf*7FaR+Gt;tU%A@e7U)F@_`VuDk|GPC4ePb56XSq>~W4^c<{^JdbMO?lPMi z9PdN)E(~!*5Z7z(y`)f7O|bapqwg!Q1l&)C2U7TtjNjVQaSR#d>Wzuw6jUw+4_AB; z!U!iMamn*a{HQYTID`p2Bt^86$tzD3M@u8SOpP_yc%VWy7@5GLzzc5Mtv?*m(s9SP z=Az4itB%{NKfj7RvdSct?6b)~DHPO6%A~wZMMAR#|1{7=Q{-~XN4;dRH5bc_i!I!C zqtU-P)+i9pPcJ7QN_WWW zMcRbR)V3PY@>IY9bCh!gI!Q$iPYh61^;%jP&DK>bX|l}QL~m_W+(^sabww~MEz?*v zlU<`tBHqmJ**RN~mOu*_%oW>hTh*4^eh1B^rp&@M_~3B?K9^U9_w^OgHG=ixMN56u z)Lk$_bzx(9iHkPcJM|6t-^55x_+&y|niXYo73S07hGBkqVby@8IIsG~bc071)NFyx zPU982<7bPrGYx939oZXyo3_I+r@p@^;8EM;n&;0l6d&j=>;kCEkn3;nQfBfOe zkO2t5i8ZSP{G1n>@ibqDJ$BhY)*x~`?Ero9%eVjD`~SfWfOYHN{)8tG@}aAErURRd z%2J>+wO5q4U7(fyNF^C|1%mEXK!Xqk>g~$6ESTN-?=0!kb^TXlndXvH4 z{6&8r)JFEa_(dUx(T77MV%3spLL{Cs97^081J?&du`xg_&dQ+(>}M|xYEfof>>?S( z2u1>qQGkLBAR5!yy(CUyfyjHIT0S*_IO<1^u7jQ(^KwNrWN`&ws0Sa{*hf%m&X0f` zr3eRkN>u()jXgZ%Dp#b)t1!__$YTRoL;x2_Mv{V5gr5~Ln8y!n(UYZAWib(%N@SKX zP=;hCD^>UnSf)=aS*Ya>;26mX|L~HRF^mD^FbS;<4)c?w1m6#dc}_yEQk3NMj5Ay5 z&S$~{nktYFBOBMq3HA(|Y0Bm5e;J_{&q2$~LY`huP2B)gR&+1jc_Nl0U6C108B=(+ylT|EnMEsz^iGSIqJ?teeejIg@GD!LsxnsY$C+VU&jD!4;#g zbKf?-i5HLJNeUpX>}7`<*xMeHv$z$j7>yd*(V~@}hXq^f;y6=L=~ITTO%`6$pw)cM zwzjtI>~1w{UEXFFxT1|`5QaO^9t?|{y{y0tq&f(*J#o4Jd@UZKs{!dwH@DT*Zhf)q zS^CA>VQ0_qr3-En)TRC(v$}yTKiY zJXJ>@@NVG*F+7$F44mFNMz+1wta!nV3Mc8AOT~Y=c~T zde^;vaJRkfYh#Cd*rgsfl;`|tW-s9el1`oRiYudNyP*>3HS&6ed@Mm{+b>WE&$s;z zZe)wQ+~f{;|G|%K@M5ET+3;vKo(a6V1KSmTRgI(t)UEGrlOo0dx2(WBPHKV=oY4!1 zG)o)aZg&T`c~RWY6!%F4{op!5#`&~!HqLR8gFNRRKX|A~u5d`79OVs!_5xU*(KADG zpE7SE01&sFe@7kXJEuCrE7xJGe_ZPb2YSg(-h=OwDRp2*z%6fh^f{83K zAniL7SeJU;?Y{M%^Bl)I&wAB;j?8m=UAU%QD!b6GT&3qSmrU>U(~+R@n;$9iQ|Ei- z-`w}W3*PHOr*x|$Yjn`(^6ZIUx@3U>cc*WA?suO&>+_CxzsDZ)dY}E@iQV>CUGQ#|gwBQ&4_Z3Djm3_$^ez2rkQsOLEI~4~y{}S1GhC+jQ$j*qL+V4hFw;Pav%wquksRd0 z>e)j+9K${|!ZNhPksCyEAVec^K?#JVxQY`IyN%~5x*9Y9R7^!H1c5AkMDHs;NyNlS zJVQTBz#_cFTSP-mWT-MI0ts}rHyp)iqbe>tMPp>S;rpdsi!LvW#aTQ;Xskq83`AN? zLQYJ#Uj#)bAVp+kMOHjURb0hQ**|4mMj>cMXH3Ux6hmo@#%XlNG<3j1yfb?O#zbtM zzqpYz0LN9VLThTpa%4sx6F+raM}vgNcD%-1^h0^HMm3y1vGX#Wd8@L+q<7)R|9&ih ztFnPtyhFZ-#DaWAN@PfC#KnbVz)eiaT~tZI>&0wrBpHMvcL5NL%*ZwANIV3|kTgk= zOvsYt$)D6kmRw2Ab4lX*FS5h8d|Z~Av`HB7zJS!D)F8;61Vn_~JCeN0g9OU01WJcg zNTYN(I9$q$Y)Wxt0i4Vvot(<5>`ARW$#;AhuFOg*GY6u)#;~-w%(;Y1$sEdt=YX@%T;Wys7y?@ zTuiroOS-hm$V||x;8e`=T+abSPpmx609DTdh0MHj&i7nU z_`C-B{LP``8cZuOD&o!>I!3}Nvt(4V{`AlPG*I*ONdW~=^<+>JJyEYK9tvc!KAXXc zRFg@)DS5?#(BCDJl&P$b<(R~xXRn?tqhCn`l8jH*$r;?MBxQ1Yx) zJmu5$1k*pgyFnFHPle7Sm6PfmJ8Nnzzr@e{Bml%L4mpif*t$_(yHiWOR7|baJ_XX~ z4Af3-(BL{#kJJHTEWJ3oH7W%(9Gtm2mDO2Y)&nKbTeVeO&DBqxR%*RQPAF6-e50k) zrP!g-^@^Voc*HGyuN-Aoa~)1Sh1O@))J+A{YJFE;9W7t&z^iboZB|exP(`Ph0Z;|)PK4JSN@At|9~}Eg3VZrJ=lcp*vp(( zFw+5hMO8(_$eV(Jem#JiyI7WG*GvsqflW@2omo!2R#A<}YJx(N4L)rO%_u-wWQE0Z zz1Vae*mW&fn2pw5)m54mom&7|65CluOjVzitj6=S94OkM-P&eITJ+Rfm(|#qEzzc3 zS|x1on2eD4O*<7TdwU| z#7$hqEl< zJh}{>TFuSDAMmuM(>AyLT-#k-pln>-bzISn+0$iO|M>JoJd0Wi1x=eQOstxi@8iLN zyj|QS-Cro$-Q``nCEMSy=Y3wdtzPO~ z&FeK=!AspFjYz_L9R(?{3q?vsgx!jzfplG8_8nk>onB@z;ID<=&|P46m0#eMU;3RU z`^{J9sW!ve15I!Qn<3u^NUQUuoeykW0VZG)w#sxxU?D_d1!iEWi{A%+*9dO6)y-Ef zptsR;0Y?ahTF8Zk?Oz4(uJm=GyV%Hnrd@AOl9I16V>kihb1&Wf)cSxmO&Tdw9`)@3xltzTB!VK!v#Mdg=6 zo3(@FRxpF}!Q-@4j%lW5YX(qHPTW9V;djnvV6NvGuvd`dl@0+4=Nksj$>eqM#%WS;!wUZ!V< zF5PbaU6)=eik@Mq-QN%}x(+}^eF|J(3Y>h=wva~XAMN9w}E5a^p!#-@pZkWTq1=C>c z;b2ixeQSsn8#&MewYq7ywkj+glU3z4EK_U19_s@h?YS%MW5@*5PHoj*ZPqRW)OKw< z$OPDyZP+dY+rDkw&h0PIZQkB(|KI*?;12HI9`4+ZZQeF+Y7Z~Xpms>p%37H|R&N4c)*1CQ%Qol-cJ<^^9y2*2bApYW%Xa0+*D z49{>4x9|(^a1XB*4_I&(_%0Iv?s}}v%uFs5Cp?PznrRvDDBJ@~5DSgwXtV=bp0?!g z+wl$WaUS>a9|v+FAMzkCaw6yOBTsTAU-A#9@EF#R3c3%X$Ce-$N>UMAE_HV~_aHn>0C-++K z?rBGNX~%AL&-QhH_I9WCcZc_Qmv?$kcYB9-egAc)${>I5Kz`?S$kyj`pLaBc@yR|P zTd;+6dC_DR&E$1<|B0V?im!M#Q5Jy@c!A${j?Z_G|M+shc6`6M7k~kiFZq)AISEaUpJgbd778ho6mV{t9ebL`H)w6OSkpZc^vALcBG$ln6~JI zCj%RoQmOaWyQW>C|9PV4&z+a_G*$Var_*gu_s*VmkO#XC5BF=2^`7TQo6qA5U;7_> zJGIYwDZY6RA9}ZUaeskcJ@NM7SN__AMl>%ac$ z&wlJDe4?*+?eG4P2Y>G${|OX-yeEJ2Z}yH~7hd09qyK$GJbcN9SeUy4oL2%@M)tV- zdj1E96aohlv;pKv6od+wFihC+VUZg!CQ7Vm@gl_^79kpm=<(x092i5A1Ua&##*`{g zt`y0#rAe3uWzw8!^X1K)M@l+$8N*47o<2J|5!xas(xOToGF{q~Ntz@SAW*Gpbph6^ z987S%&^3U71YyNemC?X~g|rgZvPIkWEnK!A3ep`xHzvxwEb8(F!51gsz&ZyLE-cva zUc?#o|1u_O)FsfTk7FD)D*19#H&P>L&FcAcfzYDCjvejwtlHEM;$mR!TET1BvNO!> zTe~3M+%5fj=*{~#@E3^}A69Jqc<~y@mw$MYl<0FN&ntJdI~}rgr&Ka~>8`mmc+TPt zkS}lA^b*vnW3zAVwmtm!@@WI7Z$CKp#&~bz??3!MfW#%o9Dy_~v=c{|ZMPtUj%C5% z84;o(;e-{2VIhWbXqZKK9iI2$XhEnK-+Z&Vw^n*4mPjIt2e|kmj4{SHkZv^w$e(S> z-6-II#wkb!gbPxbA(5D9*asU!Cb=Y&Y#@1LlTb?eorj~9W*&&8rI?mQZq=6>jwiCU z|7D9Snt7&-DyF$6oACXoA7eYpIVYVh0eL5$66U#vl2Q8UbIxPR;sw7nj0=$nPQ8a5qN@O#=BR<0$O4h?>3=0avMiNa6(Tz4+nxs2?dFFbtdApj(RgX+E)>&H( zY6>W`EX2!DAR#u{F+(9U+BvJe)64JPT<^|N_^a!aLjaAyR5BR7_a=SA@;BhOfU?KZ z&r+a62wQ8tIJu2q{dl>NGd}r>K17)E*qLjtx!EqK-TBOn!tFNCyPBS_7(m_a0ib;Q z9XRZBJZ$(7U1UcY?nmjqyHD0(jHBz_SfBmzBc%2!=VTHDTiP4_@zhXG~Vss;QFEQ171n&B7SJX7F=Ao|3Cj&1oMFhLHq9(M#N!3hR%ZWX*>|17A%dU>5U4DEZ z$4!y2gm$EX$?$kbJ?_zj+asX}IfudxHX(^Ggx~9CD6snJFjDO!zz&g@JV{DyDwBL< z1NfIkF5coAd-#GbJ}I~|IEyz}3?3JGXBSFR(l#Z#*ehWfwOPip|Cao@WD7A#wHxK} zM4RJd#(de!VFI&{egvcm=a{!7>VTPTJR~Bc2{<+su91$cfD4ny#7NHO1SiNQINe8w zBMlLfDnLO1a)8cif@O(Xw2L#{xk&)kKq|kxg|nXZJRWOoxyya3vY#ShDPd1xL1SQNsq?fcLr0_OQ z{gr@=C^ei%Ln?wK#Pg)O%%weZDgvCgaG-DG=}yl=#3v3FSOKLa0X7p)W-^nfN0rMQ zC7R5rdXTDC-JCFk*_wr_lTTA9Wc$#$x)x?Mk=$4$M|1emYr3GV`2Q2@BfpZ;kc^W%+e(Ua2E-EAxT6~h<|`=Bw5v!(k*j|nQNSh_N{ zq~R`aAwr+?v+2E0KVaX2;3VJ2nv~QZTu>}RO zOjVcE69YHC=6z{kLoDXwQkixySOeImMOZXfLCrb!a+M7nT^g%)%Nh1>bBlWhY1Z&i ztsya%f<+b{mc_D!F0^jPspv&>0j|fL7O)&UQ=pyqc7P6XLn{2-4H){O>J@4eCJN9XAk=|xIzswagiK?DHu(LuY1F-fvlyslC-dNZ7WN~ zD(dr~0Ek`bG>L7uh2Y#4rHRunJ;y9&+o^!P*GBQTx&NDI`C=K>sXlkKSA0%RubZOX z-ms~SSlM#xRKxO|L2?NTS*L1DA$>-J?}ROy9+FfE$!@l>3w~lRCn`K6rZ979)+GS`{+JjK0t^*5QjLu=}RvO zK~Q4#sc${(V^8+k&z>8yu)Q;G-wNFC-Wk6CJ@A7sd~W1^8gFtlxAcXJS}dRW&UgMK z9ua*%5Q6&YyFT_AZ^o`$p8MVR{`bS5eb3*)UMrB-92YP5^?P3nqvu2UJ1>7c(7*oi zpFjMqpMTZsfB%2*fBz+dA&?&H6`gDUku_O4eHc~@42Vh2tmDV|~}rXn2PU>we3EOKKv>Y^)pV=X2kIF2JC{-PqD zBQORdJ04>)N+ShYq8;JWE4_%xME@T(ilRPZBR^_mIDX?liX%YYVnCW>K^kN^HUc^> zm;zK%Q zOERP)R^c$dWAJ5;?;zD^;iCsiU`JY`D)yvEl4MB^WSe9i{E~PrIrBg=bQ?jF5Ql(sO zBwf~I3(V5EI1ag}g;w?@N9rVC`lMJ2W5%XzN+xaAW;YCYDTAXO6PONrFGgScE05U zRwi!Jc3vl3&L@3#CwETc^PPca$_H@v zrv;o%^AV?bf+m5UXKJb^d{SqEE~tDqXna1XeMYEXekXtarhlRcDSGA^4rqZIsC#}W zh`y(YGN^-=D1@GgN_!=!JTT5ReatvfYf5r+FUaXd>u{?x>FPsEO9+kD@3P z;^uJH<#6uji~47Ha{pzG+UQau=Qd6!bdo5QDrl7k>6HrUOR8v*q9KJQDRBO$j5aBm zj-;7RX^5t&l=>)}w&|K`DUimgkQ!+|G9Qa#jBkP|5|BZFmMM)wr-!1an)axh7Alq= zDx4;2m;NPSBB_Py>7G)dpE4()=BS}wYL;dyqHgM()~R^n!Ww*Op1$akPAa8VDynL# zs$%M@wyLYvrlitki?t}^NKAQ}s*_qOYVInp3M#7->Y!Tbt8%KM5^Jn7siT6a<3#G8 z;%cfs>#zpvuTHD67OS;3E3J;|g_f$bp6Z!GtBJCru<9zcR;#(5YqO>+jn=}Qa?G;o z0XMv!x8^Fne*Y`D9%#AdE4uFMwRY*c9xIJLqqaV(iSVhq&a1rMtHJ_nv^K26@@u~y z?4N2uwkoSV#;C+1thACVzAo&ze(alqtjPLm#6oO*f~vN->t14P#&)a6GUdbSYshY_ z$(F2~wyeY^tBWe>t=6l?zO2l~Y|z#$(QYTe;%v_1sl{IG`?)O83hl_!tkg#B(Aw|?PLhaNtEY*rF*@ErWYN@~uY?yNG&tmP@a;({g?bya`(ONCVDlNg1 zs=;=x*FGZMmTlF}ZQ$0e+p=BC?rf5>ZQCZS+`_Em{%zv|?%+ro9`fkgD0`xwP;w~@tn(z6l@Al?z`gUyp0xh=0B0q^ev!*2u&umn@^|7tIGlCHTd>jE>d{zC8shj0ap@CakD z2`?blB5?SEOb1^olsYgB2QUc7a0%Bi3EMCZUvLgjE$|kv{5E0&5ARyEaG^Hv4Dav} z%YiTrFR>Cg@d+a_6!-A-GNcCADiKGe5yxQ?TksZlF&9TM7!PpwW@ivvtouzd6K8Sf z-f$X+F&n>e0T*SWma(#0Zyc*}8|N_`>+v4ZP5c(I zW4?1CNAA0La^=04DN81-n6YQUojE_=O#1TZ)0R`0cAMF>YS*x1qy9^~_UFTeaogUl z`*!Hxt6vA#JiGO9;>MA$=FQx<=;zL%A1^+=y724DvnR(YoqBZ8-FxT%{tn!__w(kl ztKZI^JNozH!G|9&pL}`y@Yko;&tJR$tM}y>Uw!itIADPL71-Z@2{PEAYyviipMvxq zIN^d9Mu?$<5OUZaf*@8Xp@Afd65@j=YUm+~A9ko>iW!z@;fyTS_~M2(uGnH}w~QF$ zk3Ir`y4hx&eAaoVoqxW$C!T%=IwzNh0;(vXc@8ROp@t%w zXrzlW>ZqQLK3b`jeNwuqr-ELpDW;^l*XgH|nklKJpo;41r~tzM3Mr?wqMB-}wz8UO zsj@D&#z4D4GEX3k^tFg}NitMPTHe0Q=&>lN&sFz;rt*YRDtF5%$ zcAM?9;bJT9xaB76E~&t->nXG8Mr-f3yS}Skyz-)}?z!x?tFM&&7JTo&{gSKjz?d0~ zu)hHVyl}pzRxEJB2SXh3#2hosC%q19Y~sckQ{1t|9#05z$R*c%va+$7d~sDMr`+V&8_0?c^ zEw$HNi{10lM&GQqcvg>%_Ss=4xAxLy*KKy%tYM8;k5QBV{k7e5w*7b9S{I)5;e!td z_uYgeHMiS-Bc8b6ljBYK#eGArIpeq?&AH`^^O*VOq)Q(8>81yJdFrgU9{TE@zpmWq zr33D|>bK(_JI}iB4m#|z(_Z`VxfgHz@t*^)yycJEjym$s`@Veg)MxFv@Weykd-U9I z@BR1F2Oqxn<-fde#P^(F|x3JUOo7Q|oyC3wK}A@GAB%%BDr7_$+Y@Ps1VAO%B6 zLKf1Gg>y)u3sERT0LJi!HRK?rRyIQzVqy=Vz+n*og9XGQt^tQgydfKmc*G?t@rh1k zVicQ*1}Z|)idf8|7Pm+SDt3{JV7y`&#YjdnJOK=6OrsjhNJb{+aDgzIf*j{a$2!`v z3v~>F9``5%KK2m?e-wcV1UX1RhTxEhB%~rwu*g6zVF-^*!W|`<1w2~vl9;6ABq7Pk zPI@wupY)>%MCr&<7Lt*uB*7_H$x2ePGLeWJLI*ZL0b1G;mbkoSE?;TN9PrYVeq5s& z&`8H&8q*9h{Gt?>X+*aaF<7bmBMSQHf>BQMnls2IDY40cZtAiE-vnnk#i>efnzISn zBmz3I(1a~0ft~JzCl}%=271o(p7PA+IyI^P&OG+>p8yRgH`7Vbf?Bhh2TiC#7y3?f zI`pC5B*Fy7Nzn)_00m08s75zhON)B+qXq!!0!B*GlA1K63`i+TS6YCUvh<}Z^<_+F zSJ`bHub6dWa>O~8da(q^r~3Z z0YewcRit)xqF$Y7Ho;0(b#har(^RQXcM8+BvX!iGRjXX%I@h||b*|F@#UZoGSA>SO zs2?RNV1LTf!5S8>W-S3u`N`O-I<~P^jVvdp+Sff+_N$m3=wHj4(ZX`Jo}C3LMqT<= zx0bf8r@gCcSF2ief?^ue@a1LwN?W@B3YMIv73Wbs3)aM9)1Xkz=O`ZwS;^k?xU_s+H_O`8R@b)IMXh#WN`!iffnJ>GfF}_u+2|rQx7ekt5^4+F;L0)WJ_D$ zR`{CfHSj}s>tFH~H@+VZv3!w>RYf|L3*}5Gin&0C2RoR>=4RcSx_ zt6?19H2^RE?jSf(i3o?cvN;uOLJzB0Cf_)sS#|G;DJ$X;Tlv4J^l*OdtIDO47@}D2 z@IjTTS<(79#~<}_kI&rJ2M^i*$e7T@6*iDuo#J-DHg>X}!F*qQ;< zvc$C1ucDR9Wz3SZxlcx@pZ{D>>Z1A5Yi4tUz1!k98`%tW*0QL>jAwK1nZpiUuXOt? z=qXDX#DxB`ouk=EQZJOSkgl;rO)%(PGnKSY#&k)&?1J?g8>7XZv8L@BfRHk^)1EeR zqb*ErWe4`u2+%XC#SF<;$C}uMuJc&wJZ)NUS=SR#b&a7+RP)+)*UO$OiZQ(0;VSsR znr1Py=}_cPhd|tC&GVggo9$EQS>22MHmny8Zh+@H#Slj~y(KR2U<SrCqqe(a2)VRF?eb&S``iOJILlLAZ-u`+)}XbyCyBe_tIr^w1ueAG*WL57U;N%q zpRU>2%xP%HdBO>IxizCs_mm5@?suNSqZe)UzB@tId=LB%&b{b}f79@YuXuWkuK1`I zIMb$m{88-=0md60rXY7aCidFi=|WYjj0OGQX+C-phXK$*Mm;^PI&+OuL!^Y_`lC@@|*w7=uf};mA?KWgMa++e}DYvFaPtq|Ni)Ie;@R}|Jmby z_OKUn062gIXnO{Dd;d3c38+sCn0pQ=W!eXLb9ZoUop*o;*nzI+1(no$ zB4`98$O9&Lf+(1RDmZ#f@OS?wdoU<~`xb*V=z%uafFPKII;eV4Km|SMgFgs_EQo+Z zn1d~Nge91SD~JP2$b>EMgiZ*BQaA!EK!sL#g;@9jTDXN-ScNfQNKrV2Vc3Kt2ueZd zO1u^0hqXnoS2ET^ofZ?Sq;W{O}Bu8XoQkjiJiEL ztaybd2xvCge|s2+w0MYv$bq_=mdKfkRe*{8*3%S&;chkP7LI2?>r4>5sA)krMfO z6G?y-d65_?kr4TivRFw}=#gw-kE4i>sn=p@K#)V01|XPZ zmwq2ej4IfLk_d^VSCb1lmSnk+8p)A7IgUR0lh=rZlPHEq>4|bFlXO{^Ug-pAK!Q^# zm9p0aRcV#_7=yepe*1=dOZk>y8IO9HemQBDj(LA^xRc`Ofoxfs=h&7)nUqKwmz>#| zo{5x9kOn5Wms2T{+Xa{}czXS)m4*q4h?$tL2x$FRmL~~*Rm7N&iGH&vnc?`8g=m>T znT(mKnV(slb7_>t`ITKa7l70DiO?U*@=>yrho!qHLOX-7q*pT`E zhXxVp1+$r(8JU)9d6|cZne51wKR}c>AfMeypEYow_?e&jxt}SZo&5Qq{TZMFI-mq9 zhAT;$efgL8D4nZ02;#>>PhkxPN3R|$iqAlv8;1#1XDx*V0Ge2~rKGZ`x$`={x7SB+jK>DLXnhF#;q(O=hMVbq& zkqJVfq)f`Br0}Ew;s{K!2UL2cSbC+&1csrR1y9JFZUAo$2Aw*$W8B!Fyx+mamuFqpr(A|%Bkt#shbL_qB^RfDi5X_s@FgU0=k@hX`0$arqj5O z;fH=9N~xBLs<@h|^zf#MYN)aeWJDyhvHsm~g%(mJio z`l;1wt=O8aow}{hz^&f;t>D@X;ySM6N~+~Lr&1|}2Z~3fDW)H{TrwA!anPWckfxZL ztN41YdAg>-nhzY>uK?St#A>Vri>%6Oq1EcF2CJ~t+OP+!ui9!25gV=)Td@{Z(VFng>X)haaHdd$aA3vk*J6 zJd3eD`?Kb1t{OX(r}u?mDof9~aaxv|wK=OP`Jp6hvJrc!DBH6C+OjH}wF3*YF^jMf zs|z?Awqo0}WZSJkd$wqsu10HwM~h|%7oDlekck?#Qmd=`YPB4?s9MXiRlBz@>#|(? zuwFa3V_Ud7E4Vvbwu-yB=ZdyM`vMzFhb@qxNn567dVAB^2e3uAb(^@&+P7HCx1n3G zq}#QGd%A|3x}GbstLwO~o3@Zkw317?_y((Ri@72@x14LSh#*iqzRugY?TfC_TZXiInjjmpPTOK0s=eE*yu2E|BKp7nd#t^?zURxn{CmI%oWKTb zwhY|B@7ug27rDF#m8_>#)qA_pcE9+myOp}Xx!Sz}?7e;~!s1J?1AM%}P{0dZzAN0q z5A43r8^dbb0(d#S7JOh<`E_wCvij@6$_u*wE4&6vz$l!;4ot+%%d-mXyGWeGPW-}m zI>WO&xzsz7)|+feM~)^bw;b%li`&0J48SBj!o$m`C)~tF{KRVP!f3q4P#nY1E4y-B z!8tZ|7%Z3?ytzH>#ULEUV@$@y+rdSA#Dv_&h5W^AY{+i^{KlXf1+qoMUZ8@qcf<61 z#aO(wu!vRq%6bn8_6H($aGx0Z+pj0tA3ii$=LA0 z#p=mrJic-|%D()|q72Na9L&S~!cy#|t;_?j{Bu87ZA**Ezu9cfBx}a_yw6Ka z&flEM(%{eCJkS6Q!_g4Iv6;-v%+7X9P}KO&dd$uAJkf`Y&*H$nhkMKYOwa=T&u{F} z9nI0yFwzD6yyz^!vANEzl~}QS$ypo*4*Civsgl$GAkoxp&?KGH0-ez!9nwAx(j^_# zKW)w@EypHElPY~zG1k)0Oq1ba4OFfjng`P)mWX?r`*#44b(&J)ur6kVGY*e ztkptY&Ik?3PN2|9ZC7>nrC&N^QwRz-UDalN*La<{T>aH#-PdCM*JBOXd0p0nUD#GV zs=?%gY~5sIYE^`1g_As!pMbCEVAs%K*?eu-nJw6Ueb$?;*@tb?^J&zHn6ymITpEyq zliUDSIN6vT+Mun`vc1=zUE8*O+p+z+P>Ir|U0vWNib+)hslC=C-~@8b+EnNXbzRx7 zo!hhR+_>G@u+7=RP~Cw&4AULmC0*2NO^0#+5RGpob0!II#I4$cz=@jR4$Yn2*c}b- z{odN`+0sqj^c~-X{l?rKl)Y`msTa$`eTmB42k1@J>&@QKZQlr9-`B0+48Gvcec%O7 zs`s|D*nKZtgdz#rWel8#r9`ghUEb6 z;~##B=?&&fUJPsA=GIN-e?I1FZs>*oPSjjV;dyvV26$+1fiCEvF4}NTg1wjoriO72Eo2%l0+OERl+Ndu?&X+X4YV%nSqv}1@i;Z`~=3!lja4mr6tgYu(Xz8;)?bI&py6)-Ne%ZX9?VWDz-2Uy` z{_CP{?8S|Ur|wf}_g!|(>c#EqBe3TyAnib|?blxK-5&0l4(|FM>@z;^^^gh04#`o8 z)H=X@8RyoJe&y}ni_&)3xZ}I$o@1zRwi$3sFFb3T=RV$t0 z_K4aGKe-KG2eA(EwLbBL|4h4bL7qLlm7KZ-|}Kl_DcWvWsml1zxIQl^n!o*ZNJM%+Wc@_s+|eV_S&zw2oq^_~C1o{#vSZ}_Co_K9!VKrq@d0EBVR2Kk8N z)6{iWTv%PN@_O&^^ZxR>fB77L=0-01z<>G${`OJG`i}3h7PVtdcW$+Bp0_UtydV8X zUi!g5`qfYPp>O)y&*qB%5657cjXodwzdhbUpZD-?2GYO#>QDXG56tb4{e}Pj+t2;} z?(w`m@-%sG6IfWX4-iSX2ozYtppF<0MmQPKVBw&K4HSaOCa)qmbQm*g+<4LB$7;Qn zjSNXlWU-DYRc2agGMcb%!=Nd9S#zdLoH=jq)XCH5&z>=XN~1|s=((apjUEl!)TYy? zQK3qONiAwTY+1Ep-MX%;SFmBlS{$1(6OR};(lW_HC5jWbZHMR>h@wIS2q*?(;oElx z5f^~P3?9?uAVP->EwsoIr-cFs6G5h=T=^u)%bAt7q&ju;XV6HQCLK+BX=&3gRj&@s z+VktqR>gg#&GnoA?Am3?hIO=L?QgbI;BFZ%;x692#y>0{Q5SIFxz7(Lj_AcPWXZcZ zTAqwqTX)Wy#h;E1yY+eW=GCufe|~oE_qO5d)=u`U+O=_u6A$AY!haJy&_K8|x)4J; zf#}!^Ar255up!3!cr3CND5{S;48hZ|z4FN85H%1t^e{aVKkTqP`Q}qGMcsPCF9!XB zlWPkK47&h85*8z44FeB^4m$rDOz1%kBAien3%3(7!;+SC&BPN!EOAOHM}+dqtKwUc zMcnA~@;O;!Cz%|tN$>c4l1f1T#Injj z2X${tE)l)|@ESXU=poD)BR%NMG}q*@%_IUdlR*a+tCPYC1nN1B=kM(GIcSfIK6-7o+n(}drjsVQK7$!PxWD1Buo_3~ zmgBl>5^nxfqC3^&*=(|jem3nd)!x|cv;P#Boyp_gc4_VEHo}X$7v}W1bTf@^X298n zw}Gf}HbUpb5qBMO*pYU8@z!I{vU0gEN4e^6bKRolASn=jgTNaUw)EnuODycjj_^#$ z*P}<9cEuU5{qgHfh8uV8b;rD0;HZ##bG*eL2y_O~uNip@l&4}4CgROy`?7bgKYQzE zUp1LG_;&_>>`f0)bL*b>2&TC)*{yuzA_8I3wHKEm&ui|BgA@AZJo`aUdK;`>s!-Ga zKVbB)f1&{(_1Km>%cbypW#C=H#HYY67$<#tX+jNcxIPI62z8Gc9Fm}-!TXi4gBf!M z2$KPfBaYC3CJZ3I3P`^tMsYVR4Bz;6bt6h$4KJa~VF@&#Bh`Tl3iHcg5Y2cvV{ovE z0<@i;{C7k+hAG%Bi0VO%iNAQ^DVQ*tqfhdfkQ|bPaJm!e?r(|UbS{KxB9lO3;sOT(d!Uxlp%M>dZEKZ4yfmT3r)14%SRQ-Zl;&25bDirP zaVy!nZfvi^?JOukn}uU8^P;Ckh!`pXKa0k8b@;^WZy}pT>^{|-Qp&|;FI!UU1{b(~ z!Ygr$OV!Wj2f1N|DDWhT7dISec)^<|Sf@+AzNNqpLcMNybqiA7@)n)^9j`&pYu<*c zH@O6X&Y9v%m)DBmtAM);D;L9qYSOm4{>^TOJ^bIuR;Ih)_^yFJ9AeHQc#r9|s(#Rv zz6j6-wS-l!=_Y)m2c%`TecSIgINZSnjx~s#ed#gV8rr&^Du1t$gX4CNcZ7tXgm1iPRD-R3b|>n#@cAvKFJ7kS##BJ~H^=a#nryJt;sUlbhI)P^^=uwBJ) zy-Rdh3oK!4sM-lf_}>7J(74NbXIO`v*$7|w!2zajb}O9W@ve5u=Y4{wX`3K}RPA#M z8yIKeyI0|6<)g0|Dw3=G*(eWs%1dtaW0|-bU(apZ{mb2XH^~RfW;#u*lQ&jGP#Br(vzG@1AT;$8<``*Kj^r4p> z?0`Re(hYxj#G^g!i)Z{T)*dd=z6;*~!l@SHK6kAbyXRgn_sIMHdC(7D@S_)f;tPNJ zr8C{-kB2(>(o%w~Q@-+Be(6zv{9Gr0<*fIe z-XKgd+0Xv+6Xm#dc3)QC8-Ms=CO*-TZ~CF8p7G;vJo6uFZ|9qKqNzk>`>DT>&Hp_6 zo=>*+zu$e6j|2SsCp^-{ulkVN1A2qJsC&K~i9YFbzxT_%j*~yEs6XI~K>O=I{7b9* z<39|{Kn+wp@>{$;Fuwv!zx5lz1QdaZNUSvY00*Q$znee`oVE(wJ{kN!0E7p}D;~Rv zyiA!o13?0u-lt34}z!;oC{oBA7^gj+<8Uchkh;t#BYZww#Kiexo z2K1v9M8e`j1}0QO?+ZNTV+P@KLQLDV4qTM~59A%FVK0VQAdM@4AiTXT3_^<;!Y@2R zjR3=D@Iy5uLl`W?Cp<$mOvENUL@1=eHl(UI>>W9jsU)ZoIq|{ivqMeXL;&eRKIA<` z0>m&J#55#CLrg+aTtr3`7;`|x|BJ*bT&CaIIe8L{Cv$;%k~^HMusvjfKC}o?6huW- z#W7q(V>H8JOhpW2MMoSzSX`MoEWj&#J6nvVT)aF6M5bSC!C*wiGTbCrOhylD#&9e@ zhYGbJXeQ!N9~^`+Aml{hQJy~<65f*nYv@LN6i0pB$5I?dWjw}mB*!*k`VL)XQp%p(@ltEaXK6cmbxIgE?T!m-LOYnwkTs$+^Tyy0pu> zS|5m+)B%wOm8&Ka;z`RI;E@+7 zg3fMHg4ZPq7!R<_er&&)!s zi$`otN>3D4f34X6fMptECD(x^*n>4#O6^M_?KOlPL2KkxRbo=z+cJvm1WtPb=jnaR5_K|nBCZ#y;-UC z(ddBIOf^8Ctvq!xQ(-&7l+C-N4O{MeTBj}Bs5Q>1P1}xr)mUvlT{qe9ShHPPs6E@Y{n+sfB(?BT{utS~&A8cH%=RJL#>CsaC03-JT*@8THf7pl zxLm&VTh5J3wf$Vs4abFSvNnrbxqZi>4c5kd+`X;biq+hIgk7@jT)+)n!5zw%i8&oi z9ufrA)BWB5&}`hg{l&=DTh}E|+1=c}U0&uzTiXp?=zT;-na`}vDd_-S?wkVQ#XHue zzv69Om|eK!ZC>)FU7UU1=#}0Zpx&R2CTqoBPV`!E;k@rPUh^fz#v0$_)8F#F-`Yjr z+g&^`vD!EcLSgG&c+|wf%-7)sTkn-z@C{#lxdy}IU#0Eezg0y5-e3+^U-qTBd4=DH z%+TFyi|Y%;`n6yG6<7;CJpFB94F2Er?cnnrVA0xL1Z=IlASD^q!URT>;6=?7R)arG zVHM6`Bu+XQmf$4b${7w|w*6ouO%O4VFdojj==fpcX-w^Ug6`tU{JnaG{)a7_2d|~NJUO%0j}Cc zhU2SwuOy&jW~!tkI!rCrv@)0|@<8&V3 zd}T~+=H>_%W|uYOZ~j*@{!#wZ=O#wva!%&vWMl$#d=6+eCg^fD=$ArhXd+bGduWHQkz0n!Tn?9OI)E|iWr|+e7anAr zMnjJ7Xq^^kC?4pK6loQGrcd=KmRYb*T$HK!jSb1-}Gd3z%n#e(H)&>#El2sV=#wKIOHRVV^EXSTtOeOu&=ffJZ2V zTE;erF6*s31Ewx3r$%eFX3e&K*_x*6xMpk(#%ci8YMG20hn*=Y5Cl?a1zf2pAKn1o zGwZ=-RKxD(ng&^zvn_HAS^Wd3Fe|Nd_PX9+vVgaIdT z10V1*KyU?5a0Xv+J7{nRk8lZBZ~~w31HW(t$M6hi@JtZEJ?QW~`0x-P@enVA62Iej zwqp}laZ6$GAbkQDkMZf9ao-8G!dP#-pm8FAf;kB5u|8nsm{=Zb@GmHG2sd&DNAe?I zawTtaC;u!Y&wH6&jaqJJ0hwf0T3Vb2+T>qr`D|)@w`X>pTo{40WN^U35mDb4P#l zMlbV7pL9yEbW6WHefQ~gff7FtkMF^ zu4YF$-5mh+M7L40+H_Cv^_TYbM*nqT4|ZZ7c3?ktV=s1OmvdeZb$m^AXm93dKlOKA zbyojyoYERI2!N!9=v@C#W)JsfZ}Vj@cXK~?Wlwf=A9r?#^Lmr^X-9R?ns-1CYKOJs z7`s;=765q$_h)bSbys(RA9#W7cq zH^`nRd7mFGpigy?|M_S&`cXgnXywg{*Hk*R_=$IIcn^9jG~J{G1I&)`k0)yB`s}Gh z1Kf1_!Zb@uVS2Nl?6Xh%v@iNDV0*WJd!&bWo=5pTr~B)?dz&wLlurS^@B6-&@tJ1< zzZd)&UEd!F{GpBcxYv7W46U1|_NhrV->Lc#R?;faghz0OH{ANIx2Ioip1hawg&h3B zPx_H(4lF}@uXFaf|6;Fe{T^8TcsT;Q|MxaGd>}wl!1qP}H-Dwt-+M|w`N+@xU0-zE zFI>9^_2Q@d0#Q+%e|N^Gcj=FQZR2_zcL6I%gk9kPBjEq+E%d7CHv69&c4uK~oD z{o^Npzn6Xat@-7+eZ+77T`z+1XL;o}{`)8Y@COJOECv#+A@E=Z2?`D@lmH|}!H5JC zQcU>p21Xn|Y-ltH5ktihOPG)>>Et9zCoM|GXt~m*k(4s0%%o!TCeE8Y&|ndf0Kx*G zLW2^GP}C@rD~@bkfr#Lz0H_p7q;P_EeZk0K?KCJp$s--BjXV{Xm&HEY+gWzVi{J0oq5xp(vKeZ+R*oi9;#q`O$7%HzS8 zFHh6>u>t6jU2vd0!^HIfoC|O_G2*iYg%7}!FK_<5)6%Bn;+CBr#C!Pi=dW*X|Gw`0 z_*L`QzyE)K{s}lBfdv+4pm8kyr5tfD4fh~}%$*d+bJBGeS!CO7=N)?CiT5FhA&S@# zi6xe&UVsPQMpk_Ex%i@QDyE1ajWy0#pN%;x$U<_(k@VnbqwSa=V_A5SMISv`xFiG7 zImsQA8cqd6ha*~fC6-y9cOr~k7PKN;>UoL(V~!pHSmup7=H(w)U>)|AVP_mUr<`H% zbSIu^=otr}T=+BJ zO1fpF%r5Kbv(QRAZM0UJ38q?NdaCWGyAft8xT?x(E4fMHS}v^U(%MHQN&aalpdI93 z#vWXlk;4GL!n-S?794m$X&_x|=CSQLdoaQYSGzF73R9~ksI_g&ZG2!9rd7BZhf6NU zubwL|$Eb?TD!c7g_?Wyh?7<1ky12am$7N;6P#LfL_IpEEASgibuoL+kT3N?7+c41& z7k#wDLt7iMnx$I&G}KYUq4CI6+e&p#RfD`qyOo@5D7~+8vU1Cxm@Ud4g3zol4&d=P zlCYf?Ja@8;+5kY(NGq*3-+kxpH?eX9{Z|!MKw)@Oh$pTE7>qkEHRPtQ>h#5w54Q0f ztCsxrR0BLQcG+TY0t)G_u!8oz9oQ^GrJ?QpI_!S~9{cRID~y#IS|qjm;l1nrd*Qxc z0rhV_JRQ6qnWGx@T~K4Jwbto|fN~CGi_SInuxM{O>0PK^^G`bq1GEB<)#jqCsac=GT^ovPKMN1=CZh76(0Uda$Riv@xKeB4Pwk47ND z2?~ya6{H&mGdMmC3T=Z8=pY9}ctM_#FjPRGi-QP&MY7^dSQsG*ZILlu zvEve)VGD*oPZW1wT@X8!+m^swE#W9rBqzH!e zrxlixm6(1{jPnHAmRAKxgggkCfj-A+Cp!4>Db%SUCoFG|d?lC_sX zxXL&?s?S1(RHSbICBspDr7tOc&^i!w!4`)4!I&Ou5MnSV0Uc=4eV(%?g_Y`FjjGti z##LuUo!LgO+Sr3?GO|MuNhKR_S;TsfqmLv)5<+0lqb^OCPza`J`zY4b7B7Y?a8y?z z>eY!>!I^ipsWwx}*6YE78=Y|L=$_EnlM)w*k<6)ahoT`Tz6-E?9qa`#xmlyCm9cc4 zEMZSzSIzkv?3Xj ze$p&xNpMui9$C)M#ZHK+4B|W&IkDOOA&nDZ4V_+u^Zmp&+a(`%NRwGKgai&U^`8owyh17_`-LqVybm9(kTUXkg z0ZqgIE$&c@`mB2;F`iEiCqh zY=!f?2{ajVXg)r@e2v=H)LEpzmCf&+@B2_a6|QyEYgv7!JH1vF>Y;Nb-ldDyF@5gy zZ!CsDU=(ZBScaFGl>l0;V|)Tz*Sg0qUgd7fd_#{`@^!Vl?4JL)&UcPBnqv*9^76dy zEl0cC*Os!q9~|c04l2!Np2Q>zcw%Sr3lBZ9J^f0YS9r&Eyd&SMOizI9Y`^a18QpjP z$8)~(y#73`>-~7ABmD8DFTCM(3g64L1#$AlrzjKewB=z=d1;%2CUCF&v)$eky(b0l z0|7_hv6l75Z|(7sUu=wQ8lQrn80w!Ned$|$)nl4I_E&G2)c?Hoiqp%}lPdn`6TZJy zq2V%fsQ%c?f&JP@%;HwP0W8-4VJ_-oF9M@5 zzCke(BS0GCBTnNp5@bOd-U#?!18@Y2R-+gm5(keaRs&wxmAt<39>wObTQ)LS#)ILq4orDjHASPG_Re&%6H|KuTzW?35M zVxA^yBBo-BqenU>ORD2ZdQiet=4HO5Rc7X9ZlzZOW@wfsXl^5K`lfL9rdp=vYO3aO zy5?)HB5Zz#`_N`>-X&h%=5*$!ZvG~44rg{|Bx!P|cWS3-hNp2BCwV5Pd7dY75}tc5;NfdprgWNMbz0|cf@eaCr+#8*d2VNSqNjikD1oNuX|5-N-XnbW4Rq3H zecmU2=4XEbD1TPygpOw;d}n|bXhmY^e;O!=GNy5*RNiz?}0*65SUXqlerlZzLcD4%j&)+wV(sF;$enZ_uZLMo;1>7QO| zpQdMH3M!|{VWBeUmNsf{QYUdns->FhsZJ`Bsw%5y>VV?orb?$|f~uQ}s+Zm>m|mi3 zUTBl{YO1a(sabp`pB5{KUM8$6 zD`F)oxC)FII%^?3>sMkcx>~EcrfaqKX{^?#w=yfaQfrd#|0=t#E4|`tyJ{=H3MB*@ zC$d@s9K@--&TDW&>%Hphv?}brHf+DLsi1x;-)MpF8EmOuti$GO!e%VPKCDp^tDr4w zd^W4a;_9Pvtj2ci#;$C(dhDA@Y`8`X#R}}0mMq5JY_KkD%CfA^_AJV}Y`}6Vr-p2z z#B8|Etj*%A(&p^JGA+;kEXS6it6nL=8tUI5ZOvwFt?ul}KCRaJtkZHW)Q;`3mhHbX zMb%C$$tG>Cb}g`iE!>K&+^VeC0`1n`?OhUW!G3MrE-l^uZQlm&+`etyc45o9E6gGd z)()=ULMhocF5nif-5T!Mx~kqL-ZHM+hHd0Z|1RiGuIOH_+O93aOMEZZ6HHt==py+d^*Wwl41G?(5R*=E|<(nl9g}Zrj@K=?bszvM%A?uFJZioSH1~ zHtq0ksPY!?@ft7n7A55#Zrn2O-V$&0N-y~C?(~jt&SLMXD&WKVZstC(y|(Z7y07Ds zulUL@@{aD-axbm&?)B=b_s(zpCa?8YZvfjb?BZ|f7O?)ZFaN$T{QB?zX0HPau(a}O z`1Y&%77gw0F9Snx1kL8#d@p= z|1JV6unuo<1B37b&oB@Nu?&AO>=Gaii0lEErtq2a@D8)^4;yX^bFdTp@DOXSao(`e z9x7BeF&C$B6ie|G_plg;F%?&_6)WZi6HNvWuosUp8Naa`$MF-xu@J*>9hY$$XE8{$ zf$nv28#A#S-*F%d@*oqk9n*!3q1Z zCqwcjV{#~qGAR?WDFbC}%8pWW@+SjwE6;HycVQxraw*61j0OY%A^8La6afDKEC2ui z06+mi0RRa907nT#70MIASO^cUQpHeKDN_y?CX7h2;w)AeFIv>NQQ}08898G8vl&%S+s?)%MG-+=bTCZA6E378me z0RqUNe-J9jpM(=qc;JNy5?G*x6*d^*hyR7>A%`96r(uF9lK7y5BDM(PiYk_PVvHEp zXrPHV638Nq&bhebh%^RizNjRSLdt05lNqLXB$P}piDZ>c_DE%s8{)X7 zlUQDv<(OZ3X(pFthH2%RYJ#a{ns7>qW}ITO+2)&W(kbVhd)|p>oqv*ZHE3(i6%j~q$Hf!v$(Xv)8 zi=T)hYp&LYi|x4CJ|^t9)QX!fy3VHRZn@`fd+ECDs(bCb&+Z3Hy!^`Ruf6T6dTqb| zdJFEn^(H%TnF8ZW7r|TU0&&6=0~~O}4!5!@#_*mt@yDPbTyVvva-8wP`HrkIy%wK* zu_h*WyzM{+ zdf}>*?)vL(J1%?Xt+Q@>?W^Oid+CZRKD*|?=YIR{!;|>A@V&!sALhRYUwp{VM^F6p z)E_T8^4Mo@yvE$04n6JFTYtUs;g65H^WUd0efemM&%XQGoBzK2>v_Na{P^cj|Ni=) zkN*7t>>vCD$Ug#-Pig^FUjhecKn6DOfCapq{S-*S2U_rg7=&O39~ZwpRPci!4B;F^ zI4KV*(1Z@$;O9ojLKV6YT@qs+*-&^w6t)l)H?*M#TgXBl`j8Dj?4c0@@;uFbWhACX(30TZx6_v;o9eVMLp3|WLWkkg?nsEwUK%*Md$i_9Q zaR@<(qa0;G$2#^v1u-}Q9!0>%J@)aBfDGgg?C8cqw(*cjph6-S$w)Xl@{u!`VbBskdg?cB&8}>IZ92ovXnUhWi0nN%UNo1j<>9$ zE_cbxIZ#oGaswmiGHC)$783=SM5Z!5=}J~Q^O?{*<|LcoNGwp(n%LZe7_!NQZhG^Z z;8a33i9pVBn)96KOy@M$$&OeI9~0`#0fJ~v?h&wKha0svh=KnF?z z0|=C$2SsQ?3u@4XHk6?ScxXgTDbb2DKm`V!r2#Y=ON*YeogfA2J_lM%2voqNC`~|0 z3tG~ZmSChWEkQR;U{7(5Q=8$;=}o|s7sT_ZNKGnCm%2|>>XWH0 zeX2qoI#rKSRg_oFDpsw^)r)r3sznX!G7$<@m!7n$Qq5;Sn`%^b_VlN3g{xeJN>@sX zRR``=>O7&^gtYqguUz=+1OmIwwGz~@X#J{U7t2_$I<~Ro+-589+SbaJHL)m-s#*Pd zRJZQ*nmdTAXy;nj(k4@{sAVc*^Lp8|+EkoI6|7cgOWVi)%GS2HZES8IN`!5uK@8J~ zrClux)Yg)8vu8yvTbJwE(KeT~rj0IZXHZzA64tr+yeu+rE7ibicc~CSY;DJD+t-SB zx9HVvLv8w7I&eZ49T;w1H7iZbW)`!|r6+Nr>)ifI_qlj&XD4+~-OlzFoWAw0Y6Xje z@^Ux1t6Rx!>M`Q?>Xs9Vt{H^xqOXnOe^c(n*#X7@{RE{ z2OMB)2AHMCH1AJS`(1smb-z#D?qM5@;RsXJ1+W!ok(aDnZ+3VPnBc@7CM#kf_gKOx zUandriBl|8_pRaNub5@bRSqH>k3oZTy{6uHX%aZq&%W}c4uzhpi$KTCY$s3H}|DSoql^;=kPF1n(1-ZNN{ zOzI@p^we+8@I;;b=l2SC&?62tghz^KEN6P1Ezb2gAq~kbn7PKxwDg*XU1&@%RnZq- zEqHNzXHj2N$ts>MGMmlhB6AqZbO3J1Ud>`yQ+n34&T_H0%-mk<`rPTnX-FxX=KSu} z(5=6O zitic3ofc=d#Xw|>t9sR5{YazgFlU2YGw{Y)p3N`@S(vdBMT`OqX*O*D~kzx_@11h?h0k z-v&BOk#6U35B{+Z zIG}+XxPcxB1RofJEx-aWK!PTCf+K)}Drf^E*n)W^NQrkx9cXwL_;4}!NhY^jk%f9H zWpbUyd;>^<7C3}NSc5cJghyz2@CSqgD0~4JgapWh1&DxDScMCCh5qM&TDXNt*o9um ze=`_{BDjJsNQNnRh995-X{d%4z=m$vhHw~%Z#ai^Xi9ciSQzGQWtM$W=y%+wfoNEU zfw+csIEaRrf$=AWP}qlzn1F!)H-*3Dh>wVcSxAYNSc#X2iCkDnUx@( z82AF5z;TR4cm9WbUZ9A6wt!+7i?V2hU08ij7<{*wi;Xymk|>D~7zdg7iNZLHv{;Nc zSc5f{7d|Qc=^Qe;(*^`<$mOhz;K>3nLnU-p~mTc*QVbFp~$&^c$ zl2Ztkvp1E<2!d95l~~D@!MBaF*9l9gdtI56Bk77$nTcXqmd}Wf8A*_BDU^mNnUUF+ zEf9Yxcn0~1icJYpOlX%<`CE!~mXOJUrg@rrnVPDJf34`3g&CRw2$2y9mWjETj7f|} zxSD3To4nbZYghx68JuhRlb0Ds*>;kkMwieBn*aEFtSAV+v;dl?ckPqMg37+1`3E{XezUQ6iS)S`zp0E&n?-`$3xd~5beNkqg2Z#wsq@Vcd z2~E^R0O~`ybDcMVoz{7vOpytipr8oapalvuq`;ss5Y9Hd>=PdJXgNqAkjzKpLb% zI;2Eeq(*wA`S7DkiVya{q)pnRPf8C_I;B)vrBEuRGHRte+74Tqqh6X0UwRJfFs5W$ zrsx2sW-1S9x~6OzrfQ0#a0*`{s+2LHnK*Eqo#~Xe7zfM=3-_?1KH8;jst$unsD+B9 zP`VF_ill%4%BYUIq)7^?lIo;dil~-)saKk*nu@8M+NoS>sAw9hqB^Q#TB^@rs;GLZ zs+y{-+6}Jys<0ZXvTCNcU`vShZ;#lVJd3dztFbIQs2t0zm@^OjV6PyXuSz?z{MxTBYp5w3wJRI6E&H-F z8?#rNwKW@pbo!P+u&e1sm+Y#dF8Z@DOSNbls7K4F$J(|@E3!-4v{>r2Pdl(u%MMdp zwN^{F(x9~q>$hdhr zEt|KVySJbVxHLPnQZTrLi>@fyt179u*aoPK>$stdx3YVzk1Dy9d%L)6xo|7DoQt)b z`?ZBNO1iyNhl3%e)_xsi*xZ=1PuE4;m{x3inJ+}pj_JG{mJ z`?sS@rl3%|FUWv5U<7zNmlc}^fQbn{`@G+Zz0zB~xvRVS%DdU?yZfuX{`-OtL{s-uPeX9JG*OZzY~1F6`a4*+P?yTAi%Vw2de4;-&Byt(yD!4{0cn)|^aoWs<>!5!SgIIP1WJicQpxF$@JhzJ8r zaC{fGUiem7>-vp7`@(M;!)J>PEK9p@OTqv8!$1te2kX6D?8QNRrfW*Xw`zh^uz^Z! zYwhd8QcT5gEWtE8o8EXg)3tu?C2xSY#c8^)ge$)Ws#c1p@v@K@>TSz{ZOt-HvK{Kq^Rt#ll>mAa{S zd%Ya2%hs&Pyu8P;?8^)r3!;px!i>5O)^MxKyvgj!BHYaSpw8@k!_B< z$@P5AuYAw?%*?-hzFj+hrJRQ}$H1@GUXgms{5-%TEYA$R&Dm_p^t{g!JP0w zagN_Oj_H}M>AarmoWAPR(C4Tg=)?}_#a`;Z4(!Rk?5*zVMh=%8DC^?vSd-M`^j+f` zFzI8!d249ty8i9V9_~Oa;KEMo$bRm}uI%Q{;BpP?9F2XCu7f?7cMZ_jaK7!h4i4bH z@4f!%zi#5?UgqhJ?!!Lt>(1`GuBMuh=yba1JB^1qJ!p{b=Jzh=dywP&{_p;N@B&Zp z2CwcMkMPb;zV3d;(k{?#H{TQQ056#Bw+@Hip7ApOzwx3S@C6U@AMf#}zVkTW^CCa; z7{Bmy`tB{ca?IAmwQd9zALW9e?=!y)8sG0yPxCiV^;SRbKCkmUZ|Wi6^+5mXr#YG? z5AUH@aglUV#K!V(Ug2qA@lOBtaG&r|fAw_l?>4XXcrWf*ulIHT^L`KZ(LRDS*ph0F zU&;$&Yk%Zz&-8Ks^p0=XcklI*pZ9y;>|BoV3U9N5KYuVvJ zy)gHY-}ih!`IZ0ruy6T)Z{~D;g%ewLM>k2s7U_wPe4~HyrhocSuliMA{H>q!$G`Qi zpZu{u`}zFro4@%pNnln@R-}h(l2vJHzx4J0PWr(w_o$!x!_WHU-~8rJ`RBj<1HRpY zp!r1{knw(X21r0IOvVG7yt~b4TgZ~{)+<0HY$t6#2ta!O| z#Lb^ek4$}M=FQhVZ|}(ybWx*8kCNWegbUc!>1~5w(tc~#E$3mUcd~vr?-jp=7e~Ig z|NrWg6VN&W3p8-L023?_yX+e5&ZY+=?1_xf!duF;(y~BJgsRjF!ZtQS4DmhlqI&Jb zvCzUTtvGOVD?bV6;txO?2eeT^=nkAQIvsoTkvYm36w)&8_Uo>T3P;0miVUeZ?>rBE zBV&p+)_9LZ6HO3nJ^Na0>n;6^+_6S7Z^SW2G=EeRNCeMxQ@bHKBhtYbB}7s*C1rC0 zjSbO@GKDVYsM1QVutaf)5me0o?Y$6zLfIyh zcrq*qC|FI(L05&3wnRs$Z&5EB9j8`sa}{^jN`oEuQ*+-u5JF;)HTJ<|&01F8QK7QX zLlt~u;|L6A9tTT4lSd&KvKmC&qefb}I(l)}80glg}o}rZ2V-Sa5u86b{YSZMgfD`*LIU);wyz z_2zqU&o>WUSkSe;nL?xP9d$IsQS)>0tVD)f@mRg691P6szI*q-JO8}*Hw72obizYr z;%koM(*a7>q9qG~1gM%F;I=g=`S#qi=bdxkNAJ6P(SI*JaPr4z8D8V*buw%v#B$ZO ztG=3E;N!2)mMt~Jr+fd*_4hvf{{_&003;yza#lX`q0eiFLk|%+(3&SjFn;r+9l=0^ zJ?&+%fE(nX06qBsy$*&@gu^3V>86lA26D)OV|!raS~!7#O^;+r>y70;sJjswFo!qf z;SP0J!UG<#aK~HS`b4m$<}JW^N|YW2t9QW`D9wf%q@oXj*Fz9)4~Gc^;>F%^Bou

A!ie1xXSu9k_YH~W=*0QRP|Lf3`DS(Oj$!y zs4n*Zg@`O?K=+ha4CEjeEfuU+#oAf3f_9RjHEChbdRoJ#wxOnKN+^{#9|%TJu2N}@ z$To09w^-JZKTT~pxXRbh0#>xcEv;#dYgmb%Z9MeC=ZRQ2RjSgK1OL>mR(T6t(h7C7 z!VT_ki96QghPNllOO?QMatL0;7HtVks2|QrcA~?axgWq%8 z``%^AcXo?>z^Lro8~kEe#-0VMh7VfY4|5i=7jo?oW^3Z*u@|<~#Q`g~1>;Q2Sjjg3 zt}&CHtlb=Q_Q0$??;y=XO!W#`$P1oqU0F;N+%O>xN6xH;eY@oS^7qM5{_c&P9AyKC z_?IJ2o(h3%8uv^n>=Vl8yduLMy_*5e9^|57(MURv#X+N z0WfpxOJx=`n-!htgwi7y&ag!-(lI2d zVmD2LByXeBYIbvA#RQBu07KcpFm;<(O=wq>BhJ%~GpwuJG3GIGy6>elz&bjEGXhkr zNAop)>@e(9r@Orz= zKi%+#zo8z^mN>*2ZQx93{9a&NFkDR;Ei8kxl;Z9+6$W1Lmb+Z(d4ajRWgc&vM_uX@ zzc{t+Ku@QVN?KeIy0*Q(>*nd2&r2x!uTie@q}N=R#BKV!XP$PoTm0r$XI!3ojM1;> z-Hcst-q_1-Zn(!i;p#>XvfKXjz^i-i)4n>bH~urndtC2(r@VCij_H!0ygMvUe6fYQ z@TR}v;Y*)->E(WU)K^{38W+#qr z{gfSFDxRk|_J-8E2U>g<%zNQrXWxA8buavQCxhXJe>>DoFL-FTdice^c;oSHxP13> z`LoYE(W5_b>T~}3;2(S~McwX4oFn+&r~mjD|3&jltn_0(0*pNnSR?$HBKey?>}x<~ zIEL@*FnI7k3M9V?M5{880oDU1pK%>7xUKc$yY@rCP5XlCTR;Z%y!sy zz;DSw4P-9YbB$wK8)Mo*9^AFIAiMWF0Fzq*6SO`fOhLkX!Txi=C2T<_WI-9EL5g~+ z*x9$7s;fkDtOEprC|HrDu%00-!XrdMGJHTYEW;;k!Zce!{!2suB}~FMguxi>Bq}Ya9@eMHWDmJ?z0g^g=Ya zpUkq1P$a}NM227-Mq)h0QA9;1Bt>OB7FBG@%Oi1#3MyLuVF~U3gni3HS!Q0BmHWEbtfCNW@EXIxeg@Q!LV=PC7 zG)RzCN0T%#hMYnp3OeX4;Pye)~f zrF<<-lz<-)6p4(4vbP=4)O0B$0pbSf~+{=>AOFyKSn|ubl#LJ=d$gKp+z63G8yv)kvO3oZfqeRNHbVpITwA>02Fqjg?yvJ;O z%nON3%8X6R+{`!>NxkgM&*V$-`^z}fg``BwEhrEFNjywJ!Gwv#Mw*-UhWJ}4M zOxcW1%w)sN#7!mKP3&ApDR4B>lq|Kx$NTED)$yk*NzM##O?~9YV31DsoXzQ^P56Az z`1DNrLK-fFa8u-EV#7gLFPwSM=u2j$kozL8SP`|v-(2TR+ ztf-|tib5JX*wMlrY@ZeKLX&&V=0t+$JWvx&&<15t6|GMer4tlwQ5cO-36;^%^i4%F zFY@p?psO`SLM9XtfanQQ%L7qz1A-FV1q4mf6kSpWT~P>q(HM==8Kuz)%}?+2xodf! zXZgSYAbYC*eUTrPmYM{(G6lkY zBvHc>)Il9pLOoPNHC0qiR8@6UIPJ{}-MK!gQ^{k#7s;&@(Ns!UuMqsb?1 zLC)6v)29`}zeQWY1zfkq+Ekre&;8uRwKvDrJ9MQyG0UXEtOMJyBc_l;fjp~1=}O%x5ZV0I-~DY~4F1T1 z1K{oQUja_sjVxdh9^s=zU<*OMNnIiwTZM@+MzG4gh;4S9i_61=w9^yDNW8_od zft@ZapeZhB6(^1bB&p#!MxHvh<2z=F*)>Dx{bC^w<39G|Y5PDu3FKr|+6UfG9k>HQ zSOYZ>-aZ8sT?|CFgycvr$UIhGS>9t>=HpD(936b8ZVHCGkJ)dI-W`^^AWI>7tV zNmsVkN!H_9zGO_k2sV-1*KdERDv z&SHD6%y8c4hpuP;2*qVF{%6)P%w=t$5@_ROfMR+A(h11MgziJ>o9BjBQ;HVgWnSrt zcIkeOX>-8nrU=^vnBS^s14%#yOOVBVa;)MlX#%~-R_11herbCBW`R`dW^U?nmT5=b zu`{N~uz7)ZE=-%9OSXtHqlRgFZfPI(Vq#=!iT3I1>EN{K9i56UrQu9fXY)KE zA)GKk>uRTd>b%}-`)nw`-lZgqiCe&iJvgviDC|AR1W-_Hp;2p%rrG(pf+@l3$ELLq zeE|yoLF2${qD(T+{_IQ$ZP6a>(Jq5bFzwS$ZPQ-uGH7kre(f)C?Hq{h*q&|Men#BJ z?c3gM-j)LY-~Mgj4sPKdZr!Hs-LCE8KJFnfgFtC+=N2TKk{s#2-cGJ=>ptfYgzk2p zgf|Xp`I_9!wp`*K@8m{q^UiJa?rq>63ifXA9H3$Nj&J#%Z)dG17a035Q{|;~i$8Y=|7-+mk2N%q&gK)c(@bRSZ6JCMLPrauXukjnF@hGSAD!=k6w{k7V@-3J08XvnL4|6de^DxI= zL0&%pq50R@)`Ke`0L7U~w!q&6^m00{^DMt}JkN73?{Yrhb3gy{6$f)NA9O;$ZoHK| za6^)V?owe6(o~Lu8z6B&pL9yEbU?RrOg|V+uk%g+a!w~^Q15hB<}-jQb*p?FPqs2u zUv;Etbyt7&7LIj3k@Z85y-AI46)>Y_1@DqZ7405OU=Q{=6Hj8daAV&XYdm&)nnLPU zc42?^XZIIrcaOA*cI_rjYUiKwHLjO;;#zZ7A15MBb ztlM=*FM-;-5MOVd=e~D+Ps@Dochd0pBi{CL*Y;-T_8wPVt0wqvPn&9V9ES(@XJ4iN zKs@zgKX^_*cxyCfPj7gH&v;Sq_(8wh^wYTnPK^<00S4b?Y?etYcz0(2ad`*2%QJNt zV0I%lbx;TSVBgd&B=s?`)%$k#Z2tIUXAg;=mGG>2r6=}KNBRtK_*sPXjvw`3BRU&k z4@{@{VJ`x!{B*5{_@ifCbHa8!A$hYu`*cU)Gq-6aZ{vSejh5VX61coM=WnH_c&I=1 zrYGCG&-S?qL_4}8C;^Q*UN#ZUac|MDO3_@Z}s$2a`QcY4UL{K@xw%>Mwv zkNl~h98^bPg};=Tk?Ycb6Rag*SK}1N(}1_JYrS-N*R<-M9UN zr+wd_{jCrF(I2UofB(FPf5xu?`k(*$w}1RUf8^KkFb4=0EttS?GLWE? zgbEi%VMv8wnj{n|6o63iB1Vf57I5T9WWx*-E;0&8P;rL_2NqVcR4KvbO9e4u(u|qn z!_AyJcNQ$-6Ua}XLVpY`$`dEjqaXzKYi{9MrOn6~ zAWN?N_%i0qOOtL^nl&t6|t zeBUKc8cv~ae|Q)##~129?%#MsZ2SkHfCR>o1|3u+w;+QXO~8PI%`H|zW7*Y&U1Tgo zXCa0&b+{pjA%6H$PT|!CAA7HH<{pb+x!7J?_sK{`eKgv5297xvxFdo+9_R*-1_lY_ zk3;fUWP=h0u;c~?IH_bwBVG2>V>yvX8J06W@qvgVUdUmWnT0w3CYfJ`+1ZO}!e*Xc zE3Wv}jd2bqr;c?RDPSL%5GmxHed76JC4wb6Q3ooJvBxc(>|q9z6FTagV-I9d)B!AF zK%12kpw$pjV0!53r(yz7=BT8a3aY4NvbmaQDSCDv7<9@S=NduQdZeBN+PUk1x8AzR zAb~1KC`ny_q6;UmTNA@708tsHFR7B#JvQZ)r%fNfwAQ=%pKc@Y`H})23uMrA+kN^T3K!kmZr!@x!g>9B#sVrNUJOCfer8KVG!jag5l|~O}rDJ44Q}{;TeyL z#k(RCv8Y8WwvKmCGzzySlR*&Dpp0fbBN@?{#x*|f4I0cId{2ZQ{7$HgYjmLu_$Wge6!pRo6ml<%Y-G1g zNT>mTvXqOA9b|mhH>B~cm9Bi{DpP67Mk;duN=JCu3UawPHR^JgzWk*wc`3{fvhjjv ztQ|@~qD&Z~Fog#DBQ!xV1sFxS%1ra4JjOIpHmV)ay|Jo6cX%$&G?)%8CL6^V z(T94_jLnLo=SmpPfbvm=u;>H}w^Yq1Sre2)07(*dNCABk=1~nW0XdP=QbE#`o=H^c zMny@_#ViS-K3yqG1xTTv0<~HvJt{_#Ish5^bfCylDIQaL)TaJYC1krQ*&13=tok6A zToq~5c{vCT*|j|a^oyrcq$cje@yI>oMak*nF^BKHE{E#GYQsH{Ax z)sJh4PH>?#*TQ5Mw$h4eLLfN?{>C@4u>D9}XLwSZ_MoVRP3}H%z+CrX0lGt2Fk87P z+x|8ozMq6Jh+WHGx>Ak|UNJHM64s!){!-|!=zZ}zFN^{N7k9SCs_#d+u|mhx)UD3V zZy}q*1v>aBrg;r+b(c6?84{t%G;T731?pktIx)uGb*Cej5aA6!*g1VAz?CY>&3gZyIR!d1F^bq*VFY&Qw_*vAke@SrLip^NB%$c8?&U89TS zYaQ5z2#9la8=_ti2cffXrcj*UJJm*Zc+w*2GMT3wYATa=lea7NOlv&mJ8M|gP5QEj zt@Wp0ovy36b*pkj;$#;z*+IKzj!9$431C2jtYNmemS%ETN`;m9*=9 z-&MhOv?H(sOXStx}@s{rlVG^{P)#+tYNIw18ANTax(C#*08; z`o*;y;gU1G>EYHM)WHX*mQ(!>XoES{*IK#JwEce$9R%0zxh>*oH`kn-K9|cz6^Ld~25ufmFp!jK^3aX$Mx*!*BVHdXG7iOUt(jXa@p$*m`BA}rT zHUb;A;TkSNB=8^{%ApVTpdAX~5awYKy3rd{f>6weOt67KNYoQbpdnTvB68mwU?C%> zpcg)37(${XMxq%`VkN5KC8i-J&Y>H=;SPRc9D?EjZA}!(~IO-w_g5fS6<1qf>IhLa_-k>^~<2pj)GCm_6lA=7u zBR!t~B0d6PHBJ;TeM^XV<2Qz5IAY*9`eHkxBSNNQFutQgHl#8l<2*(rJWAv}+MoXA zV>J>XD`F!)VF^G&p+GvKI2PnVmZU>2Bubtn4l<-Wwxm5yBt=%FOj@KIQX@v*BTu`p5|LhCS2QS@@7!}<`|kLamJ-`E~j^XCwNZgcsgfn z<|TC6rh3vQ98hO<@{LadC3eoIcGf3%if4IhB68wqev+quo+p5&XLMrZbo%9c>Skcl zr+zA^eJ198c4vPIXN3M|fL7>){vcissD`d5ZXPItYAAz-XoEhegoY=7n&^I>sESgk zdA4Ya=HrGIrapG)fyO6je&vFWCRyqzgrcZZ`sj+LXpmkgkrt_JKIb=b=p%6d1C73C z6(p#F0;yq&W?CBQkd~;BTB(-0sD-vBfd*vm73gNd1QdYijY=txE@hhLD3-QqkGkob zUg?&yD3|VLjDo2xx@V0>DVnaSn)+#+!l|GR>YU1HogONZ-l?MUo}M}>pE@9+I_NT$s9@wVrFN)~OQK64lr$x4x^l{%N?DYq!$>YqU-)zEZ21 z9;=4}CcHK)z}joQ-mAdotHBa0>N)4S_N%-0s)?DS#pbH1K5NKU?7?2_$)YS_cC5#ONW;$O!$NEz$g9DYtjW&n#HOsi?jFY)sH4WL z!xn7BO03Ni?adyo&JHWk%4h?!EKPuFiY+bE3a!Y3Ytj}i(q66BN^Jx7EM>GTXU?qH z1})Z(Y|V;m)^4rJ;vj}{snde(%aSe4%B+h$DE{;S|tZQ~Ly{iJuv+e({w?edt?asP>~`+%qVDVFZsnG)w#M!4rY`WJE%Nd% z&gyRO`flLHrtN+r8VE1cYF*|=uh(&|_9`#)`X}9TZ}K*;_=Yd^#;h$&FRg;@r=ISF zZZG&AFZ^~d`7W={es84eEA_4~_R6X8s&4%Lul&w001q(zjxPaIE;OR=*QzgxSnvKm zFa*2r`yTKFQ*Z?fFyLaa1tV(yP8RzXuLNK41_!VR7qA99ZwVu?!glb-5>W)dFbs#V z{rWEnkMIfKunn6n{-*DN#;^~|Fc7D34hu06n=lFwu~f1v@OH)A_5N`G(r^(w@eCs| z6-O}+m#84}Fyef06Tk1~S}_=hu^3lzkp=_+A^8La6afDKEC2ui06+mi0RRa908I%T zh00T?f~sO6Ovq3cL4~UzvXV&4;jD`mC%V#@u_DHf5jlb!Nm8WAk_{=UT*)%kLYFB& zwtP7grp=h7a_Ss-kmt>xK7*bb3RI}kqcoE$UCOj))2Bso*7S(-q}8cct7Z);bt_h= zTAg+sOO~kFsA!R(`+JL#(yNF6FD1S)_VdNj zmw(Q^yZiR<*^^gKf4zO``12ct4LSQ}|Np@kAAs@|INp7h@u%N{2{w43fdmQ|;Dp{q z_~3;YB50n53@Yegh7}HIp@<`b2w{CFrl(?uCWh!@i6nxTVv93!xZ#Q(ewgEZFUm-v zgg?euLG-@ll1e7&qmxGNC?$?GLdj&1Jz81imQi-8 z=%IT?im0QB-nr;&nl!2@rG^5D<)>~=%ITwcR(k2CqlQYVsH}QwDx|NL|9azOijJr1 zs<676>aMxU>MO6EN(yYR#QqxSu)zwOEV8j0yW_ITM$7E9&06ZLt)NP)Ewt2DTPug$ zwz{pj-Fj>7wW-~zYqsRJ8*Z`R&U)&+{k`ihyttN&ZoH`W%Wq=pt~>6%1LJFMrvB1; zF1Yp*EQ-Sg=X)@{P$m3tlCT5}F}ob=yRX6@gG_OL4R74B#1Xf<@W>{L%<`QY$DH!R zpqRYz%E!KZvBeCQEbz%dlRWdpMC;XCu__~V6V{&nJ%bB;IIe_t*-+?to(x#^pqt~%(Iv!1hW zjgO8x=8(_s`scT&F8RK(_bz+ftJ@yD@Z#>Cd+4~|I63XQ4}ZMz$VV?Y@zndxv-8j= zpZ)Vf`~Exg)o;(e_|0Qa{`BE@o&Ngew{QOY@W)R!`ru!m_4)GSKmPo{v+uwA01Ti2 z2WY?j8SsDztW^L0$3Ox)@PP${p#CNpxcaH@f*6!w1U0z91ak0$Ak3fQG8jSam4V;Y|*2tGD(iGb8yAcrWQAmH(khD@X)moS7yIx++sP(dU`FiA*Sk^-2# zWCb!=$xeFmj)^2f5=1%5CX~{OSTLn3S9!`-HjGFeT{%VtUd5!^~wekJ$lc79aq&D-Y$8Mt0kAn>75@kuw zVurGuwG8G3mbuPNy0e!Ncqa#TsY@=*z@GTbCl|IL{|0{U^Pd34LO%maP*%e7paw-K z5yW6lbUO5*D2S*;CrVFs&ajShXtMOB zFfBk#V+ze@!nCF@1tvsy>d<525&}RC>Q9AA)S`xzq9vfHD^tnTraIN2Pu+q-r^?f+ zS{0*MMQTU4%GIux)C5XEYD-Zt)0Ud`tY~#61AL0ovbr^<^t@_ACmPRo+O@4loo8O} z+SQ3tb*X?2>|X~<*tyCztB9RISikC3#d>tB1088yCrgC2S~jhi&Fot@%h}B?R1DK7 zgJTU#TGH}$qj=q`YDb&YdRA5kgiWYy@hRBC|5EjqIz4P+`^s3}GB&KhmF#4Pds*TZ zSF^|MY;sep+?NiN4xE^VF-uF`U7l8+dR^^mwMyEk&i1=cwQVi!dDRNYGO`%eXA_>A z1cvUmx5f3Xbx|tIlb3vWTUhw{S|H811vm(QsW;xTj&Uj9;hn2uyF#vU-^c``Q zsjT2Z8(O9U_$-&ZypcIj;Gkl@?PD98*924=&5_RNdHLL9RzmpAa9(9kA020x+PMUl zt}alytLLtgm$ndyXpSjM=*;G~u!nB$mZ7`oJ&-!aqMUS@&Ai}Y_tl|pPPS}Ay=mR% zxx9DerU9Tn2NH4{YojTbkH*&9RO_t?a1+)Y+_tw!Ep` z+gVqe*4Wmpq>;&O6?^vEybiauft_h3n>p1Cj<>tnE#Xe5lf%o7c%~sua1^*#*ccb- zwDBs+e4ALfkqxUD=-}QZU)$er|NU>kx!g?M+I!b z=d%9!%az>mdq=t3X`OVWgKKn;rxf5LZ*dn|4sT3f8Udy*_sxL~^l`)c)L`F%MaQo7 z3V(FkL&t7j`)y-Hr}gDnkGnh5j`W;DUFvsMbD#Gf1_1Y+=coL;;0I6m4g3A@hX;3; z7r*kxJ6^+(k38ZFPx()~QqEq_eAvPMb+bPkt7i}K$y@LM04PB2E_J$Fam)phyIuxV zpDU*q--+7W{`R_d)m1k3RIJFZ}9T zU-H<`ei|lU`R;q+`=|uJ|3AI&{lJRd3FuG1Ezddcd(WN@)15erDZhO#+CK|XFai(s z&;S1K9|tV}fB`sw0$6}R@P7vQe^dZ|*Vlm7cYhE_c=$Jg6u5vEc!Bwsf$GNu9Jqi< z6h{09f*go|B$$9E$N~hIf+z?BE7*b`paC%Wf-)F`G+2W(c!ML5gF5JfD%gW1cz!_0 zNJ2P-KPd^>>M2c!**sfow>AYxs$87>c4eidR^Qa`=O& zNQbAmigDt#` zi=Nntpy-Uy7>%$vjnvqRtf-2tSdBvnd06IPg6DdJ@Cv*jj^tR5x)+RQD2=4}cU@+V z0|1_W=vY0w36oxJY=# zH;&T>U>vxJIOqV8h>?=mhm?qZytjPw7?SpAdk2}1515W9`HbpljkBnTcW99u36n4B zlBo!XWeAbN|Cf;X$B=^PkQtT*0k(a|$9R3$lBsBi*rIOlau(BVCj`3zyx8SlNsm+CzpiP^^HOaj}r-gNr{w4iH%G7kx+S; zeHWEeIeY|}iC3wTSP6?&xRr$2m4%x7e`9%;X?cr;cw&h+l&(jSt5}CN zAeWjsmvyO^ci9PX0FXI=((OLz@F{-o?_XN0r-9M37NV%aZSdKZ7Gh!I1Tfl3jnGP zjU%AeFre&Epaz!24-qP%{&}GsilH95p&%NfBI=*8 zK%yqIgz= zq>O;1dw`^lU`1O=pSQVvk!fGMd5Ol?2fQ$$`d|pvOK&I_rrUaU% zW4fjg$`4xlrf?dkayqAUx~2J0r~BZccB-d*+NNCkpMM&tf;y;#3Z{Z;sPurSii)O; z|Jtb7unyD!sgmlbk4mYNYN?lssg$Y%w z>ZM%jrg%E5{Xna-+NZd>r-bUKhnlFq`m4Yiti39%#9FMcdaSKlpy{xz%6bmW`V7wc ztkN2-)QY3hP_5XSt=+J#ncA(Knu1VDpA>nYe+7~JNqkz0sI0oK$?B`IYOA~ouemy} z^ct^)8mz|(sKaWk{EDpp+OE?YumY>C1lz6LYOo0Fs8aBxRPdNgfSYJZuHD6!OnI)f zm#SOJt{N+_?z*S)>JRhqv3Yu{9$T{Zx~u!ztZJ&S{i>=h3$Q9Xurgb)G<&c%|9i6u zo2g#_t_|A);u^7}TB`dwjHwEy8#}WC%d#R1p|@ZUce=DATdyaZvM`&kEc>!pi?3B1 zvq%fA*BT5pi?d;yvy!^A1NeqKX^U_rv=#dbM4Pcj8@EThsR#P5OADZF3bj!ywY*BT zGMlx48@OE?sa!j_V=J~~JE>N{u$8z3Y3q|trgv;{wj>z`Sqiti|I52DtGOTh zpPqZX$Q!=UE4s_uyw3Z+&r7}X+pVsvh*3bFu{*nCkiFWwmY>PJx(B}ME5B<>pyYeL z2;8^HyS{(hzVPe53>?4n%ewSyz1VAR{TsQPHo3Tau>vf<8*8@-e7*}T!43StD4fC( zJi#p7x@O0iHo3nT>|9EgrJKYmY{b*s z!mL}r3|W?q+XA$ka<)sk-mAkqjKp1PygyvLV4S{6yuwUe#;1G6W}LKROvXp-#AIu> z_In0r+XFUG#kPi@l3STL?50D!#f*BuUmV7AI<;TQ#%b)vE4;>S|BT2@oVZZ@sY+Rv zGt5%KHnI8%z#iPki_F8m>&1c0x0=kzEUd^&EXs|1$}Swokeq*Z>~^lqZ+M)?LmR+? z>c^aX%lYcbpS-svY{-ZF%fbN6rR>X_Y|3yP#dC}Ut^CS}7R$0c%b6U?w%o;=OTx(8 zrqYbY!o1Dg{L97M&8Cd7kDLX`yv)nBy|+utwLHzN>cb>^4G;>?%sbDDO3zvg&R~nr z`K-^jtj($$&J^6J3wyS*>##gv&dG(&SggC*9MHIY%YQ1*5GVRg9|6J2Hebe9U&pNHqJ8jG~ zX^mS@Z2VQyayGfX*w8H9(;Yq2OHI=_-PBO6)TlhwAYGp#Ey-7QRim_lT?U*+018H% z)K1ORQC-$%P1ZRr)hu1r9eCA3jaG7H&<8yT;OeDfJ=JQR*JrKQW6je~z1MsV*a^$l z;mUy{t!)N&n`6n8co+!lir0Z1*=YUOf~^dGec8J(43@pw(LmXi9n+Ex*^0}vZf$f3 zwNDi_mO9AT4WNU1z`NNX+MPYyv>n0_O0FF&E5O#-QB(0r_A5}4dD5W-vB+_RakOaExAnQ-m2Z& zW8mD+{o3?R-2y(|1Ww@t9^e(8;T(S9(mmLuJ%s>f+-6l?@9p3a4ukVu-YnkYE{@^% zZQ&h0u<4o@1``yAij^k3^ zNPUTY`$KkIwvOw% z{^_1B=)TVDoet`u9_v<4*jKGe?x*0UH0eeD;I0k^u?-HyKI^+4?bLqjz&`B3zU|!J z?cV-s&3p>efE(%aH5Ue(l*V?BHJS_x|hnuI+p+52R2{ zjGluXNbV(&+=bTV?{;PD>zOZdjk>E^0GeaA7y}MIX>DZ}cwz^iFT{G%xmH5B5ke z-8)}^pX!G0HA<*<^+R6r4_oP`*N9i(^+%ugct7?@Z}xuw_hk>-IX|gVzxEzkfoRrI z&aCbY!0JT*xzrR;WA{HF)z<_{3r1P-i5tYEKY1P>-O*sz#Gh!K@R>o(D1G>jRCZLG-A zV@GKkMN%Xg^5aR6C{3m;$#SI1mM>+(q#4sCHJsRV?p(*S=TDzN(Fr9guc*8|Qg-l& zfg=hPrc9t_+2W*0k_uTPOn^Y-MAH^wTg)KqB1u`aLrOHz;A8|_Ds+q}aKP3|O@()bEsj$Hx(l;e35bDz>Njx3`)83H_JzjYTlf5Y164zr&hhn#RS)_ zIy$tS(xo+8F?6^np{=8Y|82M};L5FQ0mI$H;DY-lUK}uA1&Tt-d&% z$nUqm{F8CG0BaQRMg((QX-DgNWKc&Sb(C<(?Ixtq!WdqYYBcw7D-T5UMl35fubPa{ zE#I7I@kJ@0_pT30QUY z)j$WO)zDmU4OUk&hwWBaU?+{#+i#0K7E@%kMT*m9S?~@Gs!S9k)>HYMV_IsjwAMvi z59L9_PlUuf9|;#&LZ`LQfhuI-raC3K3c@QY8blo8tVgul^iO+f$O89L;JxpS z(1b%|U=fdkLL#p4fl6#4=_J@LPo-`PZg5=0)WQJ^?EpK$ue9}~>-qa_qG&&X--U%!q4X8iqFbJfk zGj=KsgAWz@Qfg8(mQsajOl{gwoMyGBVEU<@;x)xMnkp*;z2H&L$(6LN^PsMDCsZN& zR;pHYs#c}zChw`$t@6|uT@9*J#xR5_0v4nu#ZA~?H!XGg4_a+?D_k8b*Cs-?|E`k7 zYh{bUS7DO!tQ|6GQWuBAi>VcowyMihUD{a6&UKoK%IQ^Ao7$aPHm~NmY$&CY$g?dsn@ZgN14VOJCop+ue@zsQ0O(Vn<8d z9$lLP0#eyS>L8DPo?s%r_$gXTbVYz<0UUv z&&%A49=N^ijc$W|YFkkm7IFBE<5@wwIN^>8z@=^2fC>Cz1FKiL=vDA(%`0NrW|g|Z zWLAX9(;+zW_P6gz4PrBl20I4$h#pRFh)axM68|{G4wf#FxBA}d_LVB3{~cRrnIab& zUk}6m)v(Ai@C76B7{opfvXFs1WSN1Po9@}DTf);b`=*AhIv{IT_7`QQ=D3+y{?|6N zyydu_*u-7lbB~W9=8LWvsJK1zeqn4@?Lrx?hE8dmtxV@T`3~}~mi7y1x4PETrZ%mseQiMFGZRP#^{7y%%ms*bx1CTl2k(b5 zPp@*x%Wig^qb==9XWMwUwl%itooij^JG$HMwN;kI+p5S;+*V9N|5#0F><^@S&|J#0 zyWwqbO+&oidH(ag@om|&#W!ay26ng)?r~&_JK6`B6(>toj99G@xAkPZcea;3Ju>0#2+I!!} zZg!-r9q?xlBHQ)7?~K14?sJzr2klOBwgKH5ey9A(-;E5Tr(N)Y=X}KpU*wAy-pFqM z6vqqhp2pK%^{P~n-SMvX%8T9d&4_*GDcu zUir(nz3!cl`?)d0^U|OG^C2F3&>z01zW;rR0?x?c8{p%Quglhx&-~f{-TJ%NeD15i ze)uz;$lQ-QZzW9p0t}!0$G0*PCXfEG13>k2zxEq30*pWH%f1A}6{kzQuBpGT0k>cy zi}qkV6!@p&doQH<9|@=nuM|B15KOwO z%R<>%B^Y2PB+#QD%&RaI!2#qwGZe!kG`u8CK>FJm|6tRf>48Hmu)^@09ToVPB{4Y< z^g-Dh!!kTTA>cbiF zLo)nC>H|el97IwC#Ih^JP)x%@TtJ>`z!%gw9@2^#cow^JLNXu&I>bUP%)cPOLwE@w zPYlLD97a>5wv$LjRb0h2WW@$s!dHyJDMOVIh>|(zlU&?IO5{Q>1Vdr$#@jo^*CH7( zLPc^cM_D_>&_PFLY{o@I9f0dN1kk!$OpiUdMqSLtvf0J}#4sLQFmOTawB%nZz3;0C_@Ot0ijf*eh-Ovl1BOnpO4Ukf$HG`V?^)RWu_oA|^bUA(Zl ztVP_U0dDM0)zA#R$>cmx5@pT>{VK7!L2`PPd!!#xyU*;*O$?pU-Xzdg(8}@r$v-Sn z1YOS)bo&-&9*iewt zj4lOD%OLYcJ{Xa?#Q9sSn9{p5ZQ;IHK ziZo-)xIB#`eE_ha8qV|R&vGA{fyRW z?NnS1%WBP4eZ$u01XiA_!E1y|CEbWeg)WorBz!+Smc>fk5$)pwbp!N*w$gRhs{L{Jg0w} z(1vsuM`eM0)x(R$*o+O?jUBIZEm(owNI(2jpdDF~U09=Z9#{<0l!aIeWIff9$49i- zX0_Ryg;tJD*MbdNtQA^?b+&6TSyoNjrVLw^Y+9KuIjEglw6)2dty-Ry)~$WpK;;Bc z?KY(C)`*ozhfD#jGh4aI}?QK~at|Hf0@)n#4F_1oAz-p$3`+)Z9jU0mKp!#c^<9Y{-p1JhZ|3dt4T z)g@bh6Bz~gK;#YIO}$-XAm8PkU6Fm>-PPKDD?LSGO!n|BE^ZU{t%`yUSmeatNCg-+&chgC$_}ZC(TZ zU<(l{W>MhM7=-VmgG(RXaa8@^%myumLk~6k{boV|KT}~3$$f;qtc9d_=0t^JMBc|YerJU4&v;&FRla6!reugl zO;Y(c|5(;%I$~KZ2xMyjgZC2B)E!>B6lHck=!E7}hrVWRbYy~zXL^3CSCuk zLSe0*D|ldm%U+Nk=*qORc6R5IezlaI=a$CZh_+{QV523wRyrdpzI=o}tquJ+)UM#ml=>zE!irl#phyd5-l<}qE=rK(M4!ho!%(Zu!YKO1U2 znjF0@-(%S8zRm=|eu%&xY{D*V!4?d|PHe?Sh{fh>!j|X9wv21o1G1iM%ckr-$b?(a zY|X~(&fWvi&g{?zZ95q4GU$TSR%$O^Jh0to_*j8kYz2*qV2QO&X&%?eUTodo?E)%n z|KHApG5~JjE(7BJg5o}I<38>jP*6{9Zs&IH4p{5xo^CW20_uiD8n|v+#_sLz?iJu} z4DfF7{_gP>Z}K*8^pPgMqQa0fSV2!C)1e~}2Ua0{pK1jq0T&+rZJa1ZBj5Fc=w1OoYv zx|1|@ml$DAP;gOA94aGaw9)- zAJ=dtPjV(l@+M#MC68G`ZE-5Eaw}JHHw38g1JhNwgK~Bzb9O;1a%(j;8gX^h|0oyn zCtq_Yr&%X|b2g82IiK?o-=ZsTF+88IJ8$tUw`ejo1Jb^35{O{RL{h8eD$?8-RgQ=k*)-^&bHCU>9~`Cvjsxc2+8OWS?(kZ}w$(c4tp< zuIZ<*A|`8}x@*sNY_ImIyJV@09i`<(Wx{Dramq@Z0z+5U$5eOMVE1*07k6)WRgrgk z_f>mGN&32XeAjn=@ArM@8fgi5eGm9!M@nh0%}XElXJ2@PS9phi_=ab8|Haevgr9Ot z52v$!lGfe|A((@6?hs;aRhBgL7##R!uXkTP`F{8HUI%!We|cMnJeZ%4SWgv!C-@go za-qR_ghz7`-*qHA_$7~cp?`EQ6*(Z9Thj`lu6}CtH|A=pS-v9mFNA`cW zd=VyoZ$JJQ$9$~6f=BH7&z~K#xclIre&Kg~ofe6s|Ne>>fATMX z^T++(2m0Xe{t`!i>za7?XX-5Q^YE(@>kWYC|NL2ub+HEs2?7Ta#Gt`L3Kl2nECCJsvqb<%=|5s5{O#%Ma?C{(6SrbeM!)#}wLSSvs%(DmzruwutD@M;$9SpgJG zsBrMW0^EXf=hAgscP@yF6Zi7vxY6(5z<_`5ENu8NVw;3<|1NIqI4R_)l8Zh?I{ETa ztXZjWc7ug0S+t=?CrEpit%A34SKnpb`t^a-u<r-?C=IUifGM-quik$U?1V9sqMiySmafG|l&7)A3L_Y`+G=N=X4si0uf6ihXM?@&iYKuO z21>`E+l7E=qRlS)$sTi90VTDDHUO7G{I!}Xw(^mfD!Ac_yXv^*jtMTSuIk61u2#5f zYZdUyJMWHgvBx2k2rm>>AYn4fC8zvuyjsxv|8l;X9=s|Yb zYfOH2o)O-S0Oy=5sIc3|1`0RabQ^3a7F!^w8SBL*=Q`{5X=HbQ0~m|9g9Xy%jh_$~c4Im0^GavfkHigqgDtzL5q4=IcF&Vzl zDK6|BXq+&HF)&302f0O}axf20NZ}ep^kI_@H-gP+ssXyIp$n5J#Tyw;iB_!QjIemc z!E7;#-lCXlAi}*BB<+JVAmbU)m_{|SagA=&VE5uEg_UXTU9eIe{EB7<6iO&`r0W6; zt0kova^+ek%3=%;>BK@3ppi3lqJv6+!#ybB3n10_baDWW zH03E#2_nJ;u9d4qW^0b&QiCIjA1`~q`b>l;`cE^rAM~w@-YYk_0eBDNeIl(*Nn8 z0-IB%GDpeInG#`^@?=#}eM(S&#&bO1f?#m8HlJZ`ihGb-K~;0|OY}hWp}uq%5h{w& zmNs$=XPmXV~C!lFSGaA$zHuXd`P+)!Bg(Z>Dz*Yu%Akruj*;o2v!eBU zo2VvUl{Ht%(spN(O%M3N?Zrz~k#hQllpr{2Zc|DX^xH>WjtTi7uqPdAPsHV4W<*jWs zbe2I7$T<)~XLVhg1RFN?eS=-EDYZM@;u^I(1yyPjC~MHh1~;zFE#Y4m9AE}(n8IXG zSrHs{SrA?!s5OPD{}NoV-02!NxwuuaIyYRx2({L=2C;9XT1?%7;a3d&r2~+E%!MK2 z_rUWBWfPJ*Q}RlLVjSl1F+Ds~9Vb`?D@g}_kMNT@mtxu zSTC2=r`Y@_dzZk5w7M1(acOIgColtavSp-()w2sYz!o%{V916(bc*d-+zxquPtH) zb5fgm?P>oW|9xH=FxJrxo~L=yJyA2rY??mk)~yX-x4q^%sev|WuK_&hO(UDj5>B;= zL2X=1FZfY`?I)N8yliTBIRwQ;<>?|Wadl7J;8xT&nKN$Y`;B>Qmb;uD<{$`>lf2#0 zf)>gp=v0AMyyg9VdBX=jaZk_uyyo?e%U3L7itoJHzZrGSJIwRGUg~;AKl;lk*In>N zs*mFnie_K?t?8J?*t<``hDw z22fPdlNaBn)3c}dq&F^tDZwtr1Apzo=K$$_H$2m|wQ-JLVd;@`e0LA9C4^c`PFCI8 z<#knu|IKsW4pF}Y9^@c;(Eow-c{n{EQV;g1e;oC$SH0uhzx`?dc|+ho|A6TK|Bc-u4B+h%U?UWu z?j<1aah)v)p92b?@kQYBO(60yO7C36r0@y#Vc#PpUk7R*2zs9gh9CEl;0VTF{h^== zrXUN_-|M{~494IJ%Ao(bU=7}2=>;JFDFWF6AOh|n5C)+F4j~b4T>~Cr@JXN&Djx+V z{{shA!e-b2W*tC)Fa%b;!5eU(2X0{qw!!vw9~PdV7?$7+wxAi>AP%CT8m?g)wxJuo zp%3by5cc34{-6Lt0v_7o9TwpNBH{2^!x9E!6J|{nW))OffJgOG7HT0Edf*p6;uuDv zB+}p+vSB4&A{=I-CB~s99)cWxA{~O_9vUDi@}V9YVITHh0|KHTwj%On%^P&lnv_V@ zEaC??Vi%GjBu*kPcA_t0Vjccs4hkbD5@Rrq;wdJhDI%jXreg6upeu5rRPhqVqyU-N zA|vMFE_z`vieMQ6qcMi#F^VHOj-xs9U^Av8Gaeu+4k0wYV>C|VyH#V-pa&GV|D6_U zqb{;tH|`@iZsIToq(By9IToZrp5rpAV?r*ZA2#F>MkDdXq8hnUTwr59>f_pNqd#t> z{88dT668mMq(~xUD3;5YE+HJ046B3-5`-N-Vq`{gUrz2MNA@H@ z4kSsIB-)`PQ5I!Lnxs&w*N@X)rrCe5JL>kjfwg_J4Wo&4rO@`o2x*b>sW>}8o zP^x8G8YVj?CSjhXD7K|zJ|S>Agsh>Vzk6Njqt|)bGDVH9^oZ4xfF6y2tW1gldltSvC^681PX`ohV zp_T`iW@wQn>Z5)rYno}Jo@t~~>ZnSprKakj`eBx~K!9$lU399af+?t;s;QPLs@|%u z?k5tis!e7p$H?leGAgm!seIxDwM zYV8Hzlmcs<4y&@tr=oT%vZAZEUY)x3DvxTZp$;gulB=lJ|0=GwYqDmmzCJ6Difen$ zYq`>@x%wi$=Bu_EY{4dMwQkC*PAkBM=E2(Qz9Q_z-mAr~Yo+q5e{xX6YNou(X~61V z$PVneV(i66EXr1_#+q!ux@x=%=%j!wyD(uOsEX(F>#)hlQwy4a~ti;}Iy5g+R z60OdHtGiaKpi)z`dMvC~&CiZ3(28u$R;|tYDaB^((eCWhaxJtTgwHDNlag%7V(qpL ztp%R_PkR>$nQ+*aq#@*6l)SZQDkyEueu9H0{d{Xw;@`*-ow97Ou`3u1!iy z&jxGYGAr5MsnsGbyGkzJvaPdn?B4>e*v>8CMy}bS|1ITKuH{lH8~E$f+GFENYU-}8 z=eF+Wt}X1w?%B$&>v}?}_AS^l?&fmtc0#W2f-db2F7T$U=oT#BE^geqEo8DT^VaU~ zzV7HoFY!(^IxqN2Z}^UH=nij=BCYj)g6WFO_U^9tzHjqB zul!Cf`F?NxnlIhXF2$m+Ch#v@vH{$lu57d~{BB+R+Asa$@BJ5x% zPVjnWuLVo!3YYBT+OQ1MFb4~94Fj=4t}nQxB`|aPunPyV5j$}aQ*ZGe)oG;CF~XwRxG zyLM^Qw{Y3UolCc^SFl~RiY1GeW!}Aa`=;ePxRl(&h!5LUyqGc7x{#qpPE7f)UciDg zW40W4bLGyBLyLAhnsjN?k0pO54BE5n*Oy_lJ`L0HYTLJQ(?*?oa_!!^gSXZ!J2vsz zyOZlyzMQ%5=fI;Y=REv4b>qmLH`l(MJMZ7ny(j;_?w7pv>e$f>7f*_P_wV7|XNPRQ zesTNh@2>~HU;caP?Ewhifbor&pLz(Eci@8a5!hgV|M@2&e-S>I;Drh@h+%~mLdfBV z6ix`@dm3)2pNSrJXjqCPiWs7UE2lf;P!#p^+x4XqxjuI%$@i z8oKGFmfE@Kr;-v{X{V2pnkuNGlBlYurN;mI>Zz>OiRr4c;+m_bx>8DOsKCw&>#nHw z`fIJN+N!Lv!}`iAv&SNP>$K?U@~p7LIt#6|0WRxpuqwiG%eBpBTW+-6e(NH+#cqr3 zyXk`KC%flr3op9yQVZR<_tx94zoz1wZ!3%ztZsJV(hKmy0v~Mf!v8$OX-0#2k zW-RZ-13P@NxmSkV@W}dZob0F{Gb}R38k6jDy!^V$GROhf{PN8mODwa>EAxyq$2pt) z^UN(59rV%3rcAWYHZMK&(?VOUG}BOf9QD;Z8%?#-RwIqIcv5ez^(RfUjB?p%SKW2i zVNc!j+iR+plC*!Dx4tm|Qvu=Cqn%@b#?3`ZC zd+osce!J_!=RSP5=q`RbD6t3syYtXTPyF$tSO5I=(q~V-_Qp@o{rBFpx;*yGlMnv+ z#iPG{`0SHDzWMO0AHV$F*H8cY2JU|U{`~jv|Nr*Go&ErrzBv@|fCNk+0vDJ-#l5e5 zm;hY_AxObJOppx~ES~`}I0p_g&x0HE1P4P%LJgYmaV4~%3ir^c+NCgq4K!gO^gsq0 z+AxGM#9xj`xWP5}@P|DdVGaL*IK(L!@d-y<0uq-P1SS^YiA{836sIUfCU(JySHxl# zx5z~*bn%N?45JvwXvHL!@r+SWVj9t?#x}Zfidn2871=0*I@ zyde)?xWe;^$pj@xBqA4y0v9xL29As*Bqhm65?FErn9L*vG`Y!6axw&;*u17Tv-wFpdUJ_ifMgkFFwSxkp`3x-A|C(u9CK^Ev-aG0c(+sq`m61v=0z5VW8NMQA~{c*|Hev7rsMWG>Dt9z`WcJ*r8Pj#Q*0U1l^>T2clqz@;lysZ3)^)0*1!rZ$x*MR&T( zd*V~05%8%f^+{Bq7ImK=Rpca9>CmP+m8nmKs#K>6(W>^6r&!HsQnzZ-uCjCj9Q|rM z#ro2+n)R$`O{-eJiqo~W^{p_?DG{cTgG#zHt4IAQTZ@`kzS7gHcCBhu2TRz(TJ@Cv zoakNgic!FJG^Av8tXNGT27#s(r9sWCTQ>_=&Bpb!oCR%XX^P6q!h)_9UF=Sc3Rutf z)wO`-0%UzDOT+)#_O^#*Y%d)eSldeWw~;kzSR<*|%%UK)$aSklMY~t#Iv1{7sO%W% z0E%hs0hFp;?NIgl*XF)90o;`=C883V;;|zmmyB5R4IJn z^C~u%?bUE^xys~>wwT0VJ#LUg9LxDqIi&@_tbVtAUoO9Rr2;4|foWW0=5aW??<_HE zdmP@z&Oraj6DD$zb4+LZ_HvsisArS+{9q{8Hv+0ms6M4zO)tN=$=-EwXQMk~GA9fa z*F>1)m>Z7zuzk7%sW)4xSikCSYisj4YyInBzj^?{Kt8|Lr*8E;lYQD|Py0;OoN#+; zo3C(>`+k)G_XgNq*w;Qew+Y;BHdynn-PUy{Dlz!N8=mmrPQ2p1&G^PIUhs#1MdT+> zdBh*y@tAKkd^Mkj&U=2kpbtanL!X7xdx7GjM}6v5zi(Kw-t=8Rvh2e?_S&Dl1$Q5T z)ML(Lq{lSKGZq6iF}ljlKi=`3KmF>@pnTY`@%hkqLK?kD{py?k^TzLk_A}4@?Pve{ zaR5L6_Qy~D&!d42=TCq5%g_GwH~;F(8Ya>aL| z_D%!Ve(Q&R=VyNm$OK04fDjmg4mg1nSb_Lge%_aX38;Y_xPb^5f9|(`|2KW^XMrZj z11N|CDY$|v7y~WPf-VSyF$e=Q$O1N~dpP(38lZzb*n>OxgBAdUK`4YnNQ6b0dr0Vm zIk<#0*o014gWNQHkThSgG~5>VmO9mxPr<@ejS*G>!*eu zsD5quhFUm&Sr~$JScl7Jhp#7sdN_i7*oP-$as=>e8RYi z;#ZB-7>C%Xi+K2n+}MfDXMr+?Z2*^xsTYoefQ*P2exUe@f=G$1D2AmMe#EGZ^tg=n z2#ol+hSPYB*{Fd2xQ*TDjdQ4t|A>M3c#oG?dOGL@N@R;zR)yp!dVL^~zUX+M_>M;C zj_(+cG&Ya*h>#$8keSDi0I7}L7>lFmij$a;EZLGu_>vkqgC?kdy2p?);E)0ThGveq zhR2wYTWE?ZDTC~IfxGC8MtPKwhm;~YlEFukC25i;>4)stiW*s!D+!ZWX_Xp4eKpyT zB$tyBDUn6lla2S2Q)!84Ih4{@luDVD>6mzJX_5Qrlu#L!0x6Y|h>BEMh+5f~e)*0r z=tV+_lK`fZo>#y} z=b4`Bxt>acP;dnDSq1ay1NHd>_?e%Uh?=U&nlR>?h{=ookPrG`5A|T62b!SQ zpr8y|5Ao2T?f{_@iVp=kpcZDLW+M_~h4MlpSL~5i;x};3nqyb_L znD7KLX_Ghkfc^=Q0J@9OD0mi%pc49^U>c@knxtg9qX}A}9h#>7pr&a`p(OgIBFdt2 zI;UPr4|aN|Uy7$@s;6bzr$~AZfBFo7N)3TJsD@go(Lkt*+70Xf(5Q#%sF3=gbp?-9 zYLi!rQ-&#)uvw8EIiPv!pko@UqS~it$`5S1q4sd9ZR)0Q%Brr)qI}Aya~i9(TC1db ztB)$ExvHzYx~snWtH3IzQZS`d(4WHhkT;d7pZAO=TA-ggs?Zv(cbclEdaBK;s%zS+ z+KQ&H3ahqSr{Ef{-#V_i+N6jYtmvAqk-Dy9I;=BTsVW$b`jw?xdU>AetkRmV`ud{W zimfWT5B>_U0=uov0gDf}(4yJ8vi%yf2D`B3O0y3Cd$W6butSQnJZrJ)nz1;* ze;#|OxJa*@8VAjKvirKU)C#jI`?63=umdZ#Hfyd3E2}u0wdMM-J?pbwyRMTOnDB}N z@;X`~JDK(huKN10DciKF`mfa*wJi7DcN?)28w_9TwbE(@X_t~j zTeMhsnCE!3CJU--`?#U{w5Qs#u1dFZE3=l%t+N`deM_Wz`?-AExq&;nkP5VB#Gi)? zT4!6YjMunI3%ORIU2gUOS*y^yj~l&EwHhu3$lf2v?Qy2 zruw?DTc|iHyOS%mPP@5f>bG~hx7r)Jz5BeP3%tVrE4)`=b^N&m9cu(KMrm)gx|^D| z&g;FAE4|n{zqTv0xXY*C`>Wi$zwrCJ(g41H8@?DztX6shOOU=OhH)3gdI8F%uG_y% zE5GzR!BIQ2xtqTj%)c7!zZ?v}!Ard5YXp{xy#KjasyDKRhqMn2!ROke5=_Cgo4GNp zxis6s9t^!YyTLYm!#+#6F&M&A@Pes(R_uGUm9WAr+``ctvoJivPHewwo550yy;Pj8 zI$Xs(9Khi#53~Tn#j3IKiUlBLw#w_i@0-M>+Qe$yt&TgzSp3Ft{JS}v#adj)c6`Sa z>%(FTk3%cQ!8UZz zybHRyz?`u7tIEV&%)!dasoct0yb73Xtm=!UCVYpBTX-`T#ks7@yF8>O`@k>y%ia6U z;LN|qJkEey&gQ(R=&a1i?8@w_$Hl9V5lG9moTU}Hi*=gB>732_e8}re&gbmU{|wOk zEYOtNW|8NJit+sqJ11@e5-nhU3TG9-SRXcdpHQ5O~V0SMF2Uva8 zY8};2ZPjkg)^Od{AdL=leb7G)uTk)oUp-(DmIPsqdprPoxtG+}tkr8B*N8pUUn|uv zP1VDY(~Ovuz?cPUS-TBSh^KIYv?cfal;1N#W5YFG0 zO^X0t+nlvjeXUdnKHUip-(rg0&`{#laN;K3;3`hxEH2?LKH)G<+>bru_}${Lt>5wx zshY6d8Gh8xt-6}s;UE6qEARv!oZ|bv;x%sLOa9_c{^U?j<55246)wCxe&Jrd*HdQT zTR8%qP2^XQ+S!ogV~*lxUJOc}=1d;rRL6BjSb?)c_?9+3oWzN;-&8=hY z4cJZq=8(SVrq1Z79_gw+=a-J;tnTWTj_GtP2%5f@ji}+RmQ_O@;uv7$>8q~nukP%r9_`Ei>vs;u#%k;0*y)pWfPWt9fj;U~5a>VH=)f-Q zy1Wd@4(sU-?bF`u?#}M&4)4;AyGRP zKkv-`?hGIA@}BSz|M2Y|?Cg38wGMx`DAan^Q$hHE9Z%|DTJ8ov@(v&I3g6Tc-|#8F z@+E)r*M0G0K!N-J1mvFXZh$W6AW!l--|{B!@+$xHEHCguKk+RM^Aq)R7$5A|$c^~G-W zTHp0t|Lk)g_jV8WJ}=(mZS=_oTeOz+xV>H8j`QK3_PG!aZ6Eb`pZ9Wq_jF(PjlcMe zuitaP_j~SIe?NrB%2v7 zQgRU^CW#K1C1j42*g%7mL0KH^l;9(mLvY!G3ME?9D7t0UkS%DG ztW~2iwW_t7SFl&JZVju9?Aftj%dS<+*6rJ_V$;rri`K3{yao^E)yvQ?(7z5LPNbMI zW63QqJw~8V;e>(|YsgHQvqt4*9azRJhDno$gAq8r@a&l>FxR7UQ>$j(x-Q+jZDG%r zRy#KTYT2iA?|$9ecU;*p`uxV!(3LvFxt25YXc=Mal&rp90p zPcjHoT8R#qnvk$F@r(eChw?~E;V1P%6!AUwN_4M3-t1E`zZ6Ye@kAI=^iMzl7h7bJL@l$t(#smd z#Bt2^ylIFG=3+36i3G#gXff-sEAj~swj%;F6BfkKNeZF-Pf98SwQ^7_2^F+UFb_5V zbjuh6BvVWRPt$QCFK}pqO*Y+JlQAK)qth}xm&}vN3^&ZlPe2vzMb$4EWwky;W0m#Q zM7f02QE_m6w9GSq*n!dmHLWmDP94l{50gXyXfQ2RTB}^uT5K1sbw5{U zwYA!BTYXDca&P5Tur#Bn^wKsJ`*j0&J1wF}$&^JQRGTi%_gR~GiZWb)ug!K~ZM`kH z+iepTSHE)2jdVRUfwlCai;hgp!XQ-|$wPbb-OvJO!|>3KbE!3xTY*z1SmA&Tb{X7- zQ5M(XTxs4F-5sN=7>gl#UeGb=x^TDSQAuF1*~0oQt!0>Hb{gTPV}6-ssT&6W&}ORv zvnc0wH-od-1~XpvUb0CI%~X-kBiZDVqo$hax8bgO<%6@X8SA#;!Def&+x-#Vv6UQK zhO$Q(k_nJGOk?D{*{1tx$P0CQYPlb-Jo32izWQdo8;`i}%`)SxCDK`H!i`}EKN@lJ z()QPK$1#7L^2uqBTXx(z?;LjJROovn>VOpc&azc6{OH8_RlD)njhX%V+$E3xa@%j8 z(r)Z^hdAAe*xgx#mx5eidHskxXA*`d*8{lQ9tOZOnwujUj-qUKm0Lod(C+c#k}DEwF(80Q{^LJ z0S$=27ATN`4fI6|A9#!i#*l)ws~+!YxS#uFPlp&ZM=_Q_IEi8EgW<#22#rWU-mdGL;pgGAJsz;we? z9Vb1hJSR3%m8C@GFl|`ORVLDvtvqBMgLoJV?UGY7z+_AC7)`FmA#wTx+JNop9Kk%nV~}^aw5Fqku6^Q^rk<-DbO{V)1U<NE`yH~kPfH5odRHi8PF?0 zk;+%U5_Yhj73^p&XH>(URk2NFET(q$zW{awruMYwCNB#zO+luodj)N1>-JYunwGS} zCGJmCo6?dgRjnO-C`G@P1gg?Dby#JPO`M?ByrzS<|OAB1omN&q{TyA^T7tU~h8zN#6)xXTs;ev3~jcWA0*k z$2`t*mvs!}APe%mUUY42?ddhzUS^CDJb6aQ87y~xmMwPG331aX}nr zAqRRKgC;U9Au1g+OSiVg;|xZv%YrE1cg_{&v!1;iUjJIU&v?!-p#LoBL9a9u6NBVa zIq0`AcBB(&c5~4PXjv%7`2bTMQJrml<4SY7#~Rl3rhWZuFn=1+r$>}sF`YP5GE$sGIg~WEh%LP%US~Krt_^Eg6ln7 z8Q7P;cD=D}ZF+xr+uRl@ie#pbfQtp5_k=6Dsngd-PY;*7QM&qy2MD|dIqVK{M$TO8vt-*?UV?QisYoZ#l; z`N2u}^D!p;ho;idx|im-mb?7UG-rC#^Brt{N4?k{d?~6$PV%cmTixuQ7R#0X^onbF z=3(cYy={K>K;!(G{!?Psv0m_#pPb|?NV2#+8b+V z*!JA*<~BLebvtRi=Uwl6|9fJ^4tS^!9w<}q--++E__vdf2*qkVrW$_q$V)!;$Y4D) zTK{_3yB^<{ul(3;uX)0YI_Ivf`g%eC?{kD}-3VwXI@ecEdD>&X_R7b7?qOMmcSW+f zFiAYZ)ocL87eM=tKHP@Rz&hh69WdgLJ@S*keDiBOJ%aC{`rZG2_^$vG_yEqv* z1_L?|)V>5zfdK?R)Kfqc>@E~sKzWP66+A!YOFyt{zTBff&^b0vxg{t2K5V*rT{GlD(!z|ps5p16^48$@- zLqa@5GlV%_W5cmyLN|Ovhls=an?q!?L!qG;Ho(F@6p1bTK`%5qA#emi3`IpGlTi${ zL2Sh3V?-H*!@fZfj@Y?t(t2^osh!O0s&Qv^n2 z{KaHM#Z^2;P=my^vpN*(2_vZ&KJ3F8YM&&EUaa6|4E64Sd0y*q~D5Qc{d@Kae#3zVH zTFk~9SOQKw13=tIfHcSdLv%s-_(zPqNOWYzbeumJW5>0c!cM7-IhzA{e8>=N5{U#P zU9?D~Q`^aWw#YhB}AwqzK6drg)6EZLfmK3@Sz$)d5f{EO$p7T)ISCoJfcO62UAI-86%wtIzDzPVUT4;gn3`?9U23 zPTt$h6Nt^zNk{|b0zP~Iy=;ty#4PM;O8o52r}WP5G))S%P^a`y@f0*&s7xGCpyrCr z6a&Dw+mI!)#b~3uGN?8CY|;A!&kMZ~45dkO>gX4@VwMz0QOHYDFT04P8_J~o)P3_bivxPShR6z~Zy~&DLA}R&U)`eJxX8Js5xGQ)ehwQ3VE39WHfE*LBUx3p_|u^U)uT)}*Z0 zdo9xc^XON8%~)^^*l;ydb99D+6;y)-22w@Xgk4xvY}Xo`#E|&CXeCk`kTfeu8I8qM zDD7B|WrkcRSCSoBgFV3aQP`73*=1!_U86SgPGv7y8$n z^;oSvL}K+>pZ(LmBU+)Qz&%h@eu>j}ychPI$4vY|Tcr#fkhC1Qim9zzo5fk~1XPjr z*-*^|zr~HJ9NVEq*|Kfe@_ZJVYFS5hw56q3#|uKp5E#3i+~Lexyrs%y2vx7G1+WF$ zz#UxB4c(%3*P~@w>6}hj1=7#7fTw~Cb%|TZjatf`U2%;|thHQD-P@k^TW9#&A{1Qz z;1%6rE8J)t13H9MUBXKgz@uhsHMU*d*8PAb*v257+qO9#d{t?6u`pON=yC*4kl?a?UO$q6=tViXjS~I6UfAn85I#l!?ce`Z!Ok6E z&wa`RM&R62;CD4%=cG+myCl7Bh04H01_oVc~V*(v8^vcbq8b z;(|v2TTr=TF3jTVI=%eu;>7&o6%65yh^L63|o>u4WfB*;pH|xyC!!5In-8;Ty zOx{yp<>NjM<6B-vP!?o!5ak7q;nNL4dG)1;u;2}7<)o3cSe9j(+~hxgW@wJ)&*f#& z^<@J?wCrQl%z%PDumu|~W>zLr9$sc<_C{xJAzL=nX*uUk&gD-oU$Y%%NUcJ5q2gMw z;&8^|WNtWeE@vOcWoZs*uiIsUw&pYzx$YnqEs)@EHUR<9L1flqe%@C9f1c%zWal%) zXo1dN7^UX%ouz9QP`6VtCCC9YU<6G7B1aVidHo|-hTL^C){6FF)6nRR=4hM-Pl5*B zZ5m~HHaGM{YVoEovvmk{!o#g*JdN?N;(}& zlChUQJ(>PzcP{Ij2IwSCD79W|wr*>;W^04Eg_>3>PAD`(OI*b*M!K1CCpH&TxXq4?77Ya#%^rKer(1rgT}UlOpt8LE`!U)>@U!4&hG5Z{%k1#ZPAXy z(H?DrB<<4h)Gom~QXWgYR4%Z2Eq0`@V1dj&JwQZ~g9X|K{)g25`?P$n-JXCuApsOmGF~ znw*hBq)gq@1qIz+=qfe?Lyk^y{%=Xs@cZg;59ja?4{;EWZxJtX6Blq4A8{2wy%SII z`ljv|r|(N?#o{LJcd>CC&v6|O&@I60!axS;OENRRXwpLDdO z^h&>UOwV*pUrJ8jbWhjKKXTQLxJ*(XbyK$`REH_V5SBNU*uKo901)pq$MEXJR!gkJ zTFP~})pcIq%L4oL-@P+#yW>G@q}g#a4B>bzCR+aHk9f(f50& zc6|Tke9w13SF5ln)|Oc3I}WXA4(znFs5TUwDGo_=sl#ga7zs&-u+td0&^Z zOeeLQ4{(?!HJ&FmqmNyNj|_~TZyCo!Q~UUkXZm}!0j0n6`)c~4uXwI^_NI^e!|VA! zFZ+SlaW}Xw6f@>xQG@1=`FE%Jsy`2|ulun7`HbIry$=C~H+rPE`-r!CsvrEj_r$;7 zd%eGOUpf50uXM!E`^Go?$A5gq2mHmydYljShKKxx9|4^Y_;cs;*EDz1Z(_A~b;axJ z=Vs8EUmef~cWdwVW5@W|ulqMkvD$G@A%74|L|Xb_7{Km zZ-4cNf6SkM`j7t_sDJ#o|MTa)f&>T_F^mu_7=_6a9xVhSbkf416DkoWN~AcYVnsI& zDJ+1{v4F>rA4f*ufP#bxloJH71nCmxgP0O5(41-WCeEBXHSp{S;wMm`H;ND~YBb?d zAWC5fOz|No)TmCIE*)yaD%Pw&v~umb)hpPrFT`FgTf;0>v}e__{aV$mL56W5KE%=R z?!vqc>ssvV7okRuBwvag?D8;TmMI(mILJ_QC*;VI8%LIE`E63nMw?!q`t-ABvZ6;b zEp7TV>eOvlt7ffPLqWS$_u|#7&^BG#2X|x4h$c+n!om#`Z)rRMOv;ZZFUP?Axy}R9 zrz@Zi{qfYy+B1hn-Fa+U@Zm|5cfHp9_9sYn^KLKE3;g&Nfy@8RhJG6R`1Rk9<3D&} ziLEqXamOjv!vNPQ_(XNeDcInH);S2_gv@2<7KWN_HXetdb@-ud<(((qS?-Z>B8n-d zcm|98x%l6UeaOgPj4--kBaS%&DBuP>5*Xld$hB7 zl~r1)9Eh#Gre${{beUyW7pXY^W{YK-S)-0KqM;_5X|`F(j>skU#2kCrnL~mJ1hA(8 zKpx20kVIyno`*vo(N=>SJ_+TNjb8aEq>f4&DU}{_dFiDdeEAcZoqD>)i=n!R363LRGA_P5C(aBd{0U>$TY#U7mKV(T=w<{*d}SK!H~k21-HBy3k1x+!NjEs^A+%})9( zrO`?|Ews`q=_Rt*Zh0tCV0hc>x8aKW=^A>NscNX?zIQH)`myS2tcT5NE4`%F8_O!8 zOz~^5M-o+`PXRYqB%{t6TrI-X9-Q#P4L^Hrpy5JM#l#g`Ve!QoYcXz$9Dj0OogACU z?wR(+iGZxBn*8p)tcbGz2fsn?AaKDoJ8UzD@|fX{Omdy5#tYID_pZz;EL~$F%gZqXyX?R|o87aM0HCe*+Gn@z_Q6cU(QDCl z+l_S6c<1fw(jxnvYP;wP{+|w4r%)>>TeIx7*PsXj>`O^hbX$nHQEr>Im}j1)rJG~h zIp_gq@b=p6jr`{#S3cQ$2UycOR|v-V(o|-qYsF(YN0s18wHa zC@Y_};&3|8c;hn23OPz!aAE!RSY)C7_T7h3ANaP5Pdn{BBqgSa<G`MvJ+dGM zF<3YeaIk|O{NO-9SOO77UJqR-_;9H6aKD096AWSP90> zfr8ZYlkyZ;0V1-ojg+yV8`Fp%5j21T4^Y7^t^qSWe&J3O%-j&_C`drYAdPWUU>l7@ zFq$#4k!5=%BprFk>ZCwXNOX?fo`}g!a*~r!{A3kFsXL$GFeMjcpcwxn#^^!rjA#^+ z+Dgg61StUjWg}FhA#F(qG3@e|xO63-HW(`rB%y;k*nu!V_{(MHNmG(sYuqb0IUnw&_4S)g%l-x0uP`GBo(LlP)PciQqP+&6Vppqjd9hs?7f;O|GG07-H zW0*jXYO|mSOlUVz8pv6CR9F!DB1bIR#hEs=rOsgzIV(980p?VvQ|u{F-zgm-bh4=M z48j_QfY9Yh^acBTYS+%7sfh|y049-{DQDnIxOGTLpCE6#7+ES$}-6&dHnO1|I zV5tlLMJo=wXv|~U(x%HnhvIFgc@H0lu} zi%$@)wF~;(XRiDSiy{=1stz5foJl%COLW=-!I#;5kA zsulDuX*n>_Ue>RdpB?L-5{g@k)^?fFyRK}5S;#!KRdQIU<5$<~+tKRwv+(tAe1nTy z{Y|iRP#^VzzMl!@N-d6V+#{`wrhQ187cMz1JA)? z(9LCUxq3oF@-@CWrZSOH3Dq%1p_b_b2LPipn`W=Q=)KzGrrdley^@2p$f@_D$`y@Izq;Wa9d($fNR!Kg$3Jps8+ z*ae}sYLva*+O|IVk7a%{vE?h^2G2OU=2rE+)m`iw&v{R}uBN-qch!E@ z@M>mgo_y<;$OQ2TO;CJ-H;uPzd7Hs^W4z-IML0?CEOJQyQ3hIVx4`MW?@All&x5^? z!(;9i;AN~Ro=XV0a<20%@*HY)x`f2-RPmq-z33F5IMRgT?Pj2^UqKGBxgVUf7}z<_>wtH? z^F8lo;FaHZGq*hW-4A-;1K<;{_{B565tMj51_oPl}&&2zr~CLqhWGyWyD$FnH{S7*I6wN+KY7X@ zz5Siv{O98@|76g={WdAPc(R`h8w3%%2R_ z9}Uu<7W_t8=oz6L0G)ix8~mU%*q#jzApsVl074)VCZQ6l!3I|Upc7&s6oTLdKH(IK zpaxnY7D}NOZs7?U0w64c7bb!jc3}&?U>TZW5aLW>~f!&>f zQbd-~{GR|4A?z6;0WP5+4q_Hkp&?cwA|4_aE@C2*AtQbvBu1hmu3!pIA|;+-8fKyk zsv!)zVGVNL4;syFNLI28%EdVc5cXjJ{vqudpzZx2A;w}9DqGBA-nmJJeEf-4YLMr4+wxlLHq&&hTM0(yMObjGF3ZDM9-YNlRtCQ%k8YMQ2R_9k$qCTk97Yrdvq!shGA=Gxe1RW_${=B93T z=4k$Aa8f5}Rws56r*;-6cOoY>0^l0JAra9gdDiCPz1}`PCv@_qF7jq}3g>&q=XS#9 ze0Hbgy(LCIW}`tP+EAr>=I4KIW_u3kZdzw)(&tg0rhFo3eKP23;wOMc=y$@Na`tC{ zN@yVzs8;snff}fXF6f6gsEE3xcVZ}1?&oZlCuK0FiDIaRp682pD2U4Fbt>qLI_QYn zCwxw5i&E%9q^OGW1dLuNSQ06Lo~3BgXoxER>5?|-jp}HQvLlaDqg*Nkkg6z)8fk)d zC40uGjTR}GGHIAPDVaVgj@Bnk_UK(!XwUekmcA*6mg$ztshQSklrm_Vs_BV>r+C(7 zkHTpecBz-r>6nVCozCfis$f-0LrDyin)siNwtzA8vgr>kyiuXd`h=4Yrj z2d$bXMi$_$;_9!qs;aW8v*zluu3xYMYo4~~u=;7TGV7&2tFA_Cw@z!UYUivnj1UZ7 zwkoSxf~&SRE4M3XkPB>x}s~6ervn(tG|XTwZ`hW*6U1^>bVN) zBJL}^eyO`QthzpIl+G)*9<079?7vJuWB7;%kKJtjQWJ%I<8^E^W=S zAIq}i@9<}vo=DN=YsE4`x^67fS}oFYt)%BYi@A*5}I#tYi>vIxdv@t*Acg$l8McAujF?ZSmeN?N+YMYOe9-uDU`l%pz~{RxkBh z@6j%=^4hNR?r!sXFZ2fMrUkEsithJbFZOCL_nz>ls@!teI}F96do`wnpZD(dT&Z~B^V0;4bg2C)A=a05HA1VeE5R`Bx@r}V<^vFa`Z zcW?xMFbGd@{EBb^3vdZ5@Z=qE&v@_xvvBtIZwSY*44W_w*D&XLF0J~2C$un9X5jwL zFbM~-4FfR|b8r#2YzxvbCBaU~&k^tsAF&XRFceQQ3{$cEs${5|j=?aEU(1f&STtyVUy2zfnPz6$UMXyzDd(JP z#%ZUUPGb3{o}_V!W}0f!8EBh?3gw9=i1I0@pKvxhXqS-wS!bbmQaWj*Fgm&@po~6x zr=^Nkieaammg?!HsD7H~o|0zj>ZYc`|9UE@tI{eeskq7-tFEo4YU{16k~r(G#17l5 zs>p73i+Ql*YAmL{Mw@J{z*ei-ve-VWZL!iG`)jD7WxFk}-gZlFxziFlF1OH{>u$P- zCTgg>;8JT_Wz9OfuD{b!Q%V*p-i7y62q_ZZ_tklYaW^u%{lo?5fi~yW^qfPPFNO9_~A_ z@!J0T@WJ0+yzs~;&wA@`H&49t(DVMh@yAbZ{q@*qPrdTnJA51S(Yr4F_}yz?efiv{ zul~;Bhfn_d=Er|N>gca8f9>qI55N2S*Y7|5_WNJ{GT44|IaW)8y2yLNKB#68Z?4kj6;{p`aat61=r7m~L%SdwIm%t3AFozj|3KWx=$PA_lfT;vs zHj|KC5TqEkFwJOM^P1Sif;P8_NN5t02;dB-ILRo^XpVE4y-cSq*U8RyqO%0=jHf)6 zc>#Lbvz`;U=P~nn|IB7CV4uk}KtKojPk|bg0tZbfLhoo$QaaR@rOc!n)k#rG8uOyd zgn&jjno*8=bfXEtr$~V*hF^}eo7{Y6O1-&KmY(#beuQa7b2(FY&hwt=^e0aFxl^9* zvZnwIDna4M0G zQ;L$&oQfFpoaz*)%it!fs1jfwQ94rrErvIPW=J z()zZs5n%3YTN_)IYInPT?dxTyXshybyLOd2LGvPDtduX_cvZ*X!P{?smLm z&8B=uOyADVG?b83u3R7KU6>MBwUEu{a<2=~1>4xQt`qmqpnZrxoad)E` z| zDxd4j{;8{)=Y(QQPaEY zUAsXHz83OfCrx8fo155Lg)g&}y>982Rng8S_l&hX;5yek*0sj=UQ1g|Y>s&X`R=x0 z*&J>+V7RI+K4-bly-CHZCm@=0!J#YF$>~YhdiO$ z|15YHI*?N%`?=xQWiQO(IrEo`GzqOv@wz9P*`8;ZztRr6!-r1vV^jO%2aY+&_wDhX z2K?kGCwP!I&1r;#*xzHwdKk7YV#{X1>s+U%*2f<9vZvwO1y4KL*Ut8}ll|>t&-2cI zE_7Gdz36y<_)+Pc=30-y0f6rz~2!Jnmf8}?6sz-x1NQ3H^emS^)EBJmr=!5Xrf$bxzph=MqTgouHLc!)-rh#Xjmi-?Fv|7d{}C=ZAgBhn#qcujgi`7mBFoc%OHNrig~9xQTqBbi3o{E7>lqdi?j%f zidcw<$7^P|b#2BAl~;L}cZZ!gjH7<$lngVRWj)_9HB zn2o1Mdq5<857>W3U|UVsUAy>+z9@~Gr++cPglqVQZ`h7pxP4i0cE`w!_V|p|n2+bT zjhd*6{>X|(NQ(zJkOaAo26>RRh=}lKhqD)ZqxfH0pnYMGeSJWQl~|6R2aLg(b@n%l z2w9K`IfwLEkN8NEC8>=!$d5cojQ_auhcsD}H3^dk|G9Mr368l4UY3}3 zL3WNCd3N*nh)O7dw#a^&calK)kos7PvZs=%*pDq4h_2X?)+dlS8IxSum0tOkFsY8g zH<97UWj^_CT=!pTKzbC$kyu%PTZxnwNPfm>lIU2MkVgf0>6A~2l2XZ%Rk?~T`H~+A zmhfnph{=wH`E?b@lWZVrB#4nfsgZ3NeH|EysdNOIxtWK>nVyM7U=W(AM1Rh=3CGBn zqzFW&*^qsil7G39fhm}`c$IKz1GZTMx0#zMu$xQBo4(nbzZsk}DV)Ozl4OYkWZ9D! zDVcF_my>4?x_~X#Af3~x4%E3U)oGpDxt-YAo%Fz+_%IFk|KJbgIiBWOp6GcG;%T1Z zxt{0Ap77b8@;RUK33;$!pUvWU`MIC`*`F;GJ^vY?GPFV-^g@mhLI>&+3A&(}kSGnR zpbyHR4a%SgTA_Aep@MLs8Y%}Hx}j#E1|J%tB8r(UAb(>?k!HD^x^_}&Ig*q~p7ju; zGCHF)I-~J$qcxhN?SP{+8lLI-qd=Mu{Q#r-zz;!sq(hpdLb{|%%A`*Eqdr=sQVN|^ zTBTNcrSp)bT8gDx%B4E`r8~-_;vlAEiVb6$4rZFBWx5V)dZzBcrfS-zZORGfwUc7$ zn27O&maw@da9Z-5`r}7{Qb((!9Y6M$go60F( zdzz6i3aFm?slp1Zgqo;Astd-Ntd6>@%=)OA>a3OetXe9q&l;widac9yscnyuiPunN1d;fkZPTB{{`oa*>(d3sRn>Z0x{o&!s-o?5K>dJB!JulumC z{o1cOik;W$uLQfY|Jtbri?9qEvohPT4!aJt|5~mS8?m{nt~kb=&RL|u%CQ}*rHJ~m zS=yu`Te3Xh?}_ffVPgiw2_Os zOW$Un{z!Yoo}Ey*o;@%j>+&ySnbH zw3#Zsa9h1kYrW{3x!H@iR?E4>=LIY<1)eLup-a4=SiZ-5zNVYHBD=nj>!WNNy>|=2 z_B+88Ot;;7yZNiXG|RtPsh2x@mILgk1}qN<9H%BMre%w-4E(%IE5G=Azb-7h6@0=k ze8JIh!x@ah;ra=Bd#)bLv$wWZr`4ioC%UAo!H28B3%tTp+QKqy!8IJiFf74SoWnWH z!`yeAX5?EXOt+$*nBQf()mX+`)RRqRPgL82iVF zT+8Siqo<6@tjnpZJj-;<$c_xhw4BVcip6${$Me|5ARNNQJIX=}%shIw=t;7V8o3W_ z%&@%3woA^)JkHK+#hQ=<(kziWYjWEL#*i0e*sRUk+@pkx5BYozX-vPa8_VW=&H_!t z=nTa;oX#}s3Afz4@Ep(9OpldH!ZmBp&3w=VP0Gt`&UGHOYOonoz%EH(%yTgCcRyK#sd)2*8O8wO39MvAZ z*2rhVF|UD%O5*{m(ukFD6Q z{o1QN+pS&Ov@O!?9L3(W#vc#eLkWecQ?%+i%_6A3Y4R?cA`< z+_s(5x^36IEnP!L1-~uaQvXQW$erBFJ>Jz#-O?T1=k3_#z24>xrHsf9}uhHEA8+rgdR z-!0ey&fXkO;0b=<9UkHv?%&UB*;j32m8Ed@JpwC^-=dv`dO+O#J>$kr4An5=I47p5p@!ByQjAdv(0xlkSYu@CK4(F}@9HQ?kxuDk%;`Is#augfUdCbc zedHIe=orB0ZO-bD{_3;-?6My1(mv$jfkw7%`!p6%C827a#9o*wMqS67Et zS=Qv+AD{zL7zSz%2FxDs)2{9Fe(l|k?cSd6_-^ZM4(^@)je(w`>xJ%=MM=hP0VVqG z&hF;&e&Y_m@X%iG5wGj|zVG*b@fc6<6i??G?(2VUt0Fn<_;z9kkLn8F@DMNYDnIcX zpYax7@h%_pE&tE!YY^`7cdkDe>OO@_7ZpyXj_Ah@05G2NDBlZ2Ki<@y^tJBBG7s}j z|MV-*?|AF)U{3CUM(ky--S|}UUU2dZfAnDg@H5ZyOAqzCe)eWR_V(WE3mx)=H1dV6 z;_GMui7xb4cm>iP_Ie-nX|MK9|Mz^q_Ab40yY4NLVOe|?7^zIWwrqi*h75A+UT@;vbPc;D#FuKAfC`0b{J=&}tYSkNFbgw+%#gjS5^kt|N0QgIT7>J~0D zE=1sJ0!D)mMV5>uON0m=v{_x0U|YgM1E~&1Wa(o>fm{qnWFi#^?%>~LH!lv(_!4K~ zoQXNERH>NpW5|&a9~NABGUcE|Gb=UPmvd6kd_zxZI#ufFs!7J^D6xe@g|1(Vu%W$& zjsMoQs@1Y?3zkWXw{)G{eXCcrU*yR>0}i~<`E$&dGh22nxw>`i*%wQv-krMU(BV6e z7Y~}L>EBGETxG9>#01#yAIv5*Wz8Dgxpn7e9ehOax`@lmJLZ~$&cN#ki;lnq8`O?L z?jD2?Lh`_?usjTlAq_pKPFrmU)q11ggb?Pd?Y{h!5Mwv5`hyF&x#D0#K;x9Nak*X! zL{P^F7j*DP9D$UuMAD*bpGmL=^E$k-rw*djIiD z2k@$qOi0T-6HYcIZSzu0E0vQ%B$Kpr$qe(PDh3=-Lu1q=L>!bp4^lAn&_uh0(TPgH z#4DFOgN#(0I47la(>F5}c33rmHM34olg-tgWi?dc$*7{mqKP!9j18>&>U$N@SQRy5 z)&L#FbyrJ${nglGKP7hEa(hKL-A<7eM_mj>jf#a1rFD&nt>P>3huSc?4c2abi$Si` z8V$2Ba@}qBT*Hcem)(aep7^_XGiFv zYno@y8E2crp0j6;%Wjse9Y7{ky+bM+e4m0%8|8|)4(hFBU`Q?uv`}M@lUL0+;)s|d@ zRFU2?ciw&HTvnHbzg2XI^g-~1qFl`(wieipPK26ga(|5Xcgd<&uZKP(44QF5$@8Pgi!OOx9huFgcCQyuDB;y!imqZq-5Kj=qf%QD3LqX+_ zidM{`2JNB&D0&f$yaHkv%SggMnsIdvMB@tMCz>U)kyKCI0d0~t!_2{_iVX0c9A{M= z9l~sneB5LaJt@LZ29i>RBxNZ<^)zTf@pgcMj|;3NNlI1{e^$C<jjURnSPklPl$bG9rs@#vsx>wLj3__iCj3!Je8d8l$HKRDC zYE?Pf(T~DJr(2C3Nrh@sH$0Ln+>=TLbRb1$GId50^+K5BB2$`5)v8m)>Q(g_PQ2z- zuXCjtPflF z*Fy$&i(dGu1l#hqu^LpRGQ0q{DiUQxIMI65wE9A+ zY7?7S48nG4Y@MwFQczpl?zXd-oo;TW3s~d*mZykJ+mnd8ulphPxGTjX)Xe+J*ruVm zD}Zivse4_{#&@strEgueTiM{23QFiK>S2vLh1pt^45W zM%cl=rSN@w3RFg_r#<2QuhLNKy_Z^Lz3ttGf+Y;!%br-bD#k90UHqxW#n{3=&2NT# zm#UUt(6|L1@KOnk$pgofCnYv-4)(U+{B3aiMb*c)B|+mjM-Gs6lCWg=$(U#XwH-jy z7_j!Wv)$xwYdhbf_V%#D4drnURf2YACI7mcIYooR#>?`qb~-?f?R)c^%=^|i#V6kH ze}_BZA=|KrnVE1oCOo2$V>rFfK`w|#{NfU~II&%>=3Xb--!sp68H_FPqL~}y?Or!5 zzXF1H$9vvVaQMTGZf}*h9K}j!`N}a4T#a{^+*Gi0$soA^tnb{CQVTlOqdoMZr`&Hx z9|tm+VRp3tBJKEQdc~c73Bo6DZ@H`~0?Q((2xAf0u<2fpWm&pOw&==G8l zo$NOf``R79_@=+T?Mr{T+$|q-MT7ebQa@+83v2a*6Mpc5|G43UX9EC$9rCMJJL}K> z_}9Ze@|2(a>|uWUxtF=_cE|g-_5beOqNy|VqUZVW6^_TRBVP55cYX6^AN$$38%{AJS`Ve(!n`*`APj_2L_V zEBY^h;9ExihGRYfM8DR1zV^es1C%`K<2?n;yr`Q$q2W8K)39loqbr)V{L8aF8;jFR zy#O3Q5F|k8i#`%;K=oU|61=++oG*BIKL&ikaMK0(1HNR-o@Pp>hEjsvK>_!u8~z(V z+VMa!2|*$h!Sq8x_DjGdWH|@ClNqd@fvbuUv9ymXn9=J$EX14);6MNzLL+PjFbu;l zOhF}FLbgLfGrT-Cyge7}J^wfJi);88D9kzVY9@x7oQJwX{=32d+mpn@nlUWG6)eCb zEW^qp!!%sOr)xvadx|+Uup88XPkEDVA^l&zuTAwdMiP{b!ROhiQ-#9zFS7Cc3~lfzZ4Luk@M z`#6I+XvS%i#r{))T3oAY1H@d^#a%SSL(IluT*PAJJny?Rw{gWS?1F5u4e*=AX-qs) zxJGvbMna571JuSd^u=_c$5A9k7~IDwGe+-IHa;8x30TGx*+YZWL>aInuCv8?+XZYC z!-vF2d(6jPgvVfHL;pAA$BgVLf7CA~nlq|Hs~h||ZLtm7fW|BwNFcBp0NR^Uj7XTI zNM7{Di+o9&w8)x_JdJ#vjr_Kb#JSp$#N=zn^$-}8RK}GIi%uN6c8p1aD*jmeZqXAr0Ox2Xi#+=A`giOke&5XQ-pL`<&v!8=Y0t8zw z+7Z3cOv^kvP5(z&O^7Vc*M!ZPY)#ih&Zv}4VysQ6ILUE5EIjNl;X_CfSppzCO(954 z;`~koWX^M0&gO(Z=RD6S69p+aEz%;HYALr2>Pz|TOY=HNYvF;zLzylT&bI7NoIFnP zR8H~?(DFpj$;`?NQzW2lB(cg(8$7MnIxp;OCC?EHmORYv1W)4x(6{VR0VPlm{m;rw zLIXWT+BCK>@JI>_N{{=KJ2ZQt(E~-$ z6OF(Wy-WsWQP^;(7sbGJbE|XH!W+;|=HkyC%~Bu@QXv)55hYRt^{G8jMfzb<<1;uE zAOJ1Y0RQwtm<)h7Kgm%oozw8_QZM~dJgrJF4b$|5&LtfJ*3!EDlT!IW%PQ40{lb6_ z3Nt$O(RxhIExl7m-BUf~Q#acKKYdRu;L7lGB_$A)bKAiiwE@#CmA1lC|Kw3sT~$f_ z(MaXeNgX>&EmBJDQxuIe53#f}?bJ1GsyG$0Mh(_ig;k5HRb#bP8jPEQ+tqdJLG*$E zXWgJuWyTRpRaU)LVI5Xt%~Oo5R9f}cV+A)%1xM4+RFO=_!30$}Ra7AORYhg5Y;Dz8 z-PT97Q+X{#aLw0m)z>{8*JM>zUG3ERY*QGRg6#xWYfac5o!7VXPHhdfS>@Mog$I9~ zy8ri#PeDyrK6BT%Qdn$_S68LikS)-CmDq_bHgc^${M$LRR7W?x&@VGNkiA#PtXGiymhXk?OQMW+p{IyNflf&Z62UajC3;S)Yz zCPrb+U14B|VqkcKSy(?Bo?$GW%;DY2>(vb@nik}81X9Rf{vBdY1P%cK;r}F7Vkgc$ z6K-NSmc2)iVmbx}YlPk_)>#{l-#N?S9@d>6YzjeUH@z)l9B^I1U1R4x&NoirICi{P zc!N5Y;x?EEa>?T?&SYlTV$8J79RAdm?Eq0O0Ie`%3kbR(EIBn+WH#2^MtEbXc3(vTNlzv`9o*GVH3K~`gJ1;)e8Whe_8i``Vn{n`*A1WLeQAvR@HJ{4qMOUF3K?V0m@ucb3OCpyNT1Vv&|r^W^4yK8B1o zEn7-mX5}6(XbNKP%Vq7dlMX6`_9=x1PnY&m1cvF$3d3EfXKN^lo84(#?rBy7YG6i` zQ8s`avs>UKt)*`2sa)uoX6aalNL>hoYqsR9)@q%O!(GPVP)4^7c!C@VfLX%Lu;s!I zIqR9lWDzZ3VB{Oc4kyLVgvNGk$bRfNk!*yBY|FlE%!ceS#sp93?90v(%T8O$e(V_I zn;5$VzS)DuQSD4XZP#w?*0zmX*n`-PZIc{R+Ysh5D3tkePyZWa)+*r=VQvLuyUq=? z(!myx;Y94w&TP4xkw$b{;?ZtVVoGT3hI=I-szfz1nV@D^n9F7Li>3K|&Y z^H%Tl7UlMKZ}^VyR+;Y=pzr#=Z~M+~{N8WA?Qi~;ZvY4ICK4zryVU}}IRsZwB^4S4 z|5;27zubNTJkSJzPDrx~SpU$^k~__L>+pB=@HhQ%5eIP+pV9d?aTMS16kl-_Z*d{q z@E0#}7oTw&uW=i{aU7TN5?`7GC-9UlEg%o_1sC$ZgK$=`0zuerOSs^1MQ6h_O})u+ zDUWd~uW}Ex@+-%3E#LAf?{Y8yaxj+~5N{eHFLN_L^Zy_J=DhBTE?5O5R|6bU`2VKPU7+KlCvNzLN53G;j0>(r6}l&ozf2XS{+C zc!BetUqdBa{<`kkt@m*i6Ul9c!fWBb5!_-r$b$rK>vxKc#7XIi?4W$&-IJXc)o_ki--4k zH>Y@P|hp+gaf12Ri zFFxaTp4VVYO!}7R!e{UAju-h#9C4O>dL{yQ887*&xB5L7d98PGu1}e1FAIL}%|`1! zTrYcDmnOCE>iI<12+)FZPKGf}Q=H#)?X+({N^e-bWeQBUwp=w{1D&#rn>e-XZc#EzZNL{OPac4)OdX`2Cx7_=_|DtPGrxXQks_G~KY$<-V+e^RC;}7^kZ|D)7z!R(fGFW1#flauV#E+p zLy8(eJ{AcHQY1+uOGv&*$wKAImM%NKya8h-OPMrp+RTYl=T4qIef|t8bSO|8F^wWk z>QO1vrcIxiJUOKWlYu<0TCEy2ij$KusE*j0wJX=MX33r%vxDJ6whP|EjliJ71iEn* zCcsPAp~Q_D{{r?Kcram@B@ZJWdH-}V%*Ko#|4@vhC}pRWFB7$_8MEZhWG7?IO10}( zur5PK9j$dW+9_zotcBqAF73H(Z{Ic%gCa-4g?|%v5d7kBjF2-U#|gqQ^X1N&a|WG! z`sSt8uP0?KUHj*&RL*`6pVmv3^5oB-Pov(A4%zN!%Z*h|haS4YMu;Dx5r>H)qIhD8 zE2`KcZ12G*UtKdAKwo|KwE%!$eEp{pVL%udV2?r;NMw;gBB`U2K|a@{lg}y0pkfe4 zSY?&iVQCf_u5C$Tmn5rvB1){WjTU<>vX2gSTBXlg zx73zka7t~p*MeGNwrp%0=C+@LyDh1!=0)EWpm1_ay6ful#2%WIafPjI%Ftg*Ty2C@ zL{3@x*Pg>Fi{G*X6C5zW2P^8^aa@{V#T5=itcA4^Q~R*B*iH+_m!EKZv6kU>n(DY~ zmTM~+>!J*+tgNiENB<6WlCWF8Mj?El!8hZav(7sw+jFxSOAG}PS`1w@(MKaavC>Oh zo2sWCYZxxKBA?hp$*uaCveqf1D<{3o)*QCjJo`*`!DcIZVH-#rEkqSs2~ z(@)=x@wa>T5ylC0*1E^m;}Q-FCbNj*@|=qo6hd!cLr!GmlM7t7ceX$-uXRP<$Aetxw-~*U3cqN4!-a9B}%sXcjScGAo@vR}x zomk2w&&NewiIl$-@f~;&%L%9 zbVvNQ52oXfKmXL7Cc!EJv{H?C0`#3Iyv8`X^e=%5T;TD@m%Q>kFoMa$5Ct`)JsqS5 zgVxhv_d4jo53HT`t4N9?B4_G5`)oVe#As zZTFxCpfPk;;+Gn)*sIRL(T#0n3=|!dMqQy~1$x}0k6cItJ_0h3f)peU*LOqv8S->T zh#v8Vut3KNPJkvNqT#xL#5(Eae+c;D9n;9fPEt{mO+3Ah8Y+f;q#Fs;3uDc)l3;)Go#ehsOh5lM~_w$ zLk3aeeW>`VA&QfC;oQVH0eZlC9^^(Q7@i9fcuNkzD3eJT#7G+;%f$f|s1v=VIu%II zcj6_eN;T*~cYuW-vXi2?#2Q0GfKM5e@dQaAz)+=1%&l%9qB9@}Q@y&MDZNobCs{!h zeE$VV1HBcRgUkU`1meh{=Cp~rOR4WzT7+`Kv;a7b;$REgL=#|=3xoA(Jne$3rMeWV zSOlU!D|rL)wiHq8PHjbS?mCt!m`pw-%0V3zU zm2qu@T;wA5u_oZDTE^hu;?Y65(bcLZIjDpt_>>6s#U)5wYTh}i1iVJoFA+YuLI3cc zR=wxV<%gk5-qJEowpEylRZ(};(rR^{oW-vb$9my*F3|=2eQa?5+q*POWWa$vZi=r) zqqf8|%7d7)7#v(*0%JKmwe|5x8xqLNXbyIauQdQT)`O{ND@jDHuPEn3^!q8>1oB`cpI)9gLRz-77sCwAs zVhyjYhV-ZBe#~Ie zl_WN{SOsJEG*CHB5INVxyAs~>OTddii%xm1(ah+MpFHVHHQ>p4$#h1$JnDO%`pc@$ zUZU@q;3@|JpWtUVuhZP%c@sN^vS;(x=^X88Upw39(Dt`ef)P9iI@{+?_q!)z5`NhG z9}0npN9g_UfFC^J17CQ;ui@pG>Vn$rCO@!yovwyt;bq8d`3p(D@&AzDyb3!{_Rr5R z^q@~4+4&Qh-3n4d_glS%gr>|j+=2A6U%S|6Z~Hu`o$t91#O`D7``_~c_`#Pu+=(Ce zL%;dXm_GHZZ+&i1AN$GB{xm>wnll(-jg_>7_P1|+z}F)E$uK-2 z+VB3y!+(ckznkdgKYHBXzV^A-fBx~HvHHsa{tcf10z$)}o&DV(00tle@?ZbifddMk zA?V%!QXucK-vm}*1ZH6INkRvDAo3+2^Vx#*MIQ-X-wB#u3aX&@jRg3;gas@CsaS#@ zsG#~4-ufM04#wXOGL8N5pbz?=1_mJkLZA>1;Sp|N680VuF8?4BHenGeVH8T?5=tNj zPGJ=;!Ut+$2y!78mSFTbLoT4;7^#~{rc;DQb2AP(wa;mOY%{NNob zU>iK45jr6s`e7gT-5*||AP!<65~39%q6HSaGmF4iJ9dgC`5 zA}{XZI2K|t8lyR$<0XFK2rlCoz5z4!Qe45+G*+WEYX2iP;^Q`kqdx8>K(f@DY*q(O?L zFp^{?qGUpc-Xu=mBS(5ASdQf|{v}`vW?-h}PZFkD3T0s;W?8mn zV>)JBLgpJv=K1N`%BN>` zrEeBzc!Fns-lu-@XL*w6f1+oAre|!6e~kj3Vin1|BZ3r*AN6n!;y|Mk#9cq>l1voO3p)On?k7ZajBNpsiD%TohmArt|%&s>7LH0!Tc$pBI=12>ZMj{rXK2|aw?vR=%bQp zqz25SQmUp>sd1iWr^>0Rvg)0>>Z^Jxtb*#Sim0e63#qoLpnjvQI%uNyDyD9#s`6^A zj$mt=sjaGMvff##w(6S>E1?4GNB*j>2J5p5E3vX=h$5@8EGx6>>a?0Aw07yLN-LJa z>9tyGx?(G>x}at9Mz?w^xLPW;p8u=4TIs#gE57RMzN#y_u4|xv=(Z-SpDH4`0_?XU zY`+?8wEC;UIxM5o>SOXLjmE3Io@$jotiAGUvu-TIX6(m8tg%k4q*7tUCTzIYYr|Hn z$L6cb{%eS~D|N!_!glP*rYZ-fY|hrK&T_2F1}M&gEQpTms1|H*9<0rxEYGrR&n~Uc z2JOoRtbvm3(NZna-mKCxZPRM))BY>eR&C5utjSt!(Q<9p?(E8zZPx;A-92Zqx~;?t zY|qRr&5rG^rmfRrt=YD$*H$h2#jT!&<}U0Ouh0JNkw5|L{qEJiZtPBP?H;b} z9&h#<@Ah&p)>`80dP46KZS+=e>00mgF0b^eZ}+k<@^+!7vS{-TEJrLK+LEsDx^Mg9 zFZP~o{-&@0b}Aak@4!AU`IayB&a3wR@BT6{0PFANw&egzZvFP}0VDAGHn0L~@CKhR z`YP`NLvWz7F6c(?lFaP~-){zEFs!OD2Zt~Ur|t;5E3kZm1+VZ0x9|5E0@*O8{KmY(C`2+&6_b_ z=A_AUr%#wVgZ}I(lxR_-K#z_}%G97!q&1sD#W}O2)vF^nW=-jGD^{;0t9GS2R;t*j zWQVF<%a+tsrElSyT`G4jTDo!X<}I7{t6#2w18WU?wlLqrb^q?|+qUuI#Ev6NcKcYb zWyY8hgB4|Xv**E}J8RB7xwPrgkWZ(sy!xr#%XndsHOXweR1tfwPu9 zTQXtXxow9&o_u%l-q3|hA0FLxb=Jt4H`ktB`Sa_}OHu!x4xaq$^2Eb`@9tf2`{nPq zub*$ee(?MD^QWiJmc4)O@c}3we&Q8~-hun+r{H+!B^cd+0saS{fe}Ke9)AmF$RL8$ zUFe~N777?5g&$(5A%`atn4)nbiZ~&NExK2tj40OVpp7Qx2;+`1&Zwi0IIbAvk3Cjs zWRXD9sHBiX-q<9OOCouslu1VUB$G~F38jlvPFdxaT4E_?mR)k`<(XE738tDIwmIgS zXug=HoN>+xC!24&NhY3l(%EO7U*4%FlzR@^N}+uI322#$3YzDmh9c@HiH-VcCXb8; znrNgdN;)Z~np%44r=f0YYI&e8n(3;fo(gG$t)~B~>a46*TI#E=)_UuxwA!j`uAT1s zYp=w{YAmoDdio%Lu-qc+u*D)P?X)fG!mF>?UVE&t)N)#%vk*4>EN9V*%I&$`D$AsN z;*Oi`yU?cVt!$d8D;>D;nyan8`?`xRtU+2Qu)WwZ>ueLuACsLn+Hh|z_u59&&Gxc7&mH&PeCz-1_uqgAF8JVB<2^RmZ})BY;!0b6=%a^6 zj`-J#6MnVjjc2a;!;*KtGboi)9(vsqmrgq7m!Gcs-66XTdelXuE_>>mv+j6}c_(iA z=(F$M`|7nbZl33^zYe?a$fJpJ^34ApyzM|fZdUQf8*lyf&GVl9_Rl-t{q5jiOg#0v zYtQ}n=z9;m@a3QHzWMJjzkdAC&#!*r@YA2a{`>QPfBOHM9Q*p`zXHndfCw~T$`Yu+ zwh54c4P;;hB{)GuQSgEQWD)tW$H5MI(0F_JUzL*cg!OmLy*S~ z?2(Uv{9_>XI7k^7QU`P-!VsIV$VMu`k&IkqA{9x=Nm}xfm_(!|>v%{KoWPTx3}q)r zDauk3@|1!kWETpVN>;k^m9HeFEXDXnGmO!exP;;yj)=w``ZA3wl%orVNlalH^O(mp zB`8z5%m`3`na|uK2d0UF3rzEo*vuvuhA0LsbaM>b{H8a@2~Kg6bDNcnq&m~d&URMP z1jvkMHH$gVdfF47)Wl~#QK0_>Y36eR{|u-#1v*fGGQgi)Fz5mn%1~)G)S=FNXfzE# z(TYmcq8QDnMl(uCi$W8XmJFpklR3?ZR=}hXNU2Ioy3&HSl%Ot1C_de}PAt^)rZ|P; zNO#IpdD8QqGYzUx^*L0a!c?LCMCv|?c~FHm^{E%d<3>>$Ri|1Ns#l$=R=4WWpL+GH zKRu~SSt?etzLcs&Jt{JD%GS0f;jKN5t51tc*Se~epGrL{K$GfJeo|7V66GsZt4dg` za@DYhO)O$5I)iYE0TUfiBv8Rh&$1G9uxHKeSos=M$?DXt#>6Q);VReChSigEovd9| ztJl`S7B^+u8rx_O^=Us3N~f2Rck+4tS)ja5alr;u=?=Uj;2`m229yZW64) zHNiJk>)7g|^RBL4YIdnA*xKS&x3K-LcooaYaQ2oCpzEY?;XKKZY$mtdpo7N3yGqt8Gpq+f6R;PIMQ!V}&D zAJ@xX4i^``*bQoc*J(}?m$ow)34c`Qqq$8qg{RwV|&XXe%#Tt6RSGs86liG6Px29zHRQ zGrZDDm*CBzjQWoq)Ef?SRcVY(RwuZej-IegBYj$0Puj?v zopG;uO2@$;-}=~Sb7BaQrEIe={8D|Q z^wxU*D6s`y%w30j+l0>K!8hH69UHaE*hVXt)g9Q9GW(Ger#No&3(jc6n#g_bH^4KC zfPk~u(bqWmh{o4RL@vw8KzNs#hHiSC_ih-LUnp+q)8A??l+mJa)31{p??lI@i~p^|o7g zy!b}#PwS0# zv0MHXmp8lG`>^@V$KCB%553p7Uir+MzU-$*ed$%t`PRGs^_`Es>}L;qw$r}rppRnc z=?=-c`=0l8?>+E;Py9ItU+%{@!rqtfcUc;^)j(eQJwI&w(eHlc%FjL%w$FVac%KJ2 z_`0O}d7B?I(3|7X|i5f+nbc z;)jANh=1xwdzHt2Fc^Z?CxbJ1eKvT5pO=I3w}Z$BeI;eh=B+KkA-03rEiS}ZK?mK zc{Nyu?PqqFcX^d~iK&N)nYenK2#JzthG(dQqBx3ZxQ3?qhJ%=jbhwJF2#ABo0*Q!d z4aJC`XJ)*hiDM^%r`U>jD1;pscA6NB!kCG|*om9SddAp^F=%_s_=!y9evvqX%vgKL zh>X?sJM@+$cp?&flwHb>==dvNPc(NhC1kZOb~%|Hv(+n28y_dS4A=!-4sEeeyi~I5x<4kP!bVi;P8! zTQ`npCW0B+11q_azDS868Fo~6jW2kQOzDjJc#sJ|6$p?qd6i3ul>k|lTsa0t z(2h}nlM%UbJh^#333*v~gj>0XD~NyZsFXihc2w||aQT8#*OX3al24hEd?}R}$d&~7 zl7cyy0=br8d6PCNmO5zwWm%Dxh><1;29i0Mw?vtid6~JCnLl(xn-F%>0FR#u3l~Y3 zuZMb^fQ`A>nIxH)x`&cH7@4jJh?hxAv?)eMaGSUZ1wWvhx!Ie(=>xzSoWhv{!&w8y zd7Q|Z0?G*j%h{YZkd;+AchRYsyy;GN&e4&nbfp7B7Q;(4Ct z*`4aSp4#aT+3B9|8K3YupYmCs_DP@lke~WF4f~m&{OO(U!9l8r2>Y*Syq9Q7yq#z6@nhCiuqA9AP zE~*JC>If%VFK)31wgi9;nV99YnAQn(*r^ZpP!B>nq~%GZMS7&`fTS6!q)Zy5K#HGG z8l~+irBvFVR_dNlnx#&-8m8A9pKdJbt?qH4;fZJG{inxsrB zr*q1rWx$on=LKL%of4T`S9qNkX?9p@rgd7VaB8UFxusP)pHu%jsQj>~{Sc{F+NhMO zrBiyTVoIi&x}}=hsb~7BXd0()I;x~vs?yL6ri!Yms;aBH4!UrsU0IGgnv>6VVaAko zw)mLn<(`DvtA~oHhN`HL%B%W0tdcsZm1?P8im98*o!!Z)o@%Df8m*ujs=q3#t?H)A zz^d8Gt=?*?uo|7R+K>-9mR!}Nxhi(&HK@J{tk(*x+nJ@timdbcsPw9=%-XEfI<5TL zulri9*7~jjtF7I7s^40$bV{nPil7@1Ap0^OM z$J(*?ny(%Ut?T)(D4VhXJFqONTTesGFvTIAQS-ZDe+qD5mt`ECnV~egp>krV% zwQ6g$@;a<;3%B^X4_2zL&MK;QTe($Bw|uL&naj0a+mK%ymf3h@y49mV8?tGKN@z;S!P{OZ68+`uRtzS8@^60EylaJ_qq!P`r^ zg?qXx?7>ZXydsRW2yDU+oWel-!9G01ES$SZ+PQcN12UXrwn}|JE4=>e4mXUji7LQ5 zT)@gZxhq`6LEObeyu}eLzb%XoM~tIkaJ^nhX$7#x8wHLO83x^Ju^o)VLM+BYEToHj z#Z^kO&-=w({KH=C$A4_ZxqAhognb|IxiSB+#BAKijU2pC?4*QB$9o&VdK|)WtHod( z41*lT3CzHoY{#8U$T{iWh-j4cDYx)+l}0hV9mAt=5d) z)-Rpdkp0&69N9H}bK3{DW0=zmCPz)?Z!#GJiU-G85ZEK21>GsusGZt{-Pn^|)+%ZZ zudUdu9nFjV*t5;nww>ECjnsr4*BY(O*muh{< zE!&cP+ty9ny4~8@P2GsC-IV{WwKfg7myHEIr_*hCJ>U7w-}n9CwY}dCj?1`R z;YW?(h#TH^D7N1vOc5q*q`lq=zTnTz;3!@U8gAe99pMsA;rOlJ7QW&%4&fLr)m_=) zpI<2%|0J-kIb^UglIj<2LT(P5$I;e&%T&);K<$Tdn0EuGf1VYCaCy2;SaR;KS8m z=1ZQ~Q@-Ye-sWz8=nel)r*Yl_h&+aLPL|YnOnI)3UmoTOcnjbF=wzPg7mnsgKIq+@ z;fBuXp5E!Ej_7Yr>bF3gIEv*sU3jAVRC=9A%VYuOSm`Bh!kbPEyN>FC{_CSo>Zh*e zzFzF9e$tEH=z*A%<)yRcJ#!1`5DpM@dq_|8E@RSe&9Uc0Qvs&LjUzhuk=l?@<}iDW53(ey$9}oe)NZEApUOk zuJ!lcg+A}~b`SPLZ}!&A^klF1e*gD?U-sW%^lN{Z-ga&dXJI=(_gsIXcaQfZpZAh~ z_JV)#fq(LsPx;3z^T0iSjBMvt|5lzIev}?{iP!Vj4*A-S`Kv$qnxFYcU-@AV`?PQL z1P$?wKCW$;1Ov8Ghkx;KU+ebn1?q_UB{N_*YuMhnbF8$H}{Dc4g@K4xk0Ion$zdzM+=#}pj zQHP&=oE>~|4-gMV7>Sd`!NCSdNR>keuAxJS*(64sSdo~;UKf{X+{n>mv0)*jffPv+ z+R2h5Q>s+fO`5QlFIU23In$;~nKg0dJjZiqPoFVw4kcRDC^e)>u`N}~wCTF1?&3v- zT2-D^te|oj!NDVj6DnSrOaU8aND{OaCPYv>gaQZ+A6N_==wb#Dw0Z58;ETY*SAk3l zBDA28Aw|Rw6JxB#_@mCFkb_SCDLFFb%R()q%>20XW~xp@V-+Ww^i|ZSukQIG_G{R& zVY8U!f~EHV+O}|~uvx>K_uad6#OUR#BQD^9eZmknyjbyb=O#Tv2VI@>XX}-(W5(V+ z`*iT%O^^OIp1SJP@U-@D4ZG}Y+aj7IQJXtMZ!-8f0k>-#%ig{Mb0E1fm|Ms(1f%OH zLFv5P?m_V&bPy=*BAhTh46`#2Jr2|JFtzpC2B#9hy$qP3u?@1_~j8aF`_Sj0q*xn;TJ`~?d zXhr=@1WvdUW0bK08q@H?z{99ikii9+Y;w*c>r9ePIrAiM&(fy!)0+>iERKiS#3IX! zEf6jL;WoAs*x}1CG2ySa+st%pKqfF9OaTkNe6vG1ffRLAJ4>ZdNjy>2Q`J=)?DJ14 z`OL9OD|P_P3np3^QOhx!AOV6#Wq_276jTHd1xqcRF;fi$OM}w}3OmrvPvf+fRBSB_ zl~q%_P1W00V+9pfay=B(&|C@ClCLs@9ky6DBt3Rs;{oVpT_w&ktzX^EpocTUq_^W=AGMcIG)6A@=zx=US?QLkbz0?=4<1rsm}6%D zcxH9$iY-z9%zW!ofsVtNW1xdx7G#lcbYvW)pU%jbrJI%-YLn@9J8H|Q*1PI)&AqZK zTx;GMth2hDPp`bTAbZ{sKEAhTXldBDZOY@m`*O_f*1Yby2j2U0S*PlI9IRo37>z`S z8xA*%Mfm#l#gp}t*^r+FAm6nshkJL<{q5cI&NmMp@47_?oo|x;emLvXyMD1P{y6yc z@MO#On6Ae=)m{+YwekLY-`{3>eDX6le|T-t_uTm7k+*t|znO=P;;tbeLJa&V)gS+# zgFch?FxU=vZR%e5#AiU@l}~Q!`(650)jsw?FnO~ooqGmnIJ{&KK=)f+|42~(FYMJ$ zdjqmVI0|?`Kp`-J3~ZqA&iBIgxle+lGa)&;5I+lEuwCv7RwSL=GnI^6sah>jECjmnUkBQ7zC;A0~1 zqzJ_~#&L=L!L?%<2td!*|o5{jfIunvItmPdmxGO6E^vXnx`lSy? zDMSI1kZs6ZWjUqkN_3`^j@67HJEd~NJnGR2%4!t<{K$Y{iY+dRLZv#F$xMKrGnxa1 zWkCsQKzH7fmJ@_$JP&0S8R3S1)T`$=iD}9-G%jCeASWXK`OIk^w2A&CsYl(I(1nhZ zf?C8T>U2;)i@8BegP>`dG)T7k2r>>Ih#W^tYEF>;RHOzKYDiN`O)MsLr4G%eCtup7 znF@yvl_1lJta?-X05F^w@M%Yt3e=zqG^|A(Dl1WX)EupWrJb~uTk4}0nZjrbQl)A> zhj|=+hD{A%_3B5*ddI(#wXEhuDq7#MR8AT}k7L-T^dvgOwoJAE6EnEL2C7%FoHdK5 z7}Y4I_}bUciq)tl73^R`dsfs|X{`@M98^^Y*}^#%r^3AJJ~Mhd&w`e;0p+b|O-t0$ zhR?96J#29sDmZjOG&jK#&P!?9+`2#yXq2VxMe{nKE9f?`+YK&oNBdpwDzu&ACGT*_ zflK6$g=lA?lw;vaF}e-{wjLzqUkZg=u0BS)y#;S~W9i?r64#dJEv{~rfHI~U|sv-aI7ezS|l{q9!7zU450`ODw(j@P^eR%w)`uv(sRjj@=0uXCR}QR~sO zzRBrqelxt`k@i=|IrcD)%NJq+lbE3>Zfb(x+TsTLO1dBaWZVgaY2DlE^|ToN>WzOJ z-X2dG$XgDwki~J_o7t>H+(fc`7X?b$##o=6weUx_VPzj@SP#XgY>d?n=Ez#bO9Y`+1eC9 zg{Cp084YPi?^xB|tuvM-&E-l9`O=wQFN}J*;1|Ex))VND>_}Z|?P+1ki#~O#tGwz* z9~;uJW_GhYbn*ml#442XG7S00#eV*m=7`1sDPUUjUO{NzFBde=pM zaFh={vf)6wOj{bYlWB+1B;P<_v&~JfEef7y7zw%vwe%RZ-`9EL&?KPkK%}>AKx`gPi zm!5s?J6Qri=)Pc3pL)47KLJD{=UYG8Gr;ycz`lb&N?SSVYb##KAc@Pr;H$uV6Quu3 zfdE7d@{7CybTtH2zXU|N=Ls+`$s^!XG3`$6Br!J}ya;^3d$YCJVlf${!7FSQV*p?#JLqFuiA_PPb1jSG^ zL^V7_L^MTIM7QOcKSs1Hr?MEF*(4d*w@EYrO58pyq=Fsfls)W2UlhesG{XWUMROR$ z0$W5@gg7oavH{qv7k~kzbE{gsMKVGi=IF&=%*J38#%(0VPy9wt1V?d9L~?Av2TU$y z>=KsK6c$l|X{5#*(5h=Bu}#!QVC1oG%tw9n#$rsxa!f(_1Ds@pF=k|eHQ0?kzz=t{ zMTWdZO4O-cEF^lYM|%`Rd_+V4ejLY)^v5Lp$4Wy+#bPUU+$uDrv`v@OkCfiLJt z!SqbR9L2*F&C>i$-vrM8sl-jiJS@1(CDcq8%6uK%vdM#7NKuo(zWlzhT*=(j&D~VD z(cDhq%ueo9%;Q8CbX2>_S~3WHs+^I{*<`}Zv`*}NPtZI=i|kJN98UYZPyE!qx4~m4bJ*}P|`%fG6+2JBs}{o&-+`hib;UChyd#P z8Usze+tfw{4MP(>%Le7m@5HhEEY7#wwVO=M4&|UJq#pfwM89Z83c#umMbHHmLm>6e z6TQb2bx%%_*)lnJ4 z7>%0L1C;_Vz0^)M($B=yO^sDF4b@v6)KP6k_Iy(XW7OaS8rV^?06DS!V^tA-)x)sV zAZ=D>eO6mF!ZHoi2KCfGwANh3RR-Kuxn!-JTpeKTBYXpv9g5UrZO>M%3TA~?Sf$k! zrPgfKR*!6rmIGH)<+A}DHFIUN9dy-pT}m#TRawTY#ZyYX*mZ5$>*p2tkv3#{V=ZOSVa@tr48Gs z?Y%k-BYW-H>8x6~!;!az+pjHLx+PjN_1Xwk+M3PV3?)`z5>UUT*r*c%YZzR@ja%45 zT$y!H#=Tq6^*hIHy2o|Mo{bi^9SB^xTB60=%zfRt6-Ul3!P)IxyZv0;4c)`y1hPfc z`?IS5K)W{Ts<_j&uawQe%XMANRos}3-P?U$+#KCTsX+j-r%oG$j(Gy@m0X`~TWD!r z<%M0tRi@hgu<~VI^vzxMUEgxN1?m0I%5p9vYc5aFgoI2_(?wP9?a5mR-vvG2X|>&h zWnbMb;Er@!Nc23B>>-|PAM+G*in4}Mv*6yeP!;sAzM z69xuuP-0*};cHl77QWl}C09#(h2kdV-9f28Z`Hp32wv!Nfjh_pGq?ixs$lK|#BE#IHd9W#kgasEUuEb5Y`U66o*n$kuubcv}j>t0#Yci(U5AH{^ zKHL<>1Xth&SfGWcZc9&%YQ=k|Mxxh_?bj{?*QRaT&V(;01KZAR-R5oo-~MeW2yWrd zf#D`@<1TI_IBw-O?&YQd;}(MFZf@xAfasp?=6-JOJA&(8?km{dGI%KO*4{l9@0c#{ zfIM#smA`#kV7UhG3dmtgz-M6*R-Tqr%p{}~NN)W`?&9X}?f!4$&SU`}a00)}W;}2N zr&MT3a0U+>1z&Ioe}D&{a0;*R2)A$y&v2mL@QB23(vwv4hSFUbaS4{{+Nat`-!Xf$yoPjV&4k`sTo z&aNjGw*vcJf_vs-eJ*XghB^)SaU$<>FXwVF|MCkLb29I6Ge2|xxj0AqTk|E?r8Zac z!Go;Ie5?FmafUO44j}E;+323W?=T;8Ko9gXAM`&b^g=&$L>F`)Pjfeab4PFVHkV^P zlItA>yrj$KJV${m@5wA@0m3=+B3E=nPjpdta8o~ZBVRM3RQ3FB^)!EVSBLdjpLJQM z^;);}xybcfKXP8@zaoPVU=McBo3>#`l!rTZWKVWwFWb>>Z_8SX0M}>^-RSy8x)0x) zo(Xnu-S(V46r1^W2V6ODANO!y0&*{R^FDWUC-)WMcJdbYZ#Nco=RS7#_I0oKZa4RL zr+0XtcYgnOf5&%izezTS^hbwh<)ZWsg}FAk1cwj2o;?BoGD@**uXu~kc73<^i)Ss3 zkMnkS_j?EVJ+t;B=T&s4Q&;!*RsV1gIN5pkaBjZ$Kfdr{`S%~-zncej7|>pu$N5n2 zcX%EV8W9|j*A-05v%w3YmI{MsS9svVXj@RLO|Pv~2l2Dh_04*_xiCAQhxuMdZ1%*bJMlOOTu z8hgW6^MgltreF9u?;{+p1#2LKr!!pe67cN8`%82_xtotdk4RIvcLP#Km1z( zeT=(#y~lZ;SN)Y=ebs0Ef1ma2j{Tm$d)c@B*?)ci&hLHQ_x-K^{oMEcahInsJAULx zc;)YSijG>gKmwbFgAn-De8rTE7x;M(_}fQ(neTq@PxtTddF>bf@F)NBH-Gfs_tqc& z^v8ENPyQ8z|DdNB#;1G=Om8mG=ODuT0|kf-0x?(^sKMYt7Dt{eY3T4FLx@Fy+<@`o zBF2mwH};_D@#930Ka5lyS<>W594S|_Z0XYCOPDfea*V0a22GqOYv}B`(Y-`2R1kaC_3H>`-L|d0L&sdddw278;QM!P-MVKEE*_CEwcwP9w<ilv7%HC6-u{h~AbV1n?z@Bd%Bf8<_2(Sdd^d!X%JIbnz4*Uj*S*#EUb=c&Cnb z=BX#2efqg)j&Szq+@PE-RV1Pw5gDhDf)2@3L`_Qi4wc>7;PnZ{kj{RUg?aGEK}(AufU?K8*p|9S z2e+)UhaFyy=~i5Gt@)ygcRg|cue~t?;IFXw%1ATLInTMI8cM|no;q|;qA>IbL( z#zOJ4F1pdF+wZ>7%93CIwW=v_ob~Q>XE*^Ot_)s#O9_-9p)hcPfmnDT1hXJP3Qe$r z<-6ckzJtBcfNy&m^qvPn*gfrau!Q0}AqG>3LJP9+HcTNQcYannxe1|$Hl!iZa;U=@ zo(_jOtYQ9QSHZ@Kz$--n;M4-B#KVp2cjgJxwxZaY5>PREMZ6*wiI~MLO2{)s+yfY; za4Iq0B6!@Bo|mksmJGPOhkeu7mA_#`M1Aj(mal8rMvh$%^_rZ)1A z0Iqx`0vt%5QRaaE6C}&QHM;PHFn&!|xVa)Luc*Z$rm~pETuj(D$1=I(5l+gS)8|U* zOg-Ytkc#vpHnaIhL~>J+IPB&(&vlq!N)ePVK;k9`*t=96(351^mL97(#bR=R1F)3l z5a^&!H>6CI?CesRW^jUmdPM_NJm@e(b*2F@^qvk)rd^z~qHzY(h1hw(LQ!cm5D1ir zjZ1<>sn}3zA|{)G1Sd*U8cu6&v!y95SP*p31lvio1LHv_0oBPnX}a^B7x-ENxgb!k zu#uGTbf^gQDM4V?w4I?OrSKm3QDUZYpvENANVD3|G;VdEGkv30xjF!^PLHff7;8bb zI?=Nt^aLRPeQSAOxltd`wXSwWO3jHq zeWFj%g3!qNlmL>1Cj(*Q)Xlnbs#8VkRXHo#S9-Q_VZ|o_((+Y*5@86bZRIIj*4e|- z^h*!rlP_OYQgi*W1{iSPZ$}bUgNjwH&@^N=F~Xz8F%Ys<>hjux~=Oy~*RJKM@uZM?R9E_>U%UE`*)wJ}X& zMbc;28`MD!da%M4xoh6r$Tz>LTCfrMm))i=L9sby>?SCBUF<&gsS}Q_C1t8WeVTy2 z0c9}%YyGQ))V@`&|8+2DQ^nN;4p#(95FS3yG}Hag_no2Dua9p#<9B)oXl8gX>B#U; z1(TMNY3;F)gK*y&+_fpJs00irqQWDJV3mUqyjT z@99Dqw-c>!fpCovn&YCn0L(#d6_5k1-?$!D1_w5nYbPQID3r^!PQ@>C-9y?xKYFrP zMl3ojMZh!oX2TrLE)+jmPji9_#AjBjP{m9e!+MFqqxLg79YSXx$CS;?1Tan6IRNpc z(!Q0Zw4j0gXF}U%*9_@je0M_LqYj(ch|P65U2y55K7m0I8r`#Y#{_~j`?2+eGZ9k% zUFC11c?Qj%tfi-2V7LL8)NL)0 zn#Z`Dr3MSt>rC~Q3z^Qk<}uH4p7Vh#H|(>fch|RiuAS6RFl~SP(ILr_yeYlz6Tds& zBj)tHL;Vtb@B81A?)OWm4u6LK!r|er1%;{Z=M87=(Bmih$wO+uYrj0^LlAlY$GaOj zna4bWL!Se~ABXIwAGpyKKfj}Q-~h<%IqYK(@ga512X?=Gf#aV0-3x;E*v9=G_~3is z2fp`#I6ftcKX|}1zwg|Lg)?Lk{pmZS`qsZb_PLRL?Q>%qK&c%~GGq%&WWN^Xf5iFG zuYN*ILUitbe;eXY_tl+n_3+ny-SMyg{ma3?{U3ht@gAemAKcj={{dhCCZGc5!2$}O z0}>wuKA;2|0`f&d1tuQ`N`eJy;PN$}2VMg8RUh<;VEL6`GMpd^qF^g{pHEa*yhsfh z;DQWdU;3fo`rY6o$e#}GpaI?=5B}i);eikiVGtsq5f&j4HlPw7VFWV&AQL*_1TJ9| zP9YT*LK7|m7G_~0_`w!Z;00#j267-6j^PK69|?w_38rBRYTpr5O2h#{g0zbQ91+62 zfgRQ$4z6D^z+Vpbp&tUF4+f(D0pb%Dq7)|KAy%OxBH|)8;vXcUBXXf6cHtpZq8C!) zC5E9UZXy|WU?+Ow)0rU}sv#+AAN)YUsnFF_umPP&K#~~J8=RmX&SDK4T+%BqRKz zNcQ72j$||jBub)TN)F^63KBwoK%X$0Db}Jx@?k_mq(tK5MPj5!a%4~Hq)+}NP}*Zh z7Nt;vBuSQ}QYIxRisDMLWc3BoO9mE0He^lOBu*M)MII$4a-~-WrBM!LSSkWpnj}-6 z5~W)DlU<#&Y7Uo!Xrav10rf4E2X?Egbo+fHyUmoF% z?U^A>+NEStrdDodSJozI-X?B(W@sX%ZuX{cmL_lpXJRU5YCar*Mv^QW9rzo}Y29roSj8`9)@IQm1vs z=XGu;eQGCdcIS7FCVrA9fA(iJn&)|@XJf8sYeJ`Vz9)QEXJzImB4DR@LIQ-=Cxbqy ze^#i40_1-ND0-r%PSnS8E@y#8CpjKyf|h7b&Zm7kD1}O>imvE|zNmf%XK@Z5YPRS3 z*%6F=9~uVpbDy-7HXX)DwpD^ou=uYf~lUCslSA%ni^`GR;iTU=bIub zrYb67}-+e7@?MhH9sNs;7c#qu%PM=Bl2Ws!C{7rT!_c zx+;~5C6{jMtk&wW%Brp|E2%DJq(Wz}Le8pMs<39NrUt9ACM&XTtFkgHu6Aj&k{_k2 zs^##J`&H|qS}S*|>!M;Sx3+7yeygL(t7nG)tEocixRPsZQERrkE4#v@h4$;aYAd|b zYrJ}EX}YFV_-em6>64=CzecQ^670YptiKj4!p^I`QsccwtDnXy#7ZouQtZW6Y{{A| z##*ewZtTN8Y@ZUSi3%*rUhB!$tjf-8%C2n2-s_DH=&xEr%-Sr?hNP{MtkL4^%_i-r zF08cP%DIMYx(=<-9__>`tf+P19A`fS6t z4%mL})WYhzTJ6}LE#0QA#x5<}?n?@oBghu4*k-NW#_in}F5a@N+KTJm>I)6PLG#sZ z;2N&v4({P9-@!7ihXPCA0`1#E?mB+|rooPE*_y5APHx^N?rw7J&vq`{imW33-RKH# z>BcVM)-L4kEb0!P>b5QCx~}hnZspRh=~C|SnjwGAO6KaT!yYK$%I)03uIv`C@Jes+ z<|^iPZST(R?^18|aw^zJY5>MydQWQESJzOJg{mM?JLF9Jg^|6(lzyYB+W?*lt9_X_X;6GQOFb) zf3X+K@Dz7%{a!JFHgWT!FBp%p0<-ZNOK}($Y7w;M$jC(naBf&Lt7kf>0jMUNI#nzU(9q&G=Qol4bY%#v0`V!fJmmDjIe zXFe4>HmcdPXv_Ls%eJjrw{V@tB{dhS+`Dzta^2gtuiw90wek&1RCS;Q>keJJ zaBtVMgA0c(n|Sf#On>Lz&07?6+qZoypALL@_3PGIYu}E2`Sa$`y+{8aPrZEi^W4*` z&yJov?(gB>yJv5{KK=FgDYg%^sL;eicyxFLuuBBzQF6mg2P)3QQl1WZUWs^Yqm?f4(O6gpeIL=sQm`*0A<(6lb zNv4=!T6v|LYI2#TnRAZ$rk8lGX=j`+(pjgUl*#!?kc7gS=b?A}C_=%tTl+NY+8ve~JgmWB#xkC<-S3a6hMS!k-BvYINXrJDbGDxS~*!=6Y+a zvz7{Kt*cUstFFWr%PXw38Y^t2%kJvzv5sE)YqQi=tL&@DLW|~twsvdnx6?kWt*lF` zs*AVaGMlct;%1v{w6ci^ON79#J8Zl7x~ng_K89=WzW|RaFscHhYw(u=-+S%C@9s;m zCJ;XiZ^Qnwn{UMz^Q$qx2X}07y$KJz?Z_l=Y_h2sD-80-49h6-$>YA*DpT%-4{Q%{QURtzx@Ebp8x~+zWv4T zfd3<)0`tef0pibr1#BMz7f8VdTCjo-#GvRVs6h^T5O{Ow-~~rW!V;QLgl-|<2Uoa< z>$T8@vI`*_VE95A#;}GqM4b!+v4$P`@P|OeVG#d^I7Gtro*^r=wQr#CM;RjT&VryKpK zR=3L4p`w$L$&{%`$4b_+K2xk{wW?Zk%2u6H6suSjsXXQBPNz2Yu6WI>UH2MQzWOzo z+cfD~2kTXh&Q+IP4QgURdQ-DH_OWS=Y*i;aN1R#~tgkvh5mG>ToCMG-&t3^ z_SLkg{p)E}OHVzTbPQssBV}h>)zSYN*0zhyEjyF2*|1WQw~z&Ha3xFJi?;KmFr{f) zISbdX2DP_DXliLwOI@Qnm$lek?LJSM20Fl&4#%Y}Z41lP^5T}fnN3kXaLOfdsF0Lw=D9(%v%Ho6M!_J*!J#`x0Tk+Rbi)pXuEOcY%)Ljc;eotJ?~n z)4#J6?qgAk)-AY~s>xOBFC`pPlFGHM|E=p~13cFR=asws9j$_AjL#Y5mkxu-u^T!t z*zq3rw&%6*b1lnKvT|6%>-{i%jVnhc*Auf)zG{Cxn_`i2_yRE&u!{>EWEs;~yC}u6 zj&Tg(9#eM6T^2H$!&>BN?w9|)A3kJ=oebdH-tw4t9x;lmJYSR=6TWFCFhEy~)i0km zv|Ftj0B1GO2|#S5Cp~4~@;Avyj~g>K$i=QHp`*<1o?ds<&X#AipN&`K3Kqda zoi4rq8s=-Wfydh3_JqNm=1Mct+z2N&xdrZ`L49f#D?8&DkGGdA zy>UbLThSc|!KRJP({_vC=}?ck&L=DEmRkMlCeFIer_Ob!N4>IA)}+gU{#-eZ{pBsU zaM}ZU-;Fvz0R%w6w@*HFTJzJjxEAx1tBPnHcOu7suSDPlKlodZxetgxyy6#6c*gtv z@2`Now|Q{)^XnT zM)H6M2zWc%mumOBbNBSJr6|ckPyA~y9=4_Tg7TN2h33az_RvqH^rKIG>L|5Hx?{zeF8{%z0y{o5b^ z)wf6U2YZ40SJKpH-Qu=f!6nc4)}c;_<0Ssfewg)s)q*Y z2Y<1rekK@!7MOy!SAi=C1TEMCH~<4I0E06)0ySuZIQRiNr~y03gFg6!K-d65D1<~f zghhCSL70O|Xi!PlgidHrq7_wvb!AkiejfOM%lCp>NP{=HgkH#nV0e302!dBghGi&% z-L-yZxQ1xhhHglLYB+*#SblI=hXZ(rc$kMMxQAO9gMD}dF#w1NRSEN^TH=*{Aee=? z7lw@Zg^mBHg%{X>9H@b3IDU3`i6@we^T&rQ*n%q9hoBgUei(|6IEtotiY`csomhXY z*ot}>NF#s+CiiP>$8hCmd6&0&-`9DR*o(MGdB7Nq!Z?h?h>M(OjJlYN$H^c4 z@)(WN$cE>4jhLv3oS1@n2#4$kjh%;#eGmpt&;lA@jx9xDzGsWz7zep{eCAk+>bQjH zSdlQuj)@14@2HW>xQp}HfgebX);Ndy_=*9DiW6ysDro^LiG)e0j*xhNB4~pM$&J~y zX}ACQkctO^pQwtSXo24dejPcK-ba+j_mTA|lK7a9CAo?Chmwz&l2SR9RGEWgzyvg@ zjS1O}w1s{;X?V`plTew8F)5Somyz+6d^Rn19)Ygjtx5h>A1$0kXG^g=mOWb&s(1m5G;zPdS!`$%tinmTq~B0Ev`f zIf-)_l1gcpce#&Fsfw5Rg*~{Mtl63`>6$Rfg{6mkiz$nZ$!}F?emEwXh-Z3J#7e+v z2TmlM#3=|ZbezUXMKgp6n*a;b@SM;YozgiC%?X{H;GEDojMf>Cg4dkhxt8=ensxsv zdgQrETx6c;$wj}3p1~;x?dhI(@SgISN??FWS8$*BnV@du1&V2t zH~E-n8IC%65B|Up`tYFj5TOz}p%hx7@!$^afT0(Y+h8q#atM>TskSnhr_2 zq)fUFO8TTudJa+wrBsRyO-c<|nx$H5rCe$aUTOw$qZ5`>sDw(ShI*)Q>ZOXhsEq&m437G! zQz{LT8mW}p4U>AQm1?P)x~ZHRrAcZ^vZ;-+sFiL2Q(Sqb*O#V`x28f$p^56IOgf== zil-o2qO(e?dz!1d>Zg5*sD=8gz8b7?I;@Ratgi~GnTn~%nykvote(oDpel;ZC#__9 zs;JtIY#dxtGmjp=o+ZNYN+eluID1G-fFDNI1dEaw@L0dJE-ht_^#x5WA!BDzWZ5vGHo8@|v;7 zim%PevHI$yx?l$tsfbiitp|#rIhl(DyR8O`up0}cFsrb3I;Riouo3@zt`r-u6)UVf zsME!Y;w_^LU zf@`*fi?&g!wrgv$+U2(6x1cO5w{r`pG;6N9fVFu$q7loc?|QMjz^H&5wxEl#WIMQJ zTeyY`tzfveQ=qu3#kkmddD{21EE~CVJGqyOxiy=+di%Aa8?>Q|qk$W|qg%SBTf9q( zxDyy!N-Ko{`?fj;sE|v$y_=)tdb`%UyAsQ}efzuIJH5k8y!QWUyq!9}O^UocAfT*! zTCS_0u)DIeE4_movzB|kx{JM<`@Npqz4}YK{0qM1`@aBuyd&F+>ASSe+qmuPqR|V# zNou=zO274+t}v^;!@Ix2OTqXXz~Xzl^nkejTB-`UT?hQSPW!$L9KUt@zz`h4y*j}e zT(;hO!SMRR{@bxN%)vzK!7XTdGtgoNe61urX1rj+y?esL>cA_^!VlZRE=<18VtZzT(Yp$ZTK|D zblh4a+?J9j#6%p%UFxmWtHNZwt3HasYh1``EX8cx#%}){s-w8NaokmP9LY?Jz$LuL zeay#QI=_L8yE$vfX#B~D?8%9Y4sHC#SZs0DI$4m}UG3|TmW;!SEV-Jzt+cGQdwaj4 z49dEE$hkbq`RY*8ioSse$84~?e6_x;oWQRP%a`2D&U_7HT)og7qtm>#*37j;+RM8f zzEjN0+8oQLYsy*7d#J|6$*g$G%*;{@&Fl=u_|VSq9M3V_#7Eq<%QwxoJFQTOdwV&igg||;8yyW=z12&tv6*1hNgdN-J=QRN z(PrJ%U|q_LY?fM#T~oMZ^wj{KSk*aL3Qb+qS)JE~{MBiV)@1$HXYJQ`J=l8f)N2jZ zH~Da2b&yUVgii2xMTi8VaMy)h*?le8VO`dkec34u&~ofoHr8iQ#@Iag*l`fqpg`HA ze7cs+)xtp7;LF#tz1e}S(X|cOntj{0P1`h$x-=+(JeJ8g0jjYd| z-Pb+cyFCo+UBmAU-$H#0!0fNn7ln!4fy2Gn6{&sa&D!W~-3ZR!ihAD+uHex9;L;u6 z2|nQz&e^fN-~4^FTd;DD30>lSdyY-uL@3$Hec<(7;e&nO?Tz0Mj^ZvZ;V;hO9DUd| z$>D#+O|SM=4lv-SJ>s5V;H#Yt{cPeVe!~qe6tj<(zV+a)`yfrJG%4%!PR;5^Rd zU*6_m-sgT!P0h--!R7=wiO&jLzmIuHTHDmF4|R&1P3u-fA_qWXgYXI|{aKIp6t?2Yc|ZjO!-DTaB3>5fT!bhqi&P6sUM>AU{x+^+1*PUyrQ z?#Djt<4*3#?(Jx;+>j3EI9P%@Aa{#yrjdbLeom*v+Eg!r!S6kqY^e(|VY?!`;+9N+O6Z}10yxKGWFXJyRf zrA&6@*g}}o+5UEQ;0+ef>%{-C48>6MHh=RUU+_4e^8v5%9@*EtZTSh;@! zNOyJj)=M@5wr&Bq-tsG81~FgtJOA@ofAv7$^MIc9UjOexZ}p(Q?!m0^Vq|qpCDj4v z^s#vB-6i!eFZNnL_g_!d@t;lulHRa%a_mhey`475OvT8>%tXv{*-FSwOPD1Ul5<|v&r}@ zN%fDP`9Xj8oxl0F-|?7V`?C+~o{HhXU64s&llIkLE+yb?ulV*p^}S#DlmGnC&->Ev z^SICN(eL@wKlG~&_|X53hXQ6~Q&>&&zVfe6gt89~GN1l5um0;#`|h9pns5ENKmXSc z|I2=`+>g`wCx4O#5IRJ3&;TODh=V(3EKJe_%R&<+h726xq+km#aYg{J@Bo9EZ0Nv+ zBw5nr$!g0`Ml06RB{X6$Sq`H)(`L+@%4+TuDJ0v$=J@yOl#Bj<#^;(nYL82?q##30RmiC(0V=neGXPdr#`%-=fcRPi+-MW<9VY%nJAmfZR zFo6g_BChL8y9T)9$Rm)#v+zRrGE6T%_n^`cH4r-t@xu{2V~s=+=Tq@M-0*Wzt^MAL zYdE|_P%OCw2K)#*Gz3$S2r+_m!NCU^t4N~>C8JCm44X6&MfsxSFvTfLg!0PuR%~(0 z`dHj8BpCY>j>i8c&W>E`)5*LGLaKKaPm$p^Q@9eDf#r%%000ZM@udX zb@5A3d;|Y31}}!2tFDF~I+KDE`PAP_oC0~>NvIQ_D;J0(dq_1RdXZFO2#sg?HH zK55m~P*!o>L(%?tl`)Km(s1+4G}pxBM`SyV(8w&1jLc4F>%|t;d#$ZDUqQR&_Fr6e z(-jAD#f@Tzf`1k8qULyH<5-A^{Pf5cMh)0rXko>dUw!x8xHXP#-B#qY5VfL*ws=6p z({b^->pH#;ij3iiJ0)SFiQ}z!-jFjMPh*dJ#<*jkixzrhfRnyBEpQoS`CxLXFb*Pg z9V-9Mz%Xh?LE;ABO=D*Yk1o|{v4bAkY_ikF7HN|Gy?9`$4K||3sKr=lQcCj{Q0uLW z+=ya}mmd4Cpxq7L1?gu&2^y|?8@!$R_P775 z%RQt7dK+K~%|yKGbi{bbbD-*62*KFJFoPGQAq8hBz5HD-hu&(MjKox~0GW`0LqwrM zE;70?94|WvhH4B%Yx5=fUcF42i_WF#JqAxAqB zEs~R@H&6KCzK;jG!eeNyj;s4U@BMVHZ3} zrZB?HcZLyS8BK{uHL`J)s&r&AUkS-$CexLdq~#_1=!8$cta||rUl18UNMHYY>XeC; zzzc*qOkx^SoUw$ZGMCv*bb9cXA!Lvk%>=wb#Itv#Eaki26$dJC)0^Q0Co44>J#*@i zoXE6dIt^;Fl&Q>7#6goaFA%=>xq$9oIL7Dju zPBzU~Gos50T>4VJjDeS1C`?3ehC+nI<%}6krAK}0&v6cvq(g0GNKb0gO06Mie6_|nDZpt^)v1O;h zq)%7v8q}}mb*twr=va?xOS2xVr6i=LVXz=kifj(944Z3D?+R4N78U=XVclyuD(h9e ze)J`NMJHgx^~rWt%?-u?(?Bf0iD)NgO3Iub52KOc=wmaBS-n#BU$0G{W;wgb9C}u6 zES2U4@b*x{*3_oiRINgJnjMh27Php#>~dw>T;>w=wkOpsOK}-K0mfmphn46cd|B12 z9(M(H=%{ifJK4-~@VU`FY75EXKzJO33@TvDZly*#ANn@C+pQ@WFYs3J*=Testd2fg zYu?C0H?|61uX8aw-S_UWx_)$-Lph*b(z-P-zNKngvSZvtzJR<4POyU)EYR{^(zfYs zFK$&lg>@?QslUDOb^|2Un2irP6Dx3tM@(WMGg!z&HgSVjd}PZaxy4KW)(wmowNQo9 z6uX7pa0&Vw(OL?$zejhy5nml?t+WpkU|oEs-U*}6JI%x`5H z+~ERPmwMrG7`~ihF$Y@E>^-rX)68TSNBG4trZa#BY31(X7zKy%T$b_1Wkz}7&tdjO zr#(GrLlgSUp;j`e5$)bL!5%7$n3h7Ffs`;sZSV0T4@oiD2gRr%io~Umu#( zfG+j0i@oM?q*~dmLzZmzJk8Nyp z6I?U~s!3}=ycPAXxZ*I1I zWgJKgi*ny4v^5j_-4R`n_uJqOcgF`V@PmiE&k-h97%o~Kzg!UHpOmnPX zQ|tWRcf~J$0F3Y4=VM2@&_UjGrXPLXY7hC-&5?;XPhG|}syWvw&H}G#0PH%iYuSnZ z_q3y3)NCibvEzGkCO7=PbIzsAv%dAZ4;WPc-n+)h?svdvKJXJhk{UKk1XZ zOv@Z0Nj*x_w~52191%V$ID>8TKAs!D0c-{mJV0`DKy!P*=4-+C6THK-ofu@B_?tkv zOBlkEHj26%oZ>TF0|E147AY9P0xUrjOh5!gK^0^{7i_>Kd_kt0LHLV6HnTwrgA4l` zl6>RA&)Go!`$8YIxNdq5AxwuNJi{YAK_z@bCTv3g25dhF9KEP>x!$udN?Rz0y1q1G zxyE85Geko*OhZ9*LSchLH)O*koItAUj72b%fjvxtKJ3I#97Iq=!cY`NQzXSw z^hHFJ!9^@a=ySxE%fL#^n>0EQhsi@s^eqYz6;3RQT)f6Lm?9W++?^>qNDz3& zS6oPVlt(VhxO$Yxh>Xc?%t+?LNQ=Bl8q7%*+YgZh0^WkL>2SOdJdh&r0ZnN~mAt5G zlu4Lm%9$)gntV!}oXUR`N2|QbW4swUJgdAwAZbI$qCCo^R7!`eJS2F@rgX}RtV*eb zNkTu%4p<27$D1e!~mC6OT%nS#JtFUq)WC8$OyE{ zt&EE=B8(=uw+Sf_XQZ!6>l?siEL=N-mh2{LOv{NZ&A434#OxL7BJD5X< zo0~H69?|K6xOub%$V&pbfm>`1(lpKg!(2_(j7ru#&expF-;~$%z(4JPYhMT zL|$xc`IoI6_zxx44qNb)JG2u#v27uH4M@J z4AB97DTk~-NU_Z#8i3_kApK&-2^B9=s81TT(f$0;9F0;Ph0-47(Fpue{uz@HvOt5x zH^{4}xH8D?qs@%7P~d!0HRaI%EjhOuWm760PAr{MWz*8xOrq!nJ|tBDm{QUwh|vla z0!?sI`=rqgmD0D2(<&X$D{WMjvr{N@NI;6b_;e{{WIZu7(=&C1HGR@Ub<^%l)I}9h zM=jN5gVdF}oT$Su;Nv6eL@guPlI^pcQ<8r@*rya&X+7GFrP+7wScdaGjPN3> z>%Tk9*Z6$L&V+#}K-qZdzS9|6mVMcoCD^1@T1G9~r9Io5O(<3DCElzy>io<71469T zTFT?8GVt1#1>0>jTBPk;zg0T4wb?h*g;kqWnoBE^6$7iqL_W=0?z0%Y&0C@kTgoNd zYY1GyWwXNVQ)(I#w_>Z$jR?5CTE~4_t`*wIm0Z0YTe1CH))mKkz=l|`1={sb%uUYC z#kag@D;%>nUgDJh(QVue$d0Xr+|*Uw$!%TNUEZm@++3IiZirs#orT-&*xV%}>a;2@ zK+yp4kM~?$O$5LYJ6_~nT~Fg%=2hPGg}82;L0h+y@7-3j)&MQnv&(BMvh-yXi;*zI7~ z#9vMUVcHZh0n)Z1_}%b1;S>(u9}L1I$$=QIVk^!(8m`JLHeaM=)LpOzb%9Z*&G*kl**^hCv2qLFQmXj=*O!&^6wR^hD8W zd*W9<;3SgeNiNr0F18x3X1}FfUhZT<4&`E2IU)|(BsC#?1r{fkw-tuv7?|QTo#t6y zUrp9!q00qcmftZ3yW7gUJ`>#tilXEuUoQY)xc4u0~$6RK^Lrv&-wq%^7=X#z7 zZop^%`$gN_LFR^iHPq|nIgEfc9_WV*v354-xMXNc#%6hTXpAk24Ju`qKHGi9%YN?M zskH*0g~pTRh#8S)lD=Z8+~Q9~Uz6_R%zbH7MdVH~;)ls-MJAvE*i7||+nnyQkWM2S z7-?r^)SiathL+Rj&F3rK-lU$ht2&68*4;+duUXqa@}bbrR3fYPX|vtx1l&-NX$`#2 zYpmF7zV2(k?rS^P>%h*0!6t0OK5WEB?808`#Qy8QX6(ZTg+0gwPRNAG-h<1=Y|Y+m z&9(#1c1GvG8lv%MLe$ZqV`ZtgZx@2=1@ zotf_*Z}BGY@HX%BM(^_OZtcFy=sa!smb#V$Qi6;Tr$udyK5O8@>>&wgkA7PL+t(#X z5^gH+^j2>KCx`@J@BnCV2Vd~Li*N~_a0)kY3$Jhtzwiv#@D1m14?pnqp4(N0GV3bw zY&~&$GjVa=SsWn(%!UF%sEbF41?TX>h!yY-&+!S_aUR$49{2Gd4{{+Na$WmyATRPG zPjV&iaPS`9)KYOMk8%>1^5N(eex7L*h;cm#Z8&sj>BRMHmhEWq?U*KcHX@Nsrflib9fWC2n~2va|G)=TwC4BFFWbyt7& zR)=+2S9OD+bxc8Z7|3;9UlbeofnGlXI+XIF^vTpNc1j<1c{6rpUv_41_Bt%>daJs+ zp_^%!cAaSySFAcquXJdK+UaC%)iVP%knZmV^&5{mL8o?g&vtficTInHc#ro>$3lAV zc5b(Ke82a6&lGNVc7OMEVn=p?A9yb>^?;|t7clr;Klp{`b%lTOI*)k&gqL)ka=47J z@w%JwZ&62cKVUWLy-El8gctcG*!P3^^~l8cTVHvIw|8{!KA87+5KnfRXDMo2a}aR$ zR*!IkNA}3*@zU#gSgml9H}=59c}gt#ZNGDe*Z2CG_$cQUi~n@}-iu^N22ub6*^cU~ z{S*%u`H`=6xealjzj<1Bawo5NUElc$5BpMQ@AVGfhktvtM{<;(`?t67hJX0s)qA|p z`?hC!zOVbV-}|+Xb-bf^bCi0<-z{QydVQB^7Qa6K?s&6q+@Q~V&ENc@XZX4Ie9(vZ z&{uo0ukX=Uc!pngU{8IaFLu@+_$t3|*>C;RpZ$NQeWhoorHB0g-B;HdKI6 zz$XO*LKj5%Nwnxuq)Lr4-4Vip1PvCZMy+a<>eZ}Tp>pl2z$!(NV#gLCTb8WGv>4aU zZ0mL)+_-8d&aLb5F5bLv&6ez&a_@-1BV_(HiPH%cn`y@X@Ju?i>EoqL6HFFBK?jVl z8L)2c>{;{Y(4a?CW!vzq+|+14fJni*wQJb2XREGl+ca*wdUvljJ7uL|#D{+;&Qjbs zaxco4Ge3joIh#0(*-Qvp*|Fs8*DtdmE&BUr@8G?T2XMgrdGzWZY+g$ncE{eR=f{6< z-@fkryag`#=%36@03J7ma|J?oAcE3GXW)VfHmKl(5K6d{bwEXSp?2GG2U=|6aVDBT z>ybAii5`|H-e@AKxZ+k)ec0Z9^sROujk(!nlYlwu$lw|g`lw(ZZeSuIkwyME9gsmb zreOy;bOB%v19X5LlTWdC&;d1G)Eb9Byl5hrVWOD-9+_o|Ip&#Sz8IE_ZHg7!YjMgs zC!BQ#xFenb_ITuvLoR5fgM21Q9c9~Z~rxBAn@wX6LeDSFQ zrV9nf>UQk$$055rGOV90`0l;D=E^St02{miiM}rv+~|}EKTBB;?0xC5#6A1`bHzam zJv56%Q(P*5V6*`;(@2P{1=Q@4Jn|ZP%sL>xD!buDPlYBRK@c*>40FJ+xKeP_(mL)Jww=Z_-q!Tx+cj zRv6T>V*k21uw)}jug#NTE!=R#1rB&&sjI%yNPM{-1M9HMK6~smt&aNbg$-Cb@1~FL zdvd{J9Y%A-8;@YE>mnX<;>{mnxZ=<^?ZfgP+bjGDm=mzQ%bt60Huz|VelrP4G-3Yv z*bA@z@9mTRKKz~j2Ymder#={Ax!dpm)av!S^gjUo7r@@_?k{X|U7f18JRvZ!feuvQ z10g5{<|%N3%*zw;ut$I&Xl`uZ+r-&|khUsW2mvO%Tmv*9f)ffRg?uYP3sKk;#OQ=? ziONI{=5Y%tD5?xA48aT67Q+-`h6gUp8|Ge^5hM;VZ+Lqm6qzW+6Q;0B&#M8PFrdZr zM34qvT)`H>C`K)g@r!0WV+5rqK?y#OVL(LR5aKw4?D?yMFXLSJ4mLg#QV{`?s-k(C zXhIHkZF^NTqzLGs6UI~#1@!VE&7QQ!9=wr%AAsQxW0J{3>P$+W{A3x!!>};c%`;Ox zA}8y%Ng1xNlPbI-^+GAjS(38Aqjlh$3=>P3)0-H5;(JUsJx^h!zs>jrcwp2loCvU7^h-BbDJV)W)bCR zNfVs2ZSvH?9N+g&a*ooLv4P+MW4XXvu90K@Jm@Z+B*!Ln%>{+*WA}pj!6%H+S;qWh z=$M&4Yr2qy*o>4S@#)TMel(imRMr;Y`Oxi|l%FmQT`@h`%@8OnrZV^>|F|jAnF>G$ zF|DUe>8R78DvKwd91~pPr4S>gbrn4`67i#bDC+CEPbn8 z$yp}X5Qqr}i^2nLngXT8(wrb@L#!~sA33JYvZ*{LwnSM~H}=b>fGr_JyXsZ&RdlS# ztj=rQDp#~ph6_sYrcM+o+l<_ezJ@R>>cd?hG}c!^FjBo8je@^ytfAMO@j?psV|N-jI--hQ;|OQ@on#mP2#kM zJ196V2?Cp4PSsS$B~9l@^%~dUM%KI9Q-Y8O+fwmz*vTpW?{NP!5vx&j%W(Db2GOkL zmA03Nz?>K7M44j^Ike74PUdg_J7qlEx5v~}f=WGq$Q7dSbA`GCERZwBHX|~Z z@e7|O7q~KycJWrbtY&hu%w|@#>Q2iH-8D;^JLhF+k`=9LN=x^;vigK3%ti(|BbL5R zS+QD=yxxs+;LP_eA)87Yqh$m2DPer;@Oy@ z8o4}$R;$tf+1WZ}x3uKW0Yju#wSS@E1w)S%W zEt_$kz}{eX^t2JKaMPN*;pK*E5XIa$i4$&K7QZ;XGOlr0bKDb}__!v*y>NIZT;L`XEi|#z34_KI?^wm^rbVs z=}!MR9-^)W?oPexQolOZAEEWFZ~YBDn$v={X*$X}uIz94$l`RaJeb2WK3H{o%-;@l zbVsl5YH!zkD1HT>^IdXvpt{wf;w4!!wX#laJL2zt^gA&A?|^@N`l1SUWZF4V#S zBH-}B9`O-E*A*WGMj+2&*7k*;1zw=_fu9DhpXFts2XY|$ZJ-E-p!fPWC_MZ+0pAS?5&_#g_434MZ-W9!p)2!asF`yDEU;|43 zpc6h}_F>=$g5C&LAqiUH6=oq8reF+mVHd8T7s}ul)?W>lA^zC{8ajhDtlk>#p#KFQ z4S1bzv_w>#L5FzJ8!(|BHX$Eso!3DjAWGpBYT+P);1&{MAtIt8g5el4;vqaD89E{i zieV%!LM58v8R{SF<$@c+A>vR(Q-LBlkx5f6;R5y{1)^dg3gRIyq9L}TB*J1C&LSLMoQA|;+7$Z=v0cH-+5qyHh!nmo~ZAVC7|;VDjnDz0J_vSK2-qAb?p zHDaSR+M@bd;xBgNH}YaQjw2=#qYge$6yXgidR;R%G)|-UW#cy1qc?K@qdjV) zJ~kpalA}NN;y(gp8V;m68lw@$(ioMJRQ;YaHlzbSBs`{~JWeD%=A%XK<3-*gK8oWm za%4b$BuEbAFcM@Z!U6Tn4OlRwLq;S;w&Xlk|I zqDYP;Ne*Q(nq&*0Bvr8EJFX;4x};NfU`^g+PTC|@R;5n%<5c40Rc@tE{$y8vWjPY1 zST1WHs`mRo3NQGG=3PC0;`2W9}to@}*_^C1wKVSy})T5N2Vrc@vf+k`nCR{EiYM!Q4_9klvr*ICZP|BuQYG!S! zWCPx2Zic3A{w8m_CUC0ebP}g^Rwi}=WnXTmatZQcY3Mh1DD25U!bq*+ie&~T3Xc{^KeI}@KHmHLZO$IRIiKgXaTBv9$ zCWl7nhSn&Ex~GT2r-+8Ai7q8Bpy-2&2#ZE2g%&A{$|#a*sFK?M=#Azmj_RmoLMe~- zD3E?dkdo2m5$P))DVHYcj52A5g6Wffsgv%gluqe__NW3@>6JpLmKy1W#%Y*}>6|X9 zotEjD=Bb&gXZUSkU$QBn#^~?)-ko-7p`PlcmMW)es-<=+l=^9YO6r!HDxs3;swyLY< zDXc=OsHSSI1}maZ58uI?(DlBX?zs+Fo}q-N@*3ahjNYpzo3v0f{(ChK1IYKnf! zhehjzLTe#Rs|{kSwVtcFW~;g~szD{^xQZ*Of-9w_E3sDptFf*tFqSL3K5DnB=&u4O zt$Jyt(rdlmtHI){wxXf3E^NGVYrn1t4g4#*hU>f%Y{6cv#UgCNx+uf?p2I$@kjf*x zY9+@CEV>9vq%zmrRN^HRHtkohd%O)+(X06g1?b1Fizk+PY;%U`(Ez@%C z)lRL}rtGVlt=F0!(2DKYsx8%;?AabI-m>l5zU{|Bt=vkj-V&|d*6rPbB-ReB+iq>( z4(;B;X1uln8~E*&0ZlbDg;pXk?!mjViF7M*5CyXxJlx|c|Zs+#y?sBf@GOz0@ zuk$u<@OrJfeAeG0Z|Y{R_LeQ_E${PwZ}i6Q^rG(Xn(GnN?)9E7`EIZ7hOhgIFZ_=0 z`_690o^SEm?(wqki*hgYmT&(uuKyTd(L?*(@+_{MJrb1(ug?hOj30i)yz=Nt{C6vWbp~Hp`X-S+2kzz%QtS+w7n9&u+jvz&Gob@r{ z#*qtAnlxn);mVXQTgIGuil)t)H*x0FxpN>(pFny392&H!P@_mawNkoNX49lZox+^@ zQf0}jRkMaX>Cr0JuUe^!?detOQL<>$o{g$@ty#BhNxgjvmFe73cgKdsx)-e9u3O6r z9^AHX;lqgGDqhSuZr;X^AsZ$m;dTbOqsLi$dW@lt}A*p>C$UEtH!)qb8FO} zW6Oq3yS8c2w>Q7G3^+4u-oSC|o^6{r-rUDe8yBuT_-@zFoe!5wefsj^*pp{(t{pk^ z;=g^1{~u4jy!7+iucu$ny}SGG>%+@8PrrA4=;+n!hri#u|M$z?B4Bg`&SxKf`B|r6 zg6uW;pMwAzSKx#OGPvM{81@I@h8#Xf(tQn{utzuKq6UWl1476tKy!KiwtiTFOYN!!1i|nz`{wnFP&MLd?v(ZkA zZL-u}o85D;5Sy*9RS62yvF7$ZM-(N>TkUXiaYMW z?pE6^z{}E$@V*3#q;JCpH=FLk3!BC+!w}1<@x~9^t1Z6~-=%QH_-5>I#}a=`aj^fk zjIpUEb1bvUGlRUc%`X>x^3E~i?6J>_=B#tcK;N9Q(LxVRG}1*gJ#x}L^NcdhHCNsA z&$eC+wbD_$MzYgjBc=7&WN$6?*;Z5i$=6e3?e^PamM!?2ohFddQ4n?t0*z&mKALz6am;?GEbxIw(8-4*c@U3-7$`!$;4& z^s6_H{Pn$0|Gf6ib6@@U)Zb}5*V%h-{`u&quYUO8Z*P6u=#(0~wB;M3MOK?Yjzf*6!w2KBQ$`FZezAPk`h zN4SUBmGE?P&|C^lctWPM@P*e~p$u7=E*RSIfGNzOAZDn+9_EiaHe^Ey|Ajb2BI;0w zU04Gdl$b;&HgO7OfZ`LRI7KR2v5Hr;;uD>?#V&gBi%@(b6~d^+GA7ZCSsX$X(#Xaj zV6lyFRHGc<7)LTxQ3h^QfgblrM>IxpiGTd#9HKbLJpl5Eh@96A)i=Je?LY^3G(jFA zIY&xbk_3;GBqmS5#|`W;1fHaTC__2QQW}7i6G$a0H_1v%HsO_L6eAX3IZIi_G6_>4 zf-ZN-%U)J;2Ef$8D^UKnc~9rZy=+%mRRt zn&0d|1j0$qa+lny4c2SUKFys#BNzY><^Pc!LCMTV_|IGz_la;0fs6M$c z1A-d#pkly6LKn(~hB_3XwR~tqeM!+_R`jCG%%?s#%F&E=lmj32C`d;t&|qHRq$EwL zDp!ilmAbT_FSVvjVLHy5PBWW0&8bdz%F}FyRHMh~tY||^T3jylv2@j}2}bMJk*c<@oAqi)3u{B@)wb5U*X3&hvP;?N-u9xo#VtWoD}rF2 zcB#a4ZW1ml2I8u8xWQ$qY+=hb*|^_?@>*h)BghaxgDTr zfff8$61#T6ByBJ-l^{{5Iuyba&aE@2i`ry@*u4gxsds-oT+F7Gvn6KnUE!LPY*M;N94jN{q%xTzinFOY*wtZegxC;vabC~Ii;6>Bg(Iw!pnt{CGNvpBfu52?uhaKl)$9K+R z{xpfDJXf~9chx4=va12?WxD$Ez z8qoMXGpM2MUqKVv4dS$Rr?Jh<1v7fxn+Er{pUO*l=kv6u-nFlJD`SAq+Q#o5^>LeB z0n;Wn&w55Gyz6W4EyG&ZQcd=a{jF&o3s&GL{~tKP5xZ~;A6%{t*D-LBed}yXJikk& zxXq!B@h!Vq)*>G%0{$%m8mk-K!mM!0$H?@hqnyIQHg_h=i}b3a+vxkmdd-Pi^BL5d z##7d|*vD>GosZep%BFX{I}YQ;Le|BDc7w;+9tJ_z-R^k5d*0I!2fq9L?=J@Yy(`fP z!WZ7~h)=xYr_K1m%RTOokNn;zulKq~xaqkhqth|Zd8NDD2~dx^)UAH>tV46;BLIK` z8bI_vyS~a6hrFfD%lO0F{`R(aeBXDE2Eh0J_rU)<<#*qQ#pB-i$WOfPmrwlWOWygu zhyL@WKYSrp{`!{pLKd{o{q2LX^WN{i|M;;#{)w6&{GZ2kWkah=w(DBU*pI&5na%_w z*uVbx=fVAPpnv}R|Ns3LfC7kr+(&)IH+Ke@fPPng49I|>hkg%Oc?3v*`!|3B*a9(- zff%TP8`yy(zycolfgq>>A5elMc!DUHf)=oXEU1DmXo4>&f-)F_G{}LA)dT9}WdoK( znKfFmcWB|KfV|gz54c7~n12-rfJwN76R3UB*LVjQg;F?$LimJLXobEPgya`|T)2f@ z*nnUNfk#+`V@QTfXn{6ZgEP2>YzTuM00QC^hcZwJ8CO*3b#evhh1ur>8yJHx$a*d4 zhJrYQ1bBT{SbciPd0`lSj5tP?|7V7f7>N@&iIPZ(XLyNdn2DMQ1ej=vWvF?6B!*?n~F|G9_RSdl8ZiE4B36vQ5j~{rCJJ1INrH|g`gnn0x zD7k$|xROe!iA@N2@`!j|Foor4g*B;;?MRV4*?CK_gctdfM46T&sFrG2m06&HNSTlK zRg%Qlk5MU=nD_$#Xagfyl~(D5wP$>FS$GGTm0y{TU@4Z0IhIp-mH@eyY&n#Y8JRG^ zl6c7jUVx8F$!bl>c2IeZXh4BebOq})nxtt3rg@sF2?l_~noHz}y@!ynaCp;Tc(qA* zeYXjODMY$yn27nCH@TRMxs1fgnx%OJtBIVdiJHu*oX+{2&>5XIpqJD+oz|HG*m<4V z37IdzogK(@N11_g|M!gSN0(`ElB!6J_%IFgpbPY%4)Dni@hPA5X$|vvpZH0i?Wv#d z*`Di}5BJ~?0vezMTA&6>pzL{|1G=C9+Mo&gpb#3N63U(wN}=Imq3$^#8M>jm(w`dY zq3`*j@i`#*NunldpD3E5D!QU9x(hD)3@;j^G76)@kO|g63ND%oIf|n^YNI~dqneNj zCo~7j)B@m%nVQLOJq1*$h@MO^W2&ZWst<1Zrf3SM|0$;aS*LVLrFV*_dJ3X^x~F{#re6A`Ejp;@K%<4) z4Tnk%i7E|?|JtRBs-=(`saz_l@=&Rn;F3le21&}44pw<|;G|Cqr&Kzrf=a5U%B5in zr)=7$0{Wk;x(}`jtE}pvu_~u?nx}u7t5UkFy4tJ0TB?K!tisx;#EPhlTCB)wtjS8L z=)ew_TA3Mmq?($Pn+c8P396Vtpnf{4r`oN<3aI|c590c&y6~#zN~^P4tG0@(`}wPO z>aOq_sNXuT^t!CaYOnXItjxNv&kC2?_W?$52I5(*r+BTPs;%|_58Udl_A0O8Iiw_PKvO8%CIn-pK)5T z0%Eae|9Z1FYqJ=8tL^Hg@A|Vn+p*x9tM!VXA3L%oo3u(xsr_n@M3}Pt6^)hWjkWit zFB`L1tDovht`A$WwyLu`yRj`Qv_VU@LtC_Ho3=>1wEWr*O)H-FmzhkUlqA5Zh$W2& zE2;~dwOKo}UJJ2+i?d*Ru{>M2e;TqXYPLixv})V9h|9K+i?5Spr2klimjzPHWuAP8 ztrB~+eA~EP%eC{Ms)GBs=^D1Y3Zjq8qKezPj=Q>Ro3F7Oxt!>)1W36N7gsFHcrLrQ zpPRO!E4qPOx~7Y+V(YPp>$;3ifE|`@YaCz0B$edE~Tli@P4vAtH1l}y;$16{tLinD!>Cw zzzck|2%N$Se8LRez$Oc=F^~pSAi?vSyB55=87!b2+`;n-!XYferAxv)Y{Eo*zA9YA zE&RML+^qas1Tc)0(MrMf+o@BFzf#J>9&D<9`@=vSzyh4WE6l=3e8lV<#!Gz0@cYCw zY>-l%xfgtR&!xdbjK#t#pIghtq zW@V0C0B>BidP~RN>ce;pz+XJbf4s?>+{v6A4WL}elKQq|z=tEygpK^jRu;*ssKav{ zy=$wrm;AM~yvOnR$+(=$X1vQu9Ll0x4~OipG`z@lJ8GIc2v+>cu$;xrY{z#z!nKUL zLR`Y0+{;SL%erjMq8!X{3zsoK1;iY+tlW2Utjxd6%(Z$A5UbAYoVx7{v;$1de4Nen ztPJ*i&Gnqm+Puw9EX=3;mf+0D;(Ty;i^02B&IxPI=lrM#d<_o$(C;kIWL(Yqe9sk) z%@n=R+>8U&`_17zml_3<$ZXK$oWBaK(deAg&79G}aM3RP()(=D3*E$O|J;87{n6Gc zwV-#NogjR)aMCEv&_X@bEM3$v?aN4w&quw|NA82rB6c zK>gG~z0@@d)?yvjGuqT+9n)UD%u$`0{|o~P6m+^r4T z#I4$2I_+62Dd2Cm)+ zp56!U+T{(s@GZ*&N9vS<$z7tM-JgE-e==h;eFTwx;^70 zI0rT^%2z(-Vy@+QPUdoT4(#75E|SlEXRmi+PX2iI|J~+N4v00L<8^N5cOL0n zF6m)j>3dG;n4amE{^c-?r2qW1g^p80@L|)f=rS&XjsEDW?&uv3>6$+2dH(8P-shI? z>ayPBRnF>(-RW$Nq)hOLN_xXCUh23_-KQSsy1weRzU;SN>&#B;&Cch~KJB<3>--() zjqS8dMzEr_<}+UGZVp{=z5-^D?8*-9(eCWjzUk7Q?(R$Qu3KzAMoBS?|nI~+OFi>?t_Q!p|F62}j|2Uw2G@_TcdMnt%9&kNBR?`Jiv|aj*F7!1a%` z#zv^*2q#Wx z_;7a0-7X2;v>l^_Aqj;OM_{CAv4I1PM}?AH%u=!A#!4keo@}*pQOl1tZ{~bi>ea63 zK#RsY+FNLkWq62XLe@xYv|FffQAm5j|692aZm?N1hxaDEd>j6Ks6k@HL549}8Oqps zXUNf^PtWYxI%ewGuWye=we)B3q)m?>|26Dt*ss&lPSO|41>Ct|^d>`pAr?BehYKd0 z{32rJ#G~vv?Fz)s!0Nag5J3h#6DK^a$deGYt)$q2HP_P8FrgA|i%&P+=)(jdF7#6< zFyjV8j)(se6Ocd{8$6z)S1L+tIoh4xFazbgT3 zE=)ef^z%j>|LoGn2hB7T%^$Zi|8LE)Uem7$9dJ{!xZYC0z`i<(i($$t{lh3rE(PsV zR8mXrZqGvzJ(EI3)kKf17Sc0vi?%p`b3P_r&^1##sYJ{PI`RZH(NPC2wwh&-H5Jer zk5zWrRa1>{T2`w~^TJnSh&49Z+S?%s_&&sy4NS=8^+XhXU9rU@P<8fQXakM6)I!UY zHrmswy~Wi=BjWboaN{bDQ`UAxx7SXMU^ka{LUs6Ic$tk@-exBj6JLxoW_4di{pBHx zz7B>fE=Yqoui$gjH4IoC6u`1wVj;G8-kEErxn7(#R&`^3?b@~nkZE<#R)GOS;b5eX zR$4xFSzfqdnWL8aW}B;)|9H%M?bVrQxw8Gai!km2%&+(YPN3|i(^eVJr>C}>>bGN# z`&qh^ty}J`^R~9(9I*ZQ>#*^wwSrl_C89m{3|6}Y!Yo?%uwmymc5=$EmYia^wbpy{ z(0a}gU=H6>qQt_RAYJjrKWL!oIGk|#Fds6h)@|A=zx;OG?Z!H1&U+6>(Y_gN?b{=d z9{G3=iS}#pr5pdXbu1&VeR|t7$2<4x-Q|6I?tRY}YL11s%f#f%E}vxP*M2;s*uSU# zeeC0xKl`lf-;e(#=NxeAKCpF0XkW6<#NS7cdNd^kd-s9H=`8M(}R$>Yn%Z zrZ3NN3uuWO-2VU=|3FPmsesW-f$SD|LhGUMffBT!J6edEJ$No#Xh4$rJeWKX8clVp z8R5%JsKONn@p?jJVFm5?Lg4vrLs7s14Xs2p8-6f{7rK+b4Cq53X0d@2Ora2s_(dd! zMt@)9-{0 zGAI%ZkXdLE4HO7RZN!m|lPqE-DYr*VwlIGu`(xlvhr|5n;7Nu|9pe<421Xw6jgVv| zC0!Xw_k~fGTZy3`#WuV%$fg2ofmD17DM~sF;+LZ%rs)osNFEvol?Pmb9cI_cUr17# zv3%tnt!YVW|GH3{nVcom8n(&(;qpBK>DtIl*R4vVQ<%IuCeswA%v8E@j?kQDJ*#<5 zW8l+R+}vh2X(_`tIq{q%I9|}Y_M}b(%V?#n;vz}t%x%!qp7)$5MfvH^ckE$|XlN6a z3S&mmg)(#{g3lWH^RX~Ilq(XQs6{JEM?GGYk9z}WzdDMAvK(Xs<_sm21X{tMK+T39NClItoml9(*a%O^ArPX^i6~*siq^wIRk3eX>r_{2 zRq=f^|8zW6i(V62iF1*yII$wkSfe-C#HO{Wh-GY4k$T$2I@X(1jq6omW4N&1hAov9 zX5#+hR}ljCv#cs?Xi0lkUj#R`P9^SeRl8a~Lbg$Ef~}-xOGuFx#H5_nE&D+0TkQU} zxLZ1|-dbzZ@u5XzA+4-m3@WZ%~QCF%jcK&{?xCBE4mZGBN|U*O`mw8q`9X(hQ< z0LzrM8_lVs45wN04krs;MI@z`!WWpzxUN}cgKNT5NDLYGYlG7 z1y&+AoJg-kDCzPvwNtUCfiCQOFPA7>;SOupzxuUsfCU_4LP8pB!9ah5ZqcTDGFL>d{BmUN>% z{NXu_n7L9q+YKY&f+wjLiby8TuY}5GLkspC&be``9c^h?&-u=m{xFEUplQ&N)YH56 z;9p~f>O&`b(XXyEhKX%uAkW&%K+bezSteZ;^P1PCX2BBHTq>pH}}r7d~l#Q?BrzBVAwh&DVDpNWSx%q^J1Run@3&h zA=i1hJN|E;Q$6ce7dfquL576i(wL)8dK#6!vQt0Z-ZaPf)ZMOixW^sqANRS&x4z+$ z=e_629(tGq`r4x_z3d#Afxw#Xv9{s-?ThbH)#d(py5HU3d#AkJEiZY!e`n-DpF-H# zuok(fT<~Tu-{I{d^Jlc3B#c*l-6xOvoL4^fkiUG6Gk<$@Z}e|0!zoVv|5pHpoO$5_ z1AYO3?*hX&wB88Zh3Z-F`m1w2^0KGB(2LbKxDV*0rz z{`nbiy!D|!{o2zP_t-}+_qBypHU*9X`&oLD())YTE4~mwKO9iM^=pO$bie5Pw%KdI zu3I+k+ddWOzCU6&U8Af6Gp4dLzU0dQYjZjSY`xV>!1sGS_=~^lgT48S!20t%@be#S zQ4ygU0t2W)8{|M7%!m)Hi_b$n68u3048plvKofL8Aw0oYgNGGl!PN?#-RrztN&ubm zKU2DaCAdN?41ycPK^*J>9lU`yVm=WBLNY8t6-2=Yq(ACwK??M{|NQ%@z}vqJh?Cz) zu`T4n9K6CXM1oZ+!!a~N228^xTthZ&!<)h@G}p>Zy?;1FI4zjJ&^WZc87+)AzNO0WD%y%bBnEXtre%H;~EabiljTrbg?%DCt> zG@Hxn62rT+%g4;idECpfq{Xq!MZf&ZETk(v+k)7-0E4liI$=7m>NHiHI1Fe^8kjK1 z#LLPIO7?qA%goHa{HD$P#!z%g@+zjbJRKGzfB^{3{}W&`CsWP3Y|Y4=P00+z+H6kR zyhkPCD2oi4XFE*Z6u!|pzl`{l&m5p9AjQ-SPu4`vSaePET+Z#H%uU3`=d{g}F}h7l zL>Po36F|%sP#pr8!gNH+)bz&iR5kKU&hwm1^_0yH<3w4&1#Vc-S%|6jga^PJvgs@` z`7Fr5BBx0yPXBzJ)fCVG9Z>XyO%RYJ>63S zbjCba_{y3Owc#u*(w zbR58l(9vYw&W&JJOwH6Gq}5rCR!)@)Sg=+?r3V)M!)pN6^MKK@`c{4gS8<(GS8b7d zLsxZ$*6b-*cb(O;T&ohT)=I*=9Fb3L%90)f}l^*tGRKfcc-2sjXe|sdtOZo=pq7 zwcEPg+o27)ywzKX?Avm)BScNpz;(mIVA})(&FHF6xc%3(Yh1^L+_5cM%01d9wOqg4 z+$GW_!j&!P>a$>>1WmZL(OujyP!7jD-T%kEThtX^*>qjwO+;o!*nO(v` zs42YNSg4PzO%mnv-5gL|;LTJMn%t5lUe`=s-8pW!EKkR~#PWbJK+#hST%OPXAj#1(RV_s(gSfZX_;7Vmhv48D`x``=M9-;{R<{ zCMIJKehsCBVl=j^+c43%ieNBQCN1vJF2-XNoZ~R=W7Y-a98NY)c;fXSh=N?4WT4+7 zc+-JlBCmDf-Fjq7wy;!A&P#6P^5w{4_?5Y(;H>!NAmw8vp5#ilWNubMSXO3PX68gP&Fmyi z@04cOQKf2L*6vN-VGdt-j$C=3=WRYxdq&fH&gbos9W%WJ+1Z1o@MnR}1W+JofZhXy z?mG|$JdGtaY}tSm#!w%wLjRnc=3aI_x9e4&)M(|DolN*>kPc~)2I(^R=#oBZJ17H{ zUTK#8f|hn^DF6$Z&H)RlX_~%ioVID5&S{>`0UGG(n-+qc7HXg_>YX;~D|mtwUTU#Z zYPbBvr%oEER-$C)v#I9HMdD#pyuvw%==n`#Q7g=fmNyQS0w{=Sv`*`zMryTI>!R)e zxQ=VNo@=@efVmD%yso&s*6Z!;>q9#rzz%G`=4-+(Y{Ncm#1`zoR%{aLQ5~h(tB!1{ zwd%>nYRkTCRN~}mGlNVP1W0)VXAp%e5M`zjMu27P|441sUTxM+Y}elF*RHtMo^9Gj zZQH)>*v4($-tEP9?Ei|qY~T)V;TB}9ZXFq=QG~Vv6z+nr?wg&x5Cu4ofbH$vmhIZE zZtKSG>)vkO)^6|aZt%Ws?(D{O8*cMHZwl4yt$u=pPKbz}z~?^a>5d-y4sZMZ?(EKQ z{JwAQ?r;6}Zve0DaYb(euWZX+=<00fsE+Hu2v>=FQqz9b{SI&nuW$>eZpY4W4R5mt zw^I)P@D2xY{1ovJAMq03AsIZ@6c@~YDKQt9tdH7A85bxR_arA0s13>NhOWT|zl|e+ z>zd5a=x*U$!i%Q%86;2g&0O+OK5wLSasy{_DK~N|S8^*ax+}MfD!(PC=GHBr@+Qwh zEeG>3&+;da-TxNH@(TQ;0!MR;6}(Bfg!hi``S`*YA4WVcae}N^3bX`YQE8WK@Kk>)D>X=ghVB*9ds^h^F(?-t?YL^h5vl#qM=Pw*f?#@O;;Ien0nrCwPf7_=B(ZUGGnT z7j=d&c>jHe_$D`Ze7f`r-~V!L4~NAN5lo zbDIr;>s{~tLrj@JzSER;5;yUWC;Fd9_=PX~j4yPhfB2CPdA2`$w>SH?k9)a?d%CXy zvR8Y&m;0OFcGp63itk0L5B#iN`5kYvt$$;$$9KgCcE)FVwCDK72ll$3dyjv6%Ab56 z$a~D+e9iBC%h!9p|NGE)chR5v9FN0R-*K65h{0Zb*RSv0M)4E}dvbUDq!;;b-*wri zegDn}`Om+6&j^ywNlPUA?eYBdejO>VAo&Fb}Q zSFk836etV9EZP8RBQR*Y_HB%T23bA?G5283f_e{bERyjfMvsC6cPu>EFyh0C6aV8a z>Ubv0$C7*AtW24w$=vFWIr47c>> z-+$Bahg*TZp_QXtDAmS}ASi0cZG zv7XR{6FkJQZVTUy;O@Ij#yc;)^4fc^u$JX}Z@cjNJK}TV39M4U1VeW)6PhKAT)GP@ z>~I!dP!VUu6-#_=v6&_-CbJ!@=&YwLQtPp@Ay-S|wge!svIkLl8~-lMe7qowwe68>lk$(W{&h%MDn6{YIVVCq2s% zW0mqi*Kl%84&r~22`67vPoBQjC$0hb++#lwW`zce*ZukLJO97_{QIBef_(5gF5OOd z1Uw)DX^=nz?rwK!YarWVcZ5IDtpXO@fCW`%w{T@eW}T238=Ut%Bl&H5#|d2ZEP{Yk zt*>)*5Cc|{a6Jt2CI@2~kO2k&s1nR=UP%zax~?WlWrVjdh}RkJyckS)H^ z;S8+E1s7b7iD#kX%3g>w{QU8bfFz;&fcV40&Cq)BlcM}0iAPA{@my7;025L;Nht=g zlOYh|-lC@~LtQdBgKL7^h>$8uB9MRreC6z5Da%>XQvY@7nZY{l=8-7qBz~k9PDwVCoTs!L zIaMY^C^fMv*0350hT8Pr3!lkSOC~avWdCuiI8Dk@K+1K6dgQ`eRaw`dl0~X& z%_+;sia`PJZ?1sVXIEqTRyBkP41DzyiUylfl3ue3D!pt;QE37-2DXqXJ&A&{qstEmCFRye{(p`T(a`bprfP{O+rrbmk`;I8as+l~c=-o+wR zIT>IYD)o~qtx#jxlVUX3uf=Y5W{znbfg0nuc-+0JGx^I+&L&yOrKIwKo(fbFn`^4_ zlyPteOkqGTy06VmGF2a~;mj$Rry`=sB{6MfESv02zg*E81OX#8dqNYM5DtsmOV_HSNn`=fI%F&dq{49YA`m#M7V_wH}}y zZE1T6#nOh+wN0&UY?r#*(v~1ji2p6_ao6&d?CLXfe=HzOiwKAAe(1a74QYArWI)0u zH@+(^Y&4z3WXcYp0_@kMNE6&U4gJ9Zd=Twt%VDSve|WelZGFEhZ84 zp%eWPMkj>OJ1%i(G5u4Ve)`Cq9=xM691ly6y3_?xb*f*T;~|eWrm*hse>Y27ANP9K zt&aALv;FHz_k-Kp{s+0!{SY8GIo|WW@|5qr?=7Es&I6x}!4tmlgy+W2RYC@4r{Kha zzy;)sessx8p6f?%I@hzF`Tvh$KJ9An{MkMK`O%Yp?WT|W+fSeRLabi(uCF`n;V%2w zk9_uuAbBDXk^9~AzV>|A`|pGQ_nE`I_<}b)^0`5CJxoj{#Q};0Y|tmLzk%>Zul(RI zul?;~K5d}i{P)8z`qICi{8KkS`pchw_0xX+?e{qR-{1TAFXHz0ufOo|pMSsezyIB~ zxjGyl&JCdTrOhTtPQf?@5O4`o6bji?ANEZi_e~(vd7t=Q;P|nh25#Wo$)5*u;0K1F z@130O-QNgC0t)Ki398`!rJxJ?-|%Uh%UQ$C(Vze(UjbHEx{Sw)WzQ{HALU6P1%{sm zVj%csp!;>860%+rHvgdrg5V0y-xJbb6iOirzTg!CUkhR(3C5rd)|~O(AOX^x7tu?C z#DuVPM0Q1<5K^EUw!s@3AreC26fPkhQsEuep%qpkANHXMYT*_Jq8EN)7&?OuilN~> z(CqBb?Tmm!tYOl%VFV(b<;9^K&S4(fp&o7`CzfC*`XML^q9}6VAQoaS)Sw}bA@b!8 zBhE@;K%yj0VkBDPC5GN7-XbpMqVDY>FXCP(j$#)AA`Aj!@Bv;SBBIf;POvNnmmHNW z(jpvIq7zP|F8*OJ2BS4*;V)+6HeRDJej_##<1mh+C>oVt&L@4w zr+j*6fhMSe)~9_kWp1jaV=6?q0 zg=Q#+cIbh!D0qHogBss3La2F8Xq8lGilXRy?x<0&sE%SNg8FEXz9^B#=xs8mdX^}9 zp8sf$@~D$)sEY!rkQOPG!e}Xa=v7W5$GTo#LaN7Al@{Y&D{p_VC<#^|2Ye84Z=Q*bP6-Ww0sm#Hstzcje(I`%Dy+t8 zsG_N3lIo&1s-NPan0jifx+AIDzyHlqGGDE z605E%Yp`Z(wLa^tmg%Kt$*C4;vSRD9a_hF1YmaJbx02(x9;>>NE4rHNxoYdXM*pj* z+A6i;YOY?Z-O+2j{;R*%>$<`qzJe>N*6X;!>#hRqyb5f<-rc~eCmR&3yEg2;@+-#n zs>UuX!`dsdzN*89CnmHjg~Fr7_AAh3?8Tz&#&G4{ZIbMb9_-1ctj?xf;w;uOt=g`w*s`tHzHQsW?biT>Smvaaa5Ztjk4@4jxQV(ywICK-6! znBp$)CU5NOF6{1Z+tR93?q;bj#u23MpxPp&oH(oEDvGQ!V%(_FqDPLdI)1!J zvLr;2s!pn0DX^eJkS}A(oJq6hDT6I@;3 zL`ijORjF64YJCco>rkRyv4V9-om}^F-pnBbuYK#d_1D*fM}NM( zyZ7nbe}DgPp1r*J@aWTzf2&?Pc!f1?b>_`6(Eogb4b#V1W$^C>()*Z3y9q4^k*%g&30PU56o}2x5v8wm71RCBCQ~ zi!0Vhql+BEsAG&WQg@?`K=wG}jzShWWQIAasAQ5j9?9gBO&Vz-kEKW%<&scp_@k3o zS~;bcU1k}im}G_tCXrNr$)%ZPwmIgRXvQgCf@;#arJHZ!$>y4H*6AmL9^Q#%o`LSE zXP;@Zr|6%C7TPGGhd#MXh2MmyKEwT|m9y0zM=?w?83+wHg9%KNTh`@&mgx&8t? zFSPd-Y^+=CwrlRP0;fAL!4LPV<-`g%9B{-3&k1qG3}gGrzaF1!vBM;b%yGykUtIFW z38%a;${nvPv&PW69J9(e&rGw-FW;Om$RmRb^w1gq%<_mWn+&thK@;t?(M@9v^|nbX z9W~Qe=X~_mRd;Q(*4RxAHrHfx<+a3EKb^MPS&uDssc5gw_Q!MAy|mqK7yY)?U*rGX zHr;-U9r)mT7mj4#cPFkm;erEhxWSJ{E_B>IgJSvRa{tXZl#+LjG~!`9F8bz;L$3Mg z`g&e^&7d!i`s?VL4*Tq~r(X8$leg|V?WEW4JMX}+F1+oyw?{eV$Y&n>^1U z@rXaPp%RzK#3r7>iAjv26sJftMZq#{YcNI>3^2uOq? z79=S}NLn%pm&_y{k7xo+^6`->KMAbP1tTxoDfDLR_1*_GMa&!jn6oX>F%Gk7)RjgwjEK?_IO1I{8l$M3- zTZ4K?xnlO3ob0DwA^T9gCN!KRjVLY)OIX34m9?;q?O_!=20H)PwgjPVEHz0hS+XX; ztYICkUgx<)o@VyAn)R$S>44hQ;#QpEWG7ASYFedEpt^&_ZfgM>Qs}N$w%awV0TOH5 z@(NP7`HX2%SK3>d3RkVfMXr34Yuxmrvbl7fq;&N=-Mi)$qDbX!d&6tk=%N<92o^5_ zqFY|uPT{=hb*_2^4BW?7_`RRy!hVg*R}Opl!}P6ZZuQw&KJK=}0X4B*Ppew7a^bY* zOec)L8`HlMSi#$r@PHFMTMajO5Vq}ra&^33_Lji67yd0|Q%v4;=5)g#o-cjv$cmCGG!?Y-^KbEpo?|sj&ICm9^<&8IQ_AM2^nD`2iO0?5LL3AlZ@dEdsoU)-mIP- z%jZI#7{yu+^nYK>foi__s#Z3$Rm<7jJ9qcMiav7$z6|6_(_s%esB2^4{Mwezna*$S zb64?vQ&Q*loO|@FBeTqEL9@8dW#;sl$&69$Dm0Wfru9dCtmaF5;GYq0F`W_c=?fG) z)S`}LsZG6XC+~T#DsFWur7d7rZ#mX9)^$`n3~6je`plI6@q^ne+E?@T*wr4cHS^4F z=YXp3j+($NY%>6xcVxKgS z)IRbwF`nlvY@7-lhj+AhFm$39od?riwIgqi^QLF>(0-=5)GH2WBwHQLxo!XfY+hr8 zCmf`_Zg4C|OQkTcl*In^MeGFY&(qqnO~U`-f$fGSAD{LSV5AL%#%Lj30s z|N7hi{@s7#@rNJ#{`(*O(HDTz=Y9ECfYkSXj)#EwM}I940}l9r4H$tEIDr(X0Uuz2 z7NZq_7(xA%mK*L@Lqh9U@pcxZ=Z2!BAQd&;+j zQaA=yxP4Jbh*y|}1(<_dsECWmg^vIDgFfhmkl2HdSczH~1&5f4J;VZLmuPKxZ<5!4 zuIGA6_=lwkdzIFDuds@&*ot`Pim*6}vPg=8M}$WxcX)S-fCzw!$Ao_Pi-A~(!Z?VU zh=s+7iR>2zEzpTOMs~fJcCko{#E6WHXo-!Of54}T(YTGX*p0lGir)Bpws?!=sEg+4 zi^CX<&*+Ov_>JKBj^g-tK1YJKw~TJ!iLvF3>j-?jAdPFMetQ^@c{q@I7?9nEi?7Iy z38{?>8IJB)i{&VM=a-Jtc#Me1gA6E;1$l=Yxse8`f*=`$Nt9T(M~}+Lj0mQWhzEPF z_)k6zG~T)C5DS&ryvmWFtM z7g?D)D3@cHnQ)1eO4*qoIe|=Bk9LV;D*1M-7?yNNjaM*4V9=VY`I@j9n`JbcX{1B5 zCqbK#kkg=dyP0>@a(B3yo3QYZ!U>$l`IuuFnUZ;yk9R|=`JB+{nyfjUM?jrcaGlth zo!Ysb)!Cig>76;?1LFVr0^?bpXh@#ushNT~ms!w&VZa2OSWe9tmC$&am39yLU=R2( z5B#YP?C_re8le6;pzTng=^&s8N)HOUp#3=w`{@r58le(8q4-&$4(gy4N}(CLp%$8- z9{QmldZ7<0qPlRQC3>PLnxeW;9V@D$Ec&7?%Ay21pw&U6HhQBqY7IH6qdeN9KKi52 z5Trsnq(oYzq(BWux(k`Wqer@=PTHhUx|$MLk0am(@cEBb_fe%Om97_|`dJTPYM}8j zreu1cWx5V%nxrTRIia(bsAst<6w4|>|Ce)^|?`lW&zri03$ zg=(mWYN(33s5k!#q>g$HYYM57ilme}snTGn-GHgspr)G2scg!oQqXkjM+NTb1o7#0 z1-7MW(2zhzqJ;XXkqWD^+Nf??r~H7Ya$2HximQB@r@rc|gF2##I;@OJtj1cb4CHhL?tyv#>C<6YH~5`>7iHry2XEMtihLtF-pYv`q`Oa@(&_`>0b} zwJN)^R134EigZ?LkKfv`;R?3p3J+gOwupPKXS=p|y0~b|xbW(+k^8YB8@FQGpL3h6 zbi28Bi?^QpxqKU%eme%QC0<@DcZ*5563eqao47xFv5Gn|@k5x}}?{H@l9iYp$-#vX>jQY&*TTyR>hsxwLw>{K~t&TerU} zyuts=vc+4CTx+_6yNZEVxMTag&-x;wQZ%e^Lxz29pM;H$mE8@_3J zwK4mi$%u62tGvtWcFnuK?CZLY8^5+&y_7q@`s=@zi@z1zz4Z&g{|mZPO1cG%0|yMQ z35>ep>Xl);x~<#5iF*ywE3^7RKHJ1j9K}#P!D+n1!Jx%ie8+UW#cbTgYI>bm8^(&bZ>@E{D6GCoyu_8dsTKeG z!fzbMa6H3PY{!yZ#m$PxldQ*k+{NQN0%B}L?8Y(t$dG)= zb^OXUOv$pG$F7XYd`!IB3dmPVXkymOzWh>zT*@kp%8Fdf{;JBWjJvIz#j^~{&0NdR zY|FRI#+zKgw>4m*jCe?_#KgSFkDIGzvL8jnM48&(%D}pN!4HEX?}6&kJqL z#(b!H>cSem(FJ|cmaNd{%)=o)(jcACB7M^6K+)Lh&YbL&Tadi;yrrqTif;e+2_AaL z6kXAYywe=r(-UpcCSAiyO434&(nL+mNNvi@LTK2>S^S-MrL1 z9n>X#&`PZgVExrv?bQq2)I0FdHv3f}ysF-+h?twxV;v1!&DBHQ)kkgDVO`gEE!Jes z*D7t+XboPR6$4`&&z)eZWOxO-AP;ff*g*Z*cpce(t=D;7*~1_VmMz)oI@ykmrmWzC z18l0+{M1nCU6o~lh%Ls4t#u@*1qzzda19Net=XJi+k0)>nVs9Xz0-2t+rGWqeaxO! z$pnENehhbVsExIQfY`15+Oa*`zdhHGP1}7<*}8q%){WWN9o@hk%hLb--Ow!$s{q(o z(2Ua6YFfwKVX)d5Co&GJ>e2w(d0d!nwQ?6*R5I5;2PlC4-VnIx<9L4OU%uyRF5iCs=V%V&W*+E#p5`3vr>y}Bo8FLq3tCR; zwEpSDaO+2&>$-mFhA!%zUfU}U=D(ilB#!7NiHTA^&xqz@^t4ZpP6Z$y>6kw4p(;*Q(_QA6R_dm%$Se_>@MipuI=1z>+ufnOKt3>D(;BY zU*#T(t!@FY?%~m{?(IJC^1jgXj_n7Z@Wa0F3h(Zt?%DP()q?!y5~X36g-ZU;1ENO- zu?`Ld|M9pE@*xlC4Uh2jUh*h!^2Gk|-~Pv;eSjmka|!?607$p-9WU}DkMlX7@+yDw zKF{+j|L~;#fQ=XKf+&M*Nb!B8QYv=t=F3BsP-dFq0=$mSow|B{Y=LV#1VJQ`t;vHfiGI*|XbEpgx5T zCAzcdQKLv-F4fsI=u>PtrB0<9Z{1a_S+#EUCeQ2FIXNoEs7R%(Sub0hY#GBv#{>uk zL41hP<&0f4Yld*4BZiAw6A|B1G-3gx#*PhykV?6dspH3yB|E)IS?OiSP%#JFoV4?2 z%ayA}wMtstx6!G;uI7pktk|(x(5_{|7O?-4BylO|*09Dmn>m;^F~ss$Vqhj32w%*& zG4aR7nWayr?ztLh&DgVR=U%NXuzW{!BqdWLNdohbJ0iBY;;l~Ph%6#OXd6~JtK{1!H6lA zSPP3CPI!of5k0UE3<>O`(gj!roro*|)8I1EI3Z1tQe7kURnlI0l`d0DF(tOkF4U{d zQzVrXV}=Mubk)RIQAjb%PiwsuS8jI=R#jsdfmDW~-ED_H zffJ4mL9z5NTNbzN)?7x#6nNlt$^8~!Vb4`}VTM;Dqlg}Gu=CSUM}4n}_~aepfdej@ zYp#m(4Q$r_G}!XFe-94$I)YV3xZsy#UfC&yD{Yv_n;llxi%$I%QQi>qOCpSF^Cj7^ z7Sb5QUzb^S+S{fbje2UCW3K=C>YOzVhr^wdYt}Ef1kw{#vIQ$T2y4^Ew&bL3HS{7q zQkHv}r=hMISgP~Z8*7{S*7|SM9_|z)_IMT{Yz_n)8@Ub6W_&P=m`n0d%FQTnZ@cd{ z7;~68PdID8hum7{h$qIOaK4m4b!f1aoWYIO9cTPMlP5=cZrthie0Sb^pP8x9(FJ@> zIR$T*JrNYI09M?LbE|nbWG~w=wr#IGcg?eJ*>~OR?*4o5KM$UG&;d_&@U!qu9%H!V zb+Y~CiIARzQVDhYdhEI19%ubU?_a_336FonD-!xZm8AN)4}J@TpUCJZ0@}4rIrz(8 z1-%zO3}#S++Ur~gGxz^Ln31n<0W2Ts1~(hq#7lt;WB^>|2f6Ak?QJWt9Aqr`!42Xt zhdTse4|zB&5(>{OS+IYf0UWMT``_QdH$j(Yc_j}I?sMGs=JcR>^(7tPnT zvhW}a8acsk#@56ZI&pql8sh}vc0(-=@QPX7oE+n*!xH9kh+H!wsF>hKG2#nFBST{Y zU#OfIW{HAul45|wWNtzLD~cr3-rrK`H-^pp7#eW-+yqOgpv`ncxiP%$9l1;;8@#rUN7b4~MwrO>+?QoM!Wy z&^)!Vkq#~3=2E=r3v!CHoZ}p5LGftJf{qZKlr%>%WQn{38YCgPOoKz{PzieO^P0bu zAOJdYg@gVRpd$@wKus#pg{Jf!<6x;n)p-_Yj?r22qv%`SNzcS>R7Kj113woj(vqH( zphI<}N*TIRq#7eXFm0z=LR8aJC2v2yJX9h*$`LCB6{z3jDoV8~RD*8OsOXd^Qom#SqOZCaa&%T@OmdA06!wM)s&=GVS}P3U0# z`ve&nwpkY=Sj8N;r*j$1(FaQa z1JnQYaEzTSV+z}t#lV%^l_sMLgubidqW=GeS&REW~ zfWxaa(UO;%^b|8bbIKbFEWrknQ<0L9py+Nbni-6yGNdnzUsuO@&K+h=rT1*OGp=yc zgkJ9jTwvxuPB97{Gc~FcwGCEB`qi+0bbqm|DG*PL$9QUzZWeHDY{#VubZ~CscnNAx zJ%BEGA-0-TeePtpy4mWcwXEAcY5eN=bWRxXI~knqLV8Oj4n-gFv_sWpv$H$`Oiw!%Fivy0Hzx5jt~=cuKljH+9`8awInVkY-i!ra@D`64 z%%>nAJFs2xHDA2r8Gnqen_l&hKmFthSa|^>P)WJg{L;-R?j?|VhEjif>Er*N^VF|? zhus3{312#yeH?Q=d! zLFQV1o<^f{UG915Q(yk`w?6m1pZ(V3Uq<%VKkEIz>i_#bc*6!;s0D6FhFYKpcw020 zQ@%gKydR4=d*U@C%O?-OJ=`;#845l6>%jWEzw5)g{u@CO?7k7a8eBL)7379ss0Z;I zI`S(W5=$ZqAQjzkzY;sU0@y(wJenzrKMxGT4-7z`6TuKHK>=pC`c?nqkBL~oFHwKsBYXvE=-y; za3gUnMiVqgbu3799LRM{NN0q`Sa?Sld_q`69pi{NOUy@Z)V%+X@x;3T$Ur1Vj}*vr zG{|&RNL4IJlf;yEOvwZ61YnTHhkQY#n#T~MB_t9W#lof=@QpuN16;gFjNC#*0>@zN z$d3d`BMeETL`J0)Nv4F8W_(JPq6cVn26bS=X|M-clgTKIg6O%%pTSAsSh6$N#-Hp( zp#(&$D9UCq%2I60RcuOvBuQr+IuV-61C#|vR4DQLf>Mz|noO^G;k>E}4#Ff$PUOVT z>q4|tOU67(w|vZ`T*|kE%eqv_l%xl$ERVhH%dIRfYrMwF$qPV%t4}MtO)^U~NP%)A z%En|%$9&Dmgv^tiO@xd~xfH`mYsadj$^*=Wc!bF}!eKEq_Y)7z=OL6;k=~KLXoj~i1q6;l{?HbiUHhHO`hV# zp_GQ|oKM??&Fq}agRD>COGCeLhBeF;sFX)7v%JeA8WNa2)k**@49w=d!ES5MiiFPi zq)!U9PwKo++SJYo15OSNzwq=$db_j)iXWBpwW4Xzc_BOq-NMw2&I)x+3&l_wmCYH| z&$)ayx&%+fAyDG9sEpY*XbMtjik1N|ls#dvCmF5?9W_MDiN$--7%fS}t5FTLQR}=> zEagw94A2ghf&m?_0_8Gmnm{Y;HOd(PY($7N>$m?2h0+Us(@~sKDwWJR&CuJl(k$iD zJk?X+1S8A?yrg3yWBqsGjDR>4qh0!Rj(=&wBJDt=$t<*QXR6boPKLrsn z4b(z9Q#2J!5`j-7a?wVGQ&g?ZN5wihl~g=+RaK2uOPy6({W>o_AsVE;_A`NfF_|$_ zB8ym4Q$5vGmDN>+)K>jIST)vHtyO0gzfGOC^|->#OReNdPTdPfMkQ894Zvij({HU( zN`2N$Emw0j*J!P!&ty3i-9bXb*7rQTC*{^-tyen**L%&^WhK`RJ=Yw)hIGvfvVady zEm$42mbH4n!#K3Hqt|X_)_slGV$@Q9t=Rv5Ef7w)Rf(xbU`4uoqd*i;Q2AifQq@+6 zEmnwSRz(X};A2^d9oMR(Sc{$6ehpZ%Gg!e(n~6JAG~hy0V~mqU*_5@{mo3_%Y zo1_KKJ@`_9CO$MzTH&@BD261S$I7lc{SXuZQQPX*~cAS(k)%?AJWCI7gy^+h>9taYSYDdIkWHLgFyk8r$_z~A$y(+_p4(?`W?X(` zbgtu<>db6r*YnioSnw=gV%}`ZBye^uab{+FMYU1ZX9R|1UXE7x(38wXfz(MbI=N)$ zgW}f8XMHv{h_+>ao@oDFPUq~UW_H$9iED_k=>U%=0umM6Z*J&$b?ELxynZ(4A9m!5 zj^@0@XbBD^YxdjBn<#^BXSJ1_iCE)06Jh>@yDcW?|AQ~076*O~n3YAnqE70it`Me9 z;9Y5IqLyl7pz5ktYOE#=__~E-;Oecu2CoKdu+9XoE^D&3hCLu_vYwK)PU|U|5<7tF zIWPmf0Mm`d#M9~7wAHp~69hA;!38kWk;TVO=4mLi>P(pG#9nO1UhFc+gvXBTGMH@2 z{sPOsY|G98zT9lh?rgF6Y|xHnj}~nj7;V!o?bA+e)LsGAZf)0oZPH$C*N*Mbrbl^$ zMY6^1nOsl5yzBo}HqG-jl_>#(khO$(zDU6i0p`faFo4h9yFloUZq}Y|>4xppE>i2p zZjaXP?bhyc!*1`EtnVIg@-A=lK5z7XoAh39_I_?c7LfAx0OibWnyl{#?rr?GZ{G$l zflXV6ScSQs>l1hcw07bJ(%eyX-uA9524C=UYj6mU@CTQ03RiCncW?~Pa1GyZ_wODc3!t@VkdHBPj-lb zbY>rRWnXqnfA%v^qGPv55BV%4Px7)|zEfwSOWn{u zyQp;$Cw4`bcSPfJFePqh2QYvP$7R3uVfS#E%XcD|@Mqs?fG_y+)^><$xLKlh5o>l^ zH=_SAPjW2SYo&VxTj*Xk_eOFD_&u+I+wOCI7j!{~_I29#kDqjd0M|e$zmM}j4|#yU zs+N~@M*nz^7jjC!`9IHdQET*^pKzY-ME@9%7kY_bc>w`}kSTM*adc-Qb7H4t zGm2V#+h)NDcL8t`g{AqFSNWl@cbSKHeo1Dh5Br|?`tn11od1E{Hu|y``(-Ejl81Y@ zpL@2a`?!~TM8|u#H+rad_|U3$U!Qiu-*%0zJxPFrHqUx-Z%@I0{DSxUcAtD{uYAZi zdCIqZ&4+k_mwe3k{A>q((P#F{H}ZuytHNL9WI6nwY+ZNFbJ(x!3jjrL2*mJnrUz`s0;r~1Zs@0#C! z?uUK5@BGit_S?@U+%Ny+Z+7ADBlZ9N=0E@RfB*J(fA(jVrx!J2fBL4E{`t1{)CUMB zR0O)gLIFYngb58CjIeM4#E23fQcNI12M7fjG;EY$!6V2AAwQBFY0|;TlqyTI><9zL zj~+#U%&ckircD+gQjFlK^X1Q=Kzj=HS@ft4m`ZOfrP*|7Ql?O+Qmtw=s#L66w?5VI zWQ38YOq50;%fsxzuuRgfjly6+bxbiUK!ip3B zFK+CZF~_S}#X5ZEu-w6q~gnW2O#>dKyYZSTsKy&EPrBknN9VF81I4M_dn)&-q@Zvd7F3EwCh*mEj&O69R#652m=T(0fG$uw*he$#57Z3Kwj+k-ghH?H64c?ddS>|*NGS+i6d$_9*RGW_8@vJ9+e`D*}*p>jWw=On|^uexJi9z z*!N?OLCV)(7rjyR#2&gR2@57N1h~O}2VV3+b2fRjREji#2Lwso>GWKQV2Ww~Vwh!` zNv4PwVyNbYUb^`vc`BARr;T;OH%DH0-pIzDJKEQ0e1Xp98lQRbNF-kssgUF?oY>ds zG{HPudTV-`-z@53Og&YeFB*%0v0{%XtRet%IF?tV7l;d zFl=kCa}Y~hF2&$heDQTYpbLf??0Wq1x*^9)Z@sw6dqlmv28-*ycLMtVvTXnd*aavB z^G7fyOfs2pr3=r%>BANaJv7Eej~O)5Y%0BUj4+bcG060eOfttNy9Gzdb`JY8%wjMR zFu+OLY%tC_(;{hr49`%PT4%Y<-P%Fx&w4#07=is{**BBTqt2M< zk?BK8+!Y4yTR^@$@4d_Rd*Z$aKm6cMk=FR(LnOa^;>;udyz`0^KYh!{ThFiXZ$W>2 z@$`c1F8JYNp85FRmw!I5Dg$b}@+4d|02yPu{`$@bF)UmK`Rl*`|CE0B|33ixx4XL~ zsSIGyg#jCtgB_e81^{%zZ}`T*y?rNuf8$QxGMF6-apFzXr5V2c2{z*u=QTS+>#tjjnQu9t)&PI@Yp{a+}}~ zyi&Lr41j@lJSH2T*FXlj@|C^Z%pm<(wX(`ZwKC_n>=>jnA2qj;( zvY`$Hqx)#Nts*TGpYiRAXh^4m7fVA!^)>S@{ zoHb=q8}}(yb?UURGA%3>`&UXT8g{S|Mk#U0?Y#Sh?l&pb0Uk> zu`2PhCx~SVXoWSAs+J7Wil`MAx<;o4RJH}ZX=M+R&(XrqujNeZO$8gJpf*;h#Fd#* zshGRl87ZkIIBhJIpabc$vVBV>m1L8OT8lyy3mBd4`;170;u?3dw#+U+2MXPJh5%*j za_y7Wz%KZzcb@U&>{M;5Ja`_LusTI+4|%i(o%P2f=oQRn$$MBLieV6ob?`kIn@Q6+ zv4LxYCjssYUgbqq!Nmjtatt)3wKWy1U2%iw)aeO1X$*!9}x<*QsJC@lUtURb_G2C<7Yd3l^OqLr037&LhQ> z3!cR2&y)a9B{WybhxR0-k*we@Ae5m!e$rZOi{|N85ym^N?q|o`)eWZ-&ZaJPnF*wV z6*+Xg<}GHc1Bl`ddl{X}3=cYKU8&9J!4rGv^9!_$DM1Uzr-f#;u@8MwN$0fD>7Agb zeT?b#o;ufWZYLvmxzt48dDLk>^^0}?z3Xw4o7@pJ51UImI?gTdhU#gpui~vfLFnn% z_U^NfDyw+66B+#6jEUbw91*6^FOzR#dkr z)Ir`Jr#Hw$J_nE6TjVCc1j-xPm2%wrC^8JiJB?PJl%)Gbnxrn~2Hvdouh zd>Nm&yXY{V_;xa`bbDc3=?CEBEhVnk6#~8LMgap)n*Q;Q%Yo}&-#XaC9`8Glz3hMB z1KQILh_$m_<&MY^W7PK|2yEh5%|F;L-3K(3c8N637S~~ z7mj~?Eg&EHNl?D>gt$EBXC6BLrY~JsxWXFfyZ-gDYrXWQ&x7behlka(-t4G9JJ+G` z9?sML^rd$_?o*%p-HY81hiJRnTW<*82VW73`2FzZUiY{&U-`?&JM?YQ_rI%y@YcUR zHx6&S#0w<~uy9Y~B|rIOJfi&OKY!*`um1IKKJ>O{zwYBNeBO_~{`0?o@$qke{qO(v z{?9-C@gD#VAOR8|A|wI=HlG1Pg7G0A-9-WfD&X@)pWW#j;8h>?WnkfLAo;yP9Ha+o zU<;qP2Nb-)8;l?MpR=C& zVGy3787|-vq9O1d-x|Il93r6`Mj#y~p&TmV5=NgBS_2dkp2xg_+iZzR!Nhd96bc?< z7k;54h9M)CVH!Fj8%81|N@5(Up(Q$C9p0fP;^8LdVJl1@ANC;!ZlL9OOmTG79BtGI zBI5WZ;vzPp81^3}P9p5p;w-jdA>866>S6*`A`y0?4+f(hIv+6VVG}yxF+QOQh7Z$d zlPWeL7fzll!Xh<}AurmZ9poY{UgINTA~yEoH|`=hdSfOQBPX7tIpSe5exfLHU@4+v zD#qhyNlG6>wj?q(WZ+@l2oQ}4 zS)fHq<3++jMrx!;jwBFW zD#vLC=QGYF`Vr@AvSwuVzGG>jM=!v3ei>m03zUYbqsf!M&j0z}?+Gs?w z3U})ND3j`EkNT*PLTQu|DU}wfmFDP&I%Sqd-i;E?k~S%r@~D9RsFapzm1ZcKrs<5X zDVBy`mnvzShG}@Z=9rSHod)TYs;QoC=#^?|oL=W!iYR?9XduYxonGUf=IMBvDU685h3qe3dE{%MU$s$+~OiCXHVim9h6DynuWt9~l1 z@~KODCaDT)gm`GHy6SnZDzD}$p6Y6>YO1U@rKr{_YRaCWrYfBVYomJRuQIE!4y&j( z9)00xvMQ^SKC8Bpxj-LvR*5;X6vhRYqtVxyF#nCj%t!ls<^KI zYr5X$yTWU)+H1V_>$l2l3v6ecx+|jUYrY<=b0Vz4w&@wnYpR?py5g(4DlES?tHoCA z!uqS_p`x{7ti(?2zV0f)W-Q76E1!}pN|fuV`s&A)VZv&x$X=|+mMq1B>%dm4mlkTo z-fPRgEX~5~&yMVwI&7&x?9TQc&-!f0#_ZAp?aT%(tP<_9>a5WIU(yOA)M9PT%B<6x zEHHja$5!ptDlOBJtz?#M)~0Q&LLHo5DbfClM-PY~k3htTit-w;NZ8omvJ}&3}ZRa-0E#ww1 zS4y8FfUD(7<>IpI+pg~BdT!*t?%95>!RiS%A)w1FY=DB{!(xGg0A*vDxbcnx}>bicE}Zg0_HMq z>z;4^DsTGsulhbPmHK8>h-lWaob>AI{VuBmH!%M4?*~UP1V3&-001HR1O*fT{{Soi z0000$0YCu&2>$>_c?v}km8nz+1;$z^i(x`l4kOZ<*l<=wiWoDp+IX=eM~@ameq2Rz zq^pxCQ-UlhlBGn5FjpR&`A}ocQw9~@bonx7&z(SV<`g=VXwafMX(~0kw4l?ZN0Bxi z__V6kt5aqEw5heF&#hf~npEjk?AWhp(?V6tHfme2Ox>={idAmhx_9x?rK*>2-@I?b zk{xWeaM`nm3Bz3+*zR7(jUPjvESa)Yz?lDP-pp7q;?IcU8DT zbN#l4%{lgJuDDB!(mlF2@7tnPmjdqEcJbrJlOwOKNj7b{xubg)uGsf)>ez`d*S`I^ z^6bpPPyd%bKKOO(<n_@IRnhDczA9hSIZh8kYzA&4KY$fAfOQutzu zEz+oBjWgnyBaA%u=%b4q-iV@*JDOPJk4GjMq>V&2siculA{n2IwE;P#lUM4fC6rr6 zDW#WFf{EprWLnv!m|uoT<(f;LS!R}SwrM7pbh0_3ozA6M=bL=)=_j7E1*&I^YPOOm zn!CB#=c01{Iq0K>9xAD!e>G~UqfE}JsHTWQN+@x+oOo%bjCN|Or=JQ+X_TVw=jyAa z#{bFbthAmAXsV;O3hS<$%9?AhwFZ0Vt;7l&E3l+us;sf7MvCjN&?4(=w9DG6sQJ zzVi-T@4w-$+T6mECj3;t`ua=n!RsQ-@WlYLsEffIQyj6w<7V8j$P*VV@W&EcOtQL_ zLVPmHDXTnk$RI0>@yag4%&X2G%iJ!@CBJMk!9>U0Gtd?Le00t{Hy!lSNI&iL)H_36 zbJUPBJv7x^SA8|s?%gc4*IgTJwbBTWP4?Dix9v9Aa3`&G(R81^cHMa6{5IM=3jf`4 z-hkg7_$Pu7PI%#nC(gIt+`a4gQGS=L_~bNGzBuD@_Z9ivmK$z&=a*yN_~vYTF8Sf8 zrw)3FqG#Uve59Xl`s}LH{`u>x$IZFyw)b9p?!fErI?%C~{yXn3%^W=Ll?(6P}na?6)tz`_983zuN29|Gxb9NWa}R5Px%spav7{eOc(1ACU;R$odLlgRNhc5h~4*!8TMAX@k z3}!&09@Zd;B6`k=AiSWejz~o|)UXRN#G(?Xz{M^;!HYxqq8Nif#xa)hj50vu64dBL zEh5p4a6CgCyGR8%CZUaY45J?Rh(|v9agBRqqaX*V#xxp|i-jzLAs2Z_6)CNPEBOJEY=n8-|~GXKcSTsrfaxV&XFrzy>9(z2G<%;pA^fXxfU za+TbKr8g(=O>uTpoCb*G0@S(Abh?wB?~Eq}UKxUVPV%1kv}80NiT}oZqTmWP~ zy3BKqvz+KG07A8?gobiq24aBdFuAY=ii)A4zr3hLH7ZPwK2(>>45>()Nzjs-)SA>p z={3WtQf<2Qr7vBl3mC9aa@zE!HO*;4<;l~Y`jn?BJ?TtuS%H}DG@-MEfKr!ARGdEa zrqdj$F+n=js#^7`QWdIJgPKaN!t|I%ed$nfDp7~h6sc!TYde3+*0rkjtvH3NT-!=l z6gV`ZL)~g#nHpAE@-?o1CF@?RnN_SB_OOL@<`(qI*siwJrBCgvWbevLz}mI0bH(gY zGs{`c%JY~S)u>lpKv2m7Hn5u&tV&T?f>I)uq_2%_VowT5#s3Zzms$PlU@fcK-tLpP zopq~lJDbwr8ker`yel1+O9#?A_qouGZc|N*+SDFYx2<)kY^l24*}}H8C#bCn++fy} zK6JSrB`;oA`&-~r7q~?5>~Y;n*>KKRq4x#qMVZT7>53GZ?@elT2TV(rt~9a4ENpkZ z3t;iOmIRTtrFawk(T`%+y}o^{U(qSy;d(f~t*kGI<;&C1^0&GEP4RB0Th!AMc)$i; z=|(F&(T#FSQQ;HpTC?^LjJTuWo*F0+u>x7;|;Z%A_!vact6G-nOXD<#CHwyk_4;bBfaH9b2-S^W$>c!h-fo6meQ)Ow2(JFS56Ds#iU-Vl64*I&Yt?j zbl~x-r3-5Tw^*spcJ-@end#`xn$d|)@U2Iy>(){?z5#A?tMyFox7IhbEG{>Q#~=n| z`+3{E-te>KJ_N+n8Od^CCnTW2nzC)eNaK}5)zgD+>Evm~oL*TN-HZ~XB zjp9ZVyw2WEH@&Bw%?J}V-yje9Yx_-d9m6~0m;W}cxb5BKhzobDUQM{CKWkBBqg=}( zm~_Tj&T~oMJO)3{GIuem=%R<*1CjRo4KYGan`SGBa-~(ZYdezn4Q_$Mn+7z#Z z>ixdtn1UmSfgoT{FVKQ8*a0q>1KZSY5~p4MrGEjq zgKo!u?dO0J7=$fAe?nM<`Db~y2Y^b*gSW?o=@)!X7=^$`d%{)fh71UVMo5G=aE2f-1f0MH`IUn@ScQ@|eJogq9JqoXV24Avhex=0Quv1g zcz|0th+Y79Vfce(2!!vah!fa|K>zoMe5i(!IEfMHfQd+khX_Y7Km~qwUvSuXxW{_A z7ln35imB&!uaJtWxQeXUilX?6uo!r$xP-I_h&xz{$Crz`Xp6ikfPsjGgjj%4Ac1Gl z0-We|bv01L=Zbr0Wy08ei|B}q2z_mrd#K2av1pCEr;XgWd%gILz!;91h=}d?g<42} z`=^B6$a-u5eHt)^oj6~g7f9{X^`sZ zjo|o;!6=A_n0=F&fe(3;IRBZG73q%^*p3wlkH|<|9SMr{sCTI5cI2o?4k?Y(h=eQo zccF-Pi1(B!iHk1TiwYT&TI070OltO7+pg0Ic$$M=_hf7(L zHZYfG_<$xE2UXdGqgaS}$&$7BlC`ImTgjF8H}6dIuxN}&_Fpc=ZN^r@g3 z+MymAqPifW30k5idZHtGq6oU8CCZ}z`JylyqXU`_GfJa2Y7N+sqdK~yHOiwt`V2T4 zq(M3d_;jAymznGNS?%eEhsT!hX=T&EpbC1RJzAyfV5L7wpe(wf7iys$+7Dd{reeyW zWLl;nst+VOrE0pRYwDsZ>ZWiSr&=ncbb6!bV5fB&40kFGd)lYn;HQB4r-3S{ggU5K zdZ=sgf{&PZ^Z%Eg>)nq);lQ^+}~znyGjyqih}W&YN)b0tF&6HvYM!5S($Z&nZp)b1m&Ky7?75FshYZ}#+s=mDym_s zp!-0oql&6u3a!x^t*%#HukiY<279Q*imT#E1rX?{ns%<} zI;oTzrR?gi?|QK{daur!uNyP3`MR$fdZw(pqW>DOCOfbNo3h(_uq=zPE~~IZID5Pb zscIFm=>O%UdAF|Zst@vDu^6ke1Uju8>#;+-vG{tl{5rA(d$KKxvP+w%K)bRnJGJXT zq>(tTSunE@tCs1Svwc9Z?dr2Xi?JGOv|dWI`f9dnTdyF?uW$;rLdvvq8?{thx7|tx z;Tp4yO0MS0kv7|QlzMv%S`T46wuT$DN4u=IP_}EEwvCIlNvpJQ8@F@Iv~+8?g_^mV z>$031s4!c7S<3@_>t$QZwQm==uwbcTd${q+xb%v)MftFCw%pKIE)tQ)oY3A@h=y^brr(@VYdYrWX}wA-7%01CVe3xC5mzT}I% z%DcSXJGQR-zVI8r(wd;PtGy14zYrY3^{c#rf4r5WqKCCs)aEWHIv zzbpL0F1*Ar9K#j-vWl9)Ko`C@%)tR{zFTM_ka?e9-co(D1C#N<7cK%gy<`&(5`cs5X`3 zJkZgQ(HdRO9Np0#jnE7o(t66L3q8`qVA3cp(z^W7D$UDPn}+us%ryJWD*qd|2xxf)_J7N18L9+JkWtd2nPt- z(>}e?Nd48hJh?>;)=C}HM_mkM-Ogm))%L8n3TVn{Nm0<$f$m7vR&9c238P%y)n2{U zY8}>poz_O(*C&nEhV9oNjo6C4*KIAju@}*M_HVaTf(-kPZ;97naI7x9*M!~Kp1s(D zozkH_*o9r%qYc`1+t|UY!Dz`@9))sr-2x5J0#z;B9+=qz&DW%T+PqEFyFJ#X9nXY) z*1}!fzTMlLz1c7Qz1!E?oM}-^7uy{;yjOkOnyuW}t=pfC+}v&4-v1rk-woX1J>K2j z)2dyL&dr(G5N+W!6DV(rl4P2L3l+kg$= z2!7sur~{s8V%0?1?>z|dZG!W?-}!Cf{GH(luHgl~;pN@oAb#K;ey|FDmJIIAh~{lz z=icot;qu+z7_QwM9^wuR-Ucq($Nk|sZsRmQ;)Pt|(+78-=gmsj-Vr|GKZ)Zpe&I4c z<36tAHy-6q-s4Z6-0C|8c^iFGEWGJ0SHTC{W_jdcmjPD*0x@&f`$d(>)6Y zOnsRwVB&8KWn-WN7zO4m&Z}eo1ZuwIOs?N+KHz6Q=xP4vi2p9-g?`vo4#Sm1!!+y! zhS=q56?u82;_qnNl8)qLe&~k2=%CK#i9YJ0uIQzH>W!Z2$~}xw7rKtU>9y2uy;^!n zM}m8PjD1e%fZpVc{^`2@>!EJyq%P>FKJ3MA>>{4(YXs?R4C`_&%&u)?JD2H{?cx*; z?17%`$4>0q?(5!;?7ZIX;@<13Zsjr@=aJryVK4(*Cuow5O)IYH)?R`~fbH6T@7zxA z#PIFq&hJW|@4`Or0^jfE{^sc3?025$8z)yqPVIbt?FGN@)gbWzukZdo@S$As6tD5% z-tid!@yWjH=N{8u2y5;IU=UvIoc;s3;O6-L@)N)DG5;^~ARp`=&+#>%^B2GK-w^Qt zisj7izu)z1(k@LY|L`Qp^5#zSGmrB;Z}R~!@=mYwR8R0f`{<48-r%e5j)qp}H}s`s z>+-(xEid)U0P|D-^lHEMH(&K^@AEDn4M0zf=#A%a9b%MqOPm+>Mvn9nPy=O;_PrqZ z7Vq|kpZ15}_E2y2JsG4dt=R<^-AcR7{ty z{FrUhu<@h+Gd4W0Y{H5P^b;7#fPOVYsHD7&>CM4AfDg zqJ{=Mcr;KLCyN7+5>n)nmkpfBlPFcLtk!aw%a<`>5~EqOShSltaSnsl)8I6wPF)I>>Q$^swPK^jwX4^!@xZS8Mz*Y3v}wJSWSchA6d)R7 zILX4q2wk`W$5^BY;UJ3#PuPIzGUuTZffW_&z^Kte$B!XFkW49+uPHE&fKw(4il zmN{=0U7ED%tfxa~PThL!>$9=d%BH=xt^e5DcID#LO@%i^6b1c~7!lYZnKe@|by4g_ z%nXDVGj1HgV^Cy{BWJVbE>oxK)uUa92R|P5dGqAeuP^Tzw)@)gx3_&OV|U%VzNsWW zVe!In;xc;ZxFoocXff#=gG@5*x^wWs_99GA!U-WnPdo}Ud~ZJaIK0iQC5pH&zbK|C z@I<=)1JDA&j(aE~1EGtqG3u=Ifl;xc zyb_C1gcSv3qlOmAaA8Xs7o%~y>o^LDvdVZgv&SFPG;+-&)x40+HkFhUwUPS!5Tfnk;yX83=+;ui^O!uO*_pr$T{g0 zwZl>GT+2y5{p?Lpz4}5lONKbn(oqBnElxoj#S|&WN-g#7S2R5ZR#;+vjWbweL##oDW^J}oBE!1cd3u<+?cQk$X-c;a&{;ll4k~heF^cQa)_Az_c zetYh1e_#IjjK{P(uaRKLR~(TKwY=V%N#ANnSszQ!p8Ka%+Zov3HO`$e#R^qXJ> zJ9iEJagKlB3*DdaGygua5zl}HOrP4;*MclOFLn>q#{M8^JqcQHZZeFahVqxe@A>b4 zhC)CO2S`E!CU1Fc>sNK&7ATB_q=77?VdZMr#LT_$ayO)66*pKC9SW~iBqJgay@)^~ zDo}-S8x<3!7)2+#5Q;N=VjIv^CfPC2Q*>9koYf_NzaVOIHMaa zm&P?x(2(DYqXv6+5=ZC>2OdmSt4QY`gDBxTg_4mF`-s9Lj`1-|9Apagn&!id5+6o-tZG-WXhxyVzpQkltgBpjO=OIX@L zlC%UNCl7^%T>lP2o14_562?ZQB07OaqO<@hiD`^uB6FF{T<0Uxxyn~El7n-hrd`m0 zg;}a7liZ|3CARsTft3!Nzx?Ac&*=+wX0M$KWoSYR8P5wo^BvHX<|OMGrNbqSY;;Hh zI5pV`ZE}lw-eOS-g*nP}92B9(L?;Yg`cj8BRHo3JDLh$vQwYjar);Tc;Izn`z3_0K zCEE*?W&oUkc9NvPEFwsxkW!W6w59BfDI0An)tagkqFa?HPpA36pI$_%BI9Th{#keP2db*eB$BUQ6{RlHVps$A{rL}^N&uX(3A**h<#1iv&d6KeyRSv@X@801(qtCHq>--c`1G z#o%WBdRyFHv8VL<8ezR@*b9_am;Eg3V%gf*$GR4_v9)Smom<(=K6j^`B>x9ta zbF>B-Ls^OI0434wrP{W=H*F zT!kRlyb5^+dMOLt_g0L$36Ac3>8ncXTJ*kJG^$2La5#>Z07kqVDM8EI+Q=GMy^}?- zd$HDB2AkNyj8w-n@busai%vfoHC={9)?FLlv6Bf2FnKouFe#9t93mcZi5slo1*=%e z@&7&WitWS=TyO&z&alOUuMuP3&RDcZGmv*rOOW4+^iXXz*pDBBgaQ*;z9c?!irxIz zx-!|x619bt?F?mmR5za=)G|^1TR`WSHpgLJ+i`t7U^J_F&1`NnlHa^!+S)cAccwHq z;Bjd?ePPeFBwCnB*aV1xG$LoaCE9pwmqPEc z&U0@4s%d&om_I)xVRu2Q))5-I%!tlJTVmbnXa9oLvZkx8F$?KhC)snno-!|{Oph1) z8q2~iECmwaMNgJ`k5uaMvPH71Xpbx2)224I?R{?)yA$7Y)}TEuvFj;YI^4Mhw*QRh z#TW3RJBSNk_o;300C%5x7qDd5yy;zUYw!Ek8|S#3A#LSm)I5Q+nv!@CUekJ<*j1FRa?NFw+|j^$*lM3I|x>)pwrt zZ2R1A6yJK!zrJ^{?@S*;H~ZOdk#^xJ9p*82y3A>A^LHoO?vuxR-Yb9kzvmj{U7d!V zc`=Q`-$@EcPyE^i>h{J*AmMPAIv@jyb=YIR>wWk8>;Zp!+h<<$nBM${>i>a{(DN?` zia}xN>9_@sogVcIaC`>v>pa)fKwPn>JoTG#ed{k;_m;2yqjCRz+HZd*yC*-X%HR95 zPgEC(&u$9U&3yRBf8DA#EQ{R6zSn2J`uayk_x-Q+e9OJ?vpw|VJ$V|wU`x8=GZE5h z0>(p|)T=*{xIdSpI|<0Y{j)mR6F?69zX2q`0<H-!L33-gR_-EA+~P9BYs(%f$$5Os|y(Ps~M!h4a7n1 z^S&OO!W;ZRE9AE-Our$!1|ZC`xyTnTDu{1N8g7Zeb7MCcSc0dag8!(IK{$*x^Rq)c^g%B4wZrodFhrRX)F^vF5fn)_6;#3mIz9}jyC#sG&(^vz6C5FrD+UOj5`4`fD~Lr0%$?)N3(&@Xc|k#t-}RAk3yY(#W?42YaZdaTEL%)D9LNB@89MvTnG4m7}j^vI9Q zErCqH(8?!s^P#9qM|EVzhFrr8D5*1mNO+q_nykot+(?bgNSx$Ip4`Ss)X9(xN}@?Nub=ye!NMn>`AcvO0679kR;37 z8p(@d2(y8}lhgo|Srlz)N~ffNcZ^4gv`W16O1MOS2qIE<{U5^cauQ zr-S$xbi9Id+yl9s%c%2!R(v-MFei$%$IKMWvE)o`q{OiNOib*>PMoMc7{RbYE5&R} zGFXmSS;@J4%BT#(4`86G%*(6{%+B0PzVu7o3{Bt+PXGBZ2jVPF<0Q<|giO*a$wD;8 zrb)=QOSWryNu~lMdE89R%ud|ANZ#B{-~3JC1kdn1&ay1e(mYSY6h^5dl!M!s5NMv` zTT4=t!0D_^s$`4p)Xv=8Px0){?)1<94AA}zO!F*I3 zp9u_@HB?6Hv;yqh(DC%n@BGjX1x#5y(DOu3)U+oCMLxz;!qs%pE}G0#n?U`{&jIC6 z5fxAn1<)D|QEMbmfK1K=?K6DxA%v?Hgus{jE5`S9D+}EZz<9*#yhje@(H+In4$;vn zy-_Ot#uCLXQ3z5L#jghC01QK%#JQyb0GP3nk^c&f(I}l#8r@PW^-wy^QaJ_BA2raz z{KA*3B_k|CM42?XSh*L(bQ!I7VJGE0EB~A1MQ@PuLq&yzC`z2F5 z)F`+T3%tzgkG)<|Vlz;sn*eN{lTRF7befwI-q$_@@=Qa6QyQH4`&70+KKR%89vyku5! z^-^c$7&lNpX^jpC^bc!|I1I=EGTByPwb5bqR#mmvdiBX=)zdEhL*)!YT1CNh%_Bn{ z#^>x+MFm!QrPuN7R)@t`h>cieomhVDSO1I6EocqG6UbDSM7_4T0T*OL%k0*LEm?*= z*_(XWdj;2H-PekJ*0>Vjkg8po3*LC-m8Q{n=C<+LkR^m_6F_OxmQIv883&*p$s8h}vkR+NwoawIy1t z%~~FH*{!|Uv-4V6g-xd8l7&=6Ybo2fOVsv3+q4Z_wzWu>720HlTf~i9E}RvOWyvl1 zEj{X3oTWg&ok_sG+f~FwGcUU9S;pQU8}7{&CSQ=l@Hq;R@}|q z>784dRoWlYu^SU8E*RUVeFDmjN!)-m0NPy-39Dj`8%6)voSIi1NuYPhJ?`yblv1F zUjp7o0UlrvE?@%=VFVV3_XVvmRmK78r27qC2-cTEQCPr_HhA z0tG9OTm^7P`=mR}jN;`@FaI6>U^WIzE571C%;FPnQBoToQ|FoBh1O7>$+eq~NxV*w6Y zVy5CPt>r@onjp@C$TUGyD?@12-(N1iBr;=H7G_Nj&1073SWaeTZr>~l(_JngKusPn zme3`xW>)?e<zT~c<& z4tM4XkN#+dzQhh*yAk)af;w>0fO& zmDOCRo~x&B>h%59pQ$aauB~Rfg{;1XX4`|T&IC>%Yq8FRJxFV{M(fjTYwI!txPHsV zMd=59I=pR07_eX`<>>8PR!N*{HHPX;7;M5W>`W+w!A9&dSZv1rg2sOA9MCk$9z9JP zKFhxB;p+g**6bXhfzB2J&em+v9&OO3OwT^;6(DWZPHog~?bTN8&W7#KR%A(1tFIc! z+s$5T8mrXYC})^E)ra2k&!50gOKn^j~5DLbuQt zD0DZ%l7T8BG`WUnlovhxS`9_f-D@WCve& zH*`TycU0GNQZ)8}!SDQbcYPO;9JlXdXZ7_?_8N%qw32uAjP$TUciwV#0iSd=hhb4K z1^-Ft4{Wdaq;b9{h4(|h0e0Vaei!uBZTF9-cS7%NkH>fR{`XcVRD6eYj_>&UW{`Nd z0h*7CJx^~Mf%JQy$_ckWN2ho927#|(bkjz8)AsK~H~KLTVp)ImMK^gfNB1Xxc<&{l zPmqL;3V>_Rc0=oToNx4`Z+Vxm`Hb)Pmmhnh7H_owd9#0dfY)=fkNaZ>d6kECx<~j~ zpCmuW`?`mDk?(tDKVhjqcV-`a7T3{q3UwZ_kAJ?OfBR?t`yYM#C-&+Gh$v2s;Al~B zgh7M{c`#&RaLK@k5+^blBc=z0j2bsC;Ao&D$OHuhj4Wx=p#uvVGN_c$(k0B7GGnHs zSyKT{n>SgeK=|Qj&!8ZHvVZ}k2-2iVml9k`R4CM_QeW6GYPH7I8(Oz=?ON6A*RWZ~ zk}WItEZVecj~ZDS)?t)~6dSTNk_94NiUO(J?VII{m%xJ4u!-ZQgvXN+7XMEO>DVOX z3Oaty)LHqm<;Cy5{e%FsJG(>!`$JNNM7zmqQy{(O0&KVq+E zUzaWH*H7oOZcjVD*xkDK^Y6drcXQ7v*9U7?VXuT@EVP)&T+4S*KV{ zJr%)HcSC76(-R;7&>4Cqn)m>TDXMs)c`LfuB8l|jXWDye$>`yYYw717fITWW;3ftB zIHZA`Fo@A~0^~3UCb~Gu$sQpLu)#+bZZH{&8sO&ALLQ99Q(2&m#sA`XFPeEKnlP5B z=7?;9iDQm4x@V(?JA%O@o@WH|pK^2X3FLBr`Y0%IK4L=Pkq=UE5-m_VdW(}+1far& z5K^ep4^tq;WqWVdsTHSiZn|cgWR~jYsivliDyw3;>LskO-gi`gciO7OtykoFD;av^ z*{g4d`f2O0zu{S!k%@9dA(ML;DeR$q!6f*~)iyy|N2ouBYoYq5k5*KC0&D7q*Spg{X9!kjD_<)xKkM*+H& zeHb2SS&~{V#u=ZhF~{Kwq=psowmUMsBAZ+S{iE=y@g)bA{ItkaM&*yADewu7g5E!&T}q zghgTKzWd|7`@VSMdhBe#oXLvA%;VmHj&WO)|W;bSB(>e+l^Su>ZgQaP=Q}{{BZ7-Sy8uR@egq zO*4>~#h`%#qf6fa=s?01uST;0Tm&=dw+r^I0T$DeeV~GY6^xL3;iCZwQJBINs<4GF zJYnWySiY&vYb&^8UlXvPgc(TbeXWZhEapeQYE?>g9z4$xlW4@$EwKR=$ek06_=Rg+ z!wzM@M;WdtMAdyDfKF_p2$}*$CMxlWQehg0&{)PbatBUh8~_~SSR)pe@OyL|As+Ln z#~1QZd?V}Q`EXPr)Xl6yKKx<)BsIiETF5#kpkGc%K*%vZ@{MDo0~EO^p-JtmlaRch zlqML)0Je{T?Mq!8QJKm#zUhN*loA`|D3|?#5dSWpNn+7lxk&-KO_fd*;T~ZqOhB&C zd?pO$3+vd&?Dg_G6&T|h&W6ZDGP0UHyJm%mPy#OC5Mp1nq%$$uAjJ5@nv9g>5{DoH z9jfzmWt^lhAsJ6IuCkW-qrec>_%k^2FL@zIo;nGTPgHuVGCup~4BXZP{aL4;dMu%R z)}^68PE?{kxabiwiqSI)<)I!$CJ7^y&>K<-n+SM75u^!3*uidR<|LvE>gNI!jI@vl zh$%L?Nyr8@@uij2WdAfm#(5G|o<#j$J%{Qb$!Jq%_`9hrUHQR*q0^@GjL~TJGOap> z3<{wV9afgtMK?+{Vk=~&tQuCknc)q571TTpX!8**99lO7@eJ>?aop z*3{P?l%IdC>ugm!*uGM?xcda7XOW=UfBy3dG`QCrOUlI@3O1h6y;2UF>C7Zusta)S z6H77iTG-OJIvl94U8&kvE{IdNP>e2N7dnOCu1}|<1Tc38THL;VHVAUe?K7W>T0%BK zvW_(=aqn7P^ICOE_%s6)ILBSxfXudWn&!U*I_i>VRCDFXFS_>Iy3z&9WO=Ql}cl+^~z52{MP)4lF?PX^sx3!g=+#trVS1>gfeiNO=1@LQ`o9w?`Hsi!RI zjz>D<7k~K4=K=Bjy1cAGZh8O7Z~ooiB{AeHp0&<%zU!txJ?h6^dDSPv5Uv+{>b>6j z*^^umi>N)?C6V^sv%T$kzdP`Qe|LMZ;ZtYWR91qx23#<|`GSA`Bce}u!&ANu*1taX zMIZU>mp<{{|32wgAAZ<_zx=Vk`uNjdee|Ed{p1fj{Nqpi`O6;n^uIsvd$0ff%l-HL z7d`-TgI3YXSxCSXU_!5a4je!q-btSWJ|F~gp9D@I1+Jg_nP2x^AO>n62WH>~z8?sF zUJ|P z^x0qp9%1$=U;1qVC6Utu|ULgr?Ar-D57JgywwIB<=;24r& z491-`)Lk0Zpzz_~5s*Pzgvba*flLU&5HcVV=3yQ(p#?Uf6#iir0%9O)p%vO5A%bBS zCgSatAtRoh7@A=t!k`QW9}TkM+v6S|adeqSHLocdURGSOO7(VknYgDVm}y8e%f8Vk;uT zGB)EYM&k)OqB7E=H9}%7UL!XCpZ{^=4PH(V_9D=9;xHB?^_}A=nxpzLqcg(dGrprV z#-lqzqc+x~J#PQwE!v_r=At)RA~-V5dH6s%LZINOqcN@{JGNsrDkMV=B0Mf6L_TCh z-XlfgqeWh%HgaPn>f$391jl&dK!)P<8DuI-f=0CBukzoMrNZ-+G8bd zWIqDm8+fF~1fwvHq#pX@F_xtGnIudWrBTY_G^V6WQsYv_+2l<+Un}I~ zIIfC7_9RdiWLKgiSPCTEemQKF?{66Q;G z=4LJ?Xg2?*V``*b_Mat0=44u?YWihgVy0Oh=4TdWXvXGj)~0BVW=)c&ZVDblq~=#r z=4!fTW~L=<(q>}Hrg7TlZ7Qc!nkHA$rE}_}i69N8JfK%{CR$!*YYt~|5~p{1W_N;T zaxN!xN}_W}XK>D^f|4b2>ZgPDCx7}Uc}}QuK3{<1A>aW6>g*(fb|`$VrxreGgzhMlaw(YNCX`;Nj1K6KW{iOrsgZK0 zl74BEda09kX`HI)TS6ygQYm#Fjg_kDYqDvWy6Kz-s-V&-l;UZg=2DvSDVF-_n*yqw z!s(z!>X_Ood>ZPZlH-*ws-`k(n?5R}>gbYAeBV2Y@xZtAF# zYN*aCSE447vZkXR>8j@Ht9~l4_A0Q_DutfvrIOHUX6mmZsj_yeuQsc*GAFDys<6sx zsd@q-5bL4t>5N9=t{!5nN^7!y>$C!BwPGu(tN^lxYqz#4s4}a%f~&Yv>$sXLb!z|X zwg#)Wva7p3E57n;zs4(}f~l>N3%aJOadK+ACak~i>$}41{yFTR(rdlqs=?}K6S5x0 zDy+u-E5vqe!v-vLR&1r7DSH;|vNr6x0_?(iEW={s$7Za&O01sxzyjWDt)?u_s_eMx ztjlifsLCv&rhwsz?8_!;QnswV9xcnREYJ3=tI{gaB5KWYEc+R4(;{usE-ltx?b!V& z(W+h478DCiE!9?S&hBj1rfu2!Y|%b#{CO?Nu0hy(%h;|R)0*uCa_!dYE!#%x+OFu_ zX2?kXt<@?m%=Ycxx^3bjuGZddqy}zE2(8@`Zs8uT+AeP88m{IlF5jN*v_iVBy;82{ zcJAh4uIY+y=3*|~MJ4E(Eb5Lf>7K6Ba_;M*ZBMrD*Pg8I;%(}d?&bP!@6s;t{%+&K zZm&g8T5IhFuk2dx@H%howyon@?dq~F4q!%_fC2MXFYik4^FnX*f-m_Jukkjo z@Zuln>dWpuUul-W5_tNk5=^x~_Z|?f9uHLTz4=@3Xul~+20;lhZ z9$!}8M7_q_|E8{%A}j?PZ~{-S0$*c5001HR1O*fT{{Soi0000$0YCu&2>$>*2@Hj5 zl&Mq&55`)^P?f_~4P`}~MXMr3i54@uy10>*M~)vch7<{Mq$raeQ6e>)0_9Pp*7<^zWjpKmV6LoqGA~$J2Ay zzP&qptLx9xB|rXlefr|jo7b=;phAC#2QFh6tnOdef zrj=`|>86)x78&Q83bx7Smw3{7rk--n=_j9c-YMvyWZGHgp?)G4;i82inJ1x*K58hO zls*`#qmWLjDWH{F+G%lmVk+sTqh87>qMsHiYN>(tX{wT}zW;h^s+6u8Yo@r)N^7Dj zqN;1I!2T-huDaD^et1PjeHoIqgw#KAOw84_fY_i1~d+4&(#^^1q)na?Cw$F0g zW4P#!D=wqt{z+}R!>$`|yJohcD7)FZN*^q@ybJHS-9}n&zU$pf@3H=V8}PsC9!#*i z{Vr^-!V;V6u(J=_i!r_i>pL;H04Ln)$Q8>waK$Z|pSDOy8_DC`+IGGSpFfEON;At#(PedcK_R zEqd&)@B8)EZ?1au)B}tB z_tl4wefG0M-u?OLi?9CrNCVHl`{aX4EA+&t|Ni~>FCYK-=J(IN{rLl*0QpD20{Rbt z2n?VC2e?26Hqd~?+1>*uNI?f$u!0aw6ThIhhyQu7gCP812-y(55t@*Mm^h&dL9vE8 zxbTG_grN*)xWX9H@P>*rVGgaL!^HLQhc7e(8UKbzL>OW)f$O7D3zx{mCOR>QT_EBV zo-oBBP|=D@xMC2t$i*&tQ3hWOV;8X)#WI@ljA*Q48r7HtHj?p;aLl3{<0!^D+VPHH z#A6=!$j1JuOoPJCv zI@j4pQqF*o?=)o)%CJQtF4C9y?ByrHS^rLQ`g5B96y*etDTZ1Sft3YCs6rDO%Y`!Z zp}90@L?=qd6XesK7!9UIhbhpGa&(y>4QWUj5KM6n^rR+5sY+MMPnNp$rQM9FOjAlz znr`%>`rIc#tGUyf_Ee=P-~v#8NlS`K^r%HOs#0;v)TVwEr1Bi8GO-ENS6UUOGxe!f z`?=MudKIf*4Xa6CDTab(^sH$0s8lWa(_*$&nj$?bQs-(=R?>B;PQ9sKpQ_en3U#f3 ztpG9=>Qbt56|9F%tXL7-)y6tBtxOFpW8VtbzaCSrcU7t^@yc18M$@i)z3Nfpx>3NA z60RSWDGC;=*w*$ls;gzKYh_E*g#Xgvwse4OWU1L#o>JD6za?!vGn>@n61TGw&8Y~I zsRM@URj9Xps2fDejQo?I?EF8{7o!Q@d)_ zY(tHET;xWyv={Z{a2L8+>RvatsvYQQ_j+CMhS!qbJ?&%*uwAVpw7i4Jtqww)Pt>wC zyY6i-doMdxR6=x?{H1Sw?Tba_4wt#Top1?Myy6sd*t*U1FhnD8)z>!oz+R0lfH92Y z8+$jxK$h@%AzNS#hcpS*m9cY`3|bM7n8YUz6qwi+TI!@H%*~ z+C{XYUpeL*n}EC!KJ$=?yxt4{*2osV@`U9p=ZR9c#8xbBn$g_iQump_U@bD6d#u<( z54zFYg>!c}#lS#4thLsZMLUACha zTV@E8y4I@3bf$a#>u!5H!8ir2xQjaOaswN>rW8Yf*PZR`D%iKo=B8z>N^fX8^WE{5 zt&C^wRM^H>&AHb0i=XT3ZyVgn!ya`!KN{v!6BFONb~miOyklFxTiO#JYqiabZC)<- z<1F5A2?7doW1IZsoBt;Fe6v7jDjeJiqE0y-SxoaC)coc#u&B;=UUG~2ljlI!sn9(-0{j%?3uf)cC9TmPN7~F-k3$+Dy!ESl{p(<_gxEVl_E(s_>}XHB z+Q+{3th@c~TJL(?->etz&iB6iJ@0`3yWm&ua+VwZj0vAPs!jcY2RMG{ ztDd;ksZLZbc%AK-AA5P}j`zXu{qUgweCQ9)`95GB?xs(D*ttG;&Ii8po=3yhMel~z z^SuuwWJBynFZwkW9_GIXz8je!eBu}1_{cxL8-34w5IdiuIPW#J?apzGz8?12hrRNX zk9-~g;rrk>g8%o$KYsF?-~5Fyz3SJ`{`TAc{kxZa{Cgk#^vgdA+dn_|^8t{M+7=cXZfD~AP7I=XeD1i-_fa>%D2AEDAkOLt& zf?Z&FtmOjWhFPE2dMs#rhR1&}SbzgKga229_Sb^$7k4?xda0Lv>IZ*57=+h%fAdF# zMR*4EmxR1me@xhfPKbZ`w|_+tgEl~eRcM8p1OhTp3F<~>Dj0i#_kuFWfn>OW8pwcS zn1%%?e1NBeI|zhAsDyIpcTSiFpcjQwn1_03gH%|9Sontl2#A6hgL(*shWLa__sEU{P zgF*<0afpOQIES=Yi};6tXW)HjW?*0Fcbd3~Vi#Rdl8jHq{d z&$x=L2#c|Jgot>J_E(G7=Z2REjijh{y7zyJ$cw$GgTR<|dq;j{XohEqhH99GET?tK zxQz4IjFRY$(U^+WXpM=8hlY55?>L6-IFJNckY*T=fvA6nSa$)4206Ej4OWikcy?}Q zhr!o}#mI*W8IR8RjUH)_!I+OCNsarcjl+nIbCi(?xsEK^k}mm@FiD2#_yIiNkVb%e z#Q(Hi5?OblD2y|Sk;Vvv+$VP**^$Utl=nE2(^!%wX^SX{hbkG9QaP0fxRNrNkbxKm zRM3zB2$9^ilUv6N6p4;3hjW7Xfmi5%8cCEMDSMLWmiS1D(kP8g$&dZ$kNJ0&RH=pR zm;!v+mw&mJ0a=y@Xp>$^TVUCfbLVnkfPanon2;HnlIci#P(+n^nQn;*V>b<)S$3UC zcATJ@pt%W}2@7fWnWSl&s+pQ0$(pXojry2e-&<>7L^$p8hbO^jV+wc@Ol-nfSS%_t~HF`JVt9p!$gq1R9^5 zX`tk(3kiy#3#y^y84o@hq(C~P>p-MNYN8)np7H6Q zOxmPQDxgprr3K2MJ!+v!YNb_Lp$&?qT&kg7nxse?rX)J1WGW41dZyiwrfQ0&Y}%&Z z;0`i6hJ|ONOmLWp*-L`gotW^Q`jDkUdZd6VreG?hOZub*%A@^YsEI12jQ={NR7#~? z>Zg(#sS6sZUwWy6nyGFIqiedUo%*Ro8mgiir(Ed-bZVzt#hqih3DYp2_0Xq(8mOKs zt6*BFih8TK+Nh4YtCLEpzS^sm+NGE}sG7>DvwEz^DypGMs%z*4OrQZbiJdrlr)QO? z-N~o0st@iEtF(Ho!y2TvdaL1@tB0zqj|!|V`m0*%tMK`)#hR?}8n4Q_rpy`xpQnq` z8gB zto3@YFxLW3ke!CMufFtp;1~z3+NS^;uman%CAy#qd$8oWunOz2y#Ly(#9FaBYo$5c zvlwfvK)bQC+Of_mez_R3TcEG1+L0=|vM!skUplT#I)_OQ;6xqz`JfHmkKi%AsWIvpegzV*9gw zTekCBxJ62|Lx}}S+qR|1qj5X8#R|7otElr3pm-~#U(2<9`?p}*x$P;qgbS*MJG#n> zxJJtYYfFw_IFV-dwod!FV=J*zd%2Tqxec4O!5XZZySae7wVoTepc}lQyR6h{uW9QA zs4KD&$+#%{ukcW%kQ==POQ5t{y|}Bp!rQyryS=+RyrN6G;QxELXDh8nTeQlXqt(ih zcQ*+Aimk6Jy^u?_)@!|xy0_iSyV)DLz`@P{yx~7YK=S!^_WwPz-xX%l}(p$Il zi?t36py>LdgNwiXtEv0jzux=5g-f(_ngs*gRm|(1&I`K=jHL40z_8iN z!7z-$GVH=-%fUIfl~f>MBf9`0?7DDE!XF#I+m7H*3EV?8G#D#!PI+Pwc;Jj1Go72gPfX*a@;9 zjKy|LvIXpm>}!f{yTx1_#1}fmV7$UOi^hh0$cQ|{DF3>~U0k~3d!5Z{yiKsU=^Kwn z*{^;K$k?#Svnsiid%d8Xxks$Vq@2i%%*my^#&ArdrCP_0$h*f&T!K`LzLlJJdyI9NEVr{P%+k!s>sigUY|Yrry{T-< z+}zEm?9I~*&e8nFk8GV;K*y6Db-n0|X4iSJ@V>(w4dWcn^z5_Se9z*n3>v%7yTH%> z{LjDq&Gih>PrS-#+setT%mrM0VcC12Aj|R`&jTIN6D`pAe9`$#&>CIF8=cV=?a`xJ z&gTrW=iEw^ELU?($AfT;5IxTn{m~#D(-_^+`v087H7yMNoYOK5(>&eN^SlNgSgOZd zP?5)O!epZ=Et5X#(oFr*PVLh>9o051)l*H?GcD2bunq=&!zDe%N*i-A&;SXEvSb(t zElteTfYxda4Q&0?ZaoY*{nk(&*H&HEbPd;bJ=c2O$Xk7#j9JpIELK`(RxIX#rdpUQ z-E;@22eP`>j!g`Y-PCfO*LzLVI$hP6ebbhG*<_2?cRksa9omtt)_rZx=nQYu*Ixn! zt%qHcmCyo;y#kDl)ooqdX>Hq}o!gYX+nfE_ncdmG&D)|)+CIR_z&8SfNoMv|*vS2W zt_|A|INP*++mKD&xUJjR9o)nH+r2H^+W+m@-<{o8J>8{^!(NTruUvD^?c7^80FyHiD-_>p1*Dc=q9p3xh-Qext0Iu2n9pK&#zGP4XV~x=09ae*#&MhX^t=-=5 z9pCmX*#rLH7hd25&fNlz;T^u={Jq^Ge&7jid?6rvL$KgqwPeg4;fWpH7LMQfT?{ks z;WnP(Ar9gn&fz=G**WgxJ$~asuHmB#)Fpo23LRxlci71N;tyyB(@o6i}YjxOe7 z4u!f$OKv9E&b8W#-sEkL0buauk-qAzp6Ra6=av5IvHs(lUeJJks;typW~FbSK1w36 z&Q;)v%c)fuI#l=-nUNDNIPL&c7n#S|q!+r&)eC)7J>EPb& z%|7no9_B%f+{laTtMuzfx8j6F>TAC3#oi0X?(N@B?&WUjdam#I?(EMVzMO6ZEJx;a zhHL2UZ>0w8O(*O;000ok?cfUz_CE3bUh&M%?-+0K|BlsDbmUDaSqRW`4-Tyg=)tFc z1ob}e6tD3rfAKEA@=~7fj{ok+#_Z{;J#vw^P>KGm+TQRK2=OT&^e{j4GJo$ZZ}dn{ z^jPlaP)*^Y+VSPLm40o6QzmiN=W7X1@(zH`4)1^{@AN_+_RB!;V^8*Guk=Nq^l7j5 zXutNF4)t7(i&La`b&Oysh;xp|^QNu;?q2K#Q-}a^d=f_xwRP}rfwcHa1;qE>AOo#bkfBU;1{k$*z zzW@8cAN)-}uc)86c>m`5G^tLj4{9SH??#~5TTlJDkN(y_{p+v(8}ALzAOG85zHHd~ zyv5qK$>M7c-AD@%4g_L2GHhKoaOe;wWVp~_!-o-zO(eFg7sY56D?;PA(PKA*9Y2Z; zS<>W5lp|AuWJ$86OPCT@&dgYorb?VSV^TZV(R9o ze|HKVOxSPXrH5ZGX59GZW5XB`e=Azq@@2~2l>TfAHR@EVSCb4GBSy&uuwlodA=BFI zo~>BubT!eog#U#GPUMboLf39Vy?up8zF3_2ambUC2X?GHadYI$jYr2^ee?C;*{MB| z;W>0wD&OBiprHE4nzHEU5ZOZ8?N;1zpb9Ul3A54)wF_Zb?W`x%n}IzdmJD$R6Jvfg3W^&`DB5w69Md;>hB+Km{H2 z(L))9bVgWdwRP5Nj+2zkNF!zIsW^kgVuu-sEN{-&H0&ZzJx#R$DhK>rwaQk#e6?0w zZ?#t2Z5#ErMqYRAbdZNPv)F@m`3#Sl1-+2ZqDVl8uOF!7TxaAWnQ{m z?$DhI><8QX)xryyG`w)yJ5AhZ`Wn|Ra^890+H%l8Pq}lgiT6BsU9Tp6dD8{0+0uxQ ze%;s!$hMwmd^vtH?X`2S((!~O?>+Krf!93q&5t)eYV^-{KPcr(|6S0oqj#NX>*cI| zf3ULeF5eHVjd9-#4mZR%zR9f*c?(qF`x^Mb^yN){6P%ykphp$fNzZot_ zV4s12?}WWe$4^w~KpD31g;S)W3u#zGELKoE^|OlCjK~%Q{7ipi#25fUWrP7HttTc# z;1i=*ML5P0OcK0e^XM3}=iTgU;Q8WOisiF022qe8+#c@GV3!Xlk&V8K;Tzj$ML9MS ziZ86A9V_W79O_Y6fU8aaeCRQ zzVVTV_JZBT2H_s+g;JDYOCxT;cMfM5u#Ku*r7K}6NodZnmeZtT9y2AZ!2fx2aKC(4 z?6$WjLn1L+Z`hp)E62=cK9XdyjGz{u`A&AmlbZ9qA+a>s#ru(_o4#BN9R~5w*@%sk zoaDeT>(hcXi~%=lz-))fjV~ zPF5{?*SK!=qZ`%b^uiL>VF5G=L2XtcS&BrLvbC*OWo%Wg>e!k>_W!Z5!fR!5YD=B& zRb842gFVOEKcM>aq-X2cVWXAUxGHwBlMTmeBb!>*O7^jqy=-ewX9qo{t5UA>V=zfd z)K1XBs7ECtLJ!%}m#S8-vVARc-)dLrVz!=(;{m3;h^a63QVF32u36D$S__~Sw#t33 zbC+x0RbDi@?!+yys=GH~6%@3i5iMuZ!}QM8sFHhQ|_^k zt6X3!Us=oQ6>?5@R0e&aI0{+q_gUAO<~5%GOkm80TFek8!BzrU z)>Y?;(Gl*KjN^PCT5IRh)6TW8t(|91@4C~U4zhG_7;I6Gy3j^EK^U8?zYRvXdagFl zgP8nmP6;gXe(Z}9&euvx84L&=;)jolM^S4rN z#tOpk1N6AVJ?vmldE=LkD?_^LREw6Pk!SK=2 zeM7K+|Lvzf|J{$g_{opI%!8B7L$%q7u#QnF!TSN}n?D+GgrUPf2;4vJ^FHD8KXbT1 z47|IT%Yo%{FrgxwfZMp*>!9cxrO|7^-HX6xkU;%o!Qp#B{?kD5<3KTcGub#b@|Ycf zBf$db!B=~V1tcI996}eQz!*Hj3ba5aT*C2_E*c!Ria|Bvl8b$!J+XK{BEmiXLO~+z z!WR6(B%DAPWCm-?K-UvLT`0dI!zUaJ!T+Lxf!%7I_VYnHOn@FJLC#7z`t!mREJ8m- zLS!hz@RLEyd%^&WLPxucImi%9n8P~k!60}tASi-K%fp4@LOwLTK-@&_14A+N#JMBH z%DV+a1P@Z{r~7F#MNF1P6pKf!LpQ5LE!?n7TtQoWKwSJoBLqY;9K&7=MJ61?V1&6+ z+`BXAEn^EgMO2nn^t@NJLlD4199SKnW2#!rMQr56Y}7_jsb`eLCCGp*93dgRMorYkPy9u2Bu6u}ykIOwzB)fulmK^h$7Zxb z6_7`IoJM=R4>iEYTbxL4w7iP!ME_08$X(n=jRZzv{78SqJ5CVDQJWUzD;s9yR(W~`IHvOT9fgG)jX55*uJ{t8=WEY7Y{zI! zQWk*F;$TuI?awNuQvVsPQ4Y(}EZx%B43g&?K_-FEHY3xmTT(RLPBmpyKyA}N9n>4u zP&mC(uPn+gt<&mMMt9RwpF7h&O;Z?MQ%k*6OfA%T!%`jP)HqdCCU66`NJU0AQ=2omYCz*D2js zaIM!2?bmXpSpRc<))ezh*;&8Tc*b^pRCxWlVr5u}ZOnT;S%;M}i>=s|?Mi^POkAoi ztguZJC>(Wj0n_N%m;~8b#6*&%)sjuwh!t9@^jDS@&zF79j60U9Tej4|L3X7>Y3$jB z{n?TIR+N2MqUG0??OJm^*BYGJv~$`J?24Ea(;!sQs zzN974jLkXh5iMUzHqUf4o|V7(TU)km+memjxZPTyE!4Vw+^;Q%yw$Igq&h}Rwt!Nx zs72d)+`1Jo+`~m&#>LmUrCX!*S2nrKy#gf;;1n23va@`}N23ZAML5t6UD35y8eQDd z9bK-yTmQ%{M-_lgI%VCV%3NQQXUjWtK;%(aSJu?>@-<^fOsW@NoQQ!BC z)#;_)poL%T#Z~&P-=}Te%w11Xd0FNK&F2N+hHc*fzTg5jVEF}J%1hv?aL6FFLn5$4 zAOakZmD+XT6KXU()S_VbwO|5?~;h=M249?U6 zmf;LG-3_i`8!iHvRbam583J@#@-56DMz|Ix;tDp_BktlDe&OF`V!M4EaciA3$l)kP zQ2$1&%paZt6|PCP%nmJ1V!HI=FQ(%#Ug8=iA1S{w~K$cxNeB*_Z zmn@#+BG$?y2IEJTWPTK5CWbmF_PE3O5LO%B0>D`mctizApjG{4O0Hg5zUCQT*=y)#Z~kU*4(D(tidz7ROt^(}PUk&PXN)bhKW1HcK4edk zU=?j-=*321ELvNATYi2)p!;Wl&IExTXo9wbfi8ngK zXeqGhi_U0`-e`{QXpjDAjNa%cc)Bv^f|4$2lTPWAP7-uv>C}1YLA#)so@w{OCC&0< zj>~C8TMzDt05;B^oRelZE@e1|0){?nq)uw2u4sjZ=!K?es1E6n&H*W)>Ky2RtIley z-fFJ)fCa_suMTVen!m9wYqQqF^G$2DUTd}{YqxIew??G1o@=`H>NVt|4$SMArRlxy z>ypGEolZM_xdk&oj}Zn1Ghn|_epfh#>$;9>$QJ9!jx)-hY|E}}%+74hzHH963&;L! z(C%!s9&OTY$h-Dyy+&=-UTqu>?50)9i>}O({GZB=7=1 za7nC}e>=a4Y4E9V@MC;%BZ_bdr*JRe5Dd@oMRjjSWZIk-03&5=F6Ar0+$bA3aTG6c z6<=`{SMVvU7vz3%7oTw&C#Dp?@ku1{9A9Rb?wUC4@$GHtAcwCZUlt?xaU?JDAopt_ zZ}KJoaVID8S++R~k8)EqM&-`(Ju32d)>s;hU?g3a4A<~vA#-IZ^Z!k$ax_nKHD_=u zSMnf%aw+F=AWw3LVevXI-!U!lJEwD}F!1B{a})pb0UxCs7<3gcbmZP~j6?K+gmdZw z5(nen5oiG?8*&eKZ=6Hy*adNUhP4%!^ZJ2tdKptuhp$qPaRrBTK<9HiU*@{bb678J zLErH`A8kB`5CmsHQtzl7U(56D^-%|QURUv#-t}h_c1rgSM`!kCS8vr8a{7>GO-~KP zo^MaT?niU%(5rD$2Xq#0aqULyK7VmyKY&B`_HQR`aX0bo^lTh&b{Hr3c29R{{*8Jk zWqQx|Tes_YcXU->^{{zz_I`6W*P*3#ZTHsS>QqT+fAv~_cK=3?_&$&LQJ46dmUWAF z;yH(SbkTT^5BY}|d6M7wmxgpWNBJJTa+d#Vh0k<_bj57{-UQ@#d!KlYC-{)hb(634 zlJ9w-$N6#}dZNepqQ7~hH+p&hccy20EgyMHkM@*DcsSWy4nK{V*LJz~Y_1n@bx-*;*$B-e{kSxgt)5(-7QKsy}a#Ng`GH0%&Y11W69T0XVC_usI&kjN6 z?9s((kC^~Ug8~)M)Tzz~QkQJJ;XndLi32rUgaJ{))em0BhP|o)EZVbb%dSo9_N~;m zau3k0JNGW$yn6Ry$RIXrSHOZ5Rvc?s>)*sxH~)%k?D(TeG(;oURC$K7GBAvhZcVy=PxMx=Y#0$aYq&B7gS{J34@#FI1E z3!*S|!bc7=HpB5c_Q%;h>bYEU$;T0Wf0mg_W zZg(O_=ABrbc9@|z253wH1xiu!$!Am!NZn_^Yx#jN)qe^SI3$F(O;{w7M=B}dl1VoC zB$QG1)faYBAR*$FSaP_fmL^t7BA6*ohW{dq<$34g7A44tfCp>X2-9hB!ct=wJN8Jy z5bkL}B%giyDJ7szGI=1Og;F@^a)}}u7oZ=h!DW|OLJDGtVOAGr8;+g$-I?KWc9~2u zta-(oo|#$+EOF8zN*Q1snYB!Q48rFvATy{ICw$N<1;NJ5XFP#2I6IF;$Fp zyz#~egEX?p37brC$ta(^vcN2p*Z=N^l0qSb%r%eOrH5A9yfe*NqRXkb1xG9by*1){ zbiQ%ciC+msIGqI45lbEQ${GVKw8B$oZLzgw0<3kz7*Ff$UtF6lHpkC?Jhnnq4MHQwAL9;ktePyFsU&ahM zd_jazU+}n;`7OjKb@ch`pZ~w}1Ni?x04>BD;xQ^nvD*&|3TQu3(M?_!BU%VlP(Z!3 zW_k|1;0dAGBCcm@e53JrmlnXC5rldpSs6J%0?sL*2&$6Y3oB~eRh)N)9Q|~4?#|{b*m-jKIkh7k? zC}*cZ&ajdbU=Qc^Si|ill3oED12(D2nk)7MdMunJI-BUp{)JJS>YTy6*2GLh#?y?P zRObyhnH&jz%sv;fP96Re0~&DW1sYL-zt-eD2-fYG_C%;ds%)&mpL_=!mpGlt!X7wx&d~+hKU)?=Svxd zQh8cuLr;iEQLjLT7vPhf18eF%G1r8ros*eBg~2oNUQ%jNSQKBAE9AdTpkHw@~Cep=I{eln_24P`)Qgw?LD6rX3)yjlB&!yTqIt}Vo! zMf3V+>S{EwgT1X_`=Qu*00goX&4*-Dy4lR0G$W!dZD?P*+LxXLwzIA6T5Ox!xOl|3 z!~dOaDi|8R5E;*|3E^vXvm0IR4wq8h1?!Kbu-;Ra8)D^MhIse-%82%Nzt??-UlZHd z7;s-M-)K};6Pn8U)&swnJ#dLroZu5bcEKwih>dGo55`W!$2D#Ukc&JLhj_LmQm*oq zQ`_2>c1A2lZKi*0b($v17dKVgI_<(+>8vkGav-~+Go!P8=?)4|9Y4xbeXOaT*ZPe!(n4*8^0zVc9SI_5KvDgVu1 z{_LJdJLp5NcF~i*^rjy@+fNVo)UW>Ju1ERYArbq$%YG5IZ-m}y&-dN$J<>9_hTwxg z_~E~y_?*HQqM@l2;@Z!kL2CvW-bGrjo5=REd1|9RANzx&=-J?rBhezT9i{I?%@ z>tU~c^y6Olyl=nzRgBow8h^*-}Jd10U{s^CgArm z;0r>a45r@%ir);@UYod?F;2M*uhiD2#Q6BM8z0jl65WdBMNK4A+Q zpvB!F6;`1Y(%=>1;Iy4z4|bsjdSM0v-ro(O&AovU9${Uapb|>n5;`FiMxhkKU>43H z9b%!~Y2g>*AqRrt9`>OR0wEyQ-xz|R85&EHx}pnCA|+m;CDx)XYT_p1 z;wN@uDE1;KYGNq{<0t}N8`w{j7|Ox1;?XVQD>fq}w%si9A}vlMHRj+g>Y^|1A~ybF zHvV9>ks>(epRA;!8Y&|*!Xo)SqBKV1I_BawzGF9XBRqPeJkn!0-v6V~)m$MSqg^1P zC79zmIwLw}qB~Y&?X}|;)}29OVmuaPLuw;L)}uX2&ZcL6rfk+`Lf)ogCT2`NW@NSiP2NV)wI*(6W?*inZ5AhT>LzoF zCQFW`T(Ty&;bmTeWm&8yak{2)E@xkICvARbayF-Pj%Q;!W^~%+9QiWNIo0b`Gd^8t8`7=YqayhH_|(ifDtj zWr=pBg({tj!svm9D38wQkAA3+)@VbHp-`$PfaXPt?*C|zw&;)wDUd!XjZUd)o+xrG zDTcnMhHj}fN~xDNDV2sP{f(%WT4|E%j*c?viaIHXLaCd+X`70vOCsrHswtRqsVCkk zoXP>{0T=%0p3148mS{!NscPaWp|WY8>gl2WDUmK}AC6<4a-5waYHWU~q*AJ)GAgEy zo0$eEGA0+KR%)Sksi@j#l$I)~CMv34Dwz`Krfwvbf@*w@s-?oJtkSBjrfQf9DvE~c zt=1}n@~MmBs<6hXsb(sH9xKHC=b6q0L49hk^6IQU>##;Eu=*;XVr8pxYJa3-v5I7@ zPAavA>$DQ${3;gJ%CLfL>^u_NcScYq|0(ypF58 z_Uo9w=O>)&vUcjL%4^1X>$+B}!2YYlLaemLE5ItIy=tqvTI{|uY`{{iwSKI`IvdD7 zY{Z7FhnB1?8f&C&#la$D!rmswHmtvTEXmev#m=l-UM#0(>_0l_Q_8H#-fYn3>BQpf z-(jn=4r=eMWz4o~$BwMgLao%Itk8<6&IYKbYU##qE!P6=)P60|g6-4Ntjz|i*ph2K zUafV$>|r|X*pjWt#%7FjyVr3MR eV9OE;8HmEQ`EBT$uHlZZ?WQi`?(U-n1OPj=DxMbr literal 0 HcmV?d00001 diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/images/arduino_mkrzero.gif b/tensorflow/lite/experimental/micro/examples/hello_world/images/arduino_mkrzero.gif new file mode 100644 index 0000000000000000000000000000000000000000..d896534795aee089eb57ce714fbb4be0b123278e GIT binary patch literal 541423 zcmV($K;yqhNk%w1VL$;u0r&p^05f&~I(z^@i2*2M0s#U7F>V7FIs+n6121F)N0A0P zc?Up$2nP!YQM*DJ9iKqG!Y~~5>8X_1pUm6%08W|lME=?OeWFQ_QASE;+Hd`bkB_v0ACRk`E zCoLyelP7GYCv~qVCnzX!swy!&Dm6_iKV&N?Eh{T9D=jfANop%nh%7EMENGrBJXJ46 zWiLy2Ff1-GE;BKfWHF^bGcq$YGdVRI&p)U za++v!TR(GPPjiWzcx_;KjEZ`6a(a%fdwPC+pwWGfdVia=frp8LZBmSpn2n2ojg5+q zi*}HcV3D1*lX_p3tjw00rkbU-oUOW?x!RtRh@g8?p_!7Ri*ci#m!_?jsDf3gkzJ~V zS*wRytg5TAy34bQTC|ybw4;r+vz4{T=C-%IxR7AFlw-QKuf50Ezm#LZ(dom(!^E0r z#NGDBoNLCekH)=~#>mRX(8$N0aLB2G%A<74rFqb$aM7oA(W!aTtbEe5htsWo)2@Ki z($v(ba@44E)TwjS%DvU8bk(VK)~a{bta;Y0eAcgj*3Ze-*x}f+f!O5t*|2BXwTIid zjoa7P-qpw7y^!C?r{Cej-}3z6zm?(I%;MV6juW^T^JR zw>GZ4`f}0HQ)k~U{dRKd-Mw4?zs_Cz?DOBlp9C+yy?XNK%g2|W{{4FP?(vU@Kc7Ez z`daO0pMC@m=%0B21~yrOKM81Heh@nNT7L{uc;JMZ6&T@$4|ZrFh9Cyj+=ms0Xd-|0 zrRX7t7d{l)h%GJ%V~i)V#o~=P=BOi$B=%S$aU^NDyj)_9~`L<-qtkVP)( z*~l~nSS<(N)lX(gCwdiiCV2nJ{7n<}2kW|(x=*`Sti;yLG@ zc2>0}pMVA`XP$(D#buYJG4`jReGVEYlw=x+sFI5URVkyGW?E>Zk2>1urFIx$rYfq8qMAC7tFUJO%Br8L9x9)$v$neFuI_;NgKcc_eXB0{U)sN!UxA=Z^8LSd@sQeqgSlI7)z`%y%Y~z@y8Zl z+;GNng1m8aD4Sg3$Sm(F>%<*6+H_ z&u)?|w9_;rohZ}oK3(+HEgQ{l)LU1*wa!?FE%w-CYh5Uof*Z*)-F_pUxZ(^AUby3s!@Kz8lrwI56VKQv+As`-a6-zqYk>+-k95r(Y3#>d+50LF8lAm^G^5f#4lYu z?6F&!`|`}26V5KiJTE=;%~Nmv_0v;d{W;q=e|+cO3;(_N!;ina^W3LzzVXN>4|n#V zgc8a8nADGnC-Ng=i2jPq?@1)`TSffvr$74@5Pt;Jg)RcPzyS(y6$QOLz z-k4tZHW)bRsSkV~4B`4TCqkK#u!JT=p$bRH!n*-Vg8Cz18pfc8{nep{H=F|)X1GHg z*02wHu!9f(=U~Gh4$+82JmL(CNW}gbVuJc>paB7BzX#sWfeLis{U|s=>4n0JUGyLX zF_^mwnlXbO&^ArNxd20uPwkeIY26NkveO0dEZbwETV6wyaaM6iHYOyv{-n8jZx z(3P)*Wh-SF%WS|gdS=9>Ep>UzUG`FrzXYZm>v+fZ-K`mWWF#UlNy9skfe3$0WF@B= zO=lW%nMnl0H6;m6H3agTlZ>V#iO@|R>d%sn1mynshlniD5D4X5Cn?dXPDSR>hVumF zCg->Rzfn?=eyR*079og5UeeN+xkP9%hsjW47Sny@!-O&o$pv@v(FWiQ=R_w;g>rVG zncL)M5j+aWbb2$8@MNYm=Lt!QrnH*Z#3nP(*@i;4^pPb^gGM8n)1Kb&3HlT#K7$&{ z_!%&iQ;eS!KQYD!nz5+|HK_U)nowb`FppS;VN88`(i@C)t6dGk7Cst>nTGX|F*w3a zU5e9_#(|pPTq`43+CO)yV5f_8>mB+a$&JD^r!O_BVBZ>2Ql6l%^!%$y>$yok-m|Ec zKq@==Dar&=&#A72U}m9u(1t!#VBpe4GM!mbD1daWWJRr8O$*l`Vl=hy3`1e-`pwq= z5*DW_on|1gsRz~;L9ewnBnN)0QEMKuwI79NUe{V&zoIs+B(P{Bff`cawlxcgOd?AG z*<8g+myk}ZCQl1dK=S_6ib};E8Xbxrt)AAjrk!gZ;>uf@CRe)lgsXLJirk9ox2tDm zq;ij&hu*4o3@xx|5J-#KEPQ=c(Fpw?GmR;PU;5u zrs>RWQ2j|+^Cne*XKb%%J8Izt*EPdqHzWjFg6n&iZCA;}zNwa$6D zTJ8c6e+x<4X8H?@c61vi+&(Zk+Z(WS<-r8NxTo_VL9RGW=xcBYb4R2Y%Z3ga3u`T2akD1@Q#&@Bw{ROq26$kYH{`i+85P@xb z6Ww>lF@u$P(GsWhu19b%&S|XglmB|tNH~JQ<=kJPbNab?t~Ia?&28tJ^$9R9^d%kc zunc;liEpU5(|6wQE`YnyXYM&hBh3Y_V;sZOcIF8TuJst4o2+q;HP5w>>t_P}<=O@_ zsxwVe{L*^PWPkaG7kyV|M|tHP@375{E_f=JILQ`|u+oE^*0&B`?*6VX4>@)`V?;dB z5*PKBUG482KRye2x5Mf0oBGMWJ?X-o_cUM6(X!eb;J2SR>Mb1Ha3h@b%`WofRU2nz zYFqPRk34)CKW#N9ytca+E}`i>bm7Au^1N1aIM>zr=lSywy6<}b=y9%0zK#IC)lDt= zYiifGQ$Nn{x30 zW9;X0G}mZLmS&80fM0iWV25~i2UhQPfE;jsAEr&5S3C=}PRA8}9!7WbhXljracW?8 zM~6~xHg6;aeAhRC8pva;Ms%UKZg#*@BV}^%*HP{_TpLz=y0(M+=Uh10cpDZ08pa5_ zw|-QpSNt`5QFm+mm4Fkdcjm@~)<$K9XLuNvb0MXHGAK=UbUwd8Y2arDzXyT(wPdP? z14yTY9+q-Nh-;x{Xgil-%Y=T<^n#){gx|$>il~DscwhSemvl%7hu4R9A9eyjRB82x za0fN=fS9L*i1>j}R%u2=W-ldzA{cCHU@5smOrkYu&?jXEIBHCWiMN<= zrPyEnH-my#Zfci-KwyZRHdoV^b*XoBeRy=^hlaIyg@@)}@^)*`rD>gbjR2Nl*`;6z z*o(iFg_l={zPETGhjEv9akbZee)wt+141bDhU!>k%rXhzgV{KVEp>%!D1JtV zb1nC5Www4sG<`Z)e1Dja;+9u)HUjtOXl52q^L1M1mSEvDc8HjJ^cQ~y#*n-icox`# z(I|gsxP{jzclF4B*Z7G7cxELCk8$)#;)HJX=zzrkw|vX>l35parImXiH-o=udbqcS ztr%)T$&Wn=i1$Z*#&~iACyYi$0Pgp4Nl21N>3~mYis6WqZ-xXOSYUAogxSbe0U32@ z*OLOcl|45HF*!#km_$eDfuiVzMc8So71S2 z+!L3|Xql$Qby_EZ9u|DccbIY4j4w!vNhxsqM~x#_WCN*lS(b!a$N<6FgMomN$rXZq zw~=e7kvKSqC%J3z*MOb5dS2Ih&PSA`_i3g7*?72xgFo3rt!Y*9C{a*}m3Ietlc$R$ zr*wH&XN5P1*@>8mc~>l_hqZZTJ20ANpnm`Ng{=2}p7;e)_?^=UO*`3gqxqEv30vrh zkk+OJ0g8%ES&`ay0_7%GSm~Y_XOf)BX!iJu-Qt?BNmEyuj1Ov`dPjJ5w~JLclz<7H zoq1xUL}b7Cg?Y7rOXzC4Sb_5;pbQC;TVQ_5XN`r4ZkHx&=eTX@NQDhrhB0__=Qm$C zDwP%Lf4NAa3W=nO_-hDOq862&>M2a@sfje|W5c+d_6Aa($%$R*jW3CNK^LH=)r#L} zg?H$rmnLZuS&DGFl!O`rM#`T>d7b$GxuKS~p8`j8AeoLs_o!SbpC#I&vxb`}=6*41 zp^2G(4GLg~qM{+>eMV-Wr-+?B>WSahi^s{TwkP&VOEfk_lVFpD{z^nuxVUu z=~jWrf=P#`qBnErb>LtB^ED7NXEuo{|=T3UBh zDVA&d22|>FUCVgQX|Y#YsCoIPPW!Zp_=UfUVA^MC1VEI5YnLG^L`903XQ{8Ug{#21 zn&hMcS9_{drJh@Br?qN!e}HXz3U|VXsLUC(s;Z2RdYv?;clbw_NIP8k7lueWprWg` zkoj&GH>x#QmxZgcWVx;u_OuuKcPyBY*J+*wDsp=mi1TKQq6(P8iJBl=xoNsy$2M9k zsBRSqwgCum{`$EM*|r@2roHqgbY{i~WjUzn=1tG$klS0Q99nQ*Fm5FaoNZZM{a0>8 zSYZS?iUo_fw#tJ$c9ul@d^Kv5efylsi-<)>jovAeT(B&rx}vR|ZQZ#8xj1%Jsg_>qyveJmc8IhYoVRJ)!i5QU zj!T%1S)ctYtG=73(MeLVD3d2NRk9EY6`ZS?CbY*pvA&9t%hs^>xN$<%z_6EZyVU|a z8;NSxsLvXQEntRurJO)nyK$P4c*m1>aGzP(!A*y;LrB6x+^V=Mp?qAQ^=GWBYoKB3 z#(e<6(fdZ@8lO7k6DGRmZ=!*&2f-xXqApo7E*{o8FyFWC5amcT^=Chd` zsGA6YN2v&bte_lBS`8?kS_qnwOT?0V!FODZ1Q)6BdcB60qNdskYr3t@WyXxGnq7CG zzj=$=*|9S!oBdYFMWBmZdZlx`$@8jYf6T+KT5>Ktz<~$6Vrz;1n5fI#Z;D{P$<>rH zcbU-YwWvGF{pqp8Yr0`3$E&G3uIWlFy1gLU%rY3jN6T;{>bRMicwq|#EJs#j$*U4~ zsc~?Wg=fnHIt7p`xWwwCRLW;+e3xR#!!;+m9J+?``;DO*r3mcK6AHxjdZ7&HaspXg z54cDXEV7aRWn7<_WS{HJ7VXUF*wd%%qTs2Y3cQp+$7!>hV9?2>Z5z*6FnS$m(L8I^ zC{4d@Nz(5Yr?DBKLk*>11&G(0)&9KB3w*8A$r-O~ju}R0)`a^TOw-|NQUPXXPInOd0 z()dQR6N-a6SjXH6#$Y$I?RUtR+fWh5jz+x0EiJQqSh^FqxqT+mW&p+^$frZ7ob1cD zjg11p=gGWk(9e`@y}7i^iG6+gw1cRBYfP&Nx~hAvsZ2_P4ogEP9HsATv5V@};Y-?T zJykUS37;k%m}=-__&ZXi%(st-iG%yjs#mWutj^jCs>Fq$q+6_E=xr;ks?v$a$rsJY z7{&ojtcWOykEWN_$Bm1LvXxwzMyu1${hoQs0TC!u_V-rY-QDQ}%tGyZGS$D+EvzD4 zkBi)xbVr`xowmxU&?7~av{%@vT7Vasl;Ah8y(ole`+^ew$$_k-@C^i-D0W{L!{+_N zs8`m)DQat{uUht|z?jm2yI{jtdC#WA7(PA0EZSim+kpMzu?wp0iPNQAc>imD=b3Ve z{k}3xo(#@Wt~T5>+rEXG!F{=HUybJ1{fqB?R!p7e6vwzy-l;@heE|&4Yp0pARjSkf z+fdNcShc9terw<1hRgv=dfMvbmd(59y^o+B-Dj!hK2BlvlP%iIZ07bk(MXHS@(sm#CSR5st?(p!@(ivaD@<2M$(Fu@@0_6!{>Z9s%4X@Z zxZYshE2OoI&byu6i#vdOjnD-fuvf^?SWbVEiJ9en+fON#IGcmLDB=EC-Yjd(CH}gd z4zJxVX%D#Hn@i=0Jd>Gg?47NOc50@ZUVgg#%we9bwXE%=T!<;o+#qgucd6f}9>+$# z@NnJc_dV6YJ?ERAz%NaD=T@BFw9#Cgs+En6?TV|Zy`(^1ty@Q%ewMma(0$_nN|U?J z$gF*td_0LpiMR26hy5w?hxNq>ZU_B|;Dl(yJ(#3XP4PY}ridE1sq66u?RMoX##@Z- z{2inZo2+~d*dRCqJZZRs-pnX|19N`8HaBn`hVQ|i$Um9NPv?>A2Jens?PGeJRZPl@ zNSV<*fBd^?HN1WxUx0YG%p*>xziW&q?xY|+s8(O#>RR&Ij+1U@)^sm#%e%5=t8f!e z25hfJZ8Yp;Z;yWZ+uIz2oSpNbDAtt7ifw+w?Kj*%3!gCEtbwiPkto%5>ZcH2`O91H zMgDx9Px0)ljY!(LCLfUEZQpq4^0Cb6X1mgOS>QTUd7t0iqCd(kO~*q2i*hUNxsQ(e zuuGn7?8zS&q_nN2u-fY#dg=nat9NVF1)tRr2-r;T@xNbvVS0V%4G?X*jL8DUAVDNM zvT$Ii;!BZ-Zz4`S62W1R6D?vuyf}iR4HX$VP@$NRh)6YCj+`8mM+)<-Qjd?N@=V3^Y45R|tNrvQArv;yI!FZ6M8mKypWZbDS z#>gX9sWxmAvQ3{Z9#cXMBGIB+j(APF9dm+#O1^;aMwogcrPin#t=hq2^Cn@1A!wov zVi0ZCj~E-G%n8Kd%aVEiyiweere?%szlc6OR;nGBr6;zP%-C!HWvmFpvIISsiXNF* zZ4Qsw8}nx>YqiMxS=70=q)L-gYPy)D-^~eI`&JX2#O0YY<-P%~g>>MtD4`OVYOqgn zkW^Xj{=#1Hcge{^;J<4Zr0wkFBCD&DBEU(d&J2tVFzfm{;ytGd(~PYK57R3%?ri7| zK-pwt#vVNdH98^9pkvFvl)N!3VKo$J%~G|cZ_x1gn)AtD=cJ1( z!(tnOSMvhd?5R58s1Oj0e4l_p>bm*7;HSp zj^73w-ZZt27)dTxKG0`?-Fuj7(7Q2)7OTgW7iZ))4H((p7>Cnf7!9Qi)$=1XQ>9weZk`DI|Chvg088j@fWm2bj3m3Ft)ir+$ z?xN8hHENwgmrfh@jcLFAIpY8Oc;>6hKF)@5wH(`SM~iGChQK5w{lOoZLUX2(I1fCz z^Mdahq!{Btiw^s1+zhgJIsLp%cF?02o<@WJBou(}bl$R)87T2V#^Gsx-XmcJBN(pR zDeq2?iq*n?Lp0D-O9kqCUk`hz2QWwwL6j@oSJJjADXFGwr^BIG+{LjPS_^A^+n%w+ z^&Tz3Ek8cf;OyLTs|m8`Ls!X8XHb|Z&goBuDPvKhLiDT;fpH;u^8yX0p#2w*?hIKjt4}IuEU%cyTI;7*>5LUVE>7{icngW+Hb-wioEhCM@4xS?UwX<;$ z3!Y@p&A#+6$rOr?c7TMcD7H62iRpz7`$M=S!ZtUBaFktSOL`c_HwMy6mX7>VBw6<| z918AybIaYGRN1h7iB6h_q^7sfAcrRZWOEH?(B=}nxlKuI0u|#FMLEq$PIR6Vow8Ub z8hj81cq&1j?{w!C-f2%Clt7;~uxCB{NrM?&0R>YaC_oL`11wmepbI4?4eDqCU6x^& z3Z365ujW63meEBX(?l)rmQgD)w2o$g=oCT-G&eSsp(lkXOCuVwgMPGu60PJv@mbJ- z-c+TiJZJ+Q$D7Pe@pUCam+q1l&Gk{!ebzw6FK*%iZ^{4$5TI&RtqKQDRKp$i*eW}` z`c<%oRjlvmsyk|N2rDFk2UUPWTbbZiKEze7jmQLD=W18HHiDaKpwBY1QF1vxP63IyBDCg7p6leO$OBiq-`URJZ+gsf)^Yg)hV z;S8fiEn!#t&C+f&wz7q7JV_f{nocy7mJ1u4u$MfTB2}qMt)*3sdt4@Hg0W>qM>(4N zT7uJ#a*ScE1PFRK>s<$GS&f(Yi#31xZ)D! zmG3(ak=rV@l%aN;G+hQ-=}lgmk(pLAF*%J>Rc@@)p{A%7MUq=R6>22@OsJ(DTI5W> zbcJ{n^ArbaW*9Ib*SgMU*`8M;SG6%^0n7r2jeTrnzfadx6@wcbK?p|xLfUyov~&IJ z;y`12(7fmb1`Pe}1V|#W;!JEZz%huCDxyAwv4}fW))2@n8EvcE4-Mo^N_b!4-gKan z0q`JEqp}+^d)Vdw1gAon6%ZVt28Z`+U%kQothS(WfcORUO3-0$=HlGTw_YV`v4%{6 z58M>RwQlf$JfYLf1=3j*H%N5r7-#98u`2+ncx%g{@ zk^2Kbh&Il10;*C!v&kXkiJaxpj?=rjSMUSX6OsXX9x+IQ*n@yXC_37!J=;r%rCU0J z!?E7OF&#JqP2hw@@R#B9hD~@g63M?6_^8MDzvrtAI>^D0p}tD8zVibxvIXK>qZ+KgAJC=aV-G(#gC?w{@GCn@5ke6-LXsLld4iqrtGgy74JJIhQf$AA zTevBZ!u#XGD*TH*s6`{x#S<`sMew*WGeavZz>xxkQQ$5JaKp!86L#q%l;I+(xx*gP zLq7CF900`F`@j$+#M_evN$`R~L%|dr#|j9%NIXPI5H{;mDCMJ`Oj-l8LxCP-zX<@v zAasq(`>+W>MX-|sQk1^zGng4*HvmMuyTcBD^E+H20QtMW&9J{NJdtD>iC>J!jH`kF zii|v#iUKq&!}Y)_VpIV{h_=py0Txg{JE9&}l9F4ZoiWM?X~e^rlqPC~fM#=qo--v5 z96AszL~R4KLv$w|&@pl(NBfd3QAC3axXM^l zLY7j+gVZG&Xe)i%yG;tn2qB9#P`i<86ENh&W{MAtgv*wUjDmzUVv@glx-u5oyleV8 zLm-4>TsAQ<0j3NKSQ;1@f=N4+#+u{-XCnlkLxLf|f}LCdZ*zm5^hs_sM0h01q6|&n zbAe5G2X@lKhl&Z+9IBZjGoMlnmq@a*6dRlXxTG-%cN0j4vdZ%FiW=a}9C0ZBt(3*x z%&kSjxWo%G;}jY%tF`8I&ejY@4lslS)HSOFI~$QCA*##VAc)3vOgFHqXbS_AGl9a4 zz06Dn&g{$;lflo_0MX<BtR5Ay8LiP96~Pje1s+|*LG{tKa)1CBEJ&3A z%p8CNxYP@n08BLiPVLl8ZK<~N)K9g5%S#1udIe5ERdLdUOhvaVFjd22RZ!KZQTc&Hh}u<71*n0#G+O0Vgkn;IGSe_UQU84kaA&?4%dT zFqwl2Kb^Bbb%a3GGxQ{}!!lHJB|&r*hjdlfb!As~Rn!~+&2S7X08ju*om6~`18U8+IpvF*d8BRqq>Z>amvu~VrM7XMSsQBtLimPuz*!k2zE;T5X6RWR z6@{QZ#CYIYZ&=i4^?)M%fRGhge>}YV$}t3(T5rDLk_}GcHS_41>#hQi2ss%?( z1x*!FKe&c)65Dv(T237^iH%se#Zea`&o2lg&O4sq8;7( z)PbedRKSg=rDfFKgMqcgh0E+c0Kn8vm<1vIR7_n1u*HS{=j+$0&4ehhxD;qxv7LpK z`(3g%03m(Vw6%o^sDVq}#OI~khjqf-36c>1q<$3P{`nQQYQ?qKz&kOj9kvf_yfJF*F{VMTqw6CAV=X%G+b~6uDw*u z6I-nm-UD!jPB?^Xs9WFF-5%%!Z2H>id*QQ9-df;V$I66S9b(3M1c$|49KMDcmRlpH zUb)rb#?sqMXe@L)(xn|+<>e>aN?!A|t?p%B)%vUd7a*+rb%ShrsTtx8-P|oN=^6qa zE?FQ1${hkhgs*R4;5{3K8ilqR9R(LXNfrfhe## z1Z9MK;DpzufFwxcPoCNXn5-OLG9$fIs!fGM$YE6`WnXmV>TN6n3)1uI+LZIL61Zhg z_1%1WWnAVut(>g!{a)e)W&otDg^gh)sH+*w+atvTSQTIP{Q)DjKkfajtkp|DctJDf z$lP57H7+bRCfp4)GOl6C5M7*$A(=XU+&fk(PY{9@?F1E@uX}IWWfgB4jat2{VQ(Iuhsxr7< z;RRynT>(z*VODmhWwxhIJzig}W+0`2f6ZE5#$J0e-&dYmF@A*A%>hl7r_Iyb@wH~G za5rO$5tv013B7%5EwpNf_AsB)o5QGV) z;Ig*K48SuC0I|0oF$55TMkrm;aOgn5YfdoDIM@VpD_*XpXu4`L(V*rEFkvg0R2%MB zA4uV>g6ZC^hD^YPWhUhTnCvY^X@BG5T_#=wl+_j#sH^?jBlg$Hb_GX!;!AyDux$d% zJ^&ORTWc<5nntX#O=_C1=B0IN-tKMxsQzZj3N6lxs{1Y3@guK??oQ9JnF@YCx#$lFUajD{s6&qKHqUzTIgfeg`fmGSA4xcNE1DS+wnQR41 z5Cjx!G<;SuKoD1Z4f6D?u0Zf>g;lKR?I_7kgr`>8pC--`bO%4}XMCAZa|{VksVkt;|IKhOqA~_wX22^PRSA3r}x3*YGv}@ZFAA-TrNDZmr;c zYT`a|6(8BFF1YNxRyod~QsIfGd2Sn*No)YxuHtdB!UaIYGd;#?xXw%#cW{2%#2la} z`-CTb)#%Z*=ZA@i$6f(Ol+;O`Z&r`<0dVqujcwrla>vr_GsxnUbEqn{Qz^e%U`Oy^ zw{tk>=~y>*Ixlu&M{0`pb3NztJ`b(=4)m(d#WIC)s#vq8G4%D*FcTH)R?u-tpD%l$ zbk*w=!E`R4zSkzta(Bn_cOT8(8*DYDU#X4te7*8~g-=+A^CiHpekWzsO63m#@Fw>>L*&*Ou+e3G^re_M)V_7H%S9rR8k9T(e`2aBCAeV5cbSc{IdEbk5SLfHE zhxmZsX@8q^#Y(JNZps;)c#3y=tnYVtM|O+Qw>de&!WomYL=fceFPdxhQ5jBn3E=f)r} zE0_0ukI$f20A(NLtY2kYcIJX_*sXW^&p&!%H+FufZ;Gbr+0OVYeNVq_U3%)~@%pem zw?7;f<<^pK!lEnx91iDP+;#L>mRVt>A~89}#QT44Hk$|jB{EaS6=OVGj zkI|vrvB^Hr+2{kss(b;U{Oh-HpMUo|2LWbh^YhpIuE+9=|N8ci{`g;gW_RWcUaa6g z1OxN~Ed_`kJ|Iyd2nm&@KUW0ykTd9zq(NnTpwV>6;wFq4A3;1bsMf5IscK!^*zR}ON528L0rvBn{M7}44V3ynsXV;QOi7X@c*GLc{{3g(9&d;u_HV>LQP z;#nh>X4z(&t+peOL7LTLY95^E8f=xxHk*%>d4$J844I(B51MqxTQaEhfy#u`P~#3d z?2se>OPF1X2_~9mlCuq)+r0B8oSnE*&6*R6#|c7u*aQ?6G3CUQpE8-pK?FOs(Oysc zDXJ(o{uLsKDohbLO@W~#f@d#t_A-hXLJ+d)D6O=T>8I%EjE@#OF@&_Znic)5syl194@4Rx3#0{HwqT7a`<#EC9dMj~(#7{Eq zN#CJfD!ec^*fc6g9fbrq=`oZZNQr~s+@oPQoTMAzIbsS1@*b=Y1X`^jwIn>Bv9dyaS$#Ey#;XCFf4~Y%v!0NHLQo!pK z+$Z5>+eELK59buIQWOs}N+lvo$o3;tO>nshn6n_qsH2pOE^lvsZ32g_F1-kD-eQEa z4z>8IGhumO+X1t*(<(IXhv~jMi8}Upqzk(Tzpc@{8}Bq;vnxM!UYWSU;nlNg4U$5C zR3ph{z@=aT7A~`^Hu!6oDM}xN*JsD)Fh)n_-5KGr#$W{N**Bkn&W}Ukg99A@!^Zvl zzmCI(uoHHd`ymir1GQ=?2^Sjcgb9JS z9Tctzgugk!6jFg7l!dQ~;^U2OvRIwyOlK18TT>=FRE5F3M{xa$NfzFaE}NhaQ9F5~ z3j8O>cXUG#K#&g+2sof~KxmK1xR_DeF*yhcVvrgG#0AEsnM5)Yb7zTR2RBdy_{B;D zUZ8^pGefR%5$r;R>0kcvPA(+hK+Sk%$@rl%a>qL(RarHx=LZU{ za7rGk03SLLo2n`FMA<4RGB1Z4W;(N{&}17>pLsWOQ1c8dqRAK#r6OH7h2d(hqozirtnu2Rm zcSuAMS5g%fRhEN2V2LfZdPqnvk=BCp(2+Hj5n5uYJ1bm+LQr@`S(ny@I|$P^6A>n9 z&~gB{%3`hjR9adNkWKT^_g6c!FVE=f8@67f7X7tLZC=By*mzc_VlA!sOlw;3Atab* z7~?|VSRV$|X9s@+N(3m}lRL@>x4VegqgH{45m(}D414JRs#2V#;amwJWCg2ng+tZ{ zPH+*nAfFi?dI?P43zldofr)$vnouGObqo1JYRO`>?&wo7DUcKP_-fJSQM8nLiH)+9 zhS)_MwZ8eiZx{62qCe>B0C1?Ad!<#IWQLWe2o7#g7wj%sKqqexZtI2TL%;Ua7LIrX zr-ub7u~2ocnK#Bx7T;@x!1m~$)UcSj6D>Xvz)B~M*T2a zv@QG1LQMy>+qAJ+d=7z(P%- z!~5azFgLbc>-yKdzQex03~D3I6um+oCX+DeW_UF_!O#Al!lqqq7y*2?UVft+&*#)I zz`4Kg5O*FWRtf2>>r^rxaT`yjXclXP&OaWZ1sZ99M&|p|>qFN@jBw3M6C?+79Z?kg zJ7cptATfqDUL*LT+FETW6G^B-cwYJ>)eZm;v6f(pR$C;y1Nd2;CQts_utqR%6XZ+v zcrqfYa9s4=*s}3@GLhR;(YE~MmVdXW?_6{L3%fRrR7lUZUqV}N?_9Wa`gt0Hj)|ei z7)&yL+qzaRV@WH---+xr^?LAc_36?g$9a`RGT~{c_vsE?DucDCRe)C^u?*KxRL4?) zx*oC>hQ>4VJ=~ssJY?U9HRweDa6th+gkJ6_|$X&0wtG*QLU=J=EKT1N-2{ z&&dXnk)O{s0{6{=${;`^fP)fb!aY>~L0Kf>`WcZ!giZEcK^5MEL}XvBwICyKoZvCl zuUQsMv=OM$lM$SmtbGf#M9cP!3j174+e@Wtg7*OWAPS1Zu$` zLZU1H3L=IA2S%bi&f{2RkZ~>lf~MUg5-^eys0#8qjSKBi8;%h0Fch7rB0`=G7G}+f z+z9sB9ZBqgDITFKzQZG63pmge`+`WQAHSr#fJ%s<2D6iw{@7Iv7=55nj$)a zC(t8RW=tXGS_oui)HMM`N!;#%phTP+@<1Jtm>jl<9qdiw+pr7P@W?DBA+lXVC%6F? zN+I~+qWJBi&w0T4$)yrx-z8w-Uvi8MfxrMvoiPsF7pBoFOX+kyO2xFjO8m>k$ zR)Z|89!k2zoPlO&j%J+{phFCTT0NK^<{SxRpv|FUY9`7$7UD}lWg#d+(W%&h%_H0x zgdB`xQIaUR;2)_aL0{@q%V0z!yy%Oz#h46a8EnFD-e?O1r&lVULpoW4=m>F!2E}>a zu0iG#$Up}?fRaA{fY_kX8;~Izk=Zl)7vl-o48?^D#26j;=YPHpGU|dM80Z|v86C>a zI|${G*#R0s6>PB){vf6PfGCD$Xd)VdRQ7_=U1M%eVnJBg6x`_Oao#8zXOJR`cLG?8 z5+NHn!J-o3C7?q)I*}$ssIRpIC%gkr(WLMkPh-FVfUZxH>tPp0l#n6y}?*&*K6rwK3LHyl;H(G#N79&ViZv;;QU0BpV~ zg>IN^VrUxJ>BXE&#^~uhy3Fk5i6mqRjfXNhiKqjeIA70!Yo^6=qDR{TM;2!Y5^=58&A<0)ii%q#g6ULM$SqtU}CB!!nEq2 z@Ps#g6MaM?x}mIwa_F^U>&fz|mpR89XeD;&p1+oB(d@xoy4~{KM$h^z8vyMSMFgb| z+zD*|Nox2=lYP*vDeXmFMp^O*ZRl-#wrZ+wniqnhBp@IWu2K4vm4)}m!tgrj3p^b$Hi$)~oHfjs9LfH5f^X_Ax&?^aH7PXxfNJN3@ z&IOHB=Z>9kf1=2G+S@XAlYFKSlzyI;nBQ#-a7DaLEY&R=PV6(K$?zWN2hU9eRnwnF zFH%(~-xy5c{b3BuK?q{W+0bUWu>!eGC2o4}HR_w~g`z-QO25LYCDumWoXe|3PjJxx z9NMN5wy3BP)k5$D2fAJ^8^8!R;O_|{$DaX4(OP7W#A<4ED@`?=u?XQQIw3Jd*3?2U zV5A2!daW9-s@e3Arc?vtp~3rbf_Bl+Mj1dLw9Vv&aGAoKLY`^nrI60u+=e05sh&@| z$?yyx;>zA>4u7b&CRf`e1lmp-Xwe?N)!wwUD2AM`JMaY>xXxo4Z8gY9s@V%)B!Y56 zf@(D?{2;?$2-4sbG{7lh@%`o$vsFGn@pL!`^`l3d0uk>}<_7w$|PA1trQaV6ER ziYm(_>1XMxEg)Z>AP*}cC&%>Z9Z`b8wY6%qx+WjKCbVAiBT8~qVoW8oV+_mx^Kxzy z^4T8Bg<5eg$r{IXKdDGRGp>za-8@1SpjXUvP=WP<1_%{r8Q`eKA=Zc3EdEuTbr}sDoC-p~BPM7dzg@{fE=f%>bxCZ- zcd+l$-Ybe^#&(pCBDe4ursT)X?yc1^0Rv>GqSpbqbO6S*H^-R_XXGq5bo06dgpw>E z;_Ze(GH;L-4Iju27p~#PF!(x0)HzWL_C{zCo2CZQ59m`1)yPkZ$|{)un<_4(xt?7K z+?6%uCfAOa7}j2xxwHmR65}09LoJCPzjV00tq9X0ni#7TSP)Xq zS32XZ14f`azMd4dSnAbg1nL3#9%PoB%ewiryfIy2x)lk+;Y@~gt)ZO177xUxB`a9U zD!i;HWU5%t$O^ROWt+tJ1wUqT5>lNYgOt492pD@DM>Sab^7pHE3d*RIX z{Bg{|w8q}`XnF^i+#xuc51~xh3M*wfzHnrkl!S8$ZCdqJ<0kC@V?dn0F=Bc9^z&i0 zK$k@nIOk+5S$kt9)btaTz^)V#T{+}(l#Dlc2bN+Kp(3KMO z#By+1LndFLnkOZ^P9SJPQua5;eR>=JMhdD7U&u+8WUv%v#1&jZM6&N(_n&A!ID~)m zL)e+!j*YyED>)9z=tc7)e^a~_hm`TSkBXUHC5mMiK zwXbBUX+6!(m0D@E^|FiY@#%(NNG{Q*r zSj56?orvtVHrjQUOWU;Jp*2-BueYXo1YT>pDG?O4u=B=qFJd92Y<4sIJfqB=V~Rl^ zgq09op`nbG^=Qt38DM<|x}eL&;jt0pj5N0Qp@(jzpF8msr<1vSS!mcHXoslnm)M!1SwUpR_T)E31LD>n_g*xv=G&%hZ8AH zM0M}o!Gxn^+45Kpj)(vPD(HB%?b3_`6le^p_6LQZWW6#d_~63Uy=w15N|3+_qdQzC z5|C_Q6e-C90Bjmj@)W9%8YYauNpeO?QUOt1y?ZxQ|0`3m4s>`JFxDp8vu6!ZIHZKr zmAIGWr20cw(%rn8yoT}OSL+RB!O(@u#X?*!1LsX3Dhp%{lg~*XQ z?w2^5;*wfm8pJLN7PyJLoEkyECrM`M2&|&Q8c8gca>D|{46*3zBXBfeaH#>x04E}F zqM#7Pv_xoRkVQT?qnv7Rk#P~j22(M_{B$(%{{yOe^l`_s1iNv@!y3Whk;AUJwFcHSjp&Qyp_KLJmKonDjACxD<2TF~j`wHIGV6M%pG^jX+I!Rd_W9 zdh1orP36|iliz+F(j$$4>(K7IKM8(g|BNymtB(#fu4t6fh^4x4Qcfp+>P9c}2yDm- z^!sszD(1V0j|nuuNupj%aj9*6NjyWFTov6pdKX zrM!+^ZdO?bR20&us9WB^V-OhR>_`bwq&id~f0*ikaLXOr+;h!dXS78ZwrUS|C7pv# zJBjE6?s@Zl(@o&uMow5z`u5usMfUbH&_NCU1d%z!!|c6=X`b@&YUrxvAYo^@LMFZl zoFyw!Ba>sLzW9}@EDUluLvxT!?!Xp_WWnV`q}Z-vYOK|mA-UX>gO8fXl9>PvTwFxp zf;H&Qp=VZG_EmDC6=J1WTe+V4|17Deo_1QNuYbCeBC}?B2@h5fGHf~mM2+mS&t9K2 z%@bw!)^Ou5X95vb;y?-2nW|fJ5CzYa0tXqL<`E*9 zfVf)M1(_W|cNscQi^^bu&3VmnN<)hgT+zG>+-DLS9Gdj9mm}?2uZmaPUe$_JizGBn ziyNq-*UE>A^r;Vx>^mVjDzgXqJt{nI>P?%FGXneduWt9NR{_1#M?V^Ffq^t!B`Q(G z39RIhCNn|BFvpmSy^WD&|8P}iR!F>;r3W_6n2S@YMxRM+V-JcnLn8Dri5#}96G1Ug zq!!_h%FQf@$+46HXz7Q@h+rj>yh9WGV@0&!fn!iC$P_NXG-IYkYD}A67(;}I?zzHh z$}7tlc@(xAc8OeRR3o~~m5N71Ck{x6z!T`UKNZ-i4YzxhI_u~h2>{g4W!WmaZES8Tjq$T%I4xMZwHoKT>R&`XgD7x(f z=!qg)do&`id_xv6(d$9_`d5!aVpcJd#63{;gUs3|cFek7$yy4yP{<;(IkKfU0{4q* zJS@9?#e_#<|GLZreHOIRRF+!7wmG0`^R%c%S3>|%VaLq&fAVtez2+sj;mA==Qq8Sy z?fIvLamIu*L5(eBqf4}U0dHUM;R72b$|fcXllDNHHvl1=s7!D?sBl~q?$e4v;7)=j z)QOQZ+SgW;H5mFRe-|8a0v$&tb~**(lt>Af6x$dSlLO8DO_Q?dhpR5b>f(T;e=Xp$Vu?StmL&I54u_z3h(O1+CxhFV#XJVf!#&i}i%dj7I8t9~ zLiZi{=V3y}fR1zW&Ac03Zw3b9>DHp)A%NZom|j@jGX8LK4wc9i9D;HQE$TNO%+xXKq@xK!^ z1e+SX_FI@To-{VW+FM$Hfc5kHB#mouAJ#ZB@i6O4jum8N2@J|76ffD5Nn)nL44zG( z7)(^8?`xKz^du1Xu;N-Q5Cf$^12>QYYvfGa$c^a2I1H_+-UO=p$DPj2O_Xo>|C;Y` z%tr6XFO9(O`?@XHI7OtQ>Wv$lG_BgN&->?eeP!8wN z3hIy!0o79Rdu!jxdP#8JT4vWzk@30QtNRlKa{nUUF0Lx)wjNJmLosJ_K|5>mF53wTF zAW5F-hB~7#0tyMovEw|>WH^N4gpj{7krUNN7s8<(748$0P*Nlj$v&!*M#6@Epox+K z09a%NW^d&d<|eCaeQMBx~CI{ z(iMmjnqF?J%yHlnX@Yc59akY9!mm5xu^C!n9uEmpcH>gUF&`125lqDTn4*fn;UW(* zMOtYIz7h?rh%AGr6Jn7Mc)~Dp%424vxpYy#W+W46swk4cELpC*|8ztH#i%U0CozB# zd!Pj?OtSXA@)AUnB_p#KLGVn<$!-Qs1%vO7uFg4j(h!>uGnfDZYN95Zz;sO0l9nbl zjpZdZhP67ymds}1&V}x3Q#Qu06o|z(KVw*~vK&2bDG$ILSMx0N@+)0nEy4yIh9VP6 zMJ!UOZNTBJ(gF{bv*xY={%-6L$g+9{fCrXy6J`Xn#G<+WGAw2n+aqU7!X)v8G@WD_cZ5SvH?6G~x?_D;1>prQUrDY+6b{}0Mc_LFw1<|_*$deWj1 zBZ@(wg%^*Ki-b}-t0`y>Qao|$8iZ$SYA8qt!rG;L!&aAZnG03K{xjf z?}kMaEG?{1G)1+_?Qm(V7HLqo@KB!6KrWEDvc*sZYBh0#lL~D5G*BeShY!97nGOtE z*0NLT6FnXE?W83ZMTJy@BxoMUOR`TMo-BM6shVNF%l=+0AmIzhG5zA@S<^3D!8N~_@=f1o9FuN3 zt@TgMF;ent*s#@G1@&bKY61u(FpBM~%=M9O)U7Sp zQn+QaY_g(s5vb5M5R_vaH#P)1mTj{V{qpT_qtypE z=!w>#WDBJKipK{;YTR%)CGOVsIKgrA)_WO=dpCixT6Wyb7jN56XEm#G?}B9&aBd${ z7DWh6V5YtV4Dk#R8bu+&;E=|Kfrp$|=Jrp`BS z6R8i%_7@2vl@S z*L0Z*X;Uy4d|`~ov;{qdfg4y4Vt@hKSWP6DU||(n2^5kd$P9Y76FYcp$#xeoApncF z+YU(Wjth61*O2Qt0dcn_e7FP@7k&FpE?kq4A1{X-ViNzNFJzdN`VTuz=iR+efdtib*&|5Iq^wNX?RKVfRO~A}u_;jJ2Cubo8brEPsL zzl%WXHa41JS9#x9LrPnJb?}gHEu41yZMYB+JXbW42gztlw=uKA`{au&9Jqy>xOY+! zh`=V~`na!xG()b$|2x3BA7rS5L*xFMHm{kwue)+S1F(LB0B)QFi0QT*qXlq0wCU!& zW2PB_1IJ152=v>>b$kcnJIU3%$#0+qw&1bdSha1S$P0YI-}|Yv0Ke^-vE}8QMVqzv z5X^;WFK&4>UxtoT5MDU$yCJ$xG;G5;{8q7XjaTQmb(LjE!uEvx!buCpVLHY?xE0tz z$k5rwpPUERJGaZ*jnR9wZ$QzHd<%kH$c@jx6+O#w+`f_g#`!r2#KX#)T>F%Y(*wNH zPkqdBT%=k11d=?eJ-yIkouO~Lzky(YExJ~_l+c7BxPy7PgI(B*drd+-&-Xm!Dkii^ zS-Jy#&xWO{zV;>D$|3{no|(+*SSG>U-7I z-PT@R)zvxEKmErp@U|aD+k3g4%o^RI4$|pl&Vjwou`vzG{KSuaYD_$)1XZB$n0UL* z+0WM71ZSK>J-j_zsCm4s>rpHBL@QT3zKuL4CtArP`mEJkECv`#oWS8JKzO=*ohkq0yeH! zbYOi|Lw}f&AfjkoT7%C>;twL6%ohh{6$(;%1~8UDt%GJVp5q|toXr03c;n)c3?{Sg z2FknR|AU^62Q-lqPWPaCKg_w*xMs4|4(*X}Wt>g-Y7;$kbn zANvH*-rpFX?cwC^`F`+x-{T3-4BBJ(EjXne2X}y|=7FE*r980KZ^*VA`&)bPn4kL( zzp5E9w3R+5H$ScC22AaS7}o#SLmwNJuMWWA_{aU@m)@sMc)*oc4Om~B0fHO3f!nAR zOeRpA!G#HBg^9s{!;TYFZ0yJ~qsWbn7df{0*bxUvj~hdl451?B$sit4LNU2=3rvnC`?lDyV8Ht3x&@e4EWv{d!y;_hR2#WlsNx)% zl7r;QC@EJqd(tH;bOTYb61DlD=+Okn9aIxh1sJ>xW(4i(wDDA{v4KM2Nu`Sx-MqKf zW;=JIEsj4RVZs0)Pnji1>t=&0y z@v0@Fo|t^Uf$JkaoY-*nrY>=ML0j2n8ChmidM^EO$9KLlLJ1@c2J)aH5>D9Q|0Ie$ zK|pF*n1|MK1X2{)hajd!!em>am0(KB(ZrHu&S_zybC8|o-&#b~s1a_(Q5V%1&~kk*>_)l`lT^kOe~reky0g2R32+I z+IVIWU98zA4lA~KUYfuGXds?0>Z#_L8SV+uM5+BrLPZ(|C}4@LNrXjeZ-@wJPZ16J zp-hu^su~`%YO+_S3WS!osj`uFdhMcw>Un6UD|L9H|Au9L1n#LY z;>i;Msu`;;W=D+RsiJ7LtLlgwrIppFY++lfly}Wrt&2~3yPY26QR%K8j0L)zt##$f zmzKOv?CW8M__*AhH(FHSn)~W2YoKXusHnF(m22f{08iHLw8`cPE}$&K7jMn2<=8^a zq^;OfwX{epEu+?GZ*|mN#d&--n1^PAFBwpYD1z&p25qi zH-a=-N=$LD_!VX_Q^X;AT(rmm0d~MKESBxEHn2?T9D7eq_r63Qovh9JX8Tfxpkf$r zx>c_nqT^{QJ_gx9H%{Qe|1utJ)k{-LKyI2d99)MyrnuHqH^vxc|ERB`+WM7e>&j)t zp^(z{+%2^k>c~hi`Z~$;o{X}!-9EH!<4<6n_=i^$e<|dFo80%{PWv3Q^J*r)8qAZr ztS2MQiVP9>)2j}jsY<7w{%))fxwBVUPdz2`)4w|K?0coH_Ln-j-J(S4hGXz5)aFLl zvFdfsa-AC52wTcdb69I)WV3reW701`!IxQSG)cd##Yej4@jCOGka}Ji8>Tjzlz7g?(r*v z8Jx1>ZO@ z(Y5bz3q)22V_-HQ@=tA_2$pjEq`uRo%2*g7-Lv3EKrF_nK<%^Pny4tlAOI~+No!-& z3e^Xx#V~U{lOzf~6-D5gQeZD^;ovSstUprmCP>SZC{Gec;M}D>2;61m@<=*Y$&Cnt z9OT*(vqDCSXOR`GpdE)2$tl9hZ**fNT-dn7V%4&0?V;D^fEPz=)}@9l{MjbYXwEH; zl6;+fOEO0#KW@=bU>e~7)QIIYALtr69(;UcOUs-bB{C9nwe&V z5wHh|B2EzE(vMElv=>1g}3GD5|Z^q zX|S$2QkqT@q)e=%Ok?RzM>MNeWS9n)3W~NM1~GT!8k2`Kc0|!>R5avycLxJxl9ANTt$^VT}@3KU+=e?eLNNc04lJjmd_U?X%Ym?YDDZ5}> zt{)3*;N~WHsXi8|xnx9DnjmqZYkb`>;YP%^nwCyD_S!~08edF>mUDOLZkNfITCpiJ zrEtAym5-3z;!t_P4<#db_iLoY_UM1W46v+%TujI6CuMPKD1|dNtPf96ch=gc4f@H@ zHv{g;E|%mq>36{~*Z0M;J#&LA3|l5o|8~6ybV(o0dc%CZH^xmXpVD2X!9FumPEyXUxE&@Fd2`!{J7kF(U<>ZILFg%y4=npkES8&PxR7`j)| za6RejKKN%N$Ch`lvodkODQ$=`c904FrD*Hy#1c!?h_I%vxBDFJV@a5wOTKjREDY?n zUOBv(x!#mZ3|_6a>)O)%?xnMB?I*9h*$O3~>J|(;L+3@vd3J7eb0)y=7W>1_d9#~v z%jZbLR(J=Sa2R!JEQw?H;BYjzUk$zGCtkR{Jl${{+UT;4)78uj&h>`%rla-w>z#9j zTQn$+bOcAYkv*67sMq|h8~0py{}K%O)>b<0NJ2JcA#}2~QOx9oBU0okZe?eM-5x~R zVT2@Za^cvG(T_~E(Q}pvuqF)iEKJAkz)X6@8_U#9)3@kmXL`Dqd3IcTz3rLIHo<1; zUcmAh)GTK_z^!L&!H>wCRq<-uk9%*Qf;Q^#rZyNS3qOBWU81SyT3bXNwo|j(?-5Hd zB1)a9!s9;Q4tBSgFgtb1I{lVfUy?qFGG3_*m+@p^)Y^&ST1($lL}uxBcddPc5P$IF zP+xWuC3PLA*00ya)jYS~D|;ZT|L@PeJL%!x<{jD*_Q&U$vY78gPG`B94VGj5-ClOU zW=Kc2E=i+xV3a|2g;g9=|3|EW9wyQ=0c3ue1We|JM-`DE`J!15w{yi8PqPLb>nCpP zS9`YyRV$}*o_8DDu};mSa?Ip-5QlQZ^-H?7E=)&j{FQryMiigZe<-3gN0bK(<4831 zcC;pTGnB zc(q?ScSXcDblqfnO$dEKr!eBQQ6pzpX_R-rPA_?LXk=Z8MGclhELxzbJK+} z;U`0ZhcF~X1<43Re#eYx$Y`_ZRWWFU&S-N`a&W9zh{95R&nSGQ2x%~fa3bT5O~-@e zVPmLR5*ZVSq!(d}HekVrW~`S@!x(#u14ziYYO|+5DQSsXQV3chfvFgg6F8A_xO+Tz zXuh|K)pllyQ+&3Oclk$8lQdYMxKPBGhr9MFp_Df+GJW#{T9U_$Z8l=l)`s2odq}2h zFK8b6ID)jd{|YZDCNqhR;MYe=X_T5sk&_cHWJi$XXdD}fks#@3mo|6Ol|G%vY;M?V zher`aN0tMKeb&e{y@-|qHh8>+Jt)bTUip00Qe;AE$sZID(gm1-RqQq7{ zLXY|e6aqPyFeZUuSX`tSS1*{Bh=hnd8FDgsL6TS=!lZ77w`0|~j+OUxc+r@Sd0;}; zjBcrrQ)!j{@^g*ncvgj07=?M$hH0;{mtCiPK*^in*JaQ2kAAdugmZ>pG?8=VkEV#1 z@&;h$Vw9(ndUNPw_c%Yl37jk`pJ7>^rzd*I){ruJjrn&CWLCJv1uihXQGTsh$sr6m+6Z{xOH|1hUI8j?8&B>X@ze0 zlw5j$eaS=iSXc(>mQ8A*sTLQAaE!BRt|B^QdUi~tfUE6#tGVi~wi*fYT1bP6sKA;A zg@6gYDxuS(sKwf^H5#c7iI2uAu?*P#JF@XwvhF&s`dUbzAgCy-vg}Hz^%^)}(v18{vpG7X{;I0DsEb|- zkU{xq9yyRQ*Ql0yu)yJ<3p58D=&Na3q8Zx_9J{f8YET~AY9ULqS$ncu%e5xk|Fv5C zwOzZbFYB@@TL&!rt23LnYx|*MDWP$JtOtuJyvMB5N>jrbu>tF1kSMG<8wOAdxKc~C z9jmS%yRL@|wu!5_i_5r;8@7+DxbzCSl8dkRYPP^CuQp3J_PVxetF|_axAc~qk@k`I z3b%Eef;}m6{K^)AE4YKZ49`HgS8KRf>$tazySb~oj_b9%JGs4^xWMbODLb}ayQ^p` zxnMi5CriAT3%4{&vz&Xjp|A@33MKaHy0ZGZM+XhE8@t}yz2Qr%LR7f5YrDRSzUiyJ z>&w3Fi?}6wxU0}BRvQ>2TOVihoA+xB+Y7tk8=Uk@z5(pM15CgLY`_9+|Gx7J!1Bw$ z4NN;NiC})QzY|Qs*&7ZQ%)kGu!OGyj8%(8!7;487Hq>A%)K7m!5mz*F08^L{KG*k#92GULab*#{K71p#52sq%@wPzi@`VC zzdcOFvs=YQ{Jtkl#8}M5U981ktia@^#Ps{XODw}Q{HJLAm=%1(QH;Y=d&NBby>a}( za0}hoHoDQ&E7o9;q1%fjL#xS&e_Y# z=1hCayv*`!&Fjp|)r`>ZTssC$F$K-g1U)hJyw3rRzuzx+H$BNVEz-P1h%)0L>dPR-Ohjnl6z)KtCHDvi`b{nA@)|I}`L)KtsWewx)+P1Y$5 z)lH4oW{p%cZPjTV)NKva79H0`?bTnMjAM=0bIsOe&DU=2*J>Tr0bK>)ejb+{>-p=OEtV&E4Hy+KUa{)(za)P2GX5)Pk+u z?7h~bUEK8@+jRZi;ho>@jo#@!-~N2vk}TcBE#CtD|J2#-+vg13_l@7+t>5Rc!3yr* z1%A>0ZO$mI*pf}*8jj)oz2O=D$PwP(_D$mEZQ{xO;8DEaF-_VZuHFIe;Rl|~2~OiK zF5odP;q-mnJI>(Uo#H>v+&S(H9&X$-9^yE@D>jbc@SWs=G2}O{&gOsq=5UVZMn33rUgdj^=!s6~bPnS>4(OiU;BFqcTW$_J&FF@H>2|K^o37}b zj^=#c=sbSqeopF<9_Wic=b0Yrs!rgA&g!pj|KXtS=y+D@k8bPE9pspf>bef=dhY4G zuFMl|<+m>C#BSb1zUpNT>;vxVsm|=M4&dkg+{3=+ZEozfj_k`W?9JZmPwwlop6x6y z?xU{N*lzBo9@cxV?Z3Y6ou2FPY?G3PUa7f_I1zlxxMz+&h>k5@Ate|B`@%N{_}Bf_I7X3RbTimul9M5@+i*t z3IFziZ}o;x`9+`j9gp>l-}Q};*p+|wo^SbwKKW8l@|pkIr_T1)PWVec`j;R2txx)j z&-kXF`YG@FFMj)=zxtoQ`?Rn1$DYHrFZ{Z1_!B?%y&w5VJ^bLl{Mb(Xk`Mi|kNo^S z`@V1e*RT9sJ@;Zh_!5)+xG(yzANvEJ{i(nBfS>m09RAY({IE~m#$WyMkN4Q0{!+XB zV9);CkN@AC|Kkt;Hvj+MpZE8_a{l7q{<*L2rZ4{eKmGJyrdR%ANKz*^F8nX z{D1zC?%D0%>isYN(ckL&@BjTT|K@%F?ceJA-~Rkx|Lu?XbdUJ{U+tyd{?Xs+>o5NG zpZf~${)m76^8fYizy9Cv{;fX!h~NJ~Ht$m#+W`RpA^8La6afDKEC2ui06+mi0RRa9 z0QCvXw{M`qg9!N*OvtdI!-Wqa`kP2mp+$)pD{9<$kz>Y>AUlQ>iLYKslP3wPTuHBG z%7`yx%5+JSCCr&Maq8TOlBds`K!e&8Icgr~v-Z@A zm1)zjV4a3tO4h5{vsSyNRm*lQ*SB!n${kDBZQZ+N@!p*qx9?WIe&qokthcaTw1&kd zKFs)Uer-2!4~u;J;)4@4kI} zdGg`Ys}~;~{dwo+-@6aKp8jCm_2=t{zu&R{`~dzZ9edZw*B^ff9%x^H0urd;g9gS2 zA%he;=$V8VMwp?7+_l%>hZch9VTdA5Xkvm3R#+m5BM!*oiW|nr;fyGzsN#!N;fEuP zGScYdk2LalqmV-q30{v#)(B)+M(*gOjzxl)7{wDxT&d`j#}xeq_SD6f2p?rD(I`4s>s8Bchb@{6yg1{d0| zWB>=8*}(rj1#!94R?KL^7%%MCmmE7RF}WZIO6I``TU=_!CX;Bd$Cf^Oa>^*j?6Q$7 zvmA1V6YrZR%rf(=a?L>B4C>HABRR9bBs87I&`rAu~uDZ6P zfAvl3ujlMd?U>_E`RY5L)B5MU13x#{x9=YO@wy|Qyz$B-@B8!3M=$*Jtrl;6(uckc((i*EjGp^O$U6|4@OUTu zo&8pLzsc<)foV`f3=JW}IoQyKV2Gg($v{Iy*x`mXyrB>O>kz~qDo}?sB%%=K5I{pz zf`a{XApe{g3Pe0`fu7hO1$7a{2yXF$;i%y4zSzCsK*^1Qi+dbBqWvSM;eY|5`~~5 zA{Z$~J2o&92^?S)$w*2IezB7l3}Y&d!OAkaG8?aqr7YD*%UZ_qWu{T#`&jo29yW52 zc6dS<2r0}s6!DM76r>=9$xK9!!I{nkf;5R)Lq$IFhlwoYGW!?^YEE;8M~vn*wONND z_OP7hL?h@4)TY~PVmje@#2zjXl­EJ_hiRzA?E_snPcTKT*+@)Mx++aQ#^by7sW5Jpl;_TT*Z0m9^Bw<|5x(+Ss1- z1hl2=Y;l_0mcmx8g9U7C-`d1=k`tZPZQ%b*VNVUBQF_dKpFLXZP7|v+_ty55e-6(nG9OuPWj1P05Fxu`e4C= zbf(PIEnBfnP8WCh!#_T=AwYRz6o)qd!1f(77lxcnE0YV{TITpfk-+Gc zzxd`_g>cEUuJg490lB(PDY0SxUn)eK>@N0t&KIt7f-C*JS1qUfy)ix#JZ- zea-KTUZGpG^lcS)=lky0m4}+^VQ+7{C(m}(8Wifk2e;2js0~s0&<_}9Z}IQ&2SCOE zz|hwDZbvWiroS1tq38SmXr(Us0wY`Z!6!N0>+b8FyWZj{M|q1=ANJt_^3m_!{`mD@ zd;Pke(^U6w!nb~w=X4s!dKYF_*C%tP26pBLQYBMA&DTR_;A#E#aJT17)<=5A*K$us ze%EJ!a7SL?7Jh37S~|vOl%{_y2Y~taRp0h;yZ3@wzC!SA0qce;$W( zEEZ-hW_pn)BAvs0kYfy>uy|ePh&uRw)R%<>NMSaxizRq#j8JhQNPJ27j5ct0O!$Q| z^=)P6iriCKSD0Y6<0s{GZrUi=qMt%fPe%e=q zs1^Z06pm^C2X3O4it;#zDv6K`Xn5ZkXVF%Qn%08PSXZ_9iKAvw55|*MMKW$Qfra#J zKiPgR$z!xgX7D9bS4mfY2z%MKguAANddHOR=#@kegOLZ6iHLWghK~j5ZVu>W(WQPh zRF|8!et9>D>6itGHj}^jXAI|80T_2DX>>cFi&@#0X1Rcd8F*8NG}D+uP^3g4XJ!>8 ziH!(&M0azC_;*Ekj2_u?$OwN)DQrl#g1tz6|EQHmSA8KFO=~z;S!Pu!iHg2AcV4NR z>NZjKn0sTGnii*+j3|F!NrGt!k-g=Z0I6YFMucitf)V&nVf1*A`IL}2ox&M$H@JXA z7k|P3B#u0ka0OU-!Ks=wMgsZ>lwm1|!N-PX;C~89dZZW##&&^QAWh%6me>bj#x{uY z1qlI|c}pp3Qf7{67XcjDenH8N#i)p8>6VTeNY$xXYD86?>6dSqhKd=O20D@|nVQC_ z0-TqX-l=fNd6=^3YpuASZ#I_xm1vJCemQxTea3AAD2Kx(p$Dpa%LZhG)RiregF-h_ zSIM3{gp*p}n-wTrL+O^brjQW&oMh%N*4ca=IGd9ao>EN2Jh z_@Yo6T0uyH!$^Dx=$r6)YQJ~{1&N1jNr(R@r#Oe6l8K#dsGl5Ko~${czlKxWR)GBf zcu?&@k+wuZOU7I*HHxN)q@PB6YT1HWCxbV-bsXxSjG2?WX=vOTpP`7IdWe_Zd7GSO zl^-ZyKWM4y37cFhgu+Dz?-`^Jr#l3_<|{>Gc?W~;%KaTz!uQyHntw37~rs0Y}X zZ3j^i*Mfw{nQahb+?k2n=Yx2Pr}-JESy)yB#*ucKcPHwF(`Kw!<$jH4Z7S%cXgHv| znW}f_oFi&hF#xCW+KI7hZ&m7IaFt7tnxPshiqQI_(^{@X|6?AqB3`gDVv!yw~9~ycu{Rg zuqNbBW~4`&35c5LswSz1fcl3ZI)Jl^s^^-Xf#9v&NS-iRoYjc-t7 z=jfM(%A>yrhf!LuWtyoz`ieB`OLV87{U@^Q>8p|!pscEnI@?s+!?PTup{**mVLooand>`gl-D zpJYm!SO=Fy8Uk;boeDc+f2msxYIH{{op>aZ(y^?}>XXZ9ZmoNQKKccps%u({rg^HQ z9(t>v>3CEdi$P1Oc0hUvYX%J9pzz3>p?L@T%5u_%p!YdmOp9?Hy1Uo^in9qDu{bMw z>D8W0`gV9Zb~QPr)zmw63stNTM5@}S#0skt8<({vxnLNlm+QBUhO0ujsNHL6YdWl^ zItBV`y~rhPyUM`M8@Ht9dWXBAylANgHhW(Cs&y)h2dudOTb!FaryY~`xo(Xxx z5Lv>$6um=;Y0Qb3dMIUi3c+}nzKJ-*Ela29o3$DjFnv3(Ge{ugW^XyhbOGO(bsnG+gb6uvyvK`yw{B{`?r4DTDlpwXRM5-YsM?; z#h^N(+qaRYyOpnJoY6F&2&b0z8J)DMVoW?!PM6E}RI=0B+qk_9w#>9XE?h-@`mm2|r6w$;ddP!{*i|Krs!;ik?Al{AD|e~MxX`4Ss3?B@ zb+&7lU`f1sb~?BD7pJOwx*=*?(R{u8ipqb7zlC|lA#0Sw>X9Y&t*VQIPe5p;T(B=J zXY320dpf%yePtav(E&`-`%JV8i*~ea8%%vNvud9|{$H~nryMSwk0y?$h+K1TcgaHei>Kl2k z8?FSMoSuw+_zJ^^x6ycOMisfcD$Bd|jIhMpXZCAj5lz#vyq|rH)1!!Xq#q51 zvXR?)gqp5xt%{S)&_fKjVXK-U9d*Oz($$2|Vl8ociFW16uAHsVz&g!zhf$7=q3ILZ zPP}ECntNXthu$e>V12JzJ-QGr+u+KJrPX`MJ?In!-$ef6RDTouE_uYKb}8Tzk&sZO|1Rdy*{4XC2&}46`}i zzHsep`u)c^3dt2LgQK;^18%TJ4V_--ea^^s+&tMlcGdU%sWsQm9E_9zML@d0?A*kO z?aLPU;KS^@ny1h(>)$8%Lx5bsNc~WA4SC$DkLO6szgEU}Mdmkqeu%rRH20`3>_L%@ zgr)bk#5v;IjeV#asEn=M{R`M+HMj;H#`$aB2YjwJ|7yoy0M>fE&J0at(xl8+oWFbv zmrPfjbgaE~%&#b!q+R;K!~KCp?vFff%4;-4P3`CSs_CuzyjZMNfj)+`{Ne4nxF(&& zUCe7Y44@L4z4Lf&ex1niIa&((lkUBt;>g`3rzC}ip$B+jg?>X^FV3x2gvj>0fW;)Kn2b4})VELTd4*y*00P|WL|ZRMz| zzwX#|C+wOx*GOu}4jtVzF3MtF+m$Zi+-S2+-Qdd}%i$-1PDN`eot=Z#m2%A1 zs@~B47ThM;@1)(w%AT$Ms>M1DoB0mm)h4)i|GaSkeV;RW#SebeWiCwSyQHCwq$vLD zWr@S$iP^$B^glnk4-TAvZ_$A-$ogKZ zTMEb0yUALIm{@D1;%((*7vYMH*0ao|nm6soc=%ME+={Q*d!KzbFUu^RX~+EBsCibo z8i*z-qhN=VcYygKjQf}0?q;u~OI!Un|4ZI>9%XC~((w4PjoZ1jN#TGy&{ghaAV0{Z z9rQ#V+FKlT$V%N(Hr2mxu(W31x(pBuPJ~f2hRByLR1`ib$mPjFhfsDTq{uL-@Y_YK779uBNjuGhaMi4tKL!u0F(jw53 zE{|d)Nv6-!9X)HBoO;mZn3qU&st~wwX;YCB?-U`6Qw>xbVmk)$Dl=`7H&D$+n3-qn z&XO6ww6OT5EKQGqRa#x?7s<^#K^NM68iY`u7I**3Gi|-<-Wrg%0MCoB!Vyob zusDlaJc=du;43ex**Ji2HiXQA&pxttjEOnosKZI1#`++yz5)fDZ>XgDax%h~IOxf* zwT{5gJiUOUa!b^hOV6$t%aaMl$athIJ+@LjEKAl*ER!YufCwtF8M(kvLk{!outQM{ zG2_n#l~NK-=XQJYJSbOl|L-6Nk;E%0y;O9J(G$Mw2)eWK&=HEhnoP5!B}Iy%i%zlo zDysvV6A3&(+Z2^P`c8Wb#}R{cQPR`QqAJKF(He+OpT6{w)(9?AELWkj<5APU%Ixk= zJfqdqm_9R|^M+GHWk?H0T~(-5oz`rRQV|2Cj9CH4QqwraD(#e!y@;c&SO(*p%1iTL zRnA5;0?98WL-~_0MuBq#65k^DwU1d|YtmP$>?DH>EdGYmsHz;(jWomtNkxgwah=V= zKSeJVxn14Zk`~%&z2UQxIgVAgEy9|O?kZky!l7a=9|}>~RD=E^5Jw?a=?y~v>S;T9 z;}UA65Quu%IC_a9{~BuCFkW=JMm;vwBUyJnRKDBvz2nM3Z-{hLF8^e3OJl>m)Yx;+ zRP;z)+tVu{#te#)podnLwwn%v!NgB5)O3r&!>aD_L2>Dx7`&uyBN?ZU3MPr(n-_PL zu8a^C+U>8A#3Cr33w|5a|LW$}ro^2~Fv+qro>f@6|Cl>e(;u4(M7w3@4lAWCuld}{ zTBCey6x9VeBf(qNW_s$WSEP5XDD-N*^z` zgDIw)H4+B{{nf}R9p`T-w!lc~W60~9T6UGU-?i>5l0cUEhQ+_bkWF=B`5mvk2e!`r zZ7yu6iRJp(oP7IEn)e-yciyit!RBPToL?|C!towD~xU;*_4VC8J*ln zZ&@=6!tOT%&gJDw1F2u~FfuaRHH%$=oS_*GNSvh{&66&|!fpIwM9gK3h7nw$@mOfH zRBlgs1Xi~xx&Wt8C zpLtDc|H`31N*HDjeBe!DUO@>o7-t!1V1YXZGo0kS0y?LFf>NgQo%4(*J)@RR{-B^@ zG`gb`*;&t80!bl85g!BbDJ6WKE^$VC-JepTPxZkQq7y|YIs;kHcf#`u=xUWY6RJ>q zYLpK41Su@Bwz~Kc@0IM4-kyBf%auii&w>=~4qSjaB+ z|FMn5gI_IM*~2c=v50M}U%5G1VSbaGUesjC4hfqYrk15Gg=uR+p$4<^RJIM^DIhYj zj-jFhx4PwRZ+Gikq}oERjR0#-VPJ*hGJ&d7Wdl{Q%85&4ViCv$D_O^CUF^O9tz!j; zQt$Cy@ZQ4}h{b7n&3oSLqF24?&F*H|OIGT}7nzlnZ+#tm-(%jFzWF`Fe(5V!#Re0F zOZuDZfYmVp@lwGHj;S1H`{13r;kLO2?r$x8;h|Ea9N%37glYR>5HA1&XUHlt>3LCa zg0u_{O&p6?EKV4|SeT5VaW}OoV;HA6$0rsok9VwNAY-A&V(jNfgZxFxs2ET<|BkT} zos0(9Ecwa9)a*^2(BBs;giLV~8l*bLupqIvwJ_zxAY>~8A;=&DYi9GB@i1Ww$63zy z!0>oG5Mn&jb_rM2Do~X=3t~2oQ0^q{J@ZIp?#)x5WqVZkR`)ta2a}aQ6LO&+R%t%wU+>6#@@Pvm*fE z>t73@%^8p|osFGTWGCCgt}ydX6Od;;*8&{8`01Z{5ojg~gf2AD=0Q`#&2yeEYNYh& zC~wtmQ&U0P2o>z4N$Ce~5^>xJT_+x0^wodPn*kHp1CI6m0pfmhelB9b{}1?pZFRdy z2gF^m9$rZAev^9MYxt_FSG{Tsi2Bj9PBNmuHIQ55I*x2$vm1U*2%m|;4J-h0u_-)f zm`&3K0`E`f(ao^bBqP*05k z`ul-T&6@%Ne_YjE!1d43#R_2eOVY_sBY|8M;pmxx+RJ&fqqsdveNj7 zvT&n!JvC!rLkg6C5~(_$c#Y)4*{RQU!w^*N*+2gF->Lh$`&0G6UqmCC>xV1y9pfgj z;I;L z&;*xh21P&@-ARptI|J9_6xd4w;v*XEyFGPMgQ`;k-t#@92tExotrX;gr{=$krBGCuONK@-Hjy)cR%lmb#)Jvj(Jp*X&+LIXd54L}%$i%S9)pgI!4mlQFZ z_L#phd#U@&zx}g>oKOt_6hHxVxnncH10=m>NChQmKmgc){|y)fT(G$r7`Wcz_ z%Mb9QK?}G77i>P2>O{P9C@{>#EE~TPD8GX<>S2)Qhz z9#)7jN9aE=l!G>-xHCLMRA7W+Vna7f0j+|!L`Xne(3sp4J4U=fhzmqWiaS8;!#J3@ z6EwB=S;QJxA{|6N3Sf)7)3!=v3qM4GOti#=ArkMqCEqYV9XPQT3BtFt$K5-)A%qD( zFhWJ_0ajGPhiE)NxHwuw6^NJ=d)XM5z#{bF#a`S({|L}OOL#p0Gb}MAHUca|Gqk(} zyo4Tbs+>znl{Bxtt~dmRR|vXx zlCn8LgGJMwV>unj6e(50$xtf^ZYxKw`3QpyJ-A>ciTJUEIwsI0N>&m9%ygj?xG_na z0$piMOqlemY!fsMaC%Pi@w2QJh&+ydE#|%s2b4-AeF@HkM z7R${Il(N@^&-%P5{?t#$q_XpDxb>tm`5e$0I|xT`LdBcK?^`{4gt-Z{P6-eI5o5*=0|7}$h0WtD6Bs_{3%zoTw{SbU*PKxwn*#Gh zKOMcI0tHag{LzlHQ3{eYjiS-Ugw6ScGLkaV(fX&7V$a>&&;JZ3DwQ!JILn1xPzZ=Z ziW(q7`XHTg82TGi`{SuG{O0S@iZ{|}urX%tZsRml>~PULdKrfNnCH~>AJ06sN< zJ@r#Sy#PVA00Sr%i||v>1k@CDN$6V4O?ZXMG)YZ>)D-;#LkqWx8q}!KB25j{O!ZVy z%~T=%)QVEHM3t!16VxVU(pCMj&x$M)Gr6c*JO*7fQ2RpJX`scJ8#2AfQFyRnTT{Di z(+caY!1RG;oKp};D+N&2IBix9Z~$lJfdcqbL=^#R9n?_WRs)z+MtxKjRYN~bg{V|f zO=u=ib-rsor>N{yZq-&*?Nb;{RZ$JSdX>?5%~yRI)f#P8dP)JwYA^aauUf5&pBWt& z8WET3RT5$az1-3@jaUp2gI4&4|8}t0I+VtYbs1TB$&U4g`_tHv<%B8lfC`9K$ue1$ zeXpldyRyR5#mducMSvJBfp-m5PN;=skSn<2Sv(k5A4r8+7=%|C+IN*#3pfNHxY=}t z*FD8nne09}-ttrB)uJt0=0zl}6 zZ#dud)hXN3+q?Y+|1l6Y78n8mBm`Tyo&==?rx*nj3ItI=qCRM*QIaSIHi78kqJ*%8 zdsu=NwcI^)LB>7V1xDl~RNTbf;79!c%S~K1yFm zHr;t8<(oZ}f@+o4W$mvhMqEA{uiE{9R4`o+xZp+*T}0hv`nm)^eL8NfWEUM@Q}!lS z?&hc4y{B#D|5a>0#Pw!e&fGpdWZT8%MFzZ&ZeGz60g+}bW47MIOljq^EkMwwaONEj zGLfMGj#mSM3dOZwXxkwe0znwuza4`7%g8nJNa%FmUU-L~`#BH>1f?q8qdp>8K;V3J zCmCMy^%^reK4fXbJ|Zi>}-OpkX-dggr%JHRQQPz)^z=;cLC& zp522Mjzja!k2EE|iu5%f9TD zhORXt7-#mGyJ2SHjOGPf1#KRMCahmUNNqtFg0}UrHWPwTIO@`it6reya&Wmn$k}bc zg^}jd{{dL6p|$ESKm~B%y|4D_&qGN(wdfUS;Z86ojSlSRQe3y5-DL{8?>N`bRp>mu zI^z1>1#DuQg=o97s=%(?s)B0;_!fgKL@2Y~g zo%sscIauP1Pz4d~X<>z)K49+(uW5tWkA&8b9EEAq1UHDQL`Vd^Cgy9)SqpDmnZ>&N zWX;2ELpY#kHO=q*rfBZ=V9X8iYHi?PZfj}NgbF^U?A`)ZG-ViXam(d#A3tv)2SiKu z?H-Ts`M&7-_GQVAto*KCv_h`yZD}k%+c2;Sfl?BiKyY5lX%B612d_C{%fB?;OA%v% z|M4BCSbg3aI3^!8WIK)XGpMT(r%a}b*%aq$%Fc6&{%3`rV7Sr%?*4&HFyGMCQ-U^0 z7M{K22&TG*Wrc=xYlY+>w`Bia@9nPiLk4laQgQrlawmuK{$@2kkOW$+>9XivM8X*& zd(bYwX=|WgL6Ge*2d`h9Sodv*yL^QQN7)r8ER_ZJ;|_Kdm)0tv^LH-d6PNI=PHvbT z%y~X?MHceke(vTy04IK~&1)u0ozz*FXicYfAdl~8FL(IPbQnkS`gUxk_ zDWCT%<5GP&E%&34nE5zWpC2x7uw5X8A^36!&xeZz>WLMCStkTR7`+E}RyQnh|3>b- z0H}DaK3Pq*)kXX5i`Vn5-t%EEckUMX7WZMeLSSKH^XUa_BOiH^FKkKAQ*sY%XjgJe zm-(4j@^xpdB)4yPkK%b(Y4xf&)Pze&Vf85E4bl1c2K!<$=xNnX?VTR;F*I#gqb|Bt z)+L~?iFaIMC*YIitam;gr(4z+xU*;T>g+7hmv4I?-({f{+C_NueGYe=fA1qNfa8{U zOF!K84(Z;d?-m8=ALwQ2<>YQ^te@iLP#^m8aRePUl(|2)rml2-T9 z5BZuOa^DyJ-H%yd|6|Fzxnx9zWT4%`M|=QwUTRWNZAvi{KG!AWrmouMU8debr>hEn zgvv^U*g^H}F)eQ)3?9+^3EccxD1~C6zfn-%j9lNT-)V+dxh4PzF&QwhAV>kh!3rAy z3@E@*$s$R;;Cx76@uGl?0}*b}0bqrWD>b%Y`BKtL7n6`&aCCt2CCrQt70?9X!RF1J z3O^uh=<`6rok52Zomo`j&yW>>`V{I^!_XB+bHcfH$(ANm+b;PqvVu`HO_IcN`ZMLv zSTb)Ev26?KEu*+{$BLpfcdp$*d+-cO<#*GoM}jYv%@c+Q5)vfB{{&JSV{scYh(}O9 zQTd75!+Jb+=Td?`3G&q~8|u{7Ru^kkN{mPrLT1w*)$7}?Z@R&85(B7LDmhJ* zfLyYK?*fk#X3#;Z@p%s8CX71t<7bM=>?iXmZ5Z=A#+pyZ|76;|qR#YE*Qp1Sv? z9^%VyZ$BY@9embYKbu3u4>i!?mo2XpC`lj5kVB3v+ptp(I|V)n;e_s3cp)_zYRHR& z7rui_Cb)E1PA!`ZwuvKgO!k;#h*6=&7*wqI7zZ_C2BVBmG_#{;Jnpy)Hi8IJ$S1#? zrW$L}=<-V4XM= zB~%)q945{chaDV_v*4&Bl6opS6?(X;hah&cBR310_(+qD;h+MpKmdWGuDWWb#f(qH z@M30dYWyogRI-`IJiGd0x18F9=bpjGb9ZYIq zCMm4x3Z_e?KqpGt%|L-ifcEL<0s;8@FQo5D5U{}P?OQOxd?uVQqxBtX@Sf_ql4poK z&9o4l5q0H96q$+y1qw07sE4VgqM9no6=o<4Ds_4C|B8k8I4E-_SH+qRFP<9q#jP>g z@N2L^0PEt7h|QrQj1($ODmFn1GAxlzKdU4wt4zDwI`3Er8w-_sy=fx4At(nU;mR!N zA!VWmofPc43#A{`wTqp-XlRn&z8$!rH{SuPdoaK*Ey`4W3~@x?pMm$Q_~MH`s$6=w zEE2^dxCCe^AJ?Sv#u%FdwNV)4}j4k~ex zl8nO|9a-S8nm~}z;gnZ?AgP2Tp^yB*gAb0fK{v%~I$1P;2R|6XO{Pp$tokIXY$ry^ z*hpkW0NxRfpa({v(sx1Il@F7*i!|ga{|(MFq7!qeORe#89lF_t^tQ3Nk^IMu8;jH8 z5|_BDYwrgIRJ74WgYU9mL5_GSrt2O_?}Mwuu%B z2S&K!on(r2m{x`%qgo&(jffD-oA2R0>vAK8z9_L;u{U}vbyxflM7>aTH@(*pcs+)Rkm2b7qFm1DER zS0Qmj!+6Q7UhrQ5jBt%TOrR0=|Fe%$2wH>n*-fFSTc`*<`OtrAeJ*5pAS4VZ}!okeSxFd#;n8ae?qM1IrZsZ^_jmEEFKQS!OsDUlF7<;pLW&7~J~ zq1%uZxB^6)V^6{$M-(&EVs;6&D_->~UWNu{4UKeRU}30S#rP}&BiNxU7s~(+2-Y%< zwFe<=IU1{BViH;rm$+cIv;c<-I5E|1=?c^`QE})Mie#B4Zg`-&ZpA$F5@QyfaxwiC z*8z45Di%0V3ur1%BJGl6|AElz5gqt+U~L11D-4#_{bg~K!iz3Oacr-VEi$yO$b?sj z`jiP6V5SERFGBOmSMs)uwRorkoJJBukY<6(8MRD~ZYAHu-Z#Itw8SX<%PkVEj4hy` zi+6?DWI7t?pi@lHf_j;=OEf13c|k%P&O8S&uBBtF<*IfY=tmBJB+wgvj)_HJ5d^f& z!G@xa!XlEVfk7HR2f*YMnS0|Lb6TSqrG(|iv%k2KdURwH#a)rCWR->2D*eQ8a!rXN z9VV7&SZ0xnUV2#yc{vi5aC3&poQyNqMJp^5_A83L;A3;5BxLKVwoo$H(D~Oac?Jc~ zG!;HPlLvzXj_i|u#JJ89lhfMbNirfae4C^SA-u%6aVAa1A~k*TFC zWP}4-g-nzSoI2I2Uh;TPX1tua;gMgs14oG$tSiU&Xkm~b#%-!JM?lRImH0R){wpp~ ztg^6i5xJH~NEQg6(TX*)KnSiV^BPEGY=N1Ckx@jsGSI0>pmP*`&Y(HZO_0#xv;svo zz=sW-VGj_YNXE`koQzF9%8HcEeBR{GL$PUo!n=dDaXi+2b9KZ*sqX*oGi@|vQiR`K zHO}G=LJ}SbPEoLN5qc%VR@Tsz6_3n-H4Y&~7|qw7IJ|%{d!VrZ8^CIA@>@z`#x)hj znEVl=lOW-)|FY5af-}cXA~YB8hTtI+kbHzY(wqzc2_{W1_+xk!Ai*ef;vGwn;30CF zgO^G@f)RW&@{};!h2x8tZ88c_>UCp%>o^Xq{FYepO`NPB8hwMB)U=CS)6}W(?Fg6q zsp=veF&f7T+KP4W+zVgDKyCfGqD~UdOWwLGF$DrldjyIT#rI zz$Gjk<+N5J9mE2PkF&*%KuJ*Q8QV(9jyp)*77bdT;KDlyAUND!EpQ-6@J17S2twT6 z2d;tcfzE8%%{>@_E7)EP0@nxl1uBsQ5inn;nH&5*5aMtt>(;Ax!c%->0c0x;cQ;x(B5IbJo;jLLPD=@pVD zNT3f`zy#Qt8&+B}tw8298*@xSR6*JSDxT^cS%aJy>;aAh{$2>aL!0ox@9~De)E?+q zf$>Dp0n8R@$sjHmzyX@can&3B_~5-2R>|F9O1N7L{@$ovpXd--+ws*BGGRo4kgl;w z!=;y|wLnOz-|q0+auJK(ZQ)Ic3L$hHD;Qq>ZJ!wuST)>SR-hIO{DFuu0v3tDfaFXa z9^D$c4;*d^8DS1@une;)*pXdB3dGww)l3T>;x)L0?|ona?7=%w1f%8DUx~o*b(;iX z|KbZ;m<@7bpYdaM^k5E2#`N4BF315E{8)#n)4*B4B7h%6bst0~S;B47vC#}0*j;^v z6amba-Z>Y=4c27%qT_iXkw}Yw3EMHIq=Iyy|HT{tCRnyno96t1z@$?Z;ZK?b9U3m% zvQ0=W5EtTT1S1Mv30hYptbjb;L+^n@b+r)9#26(?V(odLpd_F2d1T3LfhTT9^z9(W z zq)PH-^r;cdCD*OJ4=X{Hz`@(uQ6KH01F=b3L=;#xW!N6xL_gunxirF2`c(!@{|t3K zf*f>LQ|f?-h=?6pPUQqhSl&-k?4!LI78aD-1K1d-;8^*1fD{1L8Vnjn_@<#OM?)Uo znPq{lOhW2KBwHe!_+4K4U7q`KPQ<;^T)r1yii%(E!u|akgm~6q@}F3ooadpyWw4)R z{2!teAzUqj@9mLWQG#dA=YH7CbsWS6OwM7Nr+J3a9>yNKbqrG|(4XK3eu~>w22Wh5 z0Z2T6gWhI<bzgbWaKSoW^g$U9u%Z`(L}ju|3W`G!9{xJ z99^IpF=QVsMlF&-2VPP20g_Jm$&!AFAjU^u>{zipKnFZ1gvwfO5(^_7UzEa&BI!jH zD8xm0C@L_*qtQS^vJ4rNl`Mq9S~BM=9^8q>;;Tv7@H7vWSrm{8OY`gy7uMAF5#Ei0 zAz-4MF=l6jE!gXP4qevTW1s+$vL~soK|ehL4>&}8I_Z--foE!kK@!=1nnYb<8D;_y z;II*Rh{!Oq0z2%{H9>(vf@Ghese@J&Kz8LQw#yHs<{H)vDoolIWl*wh*YvQ$V#Y;( zk*J^!YC23LlgW-Oq8BH|=v`u2DdlB&QYs=?D*o*#kLFDOdE)=o|Le=u>!_X&QMsoC z#aIov)n8pCtG=p~@&tFaL1Kc;3?u^R6p{jtuJHAx(VTVn#y5`=8&5k!dafxtTH0}yPi z3P`3c*J2D;QUF(rrBenvW;7+ASzPQYalzg=C~WPJ7LXObG3!wxz+OzpuaTHbbV`I# zkk?#7I`Aa6{vpl69r)FSUU0%*K$%FW00XSZM{ZitXW)QtAn1i-TDllBv;VbD0bq${O_t!AG=rNW$=0Kh)% zcK`q$?8i(gl>udi6*`w5uxJlNFYc&mWZa-4I9(LD8i+VkBFd2zD3$0a?q0E4a~A6N zS)|lqW6)yZi)xzZ))I|!2)kN>Hz@50Wh&_MpXOnQw~555Woc?vYoVM$aQNxDtX2b) z4f1XZzak<+RE{Ks?UUkxC!_-%2ycYZmwNS%ff%oXd{Y^O;N9RB^IAk|z6>`!aCvx% zy&=JZl0;Ur#mE38b*jXMzp%n*YLU{;niu;gIuPK>=nS_?#dCIYs5p2)5kWD!N>mG>6$Sx_^ zhHY7SFm_y#LE6X(C}?Z-mTnc`7_o2*lL`I>oB`3?BKV!_M6&$!Fz50PByFwkqOeyG zp}+VP2VI)ysM_N$ajK}U>0#s!Rxu=B=fnXmGD@mD9BrlM=t*{9iUM%aDiqx@WA>T> zV;b!A{R_Sb=xPmuawGwrbr4s8)8gbFHB16Dk6lSFE{8-T7VRg@Y3Trf5q*7f<`${E zz1!LXMUzx--ldW!vZxMEC=dHEaqvOwVI2Uq{}H}?awxmqC_`?_h?03lrw`cgU}3M2 zrYA?@t)3WIzbdQonT0reN9jNWzYB&#$CE;o#=2Xjws38zzGiXJqNkw^x z^E9nNR>0;UT!Q8$Q}bSxzWM}$Kruk4@+V$$5W`-&IFg|tu@XA25{Ih*o^t#KGVv%@ z7S@uZx@eAMu^(#jm$kx2*D`S2GSRM_SgZvNXtN0z0R|h{-zFrb!PS5gTJCwHC{&GF z>w-7ZmIHMd?cEJIEARo4BTP4PR+sD7sj^!5Ao*++?`Q>aNC!gRQ3Iq~hu(8T0kuTd zY$q)om8;bb&~V ziC7D>zK9!4Ujb*KwQ+hQ%iXmyB`N742Lp2q+1yr4U$o(nGi`cg^F@~xYo#Eq;~r$- z(V=WAD)wRrHMoK!OIB_aP+VneuCOGkWmsJMfn(~@kwViS$b}ph!-ZQUG<>7?HNa15 zg>cyTD{lkJd7Mc)-5mze)8T<3HH1-Z0)^Wq4&26|Eo^H}i5emUl8Mmt(4;fpRxxJN ziC{;E?M$pZClH5wjRjB(K4Z6Y4zYE>33~mrdNt~0OlP7V^p>#MGJ5uxS;AD0nSvzS zTLfYXCA548#!5Cvjm2un(D#<2|I=#W9#FdN4C^>6sBKUlxcV&MhVg-p>w$?3A~@Ie z0I+c`&U7VDG7kFVDC$Y=F=d?G;XBz4P|+C>vs3U|_jQx_V!!2y`|}lQgIRz;}#A@SJ`3X4_ zGdh*+qZnNt;h}SrxvEdN78FPc5M^nist)Aa=>f)aZbDZ8ucn|S?XYh?ld@f5+#>*% zmc3VagR0PyH?G+wU<5jvRRU)C8YZasvFJ8<<}!LK*h=nrnB}(Wb<9Q68+HVWi6>nG zu%R;^a&Vb?^D@m{H2P5X@C5AH+xUM6D6dz%Y>YFxwX6> z4xe3RP?W(Tu#Cx`J4!>dx~Efq#~wH@;L6{%bZ?_odgM97sJ+|!biJzSg#@h|!8DH0 z_*7J!f1m0(_WJJgtQ1H*t9VlH%8Ka0#Mhe8wzikd zm0w@)7*CP`!po~!|2e}ANTZ|3U%*ccO+9%jJd+Iq&L#mW3$KJvak9I}g{y(HM}#D- zG}+IWd(|J>uk~bJyziIxTiQhu4x3ujy-WcNJv~zy(UHNU+PV!w1d+o?1F_7+6XEN( zSxz5-oR^jh9qwI!V_q6x_(6nmPar(g=A?ffOc9EVXS)#2zMCS`s}X=Jf^E$d+4%B5 z*$mYK1UiB22)ZkHa9~1qyijeD#A()^GZWD$p>jgSh!}JFT%;0)Oh_Ul!;pLe(j-bM zmsqycv`7;pnP4I?7?Gxss#CCHv04?%R8T~E`e@=+PLd*{*XW>_VTMi~sev>>v;}HX zB}H3BH564b|3Fv`e^@r9P-4j}UKU@?T7v+?0zb8yZDYDXKoeKnWP#HpRzP6Eg8>vM zZ1D>g#a`L-v1(YZMSdqeYY}h8E z%r-(hhwRrn4CBECvD29-osQN*BZpJfY7CJ(yi#JqTxxd7DYU*#Ty^c%4R-f_{av|x z_iAl|Biv%f6IwXV5qXmRNclKo(Dw*LeHfSfKmVDdzL1=B8ZanQK2anMItF=&AylrS z!2+kCn_@5dnk%gNl@7d$W4UX&(OAwN*fXgmhn-Kus z1{iWLT;SNi1w0;kWQpC3X!A`!78%4M*yz}CyFB&W^SeIRxo*c-#G8c^Hj*eKy_#xL z0X~&f$}gppGSbhXs`g9DJ)8t2u)t3atm6tf0wD{MQ5lO+!wPG#@PY{sXm!=uC|Ioo z924R2OAfX4V5~V&R3fms7}QJx83~(=RV0*!!bV0mNrtaf@Td|lAq~0<$SW&g(ttOv z@<|uWmY`O(ay7_OTO?QO&`KA+e6x`=vsHIjUd6m^f)xCa1J7~z{1+iV1^!c%q~w(7 z|GX>G(*i$?9A>YijU1iW(XPA#speS28|DfTBB9>(qsz2Ita1jF@ zaYf4uekj!s=wRG(M<2RHB`Z%@vE`n(&Yd?~bH`S|-FD|LAPM5GG1J($9nhg}e4|67 zU+Mq`Z{UGtIpmLhlSo3M=fdbikM|m_oZ-tuGPS;q?En|_pTOOUV~>H_>En=1mHfgo z{7B9VMNoWakhZw8-NaXz#X8>^*Qh9kMhve;lHpz(u?#)>8d?d)W&eZSt$Ufi{|qjB zP|FrZq~3meGjO!@O|J4u`nJ1?*4N5maOl=Dm$j?mnHv+LX@a*z{7waCo^ zO0X~G(V;_02#-C;!HGEJD^RHD$kcFTqX@Oo043mE`J_Svbm1=vRjI+*)Ye86_|Qvd zEY{x|05>$&&q|Xy!)U;<2Rm{JaQ8!?Xv)Nm20HM85KIVpk^{R*VXk2z|7=61yl6Q{ zG3+O_IN>H!_pw!=&1bk0r3-rq#@_wFRjqO%x)=eDDH2guM|_MZxMn??d14cHsMZ-? zX~hdXqKZ&bgblcun8ifFb(d&gd&0N|N|0)may!@W2EfL&X@Wv$G?z44v%mY@4KC-2 z;~4{aJP~wjWeij&A{Cj)b~Z95jNw2C=ktV1QnFDI%cmwYc{+b`GKHwC1gKaAM^i>K zR;o&9Dh--10Uqs@*?Zv%i~vzu(g*}*z=b`Y1jV@Qp_h!YN=K!}m<}rFmgPa>3Y&1X zzl<||9Wp~qWWtI|SfE;BSx_>`w}#01R0`ajBLRt8RKEdIOaq{S|8x%7PIo%Bijn|? zBj-a;iiuF4BAI6hP1jX`2DDQMx=<6=gv{I^lbT-(9~W<75fY0(vUm~gp+!!$t z7qd`ClQRil=H?MXYz8V4^Q=~o)N)%uLLN$04WyLqNttLSBXV>JZn8!S?mECs+i24_ z<}?$+G(|G7x!F{(0u!Ma2l>9)T5$5ufD{#6I-B}b+YY2sfY9wEGguQz(Nlzu03pPx zIs$%r^{Zet&_I9ZnEx>Zx_(*JAfBd}V&13)VT9u$1~omwdP5dhWPxqqhc+|jh7*;* z<#1UfSYO@}vBtD+j+_+}hOp=l_bL@wv>6sSEC-cds7hx!{~$N`MWP`g3PmIk5suX8 zFI^Ds8y)p`FQlg6RkW?`z=YVRDZZ1Y^3B`_v6~T_j6@?u+2EsqaEn;H7`e#}D1unv z(?Zr|I{y00&ip`zJTwbp^1YQRbMT0Zv8_9U9RLTb;5i}6zEwb@88lGvp{f-K3C2QQeNhT_H&kWvu z#!9>=X>5&8x3X-ZActn z{3;mlm&Prw656~;R28G zayv@DK7gDbT_m=3 zpOObuN}M6$*?T(dj`zFwdixf~Cg20lERn;#vVn4}&OjeWddUfun6d-9tC56qC85tn znoJDQW+`$ldw{e}0}N;iV&w}oWH$<^oSv@r8u0aO4+3dV0w*v6Y3~v`Ara!>l1O5M zGNs|RXg+R2QjG8TkPj5V3lLlZ14+ZN|7syK)?hZUDr@{)MIU~2J%N&sP^?J~i>PS1YA zPz)ci0?UvB&(I2HFAdXh57N%M=81DCjkhW-N8aO+JctBKF!^4GQ1-?TAETDQER`gr zRR|H;ly2$_=lgcywz@Anz5%zqK@yOF{7C8TKBQ_?q?>+Y3BaKW9~;c(8PYMU<}F73~%po)({z! zQ5lzJ57s0D=_3NVj`%<#4l<{n|1w4R@GuXNj~ls>#-@q|J0Tojk=hRG*v|2k)?m>5 z1Q~vz5lcbu62jvu(S=%I@KR+6V+k-w!vt(mUGRW*8ljKyhaZ0f%$SG|ri8ol>@Y|J z4ZtDsl4cEP$1)ZHeZ+AU3&ZCMKp=mSL{Lu*|3(EfYZzT%B#AL^YT-PBaT!|@8K-Ku zl;tMWz#6B~>xir09Bl-*u^W5xQ)UkWP%^3d>y(NIW?n+SmSX#WVHNxY61gKBFo6*t zQJrcHu7vQ5vQjG%DvH3tILgCZCN2kXVO%I93ic)k#If;Q0U~h$BB@1+RxuUGk~IP= zw2+2f%0ymbL+onZ#uy;C=V5{;S%^!SAIh!2Qo<3Cma>eFkJ8z-_nQ% z03)MqBRjHQRzMWhoV-+I78z^)(uS|Qy1}lRzAFJ)l z?rM;}hW{$lpzhN#|18o9F^Yc5g-w(WJh|ldbP-JK1vOl&0uZx1KR_HSvoI47Bu{cY ze?T3Bqnu<*1>?m&>yrs0CqI9W(k@7I{&PL-X-c0U;6PJA&CUx9^gwHDywWZ%614Fy z%vG$7Oi*Mgo67z#M;4Mfe+97pX#L$ps%NtF(2=tfi!e-luZgey@G zu8fp4!pEFIQbkrpNZ|xhabq#lWHRgXIT*zzO>I6P&8j?T4g$1Gm+NU708E>}G6m1A z1gO6kB?p4^QaFz*Gz}a8V?r2@W!8xXn3XGa~#aG+f6i$KlLNm1E4C$~YK@SY^ z0*uuFw(5&Yuk>(lL8yfEoH1q`^xHn8?ObJF=!8!4(Qz1uPn8wwq!nft0S&Ts0^Y=I zu+j&-O$u904khl5X(J`y8ifPhV@vDRUTFYK9YMHk z0#Exjr|3osAfi8t24nmKQ0D?yzhM;=7FaD*DuId^=Se4#)o-%~a$zMC6y|!HXc7UJMh-V|Rdy3h z<`D|2iKOLqR)})TB29}DQNEW3p0#63k96lZR*oZ?LKk>f);xgatR}S#O2FP2pbiMZ zXw#KKP7FRa&^YFdZV8dpE@TjjR~L?VoqiSL z|IjQ!tT98P7h^*hAA=`sIKga^pl|i~uI*5!}ED5ZD5AcQZNXs??)1GeSb! zm?yU}Y%utCh0BXIB9zjnhHP-YObrq$4sA==gc+z|sq)oYL51BTDi-4o_V~IIql^@$ zb&?h;j;R1A8ImcPhBCR5T?bJPL1~Zj3Rt%Eio`^!i`0A(I>gndE|Y%YHSz#*W@te^Bg$vn`RZFb&^9%OX-Uu%Q zyho>B*bLTLYFJfwNnm<=x}DuwN|pLbg&L~orpZ+ZjhJh(Fnu2FH5Qw0EY}XA8IA7hMG~uSg|3tAo3a{{3 z+Ct;8SKStnQ`llaVFvv}dKK6yCRwMwKs~mov6;nTI%0z#8zXY02tvS&B?PlKIP2c< zg#nwXqvj5hTC;_>vRC^6p_)}muHK?rk$GA_L3@(13b$(tVbHa#|>nI7tfJ}KV11QX|A;Gq@8)mc*Di=1U2|3U(V-ihjT?xRv@3E&D z+q|Q(h0AEYHCyl&nFHS2tb4i|>HE6y8@FwO0N5D?-rx;Rz>KHhwtvjCrn8*Qqy@#39 zTENMZ{0KsO$O*jqpn${UV_$h$nKzx`>3Z(PuG%~nmh$mjaV)%(gHdtwv44Kf?c*F(!3eFTpD zzAfFtxtyOt2){|KDYhJ|#T>$!e8MN4z7;07MZJ@cmaLC>2RJ(?M|{E2S_I{z&Q1Kz z@mwQnWySaWK>3_j%29B+8vu_$aKEM>wy?ih<5p*%DJ#4Ddp(%FwErnBw68Qn4*gx5 zybI7=o+;b0_j}PX5xc*=2rm8Gy9m?=nkJuo%p<(N*Imm`9o5r)$kk2`SRFT@QQtDh zqFFafu{74xpsjBm*W((+Aw_T$^r+U%XAkIc6R{v*?WK<$*#j+zr3Mlx-0hwwzArxC zSNgI+P=^To+1V?Pk6Att1-FT?p0~ig?UDMV9M&JZo?||?MUKQQoRXjR!U6i`=eqk61ZMXxz#0Q?>Gc6Lfj3yVWs)v<&L$n9BGFbw_ zaA1wFB_6zG0TgI5p1-~eIuy8jEWUq?vpJpcpFWS~qU-egC z;D$+q2LJd4uh|>BSTlP)j9Q%mpG`?i)&Qa$If3Z54Ln#7;XqkWB5(i#{vfk`B%4VgSi_KbOR$4{Ci zf#Td)6lF&;N|!Rl^5){xsZy(Yv3gZaRyuj4Qc*E8%oCkpM@W>Ia-vxkX2rTC3pX0E zVZwm<%5^KR-dMEy@`d#m@Lxn}PFfraGQ z66i^yLwhnUDs^bJV2M1YOp;C)%R?{Q7CQO%mOw$9hHU}+4bd++F84NiQYhl2$de-# zn%3!YjaapEj$<>bXFFW)aR1&lJb3M2Bm3+#N6W;qaMwa~o1K08x%BVi-Rn1h;8lJJ zo0J(67UqRO07a3-(4KMS)dtyOBi*JTdk)z)VgFf*?NA{B$+_oPg^fjkAqgB5$f0Ax z<;KAnL1btKTPbcqVsS;h##Ut~bSUDDAP$IQj;>`m192#lup&{*O{d(2EDd-RS;8^c zpj(c3r_);qUTI}pV-b>`SR;fXO)>S|hu?l?>Q#v$hauLYk{)V_C2Q`5;cRdIsbiFm;Xq=K4jD+Kj|VPEQ*KG3Md+PHo}}fX9}Ridh>GSpDTtgI zIzpnIvNr0XG3GcTZHb+LU3#O!=T=)?!Zn8;Q^gSGm}LHX=6;G4f>Cn}>iOz|5GLqo zq0EXJXG5NL*ymd=2~GaLN_j@S7Vl z%tc6^LgZ{4+FD3*!qnaiD3N5rE9su$HX1Xml#Lv(9jwAz?zLvE`XtUC1Ke+`oC-`K zk2RXyZyX8}sUoFmwdzqNI`6_Xl6boBN#=b!cUA~q|?V!t9 z?8NfEQGZGDvS6%yuGS9`{b{s`&MfuO8v1!N;Ue~JFRMwKTHxJsP8(wwEb{hp(MAWS zouge(eK$#g;*b)?&Uc*P20pyISx?SqTXW*MdYwcBT|ADn zp&MJ~Xdx}U;IE4r`|M`}R(n%xPU@SaZLCR5B3Py9rl3>VtbL^diyCAyy_Joxfy>KD zyZQh%I{55n2r{1BxMx5LVvA2>nG^JIbi7MduVn;VlHWMksiDP*3}C~X0&!=;9P$cc zcA1#|(1sK{nWr;8xnY7rhCrHC35o1#+_%PpIHElZL1OD#)`0dq^4&%c7z~0HwZ|%( zaH@%;E1LC$)~6cjka6>>S^opYqKD!H4t`l$-{g9ilc+t(Cka!ZmTGuD9`;an!LryX zDp44b=_X%HqodyFB(`MT&|`J99`GFYpd?yrj`l;A`3Sg4LBg_ zB-KfgOO|^Am!=^XXyosQK-6S1dDk{Imc(heWC=A{_t0+&CneY18mhjODik$yg@+!M#!0S?p!%BAPAW>^yyk7jRN@5bRL07&4=&PdT?4u3CGG%o zkUlIVBZ>)1eetoO$Xd#ydh@^`zO$B5eIMc~x}fLXNpkF*;HuDByi+#Kc`_np@XnV7 zKQXnU!~>FxR(d{|nl*tq6rn;_$Q_VfN@H5$5~^zW1c2g?E<_ckF#S_7lCEuOjbW^j zKt@DGHZ>wIRIKjk`md>>F>h()B+h1(y+f%;Rr1rMT}%7ciO%7oq%5PH?4;bByvUf zzL6pEk=X`>R;qHm>V>8!)yYP8>^~iuv$)3c=cul z7+;aD>h&Qd*Kgb&Ku_sIFs zQCcxuJ1ARjVzSGO{_lFiYaAHnY*V4k;s6_DY%OTMYSu&}HA=R&(~in?ZHm9~3i+(B7|UcG3u#oEXclR2Y3nx|p- z%fSFEbFH!Nqp7=m>I4PU&f+fYAro2VPiKq6PR4XsxsI$6;%aCok-335v%Yz(gb zic04eV(-mwz30TI_P)I6^nLY+9$MnU*Bopc{P|0LkdI!gTW#+ubmBiwwYc*f=)@W- zssBA^xW?jbETNP9$?f)X*F*Q>CW-mwgQo1u>-BSBx7)63`?-8CP(xQ=uJ1JO#7 zFO1e(-RAV8ll%C?4&1%hvu<8Jeh%kRwWfDIrOrNl&(~n5v`14#FJV`5?)O?CC;vVM zb$V1ZS%+6)+#*xyGHqIuXc6U5HWF=#5fq>CQURx5o|b0wc0}?wc)S-oWP?V-Q7)uuBF6==VMBCQbfHyqnL~jU;c+1LH7;g*2a`_`I6)E^bFMN` zM{*rER&OqqZOyZP*(GhmCK8`hXegM1^tXGfKr9D$Wi|Lk0oZtbw|LC=e9-q~m{(7? z#%XETWR2E&{?~!ic1JTbh2vx#;TMPLlz%Zcf=(z=b(n+7wOc$@Xtb6qsHS^mI7pTl z3A)CFeu#X@wt;-)J3M%W;5TqKcYIXhZe@praM*?Alz222eU6xjQH2&-sQ+Elwk{$# zT{Y-fb+v{{VidAhI!M+<<-uPeSBaK5CS~{u{KALCmVZ9yYL7@#NH>0_H*}+Sjqf*e zP6Ry7bBJo8fiZX%FE&zMmwXvVYyYJy)F?dJICa`4VGp)(Nf=Zq0Z4V0jLOK2edK)R z_Eut-a)2gzZwM_B$Bn}FNY4~a4J9`-hLA1vEeIHgj3<0NMvS%=V0FiizleVesZIA7 za&Gu+cGrJmIDcc9k}8NlP?unBSY)7ot1l3A&e%{YL6S&;9Rius0zhQV+!BY?(dgb`*SbSQ1l_?0n$IUcE(tcY162`gW* zP3>oJB`A{{GlTGElfWluQD=}#^@<3h4UAbIDw&d*h=XfckDUaYtN2V)2bcf1S=6S8 zC>4IQ`FUGdW@H&bT<1Nbxn&H=laqHusq`wS=!Oq@lf#)5uu_^_Q(?a8n7}z$rCaI1)Oka&dKn3MHcifczg9r$?D)QF|VcuwhcUU&r6r~iCAk)GrykOGKSkZ7Cg z_mxj`Q54C4s;O((v5y_vo*#-AE-9hAhLjViV4>N6FWFMuHI%sqjch}csHdGK7=gP2 zacyNA{0N& zEJ{5pif{XAo+oN3pb3@BmyktzjF$PFduOEoi6sz5h^R%Q0ZE-Ysdb7uisPty2nceF z*`&rWhWn!mfm#ZtfC-U63QvltQwpU~YIJT!rS=C=X?K@=Xo@MimWa1lLfM1w*P{!E zpyf%ImIk0+6L3SSh-?)GJ^B`Q`i)69KY+D9Q=$cY+W(t)6!#bsf@q3FZrG}S~qgNPMs)}={Z}UltFj|)IrHYZ+L~%KzR#~So1O`PqXUm#q z@fJzw7gSqD1z1_D>pFP0Dp0*QOoA$}r!cR;YOnX|tNF^Sh47?`>aYCDuZh~L|9Xa1 zYOtglr;z%k_Q^V4dabRPtUHsi7K;Y~W`tz&Aj^rn<9C7~dTv?8mt zBYS5iYe-MqvQQhfQ%kkGYPD7Cvil0N1Ix8ETmP(eK(JpMtUd*{jGD8Min9%Sr#YKn zIcP%S+OzVhiM-hjNgJ~6`K}d*tANV4RV%eB`?r28xP!a3D~q*-tFKdvsK4r~0o%BV zOQ>P1s55J*QTny?s<^wFxx`wwV#~HynyiJIHkf3$OKZB%khe^`w@ur(CmXo++Pbkz zxU*}xv}?PA`?{`cNcn0=B&%wRE2zNYr@dJ*{F)b~+pedJx~WTXtIN890lTt$ySY2P zvn##V8@-10y2mB8-0Qu)>${CfNP7FBx`4cQi?<`obb5=u?d!g{o4wXsz3)rE&day@ zF~01ZzY_Bc;_JWPHoWI6z{;Dxs%yXXJO8f;oWQ}Wzzxj64t&4gd%#8Yzx!Lk7mUH> z8xD56!5xgg>s!F#Cc@7P!4F)*C)~Q&V8ST8KjAyU`ui&x+`lrc!2|5UAH2NHJHiG` z!adx=5$wY(EW0W!#6e8K!b`q1Jj4Hs!vSo=1k4P~tHU`=#XM}pXG6q4ti@S;#a+C@ zTRg;I{6j8$T(XnIORU69?8Iwa!%uv}Aq>I=<_uLl#XD@rF3iPucE?^kOnjUlc`U{+ ze8z)p#%OHFY0SoLoXBp>Ls0Cnw{dx6P4?ElKMGs_li%b$G6g$2sFY|2mE%cl&Wtjx`v%qxh?(frG#Jk8ZChIOmT!pxP#9Ldid!)e^g$n4F|T*%{miTnu7 z!z{|Yyv?O-&AZ&r)BMKR+{E>)&E(w8;f&Abe9Ptx&$^t>?5xi2oV)`p(7MOL^9;-o zW6$@T&#F7n{@i;KUCz=R&vzWr|E#_Q{hJLOwd8NHS34AUI_&NmIxBQ4T8ozn%~(i7 zNXyex4bmlz(0BXOK~2a~ir;FBDZO}lb)fa8k z6fM@3jL~52)nonDGtHz_eae2V)@$vdZ4JeQoxX96w;)^Bi_O;n&DeM?*CPDbVr{F2 z?ZzG~*nyqdjeORYt=EX%%aa}1cU{M#ZQ78H+M|urt9{v-?b>O54x8=MS#8x_4cdEs z(UM)-js4KOP1?Wx*=$|eX5H4XZQQXP+qG@mx2@Wzz16y%*S#G~zMa~^&D?)&*_w^n z#?9G-z1+L~*0}x16-~#$E#2m=4A33jw94K#P25rq*vBo>-A&)$UDE2E()gX<$D7{e zUESyn;Lfeh?%mnhE&qSoE#C=F($-zz41V7MzSQ6B+{_%z5PsekUfo)a-K~w_@$KKr zZQoI>-^xwb7~bC({^9E_;?}(kDL&i7ZQvUY-yIIvAwJ?KF2y)b;wV1hCcfh?-s2AL z-7qfWFW8Baen4` z&gXj`+)|$CI)3O`j_9ep=7E0TZ2qx=9_f$n=wqJgo8IS;-syfW+?G!2l|JcJ9qOqL z>YA?VPLAhke*fJ>F6#?U>7{}{_3ut>t(*`KHbrM{@!YC>!+UN9xUOF9_-BR z>z}Uc%3kNOuI18h?8T1cj1KKDZs)At>%0E!+@9#vUhK7A=(Jwn+RoeO{_WlF?d>k$ z;EwL`KI-Hy+sFRqzYg#D&g9d*@A&TT&OYz|{?T6k+60g3`kwCY&glxz?f}2;*}m+v zUfc(7?}nb(((dUP&+rW&@eeQX;w|x}PVsL}^5(AY;f?Sr4(+hK@f_ds%`WcMZt&MG z?;5}GH$UfO^71(k@!=luAusdg&fOos+{y0nJkRg_p7eB%^f0gRKyU3ckKAAG@;#sM zP0#dJpa1dduIEm#)EWDX^-<)ANDm*>|{^xG!OP_U-m~o_h`@d zHV*Z0KjiJfFY<@4*q!g#D&O{uZ}*-b^MH={GB5ThzxpU>^sK-3s$cjvPx=gP z`XNsHa?PINjQhop`mG=By|3!LulOnd#KiCQt{?r!pZkzs`OeS$lVAHk*7rTl-`{Wi z;s5#fVg1q%`q&TGKVSL_4*lY<{pVl))1Up+uk`f41d-gP`v#3#`M@uU8 z$h0Zaq)T%Kg*o(S)vHFaN}bA8tJkMs!)_f5RjkvfXU(c5YxQf_w{YFYohujT+Piq! zmfedsuV1}@sS2h`*kRqnh~XjT%Xn_%ziA&!4qSLKWy_bbwo}Y`v**l{A%hMbSv2Xk zgDYcRty-SY&#+;io;{j&U)#7@x7NKow(pytO$#4hJGb%Ty?HB7{#!2b=g^N!M_zk5 zbHdH7M=!m7dUx;F!(0FFtQ|aendiT!r>vfOdGYU6w_i_xzWMtrv&V!fj$$fmrj1tk=aiFz zs%ofu<}@m)txEsOW~Y@}8mWh`Vj1hMv!?2+Xo*&sD^J4)n`f!NCR?kMD)P!~vB#RJ z?6f8tYb~M8CJODN+fwUoo!B}XZnolH8|{*`_S&VU<+6$^x#qgt?yBFayXdOjzMF5n z=%%}Dz5I?l@4f^N%qPI7A{?E-g%*q;oDM6Ru)p(EY_Y=e&;{|G+dkZ+!Sw$0FU1%K z%!fpF{ADF+i-U+cGhy&ZMWKY zN6mKIargi2H{E#y{&#gJ=sHgt<>Z<2WbnBvD&id@I*KYgmD2J}Cp65Ird+Darp0qc(>ux;p#V4=) z@ys_rbML+{@BHu5$Bq2Y$Y-zp_S}a*=rWygBW=FaP}X z126UWy0D^uD68<_fByZ$ge3w}>0EWVQ|AXKG^M^n2X=4}h10Vb> z$UW?ZE`#S2U;RKB!VQ{`gd-GP{nYn1Z~QNRhL}SgW=O*r$}kN$%;5|Xp$8kvu!lF~ zp$`9nI0r;bq7Y!HgAjRWLmq}gfV*G?CJea008X)iFsvd4wP-*IvSNc@B;GG5SV3cO zu#6`RUm7*IMijd7jlY3o3g>7)?Cb&&I3%JL$Y6&b^6`dYP{R<3=tngWQjm!3qY#s* z!!acChCq0N5A|3>KPtk9NL1nyC&@%ZR6-Y+P@)2xI7JLT5sFW&q5xOfMg8f*ix=GD z7{@5VaL|&CCyb>ociD{#w(*z1j3XVR2QS0j?T$MP#2pyb&Jvj0XdVDD%Gb+=f&eWwb z<*8AZI!GVp@svvxCI1MjPgY{F6jzPKErZceS$4ISXv}3qDOyouq7RhU)M*hys?u%7 z0H)gP=@){q)-F&JrE#rmAV0dzyNa~0NkuA5YZ}NNGNPl(+^Y(_nbJ4N)TeZ9W;C(6 zPMjk4sO!w8AeDIn!rC;Q*Ia5|ld0LJN)@1<+$U87ddgQSw5wc=?N?`uOE>?r5PhrY zBvPHZ1-S;ctwy!xTLCN7!b%ggEIqDKJsSt2inI?kjjUZK>)4-?fde)Ou4IwR(d?F1 zsMG~4IKg09m2yx#n*Q!kDN*Aww zmGOI9%;2D=*TT<5ah7)sW$zw1v0sjIhT&?`FWKED z2bP>%7Vm(ETiGncxx-#|tB+|)V|FgsviPlpS+x}k^L_!rBxSR8s~hGFvzV>RowB3N z%U3uzSiy32Z=g&1WilJMx0FV)UnP6k#%6ZYcD3_|LE77scJ|PDPAqs+tlSzenA3b# z?5#QN>nhiHxS#$tmMzQUCoK9-QIs;T!CYyedijv(#qna3`_{ZhInN}}GpT{f0mlZq z3k0@v1O{v3R7cvwLE!Zt1JUe2+W4pJjqbWnd|l>xSq!&%26`oLc}2**>;(Us`W2kS^NZw()_FdxLQUSKRW}HJ(E%W#>Yn*ku1Mak4+%ZeF)w1W0J| zsoOmQ28`V0#r814?LgP=nzOU<2DQUao^WMXysaYlw7+rwZIsq}+P$8z4Uj!gYzO?| zN-gunQJaUS!??l0MmeNa{bq@yTg_xgct}qU>tXYJ%YtP$6m*_f6=1jEC53b(Nh;j6 zCp_5H#QKwEO!SVsRoaWKcug4&?uHAU!OA8!nhg%?!hgKaxtpIXxSncMKi=&w)p?#T zd-h=`zNPRUI?y9u^U_b+?%{qW2}bSuNB3LpEy(*8sBY}TUm4R7mNeaC3|VA5+3O$< zLlN{a^O^d+=xr7|<+}{^4hwqWnLc~e7fx|CJdb|8#DxDG{!nVBTAq*UF7}#>tK!`! zzWosZ{c>X;{ZdbT>JPY`nwq5vWIT;$9O&`b@(S}mvm}x7Xy4nM4%T}jRkgG z7IaHTcA-amYjuA{2Y4r@fwEm&_rwQ@DLQhKFyMYn=~rh+)fO*WTm^u~bVRADoPUCIOrdT@XAr+^Wte>azF zX4XzVC|zS#fI#SLpjLj}c6m~m1u$oC&(}vII7b(>M3&Zk4VZBh$bq#sd}5b@8kljH zc74sZam99bL+E%57!CIA`WYa&iAf0D?Dk2Brw^S7X8FY7=*BA}4FE zM{OxsiEKxJmKStFM_-7=iP3k41Bixym3bu;4xrFPTw zM}D_aKyZYCSa^n4R~e;argx0ewR+`Oi|+MTF{e{a*I7jPR>o(DT*!=@Sazz{eN{#qPAl)h>KH3d&o zhcBmTrlw6PhmZ}YUVtZ$<^w&(5R1B3h0p&;ec?EF=eC8Wmx!2naOPNgWf+ov<%|;e zf9H3L5*UhIcY7pogfnP_MMVNW1&omRk&?G;XJ>yD7<_%AjzG_ zH;BbIpwoGr_sEq&`FN^{n0tq2oXJ?$(moWW4Y@^YNQsPu*8-i{l#O^&=Xik{xsf{{ zSPU?bf)}2V34kp|ez>=aApo3nwUh_Qfi{m7!(`q4Xw$ z4XKbR*=rS=nv>~~VEJbYw}$@Jo>H`ZTIqcI=X=?wjd1Cl9*6_`se!k4r-e6(EJ;|v z*N=Kdp^X`&Ps(T5*=sNudSm}ua3I+R_sPideGAf87s*1#Urf%7&g$Rzk zGdb-EKf3TkxM_(>366Dq;uEKee*LbaW`IN5bt|jTNeHjD`I-)UZjhgw8_?MN=+KGs#q&50zeK%}^ zNrl1nr>K~td@8J@7;-3ip~;DT;99B>L{!b{W@s9Mx+tr3aFs}kuH!|l-UxOs##t(d zdwJN1>A8s8Rg3PJh+qGzVxih^*h+YNIBCE}gU^bN&q#Kg*`Q`>tGp+q1M72BXQvEV zgrw;iDyp#?i%v7Ti*D+4&nmH=$FbU(sc>0_Cv}-+r=@~PtKlY}Ptc?E6{}`oQf|v} z(t42^6{ix4vek#55Sn5;imemNsw$bXi5aALN^Wr|r*mtN3i_aAN}X=SY$U1zBvf0J zYeK*9wG9`s537jR*J;ovZ4)_%^J%c3I&Mm5bK7~Ma0-!G2%oWPVzB2(w|bd9`t5w4Z%yr(*UP@ArKs;&4svUG5? z;_0B}Q{vmYC~ zpjwr9%ZR+osf^ole#@WP7kC*jh=The>yRn+m*2N@ZmTb4}Z?yxN+ed4$NxbDXNP!x+Fiy05rOk`~NZd^DLN1 zpG&VO`B4dLkTbkTzstL_2c3+Yit0*n^2c&FDsOw|!!ub%m#CYV%a4ExkkGoUW&EGW zioOgiu^j&ykzSg*+4TpVJDdPV09lH@D>$0H=Y{+`U3=?l9U8`!iLmb*osMjFk?gdl z{J#}+m%Y6@!buvaX8Om%tF@~fyFwX&Ps?~L7keWd1A6?! zFa%P;2yw|c&YtI~Ck&Z}yUMnRhw413$48m33xJVGn}552@Y{!ZS-19zuLq5KoEZWG zsHeBPy`U&`Hd>fWsCtCxrORtnOWVv-d0_4w%c#YhW?a#S*?JYatYa*zhUAgGYokYe zp9=r`x67EYcSpN0djs-{%v%h|=*XDUJiPG>qQyFamdcO*2-6JR)THRonFgA#_t17p zyd$c*kQ)Qh1ij*#OPSPpCAqzW3yxSCpZvzYf3?t$_@Wd>wqKajyKIm!3{*kv%>Hb~ z2O9wbi?d^Px4w7FV<=-8NwZ8Xp+$_OOi8WU%g@RRyoU>{;=HI^dAL=*roHsMzcj7{ z?Z}p#1tu4jkgBo`OoldK*WYN)FAdRm8>-nHkc6GTm~5=w{BW_&u8TXmzwEor?2u3( zn8Do2PFl%H*@RN+e)-#EKwYi-%d~jvm-43E&4by$)M$qKrktC`n{2>nx`kH!1swlP zhBjx~1<9!Owy0E@lGz=RQ;^Fcu#S#9tM8qX4hw~M3EMnav4UF0I!VBNTgs{!hY+-(ee)tCyfXceW+=fLmL% z!TjCot-`C_p(csZkyXOm`nyn0#G8%OA_`oV-PM-cqST7Y5Wc))e#C?+vAh3R=oWqG z1IMez&7rl;lgKCEUFey}ddPZfycf*PX>6^ETc?a|+m8FL67GDTrs(*MYksE;X0GQHCp`skv5x5r_HOYgkX+O}=`hy);wvN(9`KAG~JeYCykrC$H8sWQyu-8|wt z%7xK4ceY!-g9p*ehw$aC&>B5g+0B?(sfia0^ed0V)Xc>%@90STX8A|vjZ30}`EIwJ zZr(PJ!8GzDZ;$?X={PRE*qNV`PL>=l%nWFu{M+hr+wA0M(}v8&u700V$>O9v=nmQ1 z(V6pPKFD$`^;lito=o`a9>H^cZ&@zLDh2Yfd_l90>w$dT)IP+CYM5)OYBGzn!b$Zt zTh%&SO~VRftm($UsD%Fq=M1jik=c;t)&+Ja+M9l{T}q2>F6AJ%`ne~8hH1id8|vry zp%i=CkZtaM9!hea_+|OxbPQ_yS9e0Klv3}UvJKl`kaW&m)Mo$Z*XIh%zz4C?%(xvp zPonwec6q#IJ^Bpx{g%E~>JOvHhwZbA{xQznXMbh%&g^Uk5HY&6QCINXL30TeE)3Vu z;X{X&?7SgE#fHElBRB|*Lvi33kXh7(d@_;?6qIeg3_LO7NJW$+M25V$app-bBnS}c zqC%#gj43Uu*fKG~2`v`46!kc><qE9-mvf7Zr%&KZ>EXH`sk38k%dJsYt ziMRpD$~OALKefJKkFwH2@~A%dn1iXx+<*hnND3qH%ZxGXN|3WF&CCr&v0Npow497@I#nyWua72qZOcIn zS+ujoHjir&)KK$`$)+@YCG}M-Q&a;q>iV0ui$V?cR<=0FoNxy-QG>$Hy`Y=YL$+L; zl(21!Euu{;d7Krol(lF5fvGH})d`RlH& zgmXofj8UWAb$TXF8Jlj-uaC=G#3tbJts<%&~Xd2V?b)f_;1OBS|NwOjaftz6F=&|=r4 z&D^9|rItPK!>EgI$hRr=ld0AV|Muo?ZuX>*6JzC>+|}8nym#+7X0~fV!}FJ8w!}y3 zQaP=~d(AHwhIH4LV=SwpPEQDZhWE6swJNc<6WX)C)J=TIo0p9G5(r6fNO;m}pS!dt zyRIycVtQDh!x&f~=cI3A$+N-pY!?4L=!GyC)3YGM9+o}twXQ+o$yU1*vo9dMihN<& zm3m|t!;Yi{eL)f6?c@XrFo_L&KU3Y_j$<3g0FHF%@=3?^w#4hHs%7p`n!kOG zR7OkP)yy*}b|GhYE!rCn2slLz`7K)wYn%r`C_*^Sh81=2g1ERR9JEy{OdQP1=fLJP zFm~#RSSVf*y?2@Uc*uKx6weJK(+1smu5WobA>_VBAfegtPVciL9}lRkGZ7FuyD8uy zkF`F`q(*M-%j0HvcrP_t@@Ntq84>$vMs~UJjl0aoY2chbR9LQkY(_K)F&!144eWS#t?z6AR;{a84mpr5h%bwka;p zF-MYQz~U6hhCh7TbBUS*Q?Ra+9Z8yLUE9PL3NEuKJGRrFS(`yR3n{LA5)^Zu#0!^F zIX=$ll9zEr25ov^%ri)#3NPRR6H0p0k{%)x+#p9wlOc|6ur#JKrN%gDdQ)_igB?3f zh9*Rs1fYg=4NDkmQH^@kHQ*tsNJT?aqmTn5K=r96h(c4Px&|Ml0IMaKs#crohpb-V z5}ZJVCRA$HO-Le{O)bF%%Fu%gwAHFzRRTEuFo=)lH3i^2=3M0(Ry6p4n0@`|Q|T(z zs9yE1er-ZxtE$wi8sYz|hMlWZSrFD5DB-c19j0FM*}~J|aUdN_nNmorQI7tyqoOdx z3UUAe*v3}2vIWFS-(e4)t|PZOki=R^bw8NgGilVh2tid4{TU29~$vo=tdX1 zouF)()0bqWww;g3?*lxdEMnOgW@Y8^ZcRJ3-xv!yTu&hQA`2Di5v=J19) zyi*PHqP*#?aEUW;V)X{K32P9ISt{FC>{52JF@7;w)4GYW;y4p>AgYWTds!a?*|CVF zEN6{eyq`E*LNCI%6U^}U$J@R7q3;zD|UlP?_1RiVl<4fj2zg;uu(KW z7}Yi_5eOn*}pP0*$Y(6P%Db+33M~dYegwL93Y{-$`Szl%HI* zv80%0T&I>3kdUny+yDq*y8*Utjx(L@eC#`K_!4gJbF=yI4qK4*wu4^86_~OW|AiHo zyUDggxt$e83-p>#woADpRBi@nc2MY!09zYs5p)l=F7bXUwaT4>Y7;V6_cpW)(OrY$ zLR4!#fNcL4nHB{Ad)d`2i0i@z7{E<#S~I9^F!V?a0Wq+F5S1WA7=+*mH?UyX4Ojz) zjg9PNgBZBY*eQ;i{qh8O01hu*gdYgK4Y2xUK>SW;r~H?0Ja98?f%p!>7rJk|$h)DS z`gw5~3M~H_fCkrf5MvBbhDPY1>P^r3xw*23uG@P%2zaj6#U23>ls$H>T7lZ(d2qEW z^F~tGu@uk%Y=t`k10Sg4n2%slSC%?Pq2?CFam|Ly}AcnoaInZ;)?a5J| z@|;pa5-`{E10)YseX@KUvQX%8JhuWkSQ|K$v1=ji$OgxwSZq%Hf>RY!C z8p8j+FDe|xI$(D}@>)NE<(fTA*7<;Qe}JNvFyHOVcm4FG)OxyKSK!!>JJdYqT>1WC z5Q!KC8x+2GxDJnFy}P9%gX6qh`_iNq-Z&z#9Rn6P!p%u|L@%ao`OC9BmO=$a5CCS2 zIRs3AlzP1~h>d1oh0=pM-7taF11A`273Qlz3yGw5tH47#zS%>9I4Lke z?km2flaKG)J?cw47?h0G(}A)(74Fl%0+GI~+rHs*zQC9P+bg>#8I4Eivg3jP)q|x9 zskNFw6-R@=5K6OF69Ok#ygIl)BoMa9YXXx4K+H?F2V}WKkbn)~JOgwAG;oD#c!mFA zl7-j+1%$c^5M09gsllLIx+iHp4iG7DnwQ%9J}baI*2};YbdVFA!Qiuw1Hgl>YZ(w! z#7J~4r^rF&yS^XP!5s1&{?oqB@}DfI0^(w^)NvK?I|IR(141yXLNE+N-)e>jtTr=DKxITg3M8uofx`!!f$Awl0-%6A zREsBxKG(a(`QgBj`om{h#2{G+H+lUME!~+4vQ~U#avIM-FKWp=}h!K$Qk&rPP5N3%#ii956$~gZ^;6+fJ z0muUZGt4;38^!?SuwAf)Jm5PtjJc9>EiHpaPv|=lT#L650dB0HnLLA|%f`1*I^#H( zx46Bf6C$=y0!0K1N7TA;Ga#)h%9a2)rYwR?SOk6KljeKJRQUlQ!~%nByRiGF8OX;a z=%nnpu%$`~NdScNgR3xV%Joy9xS^GjSelBf$Tx~DCIGoKP!k{v#*h@rkxU0Tqyvvk z#${AS7dS4FS}9SG1yJZWNvn%P^GdEdI+;WatwIcIqBJ}p$4WxQtm%=s!h=hk418Mw z(|MPCd$&n>zSJb0RLZ%>Bo7~eP1k%fusXH3Aw{pWr^z}OL;!^H^RoZO!V0$Jw7C2e zC^4d2M6+B>yhN}+&0v9!t#J1xkYxui>7(1l*ROKRH71?WOH*oD6gKx6cVRJeofbO7)4PD!YS zWN^dhQ@6~TvW0w10L{@TOHh~`P#pD91Jx!W)lnPmyJQ-$Z2D126Hp>OP#qnt8H>{8 zafC;hzgD}f@935IIZj8zCHKit4TS+_^R>sD&JcyOR_MzT{Vo4!OwqwS(-t*M6>|aC z@<LH$!c9V;EXF&yKjyPJZvq61CXghwM(gPPR(e5*md zR7&+zP36-+T*CCj)IVgXPgTuIV>BgQQYi~ErIUk%ge|?pQZJGB`J;@X`~5dk);6w4UIo0z<^!tLWp%ULgB*`QLd4Y*STXjf^CS#+IQnpIb+n+2TBS)KKPtIGr$NQG2*)D#_8 zKjl-rvVe1~*-MpKXw597eOh-7RHvm^cdb{6k~F8?E2<)@`2s4H?XG`SGHmj@0os!$ zT(?}pRXBRQ*b;&hbK8iGyao6;n#tH*7zK>ITL)}~H01`pHAYr|#yBiV6?H9_HCYPC z*E)SU7Z5bP}bH8KeU zoy?bkiVODW46s<(TziB(utGrS1ze!mUf=|`g-y6%Jk5j&Mgz;O0iA_ebX`NBy#(0h+|h+#m<3wWwbVa_;c%rZ z6c*jUno)ei*_tz6!ODaU=HM>_TkZN<3%kpZ`yUH6&Mt*ZnX%r~0t7PM;OY!tU!&NK z)2YQ%5wP4t?!Pq{VHieam@R`t2HLGc$u*#1B*QUD2wI~G%l_lxuJzXh%iETDv!x)xjL1(kROZ~lVu z9E1?S0jYCi(j7nFTLcuy z-9+}?6+U4=PUzX(XRZP|n6+Qj&Efx0u7L=itl1R-e5Gi_MY4?!UL984e%66G)n$`D z>0l!!Y$;giMWkL#lDIHtj*{YJPSe{eFI*r5w@qg7g*+iB1GqKTQ5aw^0EF8TziPOL zWU^#HC<9H9<9F@Tke%OCwI{@7RwL#CI!)X`8?+TL06t!W)#c!QZr}$_fya zwE!ubS!v~ipv46d7Uer$;a9!{vfg99h6A#41$)k23cy^zUf~G-fr?h#KE_wY9rjr?PPhb5Cgd}CU=Utmz4imq&E(~#u0bel_`W(J ze&fT<@5H8G$$o6dmgjiZ*OgV??&1Sr8%V>5Aymnri}IUr3~gS@Vl}nfPUwa3eb_!D zgzjC3?QO$ESmx1{<3WJ7PN3-Ve%2yop!6){3OLpQVA1fJFTQ?m0A54Lif9a8XzZTg zSt!dZ5<;PgD?UEl`o7bz9&f3p<37%09gtk=uJ0y?Z~dO~#opX1_ixC~@&G64l0Iq6 zE^t<@AcH+ymJYMcRs;VC-=)(AhnqgH|6_unmRkid@(fVs%vPyHAWB;HgKRalSGc0r zdTn?Pa}B~@I!##t;Oh!lbVh&j4;XU2*6kwa+}^jwQ0e^M>?(!J70xz!u=E7cwDkBP=gMsatGv`twNORR9g!Fau z^ZEuQ#JE*lyxPLvE^K8hujXiH^^k?}?R0@jW)lptWNLqH2e2=#4)^4C^Z{rB(G|@h zH{?CIZbdG%rEV&4#UF4bXip#YdbfA#CUpcTbt-pq6ZrR4?{8P%a(!)eE|>LL|5q?) zIELbrgY^w&>h=F!rzT;yhS`vCV=u3a#W)OL0zrsoQ2?&Zb8P@`R%9(!Yd2OoD5%e( z_I5t`Jx1I-*7rGY^Z-~ur@t4cc|ul#uHN~Y`-9yUt85kTbhmkZ|Ma1!`A0YPo0oD` z4|s#ea>^q3EYD>Ee}ON1G(f_MSoHPWE1Q_9c#9`(yvtk85N-5ok5f|jXHs3wx_5yWPi|ywugINyCqMdhZ(3Z~`NJn?C5tO#!gri^S;t>` zxu1N0N9xoBm4+NfoCs&Jn>LIYH(JZ4@gvAxyTp)$Amrqb zlq!W_V9^r80+@0%H(IcKY3lTqD-c78PNix! zKmi3>j~dO|mBLrBVh4~dJNE2WuWHvGD68WGTDUdSN*%i*?%TVIaG~NQs@XMEcWW&2 zE9o#NIz7Dq{9&ZVpdLky>69@;`Enl0l29E6)!CD#(0)dbPL(S3rbVhRCBCBt2a*#b zs04ZpgrdPBF=Ue%=o@lFiQa}2Coa+wDNEQwdhFP7m6$6SSfW%p>iG3Xk64O1qeL#= z99aH%Z%}nA*d%as0`QQc#1y7Z=Yv0Q|7!dEb?-{e%b!A9W_{ov2n4!!;C=}z$kqTW z_}5l_3rdI=9-R!LiDru3C5J72z0#Lswj^WNh{=co$~N4jxFU<&$nqj8G0F%SFST6L zNh7mV!(wWHeCUWBvHdnia7WnY&_upPaKaO=LF7D({BI1=4S zVV>mw<2So}bEZ4sxFO3o@7OZgg@w(66I}7-bzcs6;1NZG8}xYzIDJ4N-+ZPS!NXB6 zToqIY_^ovygINhUsZDdm6)C0=QX1By28QRJ5%868fe1hNz@dz$G4cmBNlwvPfnjvqDPVFl~`fBAZ)_ zH*3pn9ak>5Wi(tmLi9Sh?)nWOEU_>F%51|&$h@u828q16o?%lGqx6!mzUb+&$%tqu zVVWkDMnUbtJhK`LRZJ2tFFscDXo>j zfE{+&t0Y1;nq|Aeg4#`p@6r!)aMBAQm~qeVp?5ZlUSmnoP+EHb4oc|E9cr5Y72|#T z|NrLnm8q1mPhp%3AOb<=xv+qsbTPq2EmHBT_gKtN6+(q#pi+jHxy%BPDADY)7Q$M2 z!ZDHQgC6Ff!lhLPg_9`S@Ahzp;h{iU#&a6t6u~qtq_A4AsTz`k7PNM>Ll91jL@cgX z3}$pG4{#HSna-3pA?OVp<(h>Y;Il!+S%6emSRKRwNEHCkUoDX1Ws zTHfb4j&UqM?8-spgvTxkzHA$bl~QDM}9rXU=f4Q;aJR*lIZ82YF(#nDm?{K3TJdLAZwxQn*QZW{4KDm6$aM=J@?z`%k&77sx6p>t)uAhm z=&;x|LodXKNkDwe?Fdpcs#)|&Dp-<8chw3j38EXm^v62*fu_V#_KI>PXBjT*t{w5h zZXejE4DGcCf`t|h{;WY}Q-irnEi9^?z$~CP(Z8UeaSCd&Km-nHx%{Bue0`{@=FH;7 z86f0-Z8UB{X67vAG9^?6NWwMP6+r?~H<6|?02)kFSEFsxt`Lo=JgXpHik87?vLVtd zeJI#gQiK6T;tgd-kp^$6l%eI?EN4N;*%`t1L!t=8U&tla91zV=tdZw8Q}Leb_CRS* z)k&w?Ia>uR!4>2Tf+lu_DEl-xc-&m-B2w_5q6#pzg*#yX9a+~!t0k_*9=#|Q3Oazj zSn;2`U9DWO%f0UM*t_3Nhd4yJsx>5GhDl(GD|v(7*&OMVhHWpj970)3q@l{lF$9)T z#3RnYutuKj#gAaMYs5@OZ z+xDs15puWQJ_a(rZFMuVIdYK4o5gd)~q;D(QLgcMt}l# zTZZ_IUSS4NjS9CR=;AUli9t1hsEn35M0OQSsI~?5VS1`5YZ!^tc3wq6-)Sq3J4et$ zV$}zi$U~}~74RGe*3n~CkEABZO`Rij4Q1}6y`z3_@cK@#;yc#;{0fBR#w@)8UtS8s zfM12bgW*Kznk6Oyaa+scgf5&l2!pLVNqp~+U*w`PhkckU#3^_kx&-kI#)$Jl@=^Sw zQ+`{u2ex(lJW==+0$kCFB?QI~YFYyXA%6z{aAf`ihX^(2MZl9^-R%b!p8RS{;B<1k z5ToU$skxnU^tTFhV+F#r2(YW)5MoQ}P^bE6C|P8InmlTUjld0`w(2IK8l=7fed5b* zOSD%)GN@pChuAdsxXb9@d|l?G-E_uMosJIk+=PgOzp_POJDJ6eYcdL-dK&+{Rly_& z;A8s(3U_-AyBY-N`QP<{g9+?`o7l+k086eA#yzN0HMq$=;DQL)NGdeo{wRTJKw3ng zkfSk|9o$YH>>Xq@M!UJ&I>DRUyup}(SMT}W)WJ%zjFOS@6<{6DkZhgu{Z*2^(vECj zIaI>*MFK0NT_p@);?)Qe?w+XmS0Xh3&4L+$Cg8&Vp%DQ7U-`7$fE>cly`LKK+*&-I zul!64&YE;oOvB)T6h`8fd%#6pbJZd@Tc#=x;&W+0-Z zK|%pWm*r{W2?k;!wS^}9UL{)p;w9$e?6`?ueP0XUz^b(u0l1geotpC$Tq}`cXH7y7 z4q+;)q7ud(67nOQTtU16jxm88{GkHUMVkW{0UC51$aT&z!j8oiqcz%vAO(Q`CE&lj zLneft0al;{HbDU1%_GQRO_`n}et`Ie8p_;6gRn~no}D?m2079o>_r+C5X}`%~)ZJr{{T}e?&P2_ikSvLkg<>n6<<)!?Vkx94Du*Q?!q`A$M9!t{ z%)rOh&;`U5hBO~FahVm3v4GIMF;Lx)y$rC8#!cCtgIHV%{g*(h8 zb?#UE^a3EeeM||6 z0&fOFU_}B%3TLO`Osa^Q4DD45g+aBL(qJVYS|TL#Q6~^u8N3)mT*@VPmWXI9m1R(Y z$2p;rCCQD&V{2vqUg%H&bcF_d@&SBm84}cl5>$g-;$_nWf;8>VIpxvfphq|98Igg} z(rFQL8A;4UL4>wx2N>NO#7t_Mfa&Q-8yk@tG4Q}*wzGe(H%(YYm5Mweg{!m6;SXhOL86t(a5_Q0BfXa7w8a#x~Ub6U=31@ zp%7REp&T>;PgNL!oc&h`^r?veVP=E^C=9CB6zU7sO7?XYtHJ>}>PC~8*K^{a)Hs>I zQ7YQ$LMXuhWfg5|CLy7xM$;jPXFzJ*M3m~ODrrpgj*hqhyOG|jw(7|2h^(R!q}~9j zdMN_=DtZ8mW9(@;0jA&-m&aDwKeg!txPYFe(Ctuyg+c*kIB7HT!DnG)-*kYris&4E z%wLocO@TrhR06q*>$t+8v5dxeRgwy|8Yo(nS^Cx0$g8CiVLcw;yyPpV>g$gLLY9)H zvS4jTWs|9P?K+)alS0?XxT+H{>}=Ws6}*fDA?;EmOCT^N*DinqDPzqE5FAm^AeBJ` zeeAK4D8-2EKXF|_#OV!TXp4qc2y9i)DXzAHhI`u1&f=TT{_HrU256OO>1ipzE?3b# z#0owC8AMENb?PW8BEr40qUa(a#x14EQG(iK

xU3NoDnRY4{+!D(e^+3s%HHmt^x zPF|u{yRME~uuDzcX5-bZTWtlJ@@>&xffYPJ=^YnR-jHR80Ol=fPQf1|aL#Q459$0z z4T$J=P42i(B8v)?<~}M~?$8gx=s}tk(-vX9W@mS%F0a926KlTaLRkRQ+#r(>FHHbW*zSlO)-9ey&?~IjX`rXxY63bE zA_!LRYpyVi9W3TOBNT21Q{5T+0Ep;~oxX{o`TneBO|QB_XWO!xVKLv*pwj&AXpg4< zt`diE50I+jn4gEBOoz?zR$w08#ZP6jEbLuh6AUPm=BA5C=>sEzl(azJp@Nnc7p}S@ zRxZ`;9so2!rU!Ev2!m^gO^OnTVjGxaMxmxUz8JMA#$&#tlNN9?)?AK-rVsY)4!voiw=kXGS$`y zn=u0Wy$l5%(;Cr)JLqXbXHBiYS#}| zD*fgT`BEnPV(?B?YuLMHLBCtbM$2jFeEldh5f%h(puvZUSCgXb^Bg8&`9a zF#hQ?XI-;CMi*2IQo3dfE7BC=cH-eSk1zrtrH=2B;Dpfj=%`LPW3T1w7Rwb76PF`9>Ot{ z!tqr8<(Zzgq~tN*o+Bm8o^mRua?$5zMy1P|hw|3&sK9kx&sB2!3jT~){6O5AwSrW?$XMCV z#!0q!;|0Qo+JDJQZdSHZ55QLg5#Pc#W|%b#GsNqBmgMytcPk_RSyEQ2^l<7Uf{Q~8 z%Ah2BVvXAO5PNPL5aV*Y^Sr8^5k@USui13xw7J#|Q<*Z#A;e}_qIS&C>8gmeq9fvx<5iljzT?Qae3kiDvz&u$9P6P~wA}4aD z7C*U-ZbuYv8)`TU&vBA-vjHrWO+<7uCt*QZ{ubEiUP=AtxGJJ7hKKMxE9PYt&fyrc zMhc&dLseEpK~Fgw-8u}o#n0nj${dR+Qm+MR%uHwVcmfGXEtoKgGXm5MFS(H$_->wQ zTmu|{02HH+oS+$os0MkwM}dEHl~Zo4RCib&*_MC#OrLXvudCCfMq*x{Cp7n^+N(U* z2+l5*?@Z*TW#0qFx|*W~2MaiM%LmJ`8=b>X8T6T3tigU%tgj9_dPovedu;%rR=VO2 zQ*L`|GS{RRljXUr79!1?{o03Q7Rq+e1x$Gk^MYI-YQCBOjj!~hFaGOrM>-LZClQB1 zS-~Q)^mQZ%_PlDc7%F7BF1H-&Ix0x@!)ekXAbL$q0FVQ;o=8R+T$K(O8$0UMj0bv> z2J{&1xtMYjxFy$W8*5QITD&o-xSJJd%X6piUB{uXzWL~Z6K8b;w?C@7TDm&V|3=se zHDZVYNL>l0I&Q`#{p-X!e<}QiH#{|XmY}Pz#7A1(h98DF%w{%&CUozLdp9!gxs5}6 zRfarv-*?L6yK>=Y#2R_rW@xL>7FIhyp`j;knW_G+q-=uWb2 ziAGu71_MZ^nZrVF0=xb)efM!$r$RNp zkA1bi9C#D^dXvQkBn9Hr4ebYO9Pok};J(J)+Ndbf=JliiqaQwbLatzthKA%diF$YO z46YDokXjun+4T)dII@@&*j@h<9RBP}0vIg*lGMQ`sDk6Kf-Wq)<@>Koi*Tnm##T|> zV!q4MZ#EX?k%dtLLMa<#e$X@)?7Ra+fCL{J-FsK9+=c}L1~P2uFu(u+6jms?auH+3 zOnfdiS|Jc3FN62s6nN-B;zR&Cq)4GKAPJr`Ep>QksZ!#B14ew{DT4;6965{147y^J zXwsuX(X5yN0>K16t`1$h)Dn`ULz0XP;w8!d)4gh&iuU>Aqr`3q5=P5o$V5~;!BIJ}<;#>u0D;^Qw5COx z7%{RlfXZgz={skSdE?1SuHAXu0RP> z5l>efO0}j0aqJov_~3*});mrr&)5=`>YYM}>mBCgxn&9#`0|BodL>IFnv^QdLuz)$ zB=$hUW0OasP~w`p$`S7sl5|=K!sqs@009punvlW_ovJXv1NZ{BqgKb4;k4)1s5e$LTP~8!mR0}3DKx!mJ}`+4HQ;%5%nIl zpz6>h3{h3jLP{l6$OVa9P?10@SOwu$4JkShMxSQ6Ye5`u1WdY6dx8(8@9@41>v=m+R5Btn><%*r~j7nov zGGu5{mCBff6BG>a!y8Xr)}@(urcia&=Oidu;#Y0nS%;}`88PReZ_T$NH&y)gtzm@? zc1L5IMu!tfm};V>X6frgNzWh)MB69G@H)AzkDS9wvd=qP)KEP^u8u6vMK@hU=uj?? zx$QozM6T@<56oo2W>^^%MJjn&W=(H)wZq*b6x#P&Kg30K%nda( zRu=}=7bv8YW_oF2oqpQ?Z>r8{mW3i_Xru0|AG;$CHLet!F~=}F^0oBUPTvr>WqTXl zNRC6y)B1ZFM3cE+L8#A0DKX9qjNn8bW(tZ#M9Q-`cP+K|9u|aTU|!41m+7Jzm631$fD>Trm>2^h*v! z%MoO`5Goa)re0-Vf{%@~EL6gfihIFdVfa0^%_xf@Gf;A9(< zMHW;@x-R@vKK{GS6}%RRBIu-%uUG;NZwWC9;S-jVh$TY>`9xsiYG?a-SgttQP@P#+ zkq8AGu!?ESV^VK(m5amdI@3~XEec!EbY^E_8VhW~F^<{Xk|E^eCKg0RCqboHbi~<5 z`}LHfR8x`v6+xj#fCU0+q*2%-;PO8^-KiG_v4@!Osm@Jua)lVn!|Aeeo;<`UV_<2- zPI5`hHAHlxz1&19YUc@R>_McEG~jj|>QUY?v4$K4W(_n(Ql%mDq*xpa=N^%W!)V4b zOhRK!aS%3cm2F6w4TDYJ*jdl!RHv+|4GUP{22SK=MU4DGidNT0KEBqUWx2p4HK9fE z2*w`CY^n)(%Ud%Phm#cP0VCjL(FTk`kkTMT%`e2_JvWqKhrP&Xt zLZ`RJ5*4$Q>n4JcE^j=-mEcKNIr<>0j)J5EfDNo*2m6Eb95%7}HDYOHQ;uY?Tp(Z0%vdf+0ceUen)E{H7H!+ZSaH+MsZhcVJaj$u3@CMsN_5R(TrwP zA~^R71aJd!pbYacI5N^|a=ZFfQ8+h+k#SdlvvM7xma{3WOVuq+lrq{yR2Slv(7wnp z2T=H{I(Czws@^N#%>A;WCCe*#{>EiU`2`ht(JvNTv=uM30E|>1W7ni*B{L?t!);`+ zgF&lPo;G0+op7LuEo=)tYBT+-q{W`fD|Uv#d!=+JlxB|MBoQS*9ZVC(H#fW?j{NeuY3xSCbUbw? z*>gI4Wp%mdCcbJpv`9{wu-zmgnY*0D-LA4|WeO-q4ZB`2??;q%cPRDRLmlwW-G;C0#_2Z#Ew&iJ79K@BylKe)2(F=wpoM^9 zIQ)mV7GeM@Mgm99_U^_f!Y|9Sgx7;v%PPetOxlU^%0$_WE) zBpr5d19Quyc5fc;%@4)F#2_e?0Pgt2?YFW*fP^f|JmPv#a02j+349NgF z#!!6NY<#@uz_g$Y&(IhvVdQqH0TCiZ8gLQbgapk*8V!I*f{L~OTuuWyun+Zc53{cK zWI+V)Xbx$h1Y_hWcm^C~!tGX~?;61sJb-xq%6d9siBcsK+zt=kp$US>D$w8{e&`I+ zahK>0cEF(%K0-h`;T;18D3UM=RPk1P#1$v95pwMb-4P0HQ6m8X(W2&EMxYBfFTwgx zNt&seiqRNRuMSEJ(R_q*Tv8%g(A9*B9Q|mJqCjJcB^*}4u<~%~Uc)D~aS+?9x1w=P z*kl^dQ9||rVRFE5xWhU0Yzdf>h@_w?4bq$R3=ay4ug*z{Bxfn#0qR7gmt28U07O8h z0xf^gCZNn#dS)if5-n>Y@@k)rh9dXaGkNN}6Ob%Y`v%12Si=Ju1@$EAcxu zbAW*CGksDUVnH;8GA=A36h>1J*Ny9n(kRI(DO*iK3}iGmVv@EpJJXT2%<>)&@+P}z zbtp7d76B$70V)NF4F{4U6rxhDfkK%`LXl7*@3I#Ed;+7?PZ>p#<6YK3ooNu#&eC*b3NyvJ>9e5B6E<;F3YY*4g$&yfDF-evK01{KfNIm zJi#|)a9g!o8s=Dfkt$dzc31htBa2F4m6c;VHvOy+N@+z}r}Yf5X3!#Q zN!l!1p^aM)@Uy-ZTpIw-^dMS5NeG_kNuiTqYC+lhy zS<_*k7A>}dl?uk`fTJSt&vGg?U6<_&`xsY)S`qf`8Kys;ZR*%wkb4dEIAZsBu zWg=)-J(4Uezz1B_Ix?m{?Z?(e!)v3acdJ$^PLVo3thM$?CX&%rp>{N$S6Ou^Zl5-S zq_6M>_5->CJ6I|Lco7i(b~+@BFsFqisjc4-H*t-W&g{T(9bsIHBsH38qju)MYSOjL z&uBVVE{s7{uP8u8msN*SZpZ+2UrSa2w!jS0O!4VDdSG@#tq8ZmCQAS_4McyRFFGRF zOfFb$(RP45lVJ}9G@<}avzBB3ui$DuHhQ(fZikDNw4!=H?Q6TJ{hSps2p~1I0CL&D z4V++6O#qB`c%>i}3=3E0G^;Ru;o5FtG1u2zAnAP_!Di>TNJ#7K;G`Ulj}A?R4y4X= z|F>vSRe-T>o7!ik9!m~gw{`odZ}a!lB4~r*cqm{PjxShxqwgYe*N;3ne??e>ZL)+j zR*&(wU}Mm!SWd&dSSSFQk%{hVZ&-T=76o*e2zvO3ogiPlH&(--ZVH$%&uC>wc?6ia ziJ|!CqBsl^pgr9He$!xeLx2SaWoPb(KDYwxzL-=2)JuCAOh?zNkg+z#APYgS=3)=P z?$>&enVJ7MBj^K~uO`L++&F;l272cv>1f!@&W~>i70|2~Z+EwJzC@WdV&w4UULrY< z)^DAa8Gd~jh%+F2rp=`|;E3rt=Lm3c(bJSs+4Qi95z@eM#UK%oqg=&r0iG{^o{znD z0D;l$GN5Xg$2fplAsx(kn?j{p!?ShSn4?=afk8T?MOvgE+M`eU23*&nQyPB*SAUN= znwfc;Q|xZ+X61~|rW<8ya+;^v1(V~MCsd}MGwXcMvpfx9jQZJ${TU?#x`+xU~jkfTjnrPr9P`$>&udZwj02NSzK;8!cA^#^QDiTiI!j5w5) zda29M09P_Hq1q7$+6^+mQ!Aj>7-pJw5efPLj6Ii^ec4O#1DGc*%!1?>&%%m549&hy zhZ(@6xki3@`->KXfNpyi`FeFfatjCutyv1L3Ac-v`?ZGKt>xvTo7t_QxoHv6xJiJ! ziMXu0JIzYRb~qAMvn_7sQnF%+V(OLq%fSHb)h2EeDZdP6zyIIP6Q~5d!i4@2?8OTW) zz70^b3&H7jCsTuv6!}sIG;gILI=5Gwq6Iu@Nf)+5=aaxk4Jv%ZNqoUQTzmS}w@o|* zHq6UETzt&@3+S57jeEBVJHt&}d&*qQ$uTpJV19u+rbV2lPhi8JSesM7!kCY{omx)@=Z=00sQJH6s+=v6;V)V-0^N!`RQ*Ne9&pt&2b zFT7Eb*tHZxw_Q4nd^_INe9ZA3WsRD--TZsy+zvE>sC_+J`255r_j^$(3mz-Qmz~!= zywK4Ljf`>$|Ci;PD)M zl%B)?x(JM&pZ5LFi5lb497@K1%&+*SX<~ezgh<(&+TW~v$8Z=<9_3ZO^h$mX%hX=~ z$P``#yYERn-NG8@Nqv)W&Y6`?xo-ev$&IeV2-TB)!yZ1Yl73N^-l2zH>Y@Ivz3#+) zzVj7jZa$yG#M|d%Jlan@#-p6XWiaxk9OK)4i*X!m2yn9fPoMRf80J3Z?VhvnzRkZp z_DA}+<6413`h{x#8%G_OyMb_vyaUE!{N$qK=IjTg<3I(IO!Z zEd&)JLC7IQ1c?wSlDNo$VZ@CR)u1XzZrdnPtUiW3Ic{4@l_*zIo5LVtn1LAo4aWIG zhE5kMI1BcCh@@ZDaw17q1hb1U8JAh~iJRlu{gu(3_5fgrv^hf z)*KhAi0WE)?BvT;5IC&*hVnpUFFG9Y9IF}DZImlTZ1D?mnx`eBZjK}NR_QeVR!2>h zXw|tsm#Uo)H~(9?Y~3n+&6pOw4F+3!05%p`Q3yVf#%sh>W*KG@dZrowg`d@dlyrZ| z#FKE|eQ2F=RD>uVh#H}XT}DqVW>|F2G53&lF>z6ZiynrUorgEpI9yXC@OYhz+f5`@ zb~QFv(~lT22c3B_G0>HfPue)qjZAhpQCuhv^kkGN9%WsM_*wCrUilK zt*HlOgb|YF9BdZzV1yG^7}{rjdJ@TgsR72_2|WVX9(xIK$fBZV8n|VO2Q>!<5LTwB zVwYY;%E697;CLxUQXV%WLQoX9q=5tscBpqZ&SfZ;G+s$%m95Iz;i6;Qgye~u%83|k zo(h@Zh=;-1*PHo~xn{7qz8S+YdFr`mpMRb0ryQ5xJD|acDa@MH?hM8_5(!Fx z3GA2vBNv(E0d9GsSCOjMhXY-<)xyV;o z)1Lq<#k!NJ(S^n{Qv0};z~a3DX_Eup*1W>7V%>*ES}dTQoJ5%QO;LFigvBHZVK}92 z&Vu(NkrP8mK4>*@lpyqD5pA$W95n@p@?#`oj08v&weo5E8%p|oNSMkIVK|yP;~Q}Z z!eqTsC_DJv$Pg7azIiEtrwk*T1ml>BF^CW(v(qN2=(0~w(L_&wBXw)F*V)&??&Ge1` zb80jbDJCW6v3$imS`cqD)%Fw?q3)by6s5)0u6p%aqd67Lc9)nCK(oEvIWI%TS2Ew0m+= zE>XQ>hpArep#?Oo9us1l^C}Cbm*)jen zNYgEkne3PqP9@d*bb-9T0RP8lQiUy9}Hj!!#B z0;{r@i+$z!^dL7_axpFa#leZ4RblN?@Q|k3T1AEIJcs-iwb7;H5phhKFN&?Vu>3DX z|5?|$ZkThUa;a*G2xIX1IEd(4TYAezS`EijXGHcXi}7nx;6Y85FrFkDk!N9We)U2l zezTv|*D(~56eysTq=Bi|K_<8VRBQx9bEBI4-TU$vj3+d7nOZm05{q%bosx8#%lcUr z@>#^GbnIibq*w-{a(!gAu9Pnviq5L^SkRqjctTsFb1qoG-jywThrR11V$wT3RRQ9mn(Cb7QM>M9?`9Qu zC#JCFF{|=mWL|lJdN-=jQt3>;oy)pzJI#wu-~xFJ=kHv+QZMeruq8R!He}##zZZD~ zb|K;&1~?Ga?Qvc8q!D|s_aMcxjgV7?Sae;7GZM~OT5OWc;2U$cRFewyCO7>RF*(7#In1rkF3Ei}C8QPhasgo zY(|Bz#d^mFZ`W6e_=jg%D1Ms=Xm%()NHuCxlZTnOC|c(`m-ZMyw`ntYb8XTr_s4ax z=sIl?c1O7XNeFlf5@?E4CvlfZicx5KpqG2J<95H;h|;!-ICX~usCM6`dil0L7FUey zhlOz_Jr6iucC&+sC|T3iiN_d=6DUpScZaTaas`Na$%BNjU?=KSiq@1*<+o~Sg;HN8 zgLg)Eu{VrtbzI3PI=)y>2a`-#WOM(fbI4X&{Z)fc2#-l5H)p4TwpNZ%=zj(0jh04e zJcW+wsE!Bt3w|R`*O!L5mU9I3go$`~ABK;WcqOK1TCjGHC1gOTw^GICkE|$+e8oNN z1_VjwN(#i1kj8&w&NC5LuH1}TMMiySG4coGXd86^cbl6b}_k+_XgCo|yK zcXAc~SzK|IFUf346E)zbbMRMmnB#|6_>W5DeL9(uLK1{JwQyS&hYY7yy%t`eXq2qB zkxv8VHn?{omW->!Uzy24u!%*@7j$g*HVJu>_K1~Sr-mYVb7l5n1!ij>l6n6H1}wX?AZZqsob%&53=S zd7O#)a^T37=cFP_2PQA*StmA~eAkdmDl#sZbQvgk{i&icC#He+KNi@PE=L83@Qk{n z4dQ8}PpJx#fTx9^r-smS;TVfS38>e}k_y^vTG^g)rEZtZE8Yw5OdHR-HBf8nHU7 zf696{X^`s~iUO*k+$W|4d72U#rwkCPrevAR*rcCnba45rz51y{mWK_+Il&r!MJStT za-Fq$Nv7(pr%G=ZMu;y%t}x@O=c=mcimrKzA+}lvh0tB+`JV@gpLeCu^sEN z>iV(j3a@$k2_%cBtU9tB8>_NPt1Zi|W(c0Q3bRd`AT(>UHw&Ml$FBo>ruZ79sYf3S zN{K}}u|-R2eqdy9$n7MtI8j;C~vjj_WL>h5M`>o&l49uXkOWR3J>$H6fwSNn^sT#Od+p2?0 zxQA=CuFAD~TC$DHwTfE^^t!luDz;t=wrTLS^(wiTYp=YA7@Y?Mm%F*=iJ?Xop|i+0 zbX&JatG9}_w|pD8uluyJYq+&*yNG+ch=1L*xDmd#Z5G$Ltp`Z&N z$*!2&xU0*$(15qoOTE(Dy0q)J*^9k`%e&j#z26JI;mf_+JHF+sz2iznnzv}xlD9`V zu`3e}@e94td$-kVy=3RQvb(%Hh}z)idV3gW6fAIUNdtiBHn z!R>1d6Rf%wjI{JC!5sB#1I)ny+`%90!66L71-!o$lE4VOStqQ*dlJDd?7%LpG8AmV z@=L=Qj9oW;!S$=d?VGR+Qzc?(#Ih?^f zT*5>w#8|Ao*pS6t?8PhmRYr`&)Y8OC?8FEt!)g4)QLMpJJbm_?w?}-%%frP*?7^B* z$8-F}dc4PdEXQLE$Yw0aXH3XyT)|LG#XFpYReZmSY{h>J$()hLugk~VTgj9>$=G|z zVVuN)jK722RfWvPfr-ckXbx`N#;J_TjhwV_tjbIK$dRo73YkpHoE*otjLEsY$-9ip zz5L1c2Fk%4!D(E^hRjt_Jj01R%B;M~Zyd{!%)YhE%eve$x9rQ(%*SHP!otkW#SFp5 zEXv`$#x_jO%{-XDyvWrI&FV}c)_l#;jK}UA&+x3y&+N&!=*`{S&B-jzCu&8y&;$Ys&sCnB4%-HNDawjnX)s(>qPlJgw6by~sD6&nyjgDyEz(vU)G0^*!B}0?{jAYWz0_k(zbUd_mdP1kF! z*mxa?QH{}jeae0P%6~n|feqGq?bmiq$cf$96g}65E!u?5*qLqGmnYLPZP`gZ*{#jG ztNq$%Ezp|%*_pL*j^$bo7Q=@>aiZ-vR>+wF4cvO>Vz)it1iEGKIXLk-=Hq+ ziu~(QS?mjr>z1zU)~)KSF6gg*?7$xE!!GTzHlmr_Sv4o$L{7?bjadI> zj^DLyJwk`J1_Cme(?LQ?#@o%4^Q(nU*$MY@&EqrF2CNh z{qr%8>kAL`_m1&Quk0#-{U&Z?__@Cc)s;r z|MXx_Ltpfq{_$_`%3PfEX|MNi|KcZK_muteX#e+YfA~D_^nx$=SI_fVpZJKc z_LI-`cMte-AM_dz=7fLq=I-~7kMxf(`H_G5-p%=xuKD>6^{SiqeNXzLZ~0}v_}@lTmv z^xyR3PyZMH`??OX$p6sq|Nrza?v?-l|4;R$uj?T{{_(H-LO%N0kNl$_`lYY_yB-h# zA^8La6afDKEC2ui06+mi0RRa90QCvnN3h_)g9!QgQ<$)!!-obVLhQGYVMU4i%9kxqs+>uaUQL@fWyZwGGpEj-FntQ`NtEQzqd<|4 zG`h5;)2B?EMpc@$=+LTGuTssbwX4^lU$=@KOIB*esZi0HRg1MPShr#0%AE`MtxK+U z%j&JWb}!q%ef{zsOcx^F!-W&yT})W8Zpx%1_~pwmv499neg z(V9P}R*ji4yw^lM%l3>q=2_CGbL+;vyY*_;4}n+y&6RfV+evvJuWfocbKk^;(}gWv zy7KGTooClR9J=f3-ocCi7tj59bMxrK>zqwqee~{$x640YK7D=n?eV|Qe7`?^=BOivCI)BYjng6cBY!TwSR{-+B2*)gL*^x6gd6ghp^--- zY2uSmX31ogR7&aPj)EF$%BHKV`V{J@w6+>+r@O-XYOTQHnyawA%352c!J?R~uF5XUY_Yzb z>Z`NA5^L?W&}K_&v(;WpEs@$9%i^%zmRsDd<{EgdxZ&zZ=(p((2XDOe>MCu!vC4~X zy?M^d@3Z)-8St@o`Ma;a2mjmTzzolmuc`?fEM&wIM|Uq(ygEFq#sn*TX}xe#ysp3) zv&-?tCwE-&$`hB#GOi}aj54|AirlixFMAxL%si9aC%raHh-J_^&)9RyMj;C&;m_~MB(Znxv=KJMPyjaRNQH;`wpx#gRGtj_0y zhb}p|oR{AD>8P8Yy3nADzWV5@cZs>`t-o&j?X~AF`Z-YR4*T!1Bkj%N!~1T0*~R0o z{PM>)?>z6%BOm?lz&D=!^~Hy?i#FPK&%O73ScfC_FbrU5A|mEs2t^bzic-{r6bpd}4H{yDI#dJ@BjG>` zvLbd#fIsKXM2 zh(#&}(UDg)ViN5@Nk>kykzu%G6fvntHE1!4F?a$bpBTv@9+3z|1mYt*X-O(hf`@t_ zU?VBGy)izpi~`JM8rAs5Uu5u&VANhOV}VEX8RL7$M20c*$j4>|@|lEWl^~-@J)p^g zm7hfADzl);O0Ke%lZ*osz3I&?fRYQ_EFuu1n9WQ+;hpOw!y=XFPIJ-|oa{Uz6Q3CW z&n;RJim4nUHU}!ufi5zaRfJ#{E0I85?h=d}{2m71Oz*Y&;)vsfek$7FMUbYikj7#7-eQhKYFsy zoskYIrQ{Z7YEq_>RHjmWcU* z7|ND*bDxO~f@Y(tS5D&7ktBerXFogId**YBqfMm_w^&7iicyRY{UtLc@DSf~~kvVBFYJtynf$>z1VaR}^PSqfZG$`z*2U1?*BzyY6r zR`Jz&zCG-69gNyVnU{SH5%QYZ1t4Wd{?Nu2~-J81Smx>>4-!u2(i~WQGL#$)u;+V=R0I{FB3}jPkScsd%@{r}+&&?`QfO64oI2d~b3@10Z6{hZn zIjn`ka-p(Npf7_7&ET45n42C(YB;ZK;_6P-%43#rr}w;9Mkd(2mBzHLiAv}6=GnqD z#`BPy8|n(1TCldZK?_dZWi6k2(U_gHczgWB$OWs(?If~!JA7VlyV=uxw)Gui&0~26 zIkKzPvkHuAWXQ@m0yHH-3)t-Hly-O8epYI+Lp*9R_mcymMX|QK-Rn2tz{^55wtO9p z;?2IZheqx*BY&M#5lh*-k9~NfA#6?E{o=J|s4InUjn`I#8O){swkNy6ozgM`8sBTS zG@8@AZ|;5pxHzXc1MW>?UoTwL=$@};kvnvH7rN$r9_hKi!19aR+{Edva)LuWXlN^1 z+$oRwcVC`$A5#&P9XBw@nND^!13I~fE^*JbP4zCzJLx0N^t~Uh+(5eE2+}GIcxLzb#yb+ZgFN&h=8y-STI{IPUX~woQp}gT6k#?w@XUxsRK8;WFOxq?G!b zp;+~4j{#!v(*@4=9NsMNo%4ysx=tx?`Ohyso_4QwXLTBZ+aGtm4CXm>g zpLwqp-}qlo_resnxWs+V?MvTy_SD{~Vhe6+wWIp-dxvNL=}o@{FY6unp})N6=VSfrMG8tgcf2rH(&5pWaegn7x!0owgmwgW)4?$ zYd3MsR!NXRXbeDN4<>K-#ayh`cX9V;z;<_~mV2Z3bXX@+=r>i97fN%NcRLU2(81 z(X(6dr-s>th1ex>U6*#{L~;Fxa54vZL-&Lm7=bVU=y+achtUOUi#T*$0ETZygvWJqF97K6@RF=hj5f`MCNl4 zcylgSi{*xRNY{KpXmB8CQyDmJhG>8lczBS=fu1&rgot@YM@qryM9g-GLx_x(=zZ4* zdO=i+mq%~ACwp9oZxJU|vshF=SZII;Q1KKSGT2d`P<<2l26KpntLT2=D1g>yQoa{- zOxTS%hJm5R0oSN(3W#3C76Zo!Yzg>VIX86ZSB2>Gc`|iluGV{lW`uh;kXLApi|1Rk zM{H9^Z~>`x5$JH<7f%)WeA`%Tuy%B7mWv+$w?h8IYL>-|q!@l{SC9rdW-AAeGHGd* zmVIU@kf7#`(nv|m_J&ROeVrG4>*SG<7+Sav|80skn{M=#0szSO-UkK{kFqnUqpE znB_;8f|++*$BbVoU1aHtK=zb~b%4tM7KLZnlYNJqnzsX==y&3Ekz6NqET?$g$rgI`H#pLo}X+*x&a7L}Q1oj&-9c{gHjCyMVGpF!x7@h6sFa05FSa;3Ly zXZCWFxn6|@h72H~@kXGsDV?T>c6WJlZ&{s}34xJ$0+gA7>?1bb-QXMN#>YR;YV)xtvAX zn$zin__u&nrg0EDiWZodl1QC~h<+~$ko75`L{)sq=Ae`qn$1Ufy$4m75>0-TNWi#g zi|3+Ss+b{`d!eOMBRYVUH<#r92zpGZXNE~!r^$Nhn4(8Yi|^>A)tP_`+Npx*YsUwa zvnG!1nSkSM>S^Q0o_277 z6KZ$mS(wcha63SOtJtXCn1M|Qdx*JqnF*kBwV*8rc&S>T$H#THh;f_fl$?j2%es`l zHBEyANXLY)b#|~>N2<*K%6DKom0~HJy19SXx^^u(SgHDezlf*)*N)$ppk$YLgqo*l zDyKo9tg~uolB%&+7n(C_vg9{w0D6%#363fFs0fLhb}EgUs&td*vOF3?#&BTTilRIF zuV4GLptoSQ8ki*;p5WSMd61mF=%Qv|l|-;}H5j;a<)LRfa8U_wjwOa;;Iw#(oQ7GV z{dN~C@# zubk?otof}q*RbdRiMj)(pR1daO**J(i;)ZKXq8KoecOhC*RHzPdkIUo7%N{H%8DCn zqGczdSfF`;CX(c}nHx%*<#?i@hn3~}v<*vTbPI%VcC$+fp7#5wuIg-UhOMz$PW9NE z(1Qxt^tPU-iD%2AuB)%YJBZ#ouOSMl6$W1V+m-MKzQg6WtmsYfhM{U|mBz`uNXWj5 znR4OUkeeESuQ-tw>B8fPmGg&;gqwLS3A{`Tbbr`=P&{h6ia#aP3aB-8K?$Nf`Fz5r zk5oE;xi^>*$$&%pf4}v2e_&%Oki(wYw}*R_8r)|t)`#!bgDr==Klf>*n+Jecv=W?h z8TyjHrh|_EsIr3EU>Dp1Qrv=BYQ~7zZ>|ZUSuDP-qm!bLOV_nyA=}7faCf!(mQF-` znFX!0ii!{HwIfTRBOuCVT%L;zp=-#Q5xl-~9LvR*vHjal5b3{3hovb7qlDK}gGbEg zd&yV2PQbj(S4_la>0{sF!JIrs^>?CbO0>pxvB^fg65EBb9L~0xzQXr=4!DDYYRpv& zl~((V^=o{hY0l0Zzn%!MG3Upq?6|)BjZy1Y)>&f(tACAY#|UeZN*TuR{Kxv5RegzuZ8J2%Pc z3$Nt=7@3qzr24F?uv)v3E1*a)rz8s0%uK;Ptf1?4MAi&V9|uVe+{64ty9e69VQY?M zCzaK?s9Y<;uo$M7x@-W~$nTZ~F%7-v{IFG7)q5qVH!Q+Ao6!m^t9l@+QD=@|{L%S5 zsTs-5-m7*in3@RaSDL)GG>B+9>ajL?gHTEY*h$wUN7Ccjg&D@g^{2?Z9C1kv+FU zmy@Zx#rz4pZau%jM#*K(e6DS2vaHL0+k2u-&jlC);jFInoM&mR&QQ?L$*8v#lf){*P0B-M=f+|F1h&Esm*{mGunDZ?kaz~NWZ5Uh{xoPIqQ*lRlpKxCKrjg-h- zd*rEef68_}3%l)W$+5x0C#)l5naz$kp1^nU$8( zxva{{H>|R)tqTs<+y=`$eQBc_k_s88iHukJZI1y~RSlh#M+jqY9m&v{zv!%qk4T_s zE!Ice-LlMy`goz#P0TVHzkjLS{rahPy}|k?)fJx3f9loi+Q_wh&?y_o6x_JW{nHt3 zSZ>I|B7R=JZLcQO$#;6-NbJvy{HnUUZ`*pM?+n1kUEWd3+68P>P78B{nSe(BwQZ95 ziPTNVkbJGpd;~A;)a{m2inF6&WPqt*0BAZ3yzOc9^T<9 zW1d^VEa}fD zwrG2dE4Tws!YE(KA=vV=>(6z4+s%o&=w%Lft?(`8Ip2dR z?U}J$@}#Wp`A%|A9@6N}uzn`tXc^Oar`X8|vC@sR=8mpQ`N@_@tXuw#(TLoVq5HmnSlLXx#50MREU#`VIGa14_7$qX z(60AFJma*fu7Mlx=e+2_ijD@a?oYk=mR#qMT<8`??R1D<^}6c7t>bx{>wBHnD8HWM zC~yS7;+bx55SQtF4WzK-7N{RScbPw#$5?DENx3%~9( zU$Trk-P8J)=P8YdiMzS4lO|*g;ZCmc>iZ39wsTyRtf@`H4G?3-*dQ3^O`L*Ij*vJJ z1Bn@hcLKdp5oREV78nBsVGyK6mLg>As3ADwL>Cnd7ZN0rvgAodeTsAp3GpPt5HoAK z3^}pl$Sn*BaM+?H%TSRQ8?M|U@kmXBAa{~%NJ7m)sS%5M41v)q$S+;OVr@f)>DeL` zaWJInQX?0$J9);n(IpSoiFBW|VU!f4Oe8?4d{I}}@Humd6)z6g*zsd+krnAFt8-<9 zuWA|v{xC}zx_-8vlm z+@GaGA zt~>tC?Xvb`#uVM(sNA~XISerZ87mMn1ShL*H3pOSJw%daK2Ak3}4?vxTvCBaf^ zkhH7v>X3`@zS9D!kG%R$CEqMO!Va!{GYQ7@fZ(vWuyo2WEx*8%2s;&p3T?ika;&Sh zoZzzy$Qyq0t|$JO`^YC8$r{q2J8Xk4G$sXVZ%ZhFlCL_yf*51K1Jg7RLCGj1QcBT~ z+g>VdM7TVQt3|UGTv9*H+Cg(o zHHlp%FEhw{bi?v+)ooNRCe4#6H&w&ITI#$j^HOQ4AmG#{liOiR`v$vf&>Gh(l|+yt zaM7|aZ)9{SF>$PM&wA7KH7iG%6Ajr!?@YAFFDVnV&e)ur@4GBViVCoGFIo~MO#dTq z*kUtg>=$Gm)=5GoFN!KcsnA^xH(EjFmZYZqbCD=&gW}WNJ`fi7qiG3sXgHq9U^NfH zz6im>3jGt9(SDNzm8f?sO;P9RP6gOYz49IZT2WmaZdO~>BCS`~9wiRjWo3g#O=FDN zd@K})z3|w^AzQn3Oq|WcQ)O&-))Y>U{>V05qF(J;iR{fGeda zS%}mdm({v`g__@U8~2*Vfo~YY@2RlHmTBnp;PKPh9?tu3i$(9uagorg{WLPr);+;$ zg&Q~I*l)hn@_NhtjI7-}k~^rxo42WJ!@mW(P))bB>F~B+>s)!9$)mAc^fEW{POUjo zIXaqbla*=nF+XS{?WW4ItJSl$x?&%(?ay|;@kiC5-Ki}y#u#8WbYY79Ew4=7n-Bc{ z)4I^nYkr~w)>>LqmQqoqMXMX%*e<94uAgCzf#5=c&lLDIBmk@kA}Y??R`emQIZRf1 zI@+mvIo}^xhNZ`aPb<5!A6C4A z#~oToee~HuqDIy@Bgu(y(_$G1f%rk(y>CYsG~BoXI7KRE@)xm)_;QP|liVI(SGOkrZVhGD^B?Yv=c7~h28sg2qyf!oz*YPU4P}Ue9Md}uP?Uk%V!Ot2hN|3rLrK(cPC`&qnI7otY7exVze&z2x z)9Dm8n$yc!pri=}r5AR3QFne+loK`TPJQ~*qAJysBn`|q*=3z?QZ0W(8fQ7ZdQJio z@K`=QgAB|H0kozytx4d+E5P9nxUwUzbERut?aGc$;GqgG;DHt3@K?Y7l@EI$EMXh* z2*fTzv593YV&Rzt$VQg`uacF*UMYLo%KqRHZ@oq>s^Qts)?y7tSb`2#fPvDU_OkM1 z0cp!hTG+z&v8$!+Y-?NE$i5b`g|+QF&*0nM>J_-d6>f0hsf3F5G@F9@)zm_CLnd}r zlhd_^Sfg--2B=lL+2t-9MsW>$$Q2#y7;kyaD_-=T7rnel1upWsR~W?C3i5SAVC^ed z{2G?AOR()^GjLnl@}LQwxCdtod|+^#K?i+h@CzFJU;zg>!2bm>Zbv&$*<#qj8jfdg z=V@Hx{x-xT1}uLIO4Y2grJ3G~q07igU9e^{vfHKce>G8#yOLMO>P>Htsj-a$!-22d zC32AyAOQ?4Km%+4*e_FC8P%db*_BduvP4<=01U%Pwv`y9OvoaFF=c-c0Fa9cwb)gG3{CBNFcmW2` zs_6fE_pLjobftZ4X&}F28$#}Ykvpy3M|9#5c;Ezr{gCEWZB)>Dd#X_sw`Qo(w5dIN zU@~LiKAz!PNFb`!%rL^#UFT$-QH2_?i%nEp3;W8ozUeyiEMJQvZIne*5C4W&Z0QzS z+pIV%1jGOca06l#ZV*F}nShE(V_MxkrYkqzq3K4P*3(D+bOTCVjZL(d9Q)R?)Mg@` zeixDsA7V}az^CURgG0(IT+Wen46x_A0fzw-*aM$YBJmGY9HTIjI0M+a)E#vE41Bm% zp9Tx^Af24pGdnD@SxSSGCv-Xnz(OPN;A}V47tcH}!A~iqOZ_ohEb(Dawh28zR7e7? zcqRlO2$2TZ2q6(XcQgi=APF4jc-`!dYcu3Z2|iSP-km=39~4`LK4d)=RE-J3e^4pp zd6okAoeN%BI{b;x?R0)3s5L{?DQ6Lm(%WzW@y6sK|Ay!2VG1ZUivY z4pIbI0pGd5B)}6u3g(AOFH!Kqt?}WN*xL)!D{`uJf?bUd`-XQbKCV-@W~jj=khkUIFHN|FL6`zXpa$rZsmgg3^Kd-6sHZ0IG2EsD`q_UTF8Y-&d zjN&wtPkO2~V6(0G0X#5HMiNUjLn($^&MmX17P3w^v$87l$2Xg*;>1I1tVtbMi|PwH zi?A4;dJwEIx}fAtMD$ETa0?bFw@oz7YG6v$lvFT1ZqntRSP)Rz`gzB&eB|>Th zgvbmzHd;@Yi5+d)BlxU60~*Rj!-J>0Piz!B3M(~#=ufA-MA8~i*ep<~JcCr&1iCw| z91sC2g{b5*vnwSiF>@-PDuW&R(gz&`(#Z=HtHvP&j^)f9T+$puNsXg5I z+Q0^tkE{qmMx?f8(VLpBJxlO5ZzDI%O-HD00yiLpo)s2vXa%3u%AN(<)no;6_)pO# zuc2*Cf}#M}<=d@It)>ltCY?ZBSl0q5)f{jIQA5T5uyxY*iq2BaS_C-P637G*klWCb zsX-`PPFUWwy#id=p|<5)3xv`t#omuZu2}T}=k;EJO#y;hgc=y%6j+2_^;oY}-%A+W z37ArR{K(PrTYwZoE~`7eq+i`aT*PIA-`fEf0-i^bKe&0K%}iiTVg=oE0|GO!USI-7 z`!-uG%3j!o(fwHr23n%6L9JxX4+i0*Mbh}?)H}3OP{PM0u-)uB16<%NPR*{}J=I45 zExSd7_UZ(tE#3)eTRgai^QE}YI@js#EVt#vRPY1$(l>zxEw^RWQeA}3T3Z9yUJGyq z(CRyY{Q>v#;ozO(PUu@2Zr-isT@%OzBR1dvYlST=o`vakO(l5VvR#D!+zI^Xe7+!V`g(28&P0FMmjN&Iw7!BP3r|l69Oj~gI1mMNqhC<#gK14J)W>Pg(@a<(ku+l1Jvc>8- zDHVbFedE}g-d&BYYjs~bcC90xpLTF_c00dhohfN5BdlN|i9Md5n8wuKEgk+W7HXDl$bLFT-S<8AY3RnYC zs|E87SOheKwB=z_U1sGqRbYP7b$VZ#i$%GG+t`(Zm`30GUDa$ZVes8yS=vS#aP4Gr&4<$b&=xXb5#HXx@Z!;Dx7-Y96f92}Np< zraF+0?A?umw`wnejpDVX;-dE5O8{b-4y|rx%vZ@(8rVULb?GVwV=FcQG@gaM2I^C7 z<7+?zqo0PUZ>Z;pNpU3!r1z zerabbj7yoMJcJ7;-ftNcZR-2K zgX~t|af9R-d^dsq23(-)^5)ml+^k+m;w2w7fJW(3n+2KLYe=Zq(<*Tz>arM~F9#Sh zEC2G^wcArQX*d1>3O`=|TK;T&Gwh^-f`N2AyM=IFhFyY{b2}$-rFQH&FYy!a^QB(# z<6f{9M{&b_=jJ9gK;W|+kB@|Li9?Nx9@o)-hCu|YbXQ16A($<^mMLs(%=?7q@=Yo+ zrEPl^biVT4NAKPk_VTX&@8WG*fK7rqK6CZWZ#7@LOISck_ya-f( zuKXtr;>%9>g(vX};CemxdY=FMo$m96>gHvyYSdrn(6gs>)_79m?EX6Y1NSV><}KeO zqEhO$l0VA-$)&FCK4iBJ@3$U;K-FF0RsdF~Fez}Wy@vP04^=Eio+$%|Sup$~bLDzF z{CfxP&8M_9{ihv_MZkC`NGJ z3j_#AWm5F0QDzV%gaefU=?ZM;66|Y&TE@^DIW>Fw7RCd^iae@dLHAjf3 zIN2u5$|pBin$!YQrIwoEcJl1G%b2fNK(lEhTJ)&VQG_rEfgt4Rkf>1wxfvp+304eR z5m1=GwStQZUp;_;#BtLC0SVT!br67T0k9k78WFkW3tqcw{P+Rjmu&~NZ3h#M3-_?t z#9asfFh0;2F=WM(6-+j)xUmioj5Tw%T)FFt&WkCK4UG{ZC0n-EnE0XOm92-ey~KG4 zX-mRMlxo}dB#9DmQ4eML*3HN`DBB*-s@@oJQzAV%NuIFNCkxFnFM+g-FybVK5ma8@ zL~_DgGxX`zqhBRR(5GD8jMBGe{}-jDESN%l3TW3i-vHPPF1-i=2q&+&u~2ABtRck* znO!hL6h6Eq7={&~fz<$R`L)7FdgX=Jh8oQES6~?u*4BrVffnIsj>(wWWRcluVq{}s zwpoXsfp8;fqHRXw84sdCgF#*?;e#ZwWt5D9nv{diHqEdzWjU%{xurW@ddVd@SZ?$G z4w+td;!1&3vg6z%#D&xycjcj!(n!>q^j!>LkT>UgeXa+TAcYiS2{ZE{b>DrJxG_qb z2?9dMQ@!*uN-L6P2}pqNtoBDDPL_j<5oyVxkc)!7AOHnPB$$~LXOwoP8Bky`;ud{e zlS+dq>bjy}E&li`us9A0~7Ng5yep;D*U2r+1jwQcQDb8$5Pi)$nE zaqSFK+(j!EB@nc-ue^GA;jSnCEUaRoEi0@9X(g>Rj!iEOtYo@IO6d-nU}dB3tV|6qY`dN%84E!t9=hRP2e~PbWHoT zawf3F{K_KNrXu2qAVuAQ3%vLK`v$%X4YZ0;H#XzY9nx4l^R6pxyzx-m$cNTzT(bw) z3;cj4-?m;dwrf31n2Nh^E8e!4?x3Q`z1X?Gm$rY5`jEhPTQm?7DeWo$coEu_UMHR8 z1Oz>`1YM{r(>ECRojSy4VaJ#pFf11_0a1`*8gs)V+LS>*U<(j;h*iz#U>58-uWK#) z&>I**vnAMWRyKQKt!godWc6%lEPLA1b{8aSIjx6*K>!>O;w<8|sE9^9qSTOeG+cmV zT`bIAku))dGF)PYG@AeoJT|Z`qR(wuLIoJZ7`NVNA~*x1NEPrGo&jQqUhN|TBd((_ zI@JVUCpjP<4`?tv)u9l#Fw{8$i72HwVp0S8m=FY!sRQSg(h6379ZFRWp#D_U9P;@YCw#dE2vnqKULB!al1FSNirUDy*iwe!Wo&dFDkrNXTr6hGEvU!q+K2j0O(0~z@MPZgDIMI$| z#JXtxlSd8(fbdoc2ZOFMb}BoT4b|cpR>-26&1=9dftb2#bjo9!m?cS3dA(}{f~D_Z zUF}xDhu?JqTCPaKXs9-|W@2#>>EXutf+|$=rGu!_6c;w(z=SfWaV2z&!{OdYIC=SN zJ8)7b9Iyi%W+Jl7}O3D}|N)gBdRSh0CTDd;@^h z53UCpBK<&l*D~5cehSo~isPtWe68kS8_f(>p-4;W=Dh0Uxc`;&oZj?pSjjYsI>2Wh zVm+?9f@@7rp_Pi{^OU7nY6Mwsgh}>WLn^SDD!{5WAzl&E(SY~ZGhE{x*Wm@~e!#r( zK&@|HFfC-46@UX6K^CugVp^!yq60AC0N?B%fMo?k0S}nK29AsMYH zX{*}Gk=BU8rbz;G1yd6w3cmH2Qz-+b2&z@0B>uC6Lvu*qQqu?#6i-$6MZhIv2Z^{Y zuV;-A1uxj3BL|50h7$mhdvB&@oUQqNwCZdu?Pkm)qSm7(aEu5iOyLS!xV3(6#&4R0 zMpw}wUxA~ezDgV(Il&2VL|8Fb9|z;PmCF{2I8GSRdt};C!lYo-(vDL$vAF4sb(5G? zam;p*KTH4(3Yj7&qZc&5)G()sO+cz-Czljx8NS1UXo{@h*Npnbp$0HPDsBV7HvhNT z+2j1*1*e#HW%(7KXdpzVV*IoeBWVEdFx)#Aa10F&5gW9D zQ4*OAbQFFbE51NT=1e;cY_RrG!BU2nN_7E9n(zn*R|F2MvWQY!4fqPR%2h@y63;k; z-ERGBj&X!GROQ%aTfh}Z41Q(BvF#ZDKcK9OHpjVzQ4#1W;-R)_Ry2yAz*{%kOSk+R z3DQMfi8;BrII{R+kvk3DWbnQkJs$l+zOHvDCo>A$#B>i##}~bUy&#)KuZ4>Bk(r_4 zRzGDa1>i8#*Cl>ve7MIW61)fSGZER0L_j0(H~}V-30LSk!}`lq+D^LJ@gQWjI8!Kz zkOVN+W3G|(@RI@1kLP*Y=Yih1>>n_37`c(y2`t(hIhq|H8oxN2s3D!wF`XqK0wYAs zlspNzL|5;Ppp)!h*I59bWkUPqfs!l&yt!KVotZzmT$0>O9@GxVVHw?Mij>HLF8HAD zHOV_@0%Fh|1*D&-Wss(T1NLc6FI2k2j+X{dkYX0Hran0vU!7NDA+l-?g=A1X@C>T!MdPdF`|hHFi}Ss!CP$tGMZ$8fM2PAqMA(=|CJ$AW#VLxAsOaC zf{COkt;OH%(@p{ZB~aa%mXHHco+h8Q4KI`t+tpTWbyg9iUKUUwzNB00No5R(8VS-( z2I`|(=%%G`<&kw|GALjmqM#UlSi^~s3ZWtr72EM#WMZu0SoGi+)fEu-+stLp2ee^m ze2o;iLIc17F1!GFE&%li9Gc)lO`Q_=d;o*7rvY4zXoVbkGD3g}*f@>E6$H-%yg-`m z=d*oNY>dqwKm=epS$KNHt|>`IU;=Q=L7Z4gETU$(fg5mWBI2b{TJ6uF>4{Z2V+9)G zNT5^7Y$ZPS5mt5}KMLn?_FnGU#tFI;c&w3g@+1IGQWY#1VHN;j%Es3OsgH@m93Dhi z9w~(=fpj+iPXR1RVAYz7iY(o^B<2;U`yx!XsuSpuQ*xj?q_6 zh;ru7Qo;#a{-4{GN)q4#>1oA9WDpx{DjNuCmN=6EAe@N-P)oeRJA5OLB;U5kaTQ!GOs;DkN;^hFRi*BVD1wu(#fTP_>xF0Ev>XF1DGE}P0T|%OYdtw46REeyDMSSgNm8#x`b0J?erupE9S0VwZkX%21}D^AqPk)M*Cp*5eMfkp;F3Nk67j%-Em&A&>b`R7UM7!Z0C+(ViryS}f+0M?o3^Z+ri08bpy`QE*U~6XRAX&wfpoAD(NV#n3aX2ZtD*M) zgK8utU>*K5LMkpHf3MKQs)&2htaT1T`@>AVZo-gVsgN&;x6ux z9Rf%6T@@6Bp3>uP8QtXK=JUPibGZZOt|*wT%gPRI(O#mUIbR($ooAl&X+8UT{cgl?hW{AJNbAXrX{ZDCZ@9&E@$$<+ZYp1?ZQ z|9}CB0HRctp-rUH#wLh#wFxf_kqIrr3?`WwDAZbNFcaL!mP5rUn5$yyH3$bBKXZ{FLOS5?1kfDA zHL6R*DyW)m^9@9;C1gWj8WH{)3P-GTk_RzLU!1<;ilI5UAi7lP|b@Vo* zqeAa^$Zh^Xqvf9E@m=0x4@TyN=7|@Sape)_c5`QcI)2$W8s$o{Y#n?6k0)?QB zT#YTrp-MIyt~VsMY%}Q@yh8FeUUEO4Pm{-GKDR&;s}6r=%<1ToRkLG1HaSKsA2zj9 zcXw>o=%~tOyWZv@YC#E#uBlFfjc?%RR>>H_4MRiM=^O%EVzjEs0l^-*5(pz&Bsuf= zv76YJ>Q39(vV~iQwA~o4Cv4>E+VscO0A`KE4Hlw>W2VW`0RjW_f(`4L_#O;8$|f9y ztMZM5#`%```X^&pwhVQL=`=wv&?y_;QeK^D)9+P^wpQ{&+g_TCfnb)f^0#gw>E(I| zaY}Z*;c%nS-k@m$`*&j>M6RdxmMhkV75L8>-xWOnt)&y2U&xZOr4XVqTdUjPPUfTw zwtM_A1j_UnMtk?M5$8~jIEiBk>JV{Lodjp^E^V$OPVn#OZG$H;-72&grG0O;lXh+I zU|x%nfVbk5%vpqM5w9c6>tJ z4`AB0%W>U`j{)?s0Vce{FT5xYX3zCayMjmINc`1HE+MFdX_HZL){&sRwO7;llc4vX z1G=W=M%=_0N0krCYjOznpJhY3<=Hrk@twtojXXe*X#gvK#RZHqB2W> zC?dLl+C0Z{{6-PDqc_FJDX{%*`(EF|+eL5p?L! zm>+Y}G;7wLGpox?nQ=nJ2r7E?5V0KwH<{a1Y{x)?d-s#umxvc%lvb^p2n=KwA_c3V z!&T8SDV8KtQ=scEk4!P682ssK$xuyVSjm$@!HlM2T;Gwxl}y+2l(S>Nd+u@N5jX(50teVsz|D?1)BsM4GJWgPOB~GXjRW3{d$XW7VvC^;=b)p`lsrHA zPDSzr6A1+iXqaJ8?-o>|3_2QojfU^cP=YawFr-w~UU-9ZBiKIq6BSuc6kYkWFQkaX5K{h)#N_455ax^DpwXBmYn`kWt4o0x| zUg3Vz;!J(hd_p%eVbcQ6fh18Z{|zAO{6v*g?$q-+Ovb<>yzBxQNzjY~y}*FEI_j9? zkl{PlC_?3$phX-%A~}grT@t|o7fb`?)FwRCDXaJj-4_aq6FKoQNnNZ3-V%8IDP;rp zz^`LbWLapTQ<1Vv~HHgrKshe>ji$TP^z2BR!XJS4fvs4DBE^B z?)ca7%yU1ISS^VMFAmr@Flh^XaB5IQ99~7ZFz5kl2~42G(olzpG46q%K*R(cb*Kd$ zDNGc)hd^dGyiJUS7xkgpE2aekwJk>RgFYt^KU~0E`9Z)1zNILXG4Cp_fL=kSm!6<)4{ObfS@+16J?!~Jim0(d z3Bgx3uN{jHOmd%P;0HJPwTl;qQH{6)^@cv^4G41@gXDB08vyd=Uqq13bOuR_hEykl z4r`nQDJZg))u4%p|Jg)Z`XYzKB?eMGD1#pwbV3(?as_d$(l?rsov_g*SZYJZOAvs` zN$jv$otTL%NQ6XoJdF}r=+5tY=g5vF(FAEz%%!sUfjyYzmjoe*EVx89EPio&Tr6X> zjtLbVXfbQ(JL8LrhZ@PeuZ?beBW7kYNwRdokNiW^m+;t5fdxm9hLk4+8@5PALhKhw z62TL>xw4UY#g{_a=RZN|wQbBqGMd0bFiYsP>#&j$xLU-eSU|Li(2|Mcfru!8*U>IO zh%X}&lp+*yJd$$6nN?ApM+{<~^jHd7DTU%Q$F{YuaZ8KOyk^PzaG!62la1mWryI{@ z97WBkUrE(h|C!#Y&fGj97W0ITRP9;SQ`CnjOGKGJ0ou=ztxkOeCD{^%g-~zoa7JZ? zh`64G3YK_cqA4i@E@qhs3k0GU8wG{$98|#Ixr|Sq7#9leGq-}Kw1?7r>H2cUnoTak zMjjXuq9&^q@72_cnAPk~WwbJ%t}&cLmEW*72MCNIRgY`JBftP>n|%2us`Rw#Jr^k_ z7GP=*nY>X^)&Q2B{Z>hk^$Mkg6%|c{#v8ksXj<2U3grRCp#+)1v*>V*x}pHDdfjDT zf!GB~l%yORd578rV z?4k#f|MCNrm82P<6{={*SxP{}C3S-fP6SYkuipG6Up>IDli6G|sMI_Uh4(cHv^DAxJqM z+v61A;SXk5X-t12%R{(Vneeq*6hNWh{mQ{~7Mp}E`g_^TN}vPg+ibG7nIi!;3C?e< z(wrGQ8q*vi%CG4oYg#UUp03%3|63J91kEdWY z|DB;glLG~7V@l^t>}gibUz5Ua(KfCh%y4?8V)X!HB`Z>EBqG?>wxiw0wnl5aG>fQD z2q=uE3^&zpBw#0SP5`)D6dJ_wK6LNCmX~f1OG&L_YM-~D2ADqaIX+Q>7D` zMFo@7chcN<1Zo^_~r!4pmola>5Aw0~}p@~G{>i2`K43@A80&tZUowpRdy zV~2Me`p}40jkUgQ?UOYILW8z+5fU}rBB>BF$z}L1dclori=*KYm;3zITH|OF|B%I% zPIT&eTx*-I3M+UplE`;G8D||6Fui4KiYh7G|FXR07h;QWBWl!+K)rW8R$*j{bToQ{Y|(p@kY>Og15Z0&(VN3JUt&#>R zc<=Cg&{5_%sAfTlWUwp2^OJrXB~BV$c%yWT*%+{cI7A1fagw!TE0v3>z4>fneWV__ z;ict=3O#LDmO{g+)N_rBJpRn8a`^yW+^7^73amIw2R3DfY|irz@bofeTePGK@Q3ss zumK;CNLp{}P|e)n$@Z9vkD4l-U~Bh!&z?xiw62Bg1dnu7!uTxh-1O}V|IDryXl6>( zj=_>^`m9g>21-zRChrCZ!354fXzk3f;%x8?iFOV=1n%(ALJ=q_N7y3EIIH|<>V`mI zr*3dlfF}UG27Q1c0kx$IArK5BPz)V#49D=8&WY)k3hJCL_YUZe_$IY*;{!o3k=E+` z?5#p7K(+P$Ob+7x+Q6NXZ*&kTMdi75>eiTLRS* zLv3s!#SP!`LO7HiQ8Zt)i9hy~1~ zO&kI56ow6*%713>spgOleNQ?bEM6=MDo`XvP|4W{W&4EaHJ0ym|AvPU=iv~IArajr zxf~IoG|n|D=!l@@3<~5fv`9-R;FKEcg~TOi@Zx!xMF$E5NcdoRdV&vhr4ddLX`W&U zdZOh-1Z`T+^ABJ~AW=6BG7?4wyhO zC30ILs}h$*t9pQwyaUgQO$p9023J8H4zUzgK@ky=CmBksLWbNn&X+zZE2Ib^ZHFuB zu@;u36Cm?G#LX(LW-I688g$23Hpu(=bDl#310~Q1dZSb1YCT5Y|Jh zK42`SL>DwuZp0x<^5ioav6%*HG+i>F;wvj|jXF_sE70;99sw%YtN`+G6TktMmPqD$ zM3w?_D}Yl%Kj|FBGB~dSD#j8)l9DS4EGR_81ltlO|D=FAO@vztY#;F=E@6~9r&Bw% zGcWbB8ker!e$uZ1>74}AJkK*?B#1B6Gbb=$JuOkPIsrb-!iHXCuIw`w;vqkEvNF6O z5qw6KUk~4)yX(j|%6KQNwUC|L27o;pwUT5?hBcQ;F0_R{%+yLJZKL z##mrpIu2S^fOm)utbink6meCrR6pl+9b^F$xRhSGA-UkHmwd8TOEXM&)nnizWHKqy z94QKdjgEBnW4g+~aus4#4nQS#u(nERGT|97FB&(NS#OME)zk}VaRDRH2^uiZR;$AL zrZ+UqopPgFy|q@5G!X)!G-KlmvL&XX;QVxULBZv7vhQou=qB;=N%D0OU-hT-b=A-# z3v9J!EdXCKCf}xZb)anXhOhD_Z%~|V0@JHtBds;Qj$^DATzr6In@DUwB2&zkVYh^< z$}TO~_H4HnKB)F%HMT4v5XxF(?-sLU|AlWn_vX*`O&C=+OsER$3bUT*LS}t&Icil@ zJCQ+A0U>4X340Aj=)m5v)Zu6{UFdZe?o}thsTCGN5stuV!}DpUvBn72Aa%u&Ft!B3 zMQSt3l3+JsnP`%o%LJRaYqv&FZ~t+AoceCQgFA`X2rxe1nue=CvkbtQz4)U;D8Y7AZqQ!T(#(1 zEXM~RhmQzKbE(vGId^pHRdh+05kX;s0F7?%rGBAy<60MC#bT3;*FGjnEsR$_66J%U zS9;}Rxd!V!Gnj*i_c;EuF);Rn|BrP7CToAMx5*A=b-)&TVfYH{M?&JFHDZ|WkT5Az_7;MI%=*l6dqjDb0g4IzLOrIW&r z8X+X;XxNn;nVB_dBxv|B|M<66i0%&?1xmByk3VRT>GqKEHjjyKYWsp}!}gM6n2^cY zf+CIahQghFcqmS*le45`nP32`@wGU418uL|P?mvTOO*+eeIdv|s5k?xcz*vG2XGdD zCuu<-;}=^%qWPu|p3g}-EopxlbPIUx6pPQUb)qrON4uD$OWLH9*{N99mQfm+TiOpH zP3dx(nq!)OH*u!-PM!IeZXk!8d%C8n&6DdneDOJ-|HkS5**jM@sc++n*Y_h?StY1g z2D~@jWVu$3=0_N>u++&0Za}2*3V?lCqqVe`J315;qR(QlW>Gf;yjpcn*)`r8pGQKj zOOp!zRg}L#qE|Ph|M?oE1Iepjtgi!`jkyoFURv)eNU?7^=o;IxA!mq#x`=xwW#5=Q z|LFFBA)uey7{?_^tJo3dHxBH^<6bcnP(!a-?80_=Cks)mfB9)<#jN@3`21o{ESnL$ zpne;K+`NEhciV`M>T%ly3O1$>d}p}(74MQ;4UD@8{#6=x01D{9t<{>cVGp=puekZz zRn)qzjSR3GWKp`Byes9B2HXGq0xj5}T6a;qWiL`w3ziRP-@X%IG<&n>@FN!kBV=}5 z#UKNM@Yv1(RduQRL>dXe`g8AXto5~{MPWdqxdr?Jov>K2N#eIR`M{}elOsT|+f7b7 z{JWi-1Vmi9|CjrS<9fpBI>iSC*v8Gm1zWAp`U5$efPCUpvs_8bXdy{Y+$cK;1{=IuwZR)a z+g?W`dWphKTzq#R&)EEYl>^R0fO6(M2*TuX3E;vLXA6>B#&yHa6(!Gs>tf*Cqu9I; zPr$@2h=Bb3$90>zF9^kjT+;g(4w8H}_ENq#EM?1AaD`#YsT_*+dkoZ#D74%ZZ)Rgb zTEWS@%rkoJl0kn}0AT_>!}YwE=)BP9yv{p(&*fahW&94HK*Se#(p#Wkj{whoodu5o z(GQ*3|7*L{7af_Y9V2PnU>!x13E;(f+HbRkUNW6u0K0)rxu2OD)J5H?_4{Gix-ndF z3a4p33AS!^8Tv@+)y;g`a(cpZ8{510)}tH5+hA~w`?&KuQ0_bqV&mNTN^%sv#U(lp z8a>$Kdb)|cyZbE&p8dV8JquX8;u(F=86D)?gr@sk#69nV@W5~38L0x37mGS?(wg?d zU>HJO-K!klo}iy$O2D@~2#fV>IUyOoO(zX5-(efu>|oX1{10~UyOW2n-g{tj`**0@ zW&=SS75>1(9V5e>y8^q;4O`D9e%xMM`!v41n|-@YK8HWPkEZVAf#`7|>EwfaruDk+ z|AP+=SpF7=nn7;7Qa$~n1z${X-WS$=_dal`;FxWBm4fM=ksDT-C1Md{`?VR|!C9@* z;dsTp&yuQMe=aSq?TD8wP}hwN>&b4VL4xj0AM__W4+y^dj>@aGUb-E=Ro-yB9QpMk zTBhZM#d12e%YN?n+U_f;>wWt39yZ|(e-75E(@_>%iQ(oEf7Is?47T6AN7`T8*o`%& zf!Ep=KBF0A@O@xCX=UNy8y*sOX~aue#W!5wNdU?Z9Kq$^2B@F@0pbV}C<3Qoxq;#c zoq_{rVDVz3#TPq82wF6A(BT_~1Q#02L#ATGDIUpq6sZxT${j2betdJ%Wr-_g|Gd=n zLBpVmgk5l)7(sx@lMzp-Xc798=uV_Djus_4hp7r;RDt;tmMbf*SX{Su^$HeNs83^n zuw3bLr_QAz+PY<60cZpcZzIfL8zZTjSMJKOYbMK^EOdSelRJz!@wml@C&?AK6Ru62 zBe#g`S(#&Ii?Wt zP`DXvSC!uE&9z zVB@uz1XBMJx5~b*x;uW#rMzJlpS_l#r zdgFKNbruIBT1JVdR5&_E=^TX2k>nG_1S#Z@*AZ!?8j_yo9Zw+;N@;?+&d8i(l_Kh^ zqlVHs>|7uE8Chs+9-8Zzf<{E9u?r zjXoPLXXRd7C$yrjm}-~eQFL5d4vyN?G19Q=>Z<}5Ip2k(UCJ$*pvrh|q6$hGEV4-9 z3T+rz3i=4?jnJH8CszuSFM(vCvbfo4ha~e|t95Ae3ZHFX=3QsC@uJVf7HO2Az8E>Y{ z23ne%UaK2)p-HLzF|fxT+c9RuMsQ`g7nAvCMc&p7M$Q-4?I(9>j|{Scga(3V&jx3C z_zme!zOTEFm3FVaP4i2H9*q}8b*lqg-BrOHf(LY|odvsN|3q8Q?fA@8MxG;{&A8RRdvWE!n}Y2$W!;Pu%8V$dC1hlBFvNKOKx!f{jyAr+$Pc0zf^8><#WB#~=Er}>^9Ro5B}A}cBdjFKg&A!-z$S1(0`W9F)15cFd#wuxrcA z)a3YOBd~RDdsG0@bf|;NIIhAJH2kFv8x=zHxlU>DOHdm9sYM+fCxiOiWDpg_vp_;r zTI7iwLoXLaSRN36_=>^w45UzQ_EUGQY$v*Q|GCG78SQD!@!SYZ^#wQHl6`V4U)u=U zNCbg&AQlRq7SuV;KSnEScI?#-)5T3j@~>lRY^pX*ILh;Bbd6ATo7kKtO54%WbnzqG znf{5xO=S{SZz>oaeP=H$DFS;AlIcCy2gA7XlZ-wU8p1}yBOlTVm}I!A!?aj9lA4ie z)F#)UZa}Eb;DCOu;rat({e>Xh&FD|2feUvN>9?&i$XjMvf zbZBGXj_W?v98?y{8pt5AztEdr?WpsYq|BZ)T}jmi@^Zf^-H?}Tn^8h;mu-;CSy{v; z)V3jx1A0iNGOJ0-#1=Bh0<;qn*TmMdM3ug`WwK{!oU|x^RD`qQ+`{DZs1ca>#508z zS^OBPlIAgl9I~lj|F~k>LhhhP+~Hppo4e><5{DZ(CFX`~XEoDUuWj;ehFJ!MHyruD z^@8zb2Z|#{TbR1geRLMM+S4;W|Ez4G&DVKR3`a9R?YlQ5piTwnIQGxVwUe+LJGJWOi=LG{v5D4f!=%cBaP0-6LP^ z-0Ip;hq}GTq+~%&I^o)a2#;Zv^IiWsO9JyZK_0&HGj+|_ULV@CmVPiq9jc?P*-p4i zF1BToZ0_J8TEqDcUtQrlrCVlu;aS1jxWLv|X8To{;_J4Qy=%FuVjRwQe%yWmOJ`#v zdS!zKY*vv-V?KkMBDk}ydSKIOO&ge&`d-aZNiLy})()NICPHhm4QhsSWy2Fb_^tE( zD};*sEXm}!mm<~2HOuml|6{xFw6z&+aWbr^*F3ACjd{N%?hoJW`X@__yW9F%cHkn` zLr3i9|3pCS^F1)1Uj&t)_^SO1h>o>)jIMyLO zxil}VSsq&SmC@SGS|{oK0KO;z#h1`*k5ZRCQ(m+HNrvB1L}eB75Bpk zh2|G@S(t^0fIdu;A$!DoJ9UZGrHBXiNl=JPG8bs^w??B#cZz0Rg9cm-*nAi^i0h_T z*~NSi!vJlt|A#2}IaYTepyY)RQeWeCSDA=guh)U zMp#fXlZ2&rf<1V9E=XEswO`N&fPTna+Ov(!1a+;bYcvFRBh-a17kq98UuM-+a58N? zWg?3hi1Jj7KzEG4KxSAN3TdEA?)ZM8h>LkRjc2rKUKVf=XNu=mVAnSk7C3#;!v#9_ ziFyHqJtvDwXib18Xyub4&IpV1(}0tgR;`mg%gBhi@`!iWkN=p30cmyf<$T`ekh>>q zp67R>hlUwed720*2N_6>RBtNRhM3}qzqpbW^NtFnf_S(>1f-3xfq*KPXr#z$6^D`- z7z6tV|A{gwiF!wqI`wy(7=qoFkZ~u4i1(3iLuk2Wm1?JuDjAX**?>n`V?Ib&oM@K= z^-58vWG{wNuc(yr#Ft`}P%H?1YzdC&(^)SWmSPE$b-*<>#*1fRdjCAvtnpbe%_evqzo_V1S!x@vr8J1;;?q~|mS??XfD{=^wiuVJ*MclMV{~QSSXm?UbbG6@Upg`DV#jbJLKC&-bTDKXSn zl)VU^vRRjO>8MEqh=O`_f(fXpLY`w6n@zTnsaSkB$$Ld5p>ei=JK~7-DXUjn|DrQV z3Z|e6xr(ciPzb#G3B1~?zzVEDPzQ?&szR}aj%uo?LzJMIs!}ZKG3YX+N?2otTx>YPj2l~H+ggo-rn6$ZFjshF}6Z#qaZ7psTr zv9$`XawQhJO0p$;t0p_I_o}ih`>VlPtiAfOX~3_AP^>K*vnTqgESas<%CjX$UZmHs zh?#>@N2vlUkgkblF_(=z;;&t@IkCpBAKR|ymro))vM779xSFyoi?vzn|FvKHt6-b2 zV{5Pa3bQY3wl;gVSktd`C$s|#p*{PeNE@(6E4OR9dIqUEmAFN87qwD5wYIc(@oKeK zYq(vDxQVN{i_5rTtGFuLt7Lnt`$v9Yy=L~chr7Pw zOSqq)tD}R3vRWPRJHM}+z0jb&_O!hzHoM~szUeE#=nKFEY{0c!|G@tX7V5=5xU>pm z3Be2plkR%G_?y4{*uUJHzyZ9$2VA}#%)uZGz#8no47|Xi6TvC$k1LE`6dVo}48yJ~ z!}qDbR7=7ljKd|H!#cdfJ>0%was16r98}T%xqh%(m>wx2(*q9L%EZ71Qj?*DT4ytj*aRFs@w8 z-;B($EX(KB%oyCq<{Zr)$j#ks&F%cl*u2i}EY09d&&E8?x)6IYe9z^q%+3tWlu5(5 zEYI;g(DS^_1YOYH43pCw(BUk`;=IrL93=hR#1-8Q7wyj({m?Dk(CXaL2u;WdEz%$@ z!5?kV8y(Rpea_~U$rfGFZl%#Njneh}&F#F<3LS|zjng#E$sg^|T`ZF;ZNV5#)PU^L zMqSHAjlDvB&m29|0PWE_z0)Sm(@*WxRISo~JJArG|J6!;)JRR#K*!W$ea2Wl&Qo30 zW}Vh*-PBOs)m%NCW&G7(J+)p<))Xz#Kke3Sozqr*)ocyafGyR6EZBby*ZdsUay{2W zZP#_3*Lc0vdL7w&E!l-_*=tSNJ&n#Xeb}?y%!_T;biLGfeZi#7*GPQXn624<{mja+ zzyJK!vCV;B?b)Dh+oPS-jg8x;o!NuE*@n&8tex7uE!(w?*r5&9w+++A&D*&a&CT51 zt=P7W)W03p<_+0p{oYtz9l#yZ06x+KuH8=!|G3;u-`@S*pdH^0uHQbr-|Q{jt^LVS zt=0i9;S}!Noju>mUElV-;Nk7yBVOM9ZQ>bT-2&6U75>`rec~?u;{DCo2wvhI9^xYY z()fML`F-6i4&y!^<3BFoy6oC5Zs9>*rQuHKJ9_WSM<6i#gzrDtSPS|oz;^chjVt(g}e&dW@ z>5x9@qkiS3PU@zP>Y|S6v(4k0&gqNp|LJuu>6RYps&4CXe(UnR;meHZz3%IsuGOht z>$FbX!!GQ`KH0e*<{O^lhJMuf{OXB*?8Ls}%8u-(Ztc3R?NQt8zwYg3?&#Mp?$bW& zSYGYp{@}sg>Kz#EoDS`Xj^iR8?&wYJ=w9ygPVTFIPr1GA{r>Cme(&|3?bzPv_HOP1 zukQ)(>fg@r-45ISzT^jQ?b}P~xGwPqFH;cD>afo7qMh%{KJcd<@*kh=mdZ179a50?(i$m@-8pm=Z^53tl|X^^Cw^NG_Uau@73e&4N4E{ z%YF1RPxKdm^Y|X_M*sBEKG#hD|LmEr@l?;}p>FkFFZM&v^P9f#I=}HdPxdw+_A^i9 zLhtrP-}X?y-Dj`%Xus}kFZW~5_f_xndq4GS&-MR^^kbIuf86wG-|{9O`0YLSdN25l zukMI{_*ie}i7)w(-}pZt`I%q#-fs7W-|>C^_@(dpoloqfo%g;z_=+$36>Ri-Fp8JMx_OkE$HDB_ZulZLG`t928womWWj^RU2`^_)>#1H-HKHs0ekIQfR z!(aQmj`Gxh_4`iutPjjG-rT<*_t?+QiUQqeg)y70DE7N2m5eLY=x)Y0#opv1*-owQJU|M7Lgb`m-rj zuvO1KH7k{^+O$c%ik(Y0tlYY2@50;5H}6}xfCCF|>-O(pzjpO1M!XoWVnnLYQ8K z6Lv_UhX86Q;)t<7;-^S_xc~Q+7$@idXI!W?oHdIp&F9rs><6W-?Z$ zm~HMxrJ7spOrFN<))I(R4S>ThN`NlaE{q2PN#bR8mO$TvTAFsSiXv?tF!jHs*;)B znrd{q(rPTIzY4qSh{gKq?6R>UD{ZjU;yUQF*hZTzw3{lMte=X83zoKtk!$X@+;;oz zx>2SZYqKo6d+c4_vJ0-YJ)Y|wys)C{@4x&aTJOE$+KcCl0qfhV!T=`>Y_$f*EAhk} zHhe9@ij~H2lNdX^@pJ@7Eb_Aj7y9vV8h3p1%3nb|^1!@W>@mkG)6DXAFuP1~$VAFJ zv&}W%EMm??>)dM4GN+96&qEizGtW;)E%nq?S8esxSSPKu(o6f?-pf#DE%w-CkGiwh zLZe;w)nOaG_S+4*NH=`%e1s#HY@U@W>~xyzaX*e@@rVFW>v!tQX(B_0Q*y{r22LzPf~Y+S0x^NaJE9VoXha4!@rh8RpcJPUw{Unu zgp5do7Smt{6sqBiWS9dO#t_CZY|(>W{GuMZn8pxhL5*!ZBN*f8!V(rDjU-f|310}q zJn~V4IaEXqJuyR1yl;S*$X^YEILQ3*h^&EPmjZlAqY`ONJRXx5-cpF{UQ+w8hTR~{p+6(#aX`r^01QVjHEh2`N=6( z5MYx)W*mrlOM1T2jAWE$ExVxq&w0Kf4{xA?F5ehNBCK+t^<*P3pU_NNnsT7_Yy%xP z7zjNWQ=t#N=r0i((PDNql_RVJDD3FMY^qS3;PmDT$(hcUu2YkpT;~I+*Enq~(+vGQ zWf1bIQz)>~pJUi5621`7E%>vQSuiLUmdZ_xQawJlAJaCg-f%+*S@;+iSM+SO*x2E zG8BQUcJL`;6Dv@o7PhEnjp{DN>eR406sY)Is$0Km)ny(bsXHyJV()-euYxp|Q4N9( zY=F_8nii~PR74m~5L96QZWg4PMI~wpI>xWMRieg3=1p;nMU-mO5`_e0Iq9lbm^zo8 z<8#yXjN#kKhB1qX%`8B9`cL3yx4Vbs?HCDq+RBPHx1as4YXut6>ZbR*w4JSbC3;WD zTKB46?4?$ZxyoL~mkS&9;x9)V;H6eKr^UQ1f641gddhLUWObtnxtYRe?og2jZ0`Be zqd!tIn5l16W@JA+PtKwjy#-eAb#p7(gMK)vY*Yb1?|ESRf|s%*o-aQ^+{N8`_q->D zFn3wZ*wtQlw>wpYh;!^(%)+;^8WrmZI?H4i6T!%Am9UnP$;A!Q5J39-@sC3sh%Q|D zyj!JlW?gJ!Eu=R8y%AO_P#-&nGRv682hKBs*-X}oGMKdLeQ%B5Y-TFRILIo#Go7tG zU;TdZ1P#?GYo&bV{@T`_3|;daFYVpZ3YiQ63bREOi(@q}mc8_C?<;-WWYzLn&TV*a zXs2x7G^-KI24yuB+^b_cBRaNkfJ7V|O=N}s*~vu+ZJR;+>aNyPyh|4EjyKKFNb|M@ za1Q2J$INY5H#@l2Ht3bmSJy>v;n>6Nu3^qpzIz0d&E9%D7&W(T%o2n0*YR7Grf&!Cf~r#5;w23&01+)Z=2bKt~Phm zedF*t7~JFk4W;As$Zw*mJKov8FQ4(PhC^>$<9i0Nta;kZ#}*djuI;ybl?rhpdz#T~ zE+c9CedUQK+tNBG@h+KO=s?SO(xqN6vGZGOibwX*ux53AH4A7gM^z8yQeW<5!E!pB z+q{J}d#kGsbBy;p)@E;dkZsQJTNB*qk1jgi?QPv@C;Q@`#je4zZeOW)9m{$U_OFl5 zSXn}P)g}k|u!F$sp%%KSJbwDkbKd0IA~dU-e!H=CXew;z<2( zw;>DqH(Xuxc~=3X_3d53Lm}5i#&%Kf{^Z_6;oyk3^gT0gODOj}>d3D2<@eiZK_~X$ zgKjYY_7fi5Wp`iJPp$3E1uA@NTN(YB75SPO>XlDsko&V=_jXHvQ|Pn1$OZLZNLF+s zV0@tEb;?EpxTXg5R(%rn2ObvzGX{LCRe8$iSk6a&n>Tp4#sM(5d3VNC{-k)S#Q;ZE zTidsAwnkZuS9vP}JhL3H3s5V(AV_h!sjaxHjR zF8Eo;w_1=_WE&NHdPioO7e)$LggWPA9T$SnH)UOjgkZ>Q9w>zQhJ5@K1Av>Nhi!;RX@^&b6=;E{7mnVxa8<^L=9qwe z#%pvaX|4unF_?DIgGnqTe(`3A;3$skMqdqRV!3y4c(-VrwU3-QS*q2D9G8LbH+-Jw zau`T-1U64b_IJp51XMSRr$hk$S9lPKb9>Xahl7wvl`@x+&NpARcz|OS13YJBD~Xa#C5m|HkFjS?cLaj# zD35<=OR+?PJOyuLc$PmFOK0{6^M_SjSb9^4hJgu%{MZK@mw6iC79`WiLPda(TAGUrg{O#nmt*TdKq%{cx`+41bhiy&m(*4c$=fPj$K!f zKV^&pw}B05m^nz1tGSxM6^+BkZ5o$w93YUk7=(!VM#A@Z*NBz%D4o9lMwZa`gsM4s z5}BUsH&x&1ZZ{Z!ROoRa7mrh@j@jayh1g%17>o3_ZL>L@c$u8*=#`^pSOmCqTWNg< zs)$L+Rp=N3W?7U1cY!&1lln&l5g?zCmV%#Gnw%(#P@s56C2=B2Y7B^tA=(EgK!6~r zMpZSB-#Cf&xoqL~dc7%AO!NyLhMjXcWRMw-44RY}Igu)fg2D%%NbsQLS#$8{d3_L- z%;$aH_o8%%nXGA4xtVt^sfR51o!hs5?igg%rDS-wZ*T;lLwa?z#*<-ZYXgd#TB>h0 zkcNU-M4a@S8Pr>sS%oJTR96V3^vI-iNp?Zkq*OQ)l|uEW8;WdEYJJcts1%x=&gN`5*^9Y(k*|58&Bv!3shdNFghy8!F~v!Fx{Fa( zs9G9}&e>>k>1M#^lm_aGZ~2aX3X3J0m3AO`U5S8su#GTgdtNwsEC{Lq_LTxjt!2ri zHJ5(?I%px*j8>+p|1^!RxTvVNmPY!5+OBt+qwq?p>7{G8Cy6EbY#=(2y6Aw>ca8=BivukydbO30Ig6~AIs*TM zu_Bt4f>x|Ad#p^irt9X02X&2tig6wrweHDV09B3ADurNZncirXzj%thN}MKpvH+>0 zed?(+i=2(P1BR-Ogh_I-DX9pWs?N!GF4~e1JA4qgjQ1vdYp7$TnTmI3e5$ydUC^(v zd7Magn#hQ+Rr#5)Xr)w0R^Eq$CCUH?YFb7+w~`vR&xNBI29s}>fxp7LhUPV^w93aT>SBr>!n$m#XTh#g?pU>$t&sdu)52M7UdVX^njHBenu0TY)yM0a~wBnXP;OxPM9;rxQ26f~S-#NOMaIs7t7X zSgM4YXm5u#zIk|F{zZ918eqC=Xb@+f>n5IcW`x3eowgdPd8xFnJBFWCo?Xk8Nl2Gs z^p7!>vI$$A(~wMpWKv2FC`H5__G;m$w^h z2hzElo_U)N3$lrbnJT-QSed3yD`l{|qQ9qr|5buSnx1qUhD|z8fr6hB1e}zLxNRA~ z+{(1dRI^SPaAN7fD2&2=_g@+f!JIllM%7hQ8XJq`o09&;sl(5@v|Tl>EZV8I z38DViosx`Ym;13OAY!K2mHi8FXbZVD$B=QrhDO(~A}f~ai-M#ki8bn$Ba5V!%*Q*( zp01gFE+&jkB*jYUzH_!@g9&TK?8Z$@RIRD6!TYNr_qVXB$B;?D>uRs@nw`Ryw5;`@ zAX{>7C(BuwwuqQyn~aK}`pu2YcVxPotqYjOIINGn&L?S+&-{*NYQ3C`y;A(fdz;VS zi@>#vg~E%CBzl3Nn!DgDcp`ej1wFfFiM2#pq%qsEF37?uN~J_RmE;?ILg!}?x~tub z%Flbw!y32otAhhpgGs!#H@mrryh)hBpIg?f02K#ByXS`w?3$&13pH27L z$%?S&m(Vz%#T7kccdV_UJ*#B@E6}kEy?e~Omc6e5yoc+1>FDGm2XO#>uPQlJj(_ za%|1q$dp6drf9mjz@4OK z?5fJ-L{a=w`fbkZd%-`Or8j516295iT90ZO%?V6!i=AiSiNp=P)cD=bpT6V`Y*4ffmKN*M>`llb>()aR2*MVBNFLUY2IYyK zM61B)*A1|!=iASXx<=fskUEY=?d4mykk;J-pZ=s?UB;iffLhMtmFt$PTH$gV=nTcc z56gQd*yLcRdNE16fD6H`2*J0zlRCTGC;8BI{-I$|WPJ&c?+Ut#%cuYziD9kn2r18d z-r6_(-o(iPT`c67Ey7t4$ShdUA&~8?8+aXSjhVW9RLz#EZ0%~B=k;CJ4BzWyT-P6N zm=4)q@J+5qYSn@N>Z%6)pIz%_kSoEoO3W5&>4d%M>rBY{?9AbO=FMDWv3<&8SmS2u z?LlbCcI|)?&Af#h*++fCXzbvD4CfMVg1YUax~|YZMY?i~u0g-NE{(Uye(Z$`*UAmZ zJLp^eo>#e-ENR_D{+zA-((>42gii7d$$oZ@>#Av{Ay4uMO7AgD@fu3y-8cB( zEP;#vv@d5=eg5s|T6dAJXlQQsx9+PVKfb&g`s1qMLTOI(%HBD=iX!d9mRFImYuc(F z;v#?Ec=zyB0I`&d=2yAc=S-Sqo^MAUwVr(Z%li7;3U|ggebk8EV?~ORUUoWN@K_GX zMZe`+8f~xti25fx+R+XC?asjizP5pF;c3XRqh94f(2W`I=^@I%{?zQ$DF4lV^Yka?4Z%7_yLB53Jygn$i#WMUv3s3OQ0HFltEwA0bxku3!!w&1`a;>M2| zL25}DfkZ=sD9ylB10{q>njAQ`n;+K}l% zby?48-P+Zh*RSCKML@d0#D*QK&1|SV9)$v7pfCoPF-VY5H7fE+)QTA465(+1PC*+# zsla?mLNH(z3gez2sF86)ih~nNCc1bhUbt;7hxF-^Z)A;-0}CEZ=S|ZbipL8xJ)EXsSYo@tF*NU z3@SgMP!vs||5jsw3o5zH;*KOcYwkXd^rGS~A2)+yzsVxwqNg!!Qt&i3M*W$Drb1l%{X#tej+);rXFx+C-tI5pG`n{K_W?AJp*!dKU?I!n_UGQDi zdPH(kYadnj+b)__30t3D<1=N5lhl}DF*jw^(e)OTPGM-3Eb>Ho3w8L3fe9`+%CT56 z|D-q!Ne*~ffm)MH&<&}DSEqX;jt;WiQgTb>Exqn(C`Pv?XsRXI#=$eUy>2Sty_qJg z(lm_=JI1{Q+qu4H%}DgO6Rd+cT`cI97e}%uH~L&oU7k)&ns%%=K;Nboz3QvyeQjf5 z8TyFQd{uW&z!!yNDddP#yPQ;{>W2I4n{J+&@v#%fC=lip&l}*7Y?XOoB)Y9q_TA$} z{LhYOoGdt#sVjGBWS3tz%!~O<8||#I#IMT5z)y$pn_miP$S$*$=3?mU4DL?H zx6euBg<+T(au~NY4-qM27Su?lTF5)$^rcjgXc*=?n6kNWhGh<{kIRads{Hg0JKK?* z#0dC6Bz}ozSOnt{bEZ3G#f*DtNzLEF2q!4uFo%QOVf=bXzcNe#kzrIMfXE0%FL7>( zknxwtydWb>%|wKiiJtK?MKQ=R%~y&d$=WbuyXLuwIx|`b#NZghniLLkvupufP&vW$ z<>^{Ql+Xv2Mzc&pZw4M5W+QDRz5HB=k^5_)AM1z&K@zf%$T*}Nme2$#|E!@3+2rOD z&ftefpyCy>AZIzvc}{VTQ=RKe!g zO)2q{2UCSWRjqne2%Mo2y|{-vy!zF0utTh1CF@wt3f4^E;S5R0rW4ozhqv-Uu8o*$ zChU4wyfz}Qe9&e${YurTZu75f6|59o&;(a>6&#CY>|zOf*lR+z|FDmZ>{=_UgA82L zvVmo*WGVaC%?{SDZV>DqzPZ`Ho>sMLO>Jvk+t=4xbDk;iX|bqPo6OO0j2L37Q=w{% zr$!Y6SS4-)s+t5q*y0`bSVwcw@m%OeSGvrOWjccT}urQE8cfHF5D}-0PaAmDp z4>1VzqSv0X#b$fmds*1Rm#vw&26pXx-_LedzWTi^X7}sg_ZlIgrGkm#lt8or=RwB%4^|r)NMTID^$^^G;D{idg97};3Miuotu?B7WNQ%;04~3TB4J z=OvSfv61tfoyrMDSd|Hc07NCIP3=DS84llAuAm3)Z9pf37fjdyq7$72ZrB4Jzeq>7 zdGW5Pv}Yk}z;qcm>KQ!ja|a-^;zfU|AxYah50+-Y{{hsR(TrHJ;6cVTU97kQKA5yA z0dIs41WFZzWJ(GV@AC3WB3Cd+=;ItmHLU5{;8eN?p zM03Q1d_)tfDu!wkq7I$`dbTm(hF5f((BNh|S=)IJaasWc=Em^~RuGCwU_%x3@JCm} z5s5h{Dp_pK_sissVPg{^Q1m(YPC(=mh@&CxX(>rtbii@*SF2K#W4@^8Ny8hr8oZC_UWY-n~6U|6%8;8#e(R7(R~ozJx*)f)v=$2XpVy zj!)Df4QrguMOOeR-ax+WxOxiXCvb4sc)#$9W4sH-&-f6))d+$g3gg*OarTS9@GW^r z;JaM+#T8c5ID-$s zzO$n@zvw=N_&zzHEm9Icxg#8xPzdu&qyu6+1~|U>bH9iy0SJ-5A5bV%K|K9Cr~tge z`*Vvj_`iNLKtCA>JXpEn0K)|2JVl7T|Cob-CMY&Vapv;HLZzSs`s)GvbGXWzMJ&KS%iF>^#6>X#!(N;>GQ`2TVnb?s zKp2QaA+i?1XeP!Opk+*wW^*xpg~KVzFS(g?R$f#ur!vdrFm32QM{45`vJTwLYLS$SM;5cTg8bZlE90iPok^R zLyU#s1cd}MU=zd9h=Tu|G_vz0rVusSVKSHKG}OiIkZE<0hzP0z&rVaUx`L( zd>?9rfe#G9oj|bLJG!Ln#zbT~Ml3L+D@Sw8CiE%=U~sN`&^j9oyDss;8)y;WP)bnI z2_+ap2m!XXv%?ekfOzaYfMma&*-C_z4X<1?MWV<#M2vOWO8%p{C7eUGBssIB!fy&W zqbdO-*u0Mn14nR#x7tHZyo@e+8dA~~mSD+ga;ldE1V9i3T@nH`sL2dqf?Y7doYYBh zG_Y~>NuQLgJ!m&;_zFvK7d8sbe^bdZ3n^bAGm2uU4x6!>4CsV8$OPg63dY|VUB zv(@~evJ5kwGK1XIB;M?`|B3=LI>4wjaD-79J#uS51W_iZ`K4GQ%$9sfK6C`rqe%={ zfZ<{R$mBMhoXqNqhn>VM?UVq_%*?|Qh2{DNStv8$bkE)NH(iTQ`0O~;kew&v&HQ{% zRHM(|)K36aq^Ic3;S5grbWhThPXtX12BR?4R6946x$DEfU2&Z-kfe_YB0G>yX?)2Y z0|YT(%xug~Z6pNm{7&ABv1aIsRj@kqYybg(Q5khX9SB1p11Nej0op2o3Aia9y?`I( z(Nf#AQj-HskOU-6(m?IWbCv?QJAB89@VK(i)cwa)ddf=D)%DM@*J+@LeUgeQLXR< z7yVNQK-BYGLd<$O^0c}Gh*T?;R0F6~OP$n9%~T+*R6p>sAN$lzfVPLagnmi|^x_0f zcqj{~RGHFL1Mp26_)(W|Ra&i8hRV_}eW_Rt(l8a%RWzp3vp9SLu*y;^nqt#7b<^F^ zC0IM4pK-g@%hTU_sy*#V$Luo`71VSP2kN4(c}O?w0)<1>0CIiAY`OkG&BrL9Y~xQM-}0(;mUom-z$JTeVd zAFxc1^(>DK*<+N$Hqx>;k_|1R3@wn>X$7vH%Yf>HHfUp6s$u~QKro=IS*~MG6~tWU z+S%r++0At>o&{P1t5*>a(gRz98YtKD96oNUg-%f2a${P9bKOm7f=i9mt-9Lvn^&mq z1iH)w|4Us2tzE!Dz}i%ufK<4sT3Axo?a>3Mfm0m>ho!C0;{+wp)G~NdQhizh>$&M& z1RwiIiJn)o7 zq61B+xBw{6OJJTKEA;tye#2Vjp%`AjSoN^SbT&h2ixDC-}8G%lo{1J6$7KUu3S)1M`7H=!Ej^*uJe} zdamcd5`&u3oM}NyXyuuEdkYSAHV_qpOGaeH8U+Rxg5;89J)kx~pkxHFNp4eK|EM#A zq|*c}(1ePX1g??@gN0=R)gA*|D_Hgb!=mHy+As>Z1iP}aYc7FKcv7#;;pg?;9{^t5 zbzM`nC!CI_B{tiUF5;Jtf++Vrs)DGur(l6cYWS^#V2u|Ub$`Oryggl-qF!b z-N@p}%d+F|vS%hREoh^uT>~Wkpudgs%YdFU*(2y75CjQs1xse&1!e&Tp4^@UgsgQ3 zH^_q$wBFnsV(hvCNVsUKu2HWo)399wbe-430)PULY|5VO$sS|hz2;63Sa4VZrKa6l zxYtH@-9%bacLpbuIMNbE^kaq);CJ9E?GQYhZ*#wQb41W1`(yvX#`d^1KyL>ODZ^ z6!_t+O{?@u0Y*5|GYQ*Wm2Zd>LIXzw1K3{P#pW77Vatx_JpktdhtzC7=Mbmx-%fGi zUh&~>*W<3=lNK)-$KStBxyz8#-!YO?v2G{ zg-&R1UPxXUeo~VjS6C)O9fyL_bpdmQgUd`;Fb{Lfe(8DLgRM>7|1tM~yTStn4`LZc z1L*}Fd0SO7-|ZKc=CzGfT26DN_H8ovZ55AO%Ovjq>S{uV@#V(f29&r6I!Rsu ze&HUL;tZGL?1kf1dr|}$!T_&xVSic*=W}BxacM^Jt9Dmr=h#CpbpAqaqn+^?H!b7j zU-jvc8tP+)m;;p+;76`>OYbUscuBeb20>^v>s?pa4Rb4h>_p}Aam0hgZn8WWaLShV zcwbsEKlaPk!B}qu$DZ$87v^prSPNi;q`Lw=k9CJ9_F3L@|EA^d0=M&IU-o7n?m&Nb zX@B-cP+r4gnYiB3rmE`$}M3xB}U#H6vwshgW)Gk9dE-a;-;pjNf{V=V}9s zLqQ+({$l6-n)W*;FQ)9@w&C_=K}x)-k(A$|A1CPV76fb%1icr8AsB+YUf{771Ap>^ zBw*K8H`W+k)bVEZ3bXe=sOA@ZfyLkNMfLftzkJ?>gS>)gs&3e-zjKM_`T@Xto%i~R zH+^AcUjfriWW#HaXn7xQ=T`Y9fNc+Ys%&EaMr zebK*jcD3ygj_A`T@m;PnaXaT6OfPGS^V0?hMgldVf>dTojUHwEKsq$&pprqCCT^-| zvEm*)a%ORyb*WaWMUUhF`h$ha3>-jC5Fz7oB}+A9iae=Fa9LpfI5T+6`#aE=i=xRg(n? z1kjypSH}vr0PrS6_~oQamwuE0D16uO;lzkR|1fUs*aKv<2L@1{Tp{x1%$m)9;Ji8Q z=gO4vjvkO#N9oU}SLdx9xrVD$nF_wLwWe=vg1XI^A!tz1q~LlOi7F0C5+&riUmZ{G z{5kSPcoucc{pk9pM0#|pJQ1f4m?J7wbZNt#1A3MtM(CWw(;NI|@#h1zB&bj+rB3(% z`b8+tSXc?z$v5BZQeZFOl*CCdoXF9H4@NA3#u9by0v98VZ26mvmbI z7ML1+EOLbl4lLH;T^%SEz+L>ZnJ9^ZMY`~SX^Gk8;fo;lnA&N??s%+-$ts(f zWkaH-D`t&J@xuzYAsMX^Jv_N#w$b(BDR_*OIWD&nMr3!)RwDS8_1~EM%AoH|J^I94sgVYHBZ)}VK4p)ER8*<{3FmnCkZW(K}+y+ zk4PiUY_mJ!3@xkG;)2JuM(`oYB?mE-5Vd|yVMiWdgJNd6WrGQ(EwnwyZfqaLD{o1> z^}8=h>fOteO?ZPn2HEMhv&*436%xrU3=`upAeW{&%2;Kc^|%RTFes-Po`(8mBc(3% z2&#$s0ci<%)Ru%8syI3^OLxYB+vCjIuJMFyN4npq3KI_;c zt}r>ZXL9}EipyKvQ4K0w=T!m<4EAmI_366viX!f1k@p;7|1rKqsKAR(NEsc}(;fNF z#KFEP+QF~hVce}I{`tSt{|-HY|B0V547UnLBm!awI*@^|5+~U_>OmxV&=oS|1v(I9 z2_<{X%H~oS8uY*<7kE+`rspKoorn^(@PTU_@VeQ_t^q$w5?Fpmtthb2Xhq8i=Z4g? z*72~19%veoid34E;X-1yuwe44b%ZZJ{W)+Nq|y(y+k7HTt2bCly8f~5~ECkcT{ z>NXxOHT6}5M^brJKI#(V@Em?VcU;tP%6;{v$e z7sh?rtBr1iqrK`N2w^ZKK6(5mJ?t^DO%-SqcWmAh8L7%-wJ?JffEg!3H@XiM28g`O z1l4#nJJF=Dl8u0v71%O4N!HE?s9?qK*jdAcB7hIq@WM3$;WAlODHR?G$u)5Ghlswz z53|T8A7Iys&cMZ%E4Yx3`a&IF22+^BOvg)&`L?&Y0+~!e!}{*#zV_`Dd^nLwovfKZ zZy+pD2J@y+$GIstRti&BI0TidAytcR3J%;VmJUeQtGq}KpD9@2@Q5VV8h)q|rj!l_ zHE04T(BXC#{}80^_IZFP$Z4#k9GXJrXOeKy;alT-gZSjsI7z+Y>`SJ>z#MnB;sscQNL!VVn|`DY%nFn_}yxMoLO5yxDS9o7!}nqb;^bx-`H_Co5l=`pvTFyM!N-AcZR2R0(v_ z7yRBwfM!zDUJ+mbd(>#!?WLi?mKY##Kyf)xQP>vQ166J?Y(5KbVzy}`NQJktqCZ#^ z6bnh)|3g9*QOj8Hkhs(#L3yTJ{Bm#;yf}mH)XJ(?D2omj4WW`$_Ou#yLK4Z4g0mR= zqY3C>FaWDnd7+%TG@=NC2Y|zFLm5}cPOk(MVrgaV>r%>wnHriHNJf-NgE9Jq1Tpy+ zQ1|37o7Blq3zo$YfH1g9c|yWoG7~R6;VDPC>BE$uZER;trlppHIkE*Wa<-F)<3w9$njy_5=`bAmFRGxA&7QtkF z3Rzmh`t@0gE5$O;biTw+7cSC=Q^C2fJT38_-9Gzff4O$i@7P5h96Sm_eM!T{;jqXm z|Gb=vKn$J5g`nJ^^o2<8;19)Fb4s*!4@iX3aa&esDM=$_9*nYvR%i>5F3_Y8B9}7t zOmzyky5+OPo7UsCHIzMdYbeh!Ex|3g4j(%K17Nyk#{M`J)dWf=9pXFld!|pt$ZRpb zlP)cje-THoe6ine#O7)Vg0+E%#+BN@hkN!{6!Th2$|*roJnrs|ip?ZYVbo zBPL$u(Av66)bf!Ot`W+GD*2=};ALbbTWpY*OKq=^PJfr(ojTuI&ID#TPpsXy|7u)Y z=6mk5`J!v+Lx)OnAwDRaIu*gQSgnDp_c8=Y0!kmaJ8Kp>JjeY)iFbS*#vs@OGC!b+ z{R!%XKQQZQm<#p^6p`xpSOfD8u_g)>QB98+K=to2(xu`Sf4HDl^<9Ca$wc`MfzbOH zWb)TGH01nM=JczP)g#AE>CeYrJmbkFiu|Thzx!1J^-!6Cog9Gm35pchO>CZ2Eg&j* z-h=U+3F+L=jm!dyo1=9B(h;0TSsw>6S;yRi?d20nbkJpAzFNZSVSck z*tx=*G+!sA7lb5R4oHmhL|+cV#VO6()=kIrEyDAq#y`0qcLV|Z5g>U*|5MMIhZ*FW zge*bu9SJ9;lw@$g6%5<`T~_{$3o1YgNB}}IEt|=yoK7X7v`tf=RG=G*Oap2ggpEzR zOrRZBp!Us+c2LlYL{1HAUxtW)jt$cac1Jwm(81rd^p5lGCY&|V%sUnaam3ZMaQF^$)d5OA18`SH{l)=C~&&>k4k z5rJKMby+XrAC|xi7m68~5#yKKL$dLK7tw^X;ly|#m<@2p0p^5wz2WHzP9(H~B_Kl0 zCCF5T7&wMwK_o}0Xbe<=QXoZ^Rn5q+h|c;LofQg6DwdcP$wxhw|DKxgTI(cZDr#a8 zjYaq&852-qVYG{Jsh|-ZKn>P_9Dra76;IyLT13ub*n9^XNnvaplHi3MS&bhraRhx0 z6PN*GNEV|P)?+#3VHx_@4j9-8pa%g!v@uWM}2x zTRhqaHC>|(Uhk;YE-hWNJWKpJR7Ig)hDp&?0a0Bfn*l7r^F^QbeSkqaWD#*h=19!w zsNf;g0yyMAE>hKc0Z;A$h9a_$UR5LwK!iY8zhK%U>|Kv@+$@($E1%MH)s0RoA z73BC{Jz-pAoI&Qi8XOvjNR$t_{Q=XV;0GX|4s?PGu)#5{r!OuO#w%+!f%WLD;NSR1mk-y|E9+}nhuUz~hoQ0U5Ff~ff#MkOY>*L9Co?|W87YB`%BVBdB6aOwD6ebi5DJ5WmLFmFNlv})Tn=zWCOdTUKp68P}<9MwHPehxo#zCNHW~0g6 zgM|XkdBT8_=~Rv*iKPM>!UP+_)tfG*){O+I%;`tnskMrPJM1YhDIIG57n;6a5bPo|yy9CwHn*O$3;h zwohfYt+c6uX1)T)>LfRMW3h&8ZkT4uriAnCP03nXEOwr6U(jN$;A=+A6{mU`rBQ&-cnhgo zpwQu;!!oIR9!L448Cax51YlkorqRj?;J4(rF}tvHHo0c9;<48)DSl^hhu z8Zcs7C_%;jPCAXqYyjWUd<5g(DI#u1Q|hW@+Dn&}L2OtHb@X10y$D)0i3sjfrnMyL znl6R>!2R0F-oT;-)nJ@BC(Muz3+V=w&r^ zY2Dt@k%Hzo8Y?*NAB{SaK+x?VfLBD=2ITx+c92gSp#_I+|Hz0gu?GF&K}nUUHA#Ld;Y2d+VA&{@1_m#X$~;1G zljMPeB#qEKR`PZs12^!%Y|0oWL8Yj-&s;{kq^ZrNl2I+BrLLzG{TNfsLM!)GTJE~Ff1ZG3CEqEo3%hpfRRKL zabCS17W8P05t45Z$EOO(bkOk*&@1ZD!84&B30c|%c-3=$z_0OQ!MdRtlO!5r$trV# z?yipze~+Yj4^2)mvw{!tGFZnF5Z?0ffKp{Da2ls2l4k&!>oRJHJjvC0z$ll%6cEId z7|8;d030Bra5y0%B5{&rl4y7oPp~Wk|EnQMq&$%BK z+ocv;m29S&zA04fjG$qXt$hiz0+(5*tid|*R~x@^{WP;P!laeDL^OM;u-@$g6X@ts z@3%H+Q@UyEm=sKNWDS}rCge2DttZ5w0-5;p=C&!%t}E7*0pcZx5;Jk;b{b_!@uQON z0j6J1RVpsmpbZM9uUsu59{|^SVK8I#!%ihxCQ}t?i+V`499t}DjTW0x0VHY|hv9Kb z^JG-9^hLO|8E{-TcQ5UERk{K+?P%3E|14cN6zZ`8VkfpPpkz73V|VS-70iYgAYQ9H z>tmBr-97cu%CZ%=|Ihl>N;&L7&D?$rk|gS6g%gtBI_z0UPJD1W)c|D(EMF z)^=#AZBa?k{i>d2VY5dU3Pb^YjrvtRC_1-Ws6mRVsi!NzG+;PsP<|PGrBN#k*UuW zoG+iGHQspE}r5ngPI@|8koF=&e-cKq9ECs&JYwU7eS zg@HAG2bx1L{{)jT$st+hZT~lbKby2=Sj3VWgZ)n>M8Zy57@&BPD$IzR_dq_rVLUFDE!B8vNC7*;u0;*_8J8#&xygS^A|T+kP+mp9*-+ zVy5s?|K?qvnQ~M~w&4eWYTK62Zh;dwT<4*gD)xg@44N~lw(v^{fJXcE=!duRg93K3 z9~@K$v~ZcX#fu2}y<)9RnW4Mb2TU#VOvp`h4l9%908hbb2KlQhCVo5ctR65kjWhyY zX@a&ygy}}9*9I8i>frRei{E3v2c30vxt4Q}ZVV9U3Ci@HH!pn+iX{9xw*~bANrOi; zir2YX`FX|>@YP5Bs~siE&rP+DJ)%!W`^p9zcRM3Q?E?DoA%)f-Q92loWXtFGa%2KB zxdFLP8=m}^cf`9|0}4e9eueJ_f(019XZN_}JmW9Ekq11S%Lh7qr9yoU$z(i>G9p4| z{}hlQBD#(%iK2L)FW>0zdcejv(~NH3$=0bB=@ol68RWs=Hd0gXH>WniZf)_ZJPMas zKE6^919+S;(S6)?axK7LdwAOK7bj!ETRvi61j*@jq6y*N2ioV4I#DD_^er&oj z%Zsftuky@d#849?NfQk+wTvX`8NndJ*H5ZK!CIwjRb5kxBGGwx4Wg!Usix4tlBZ9g#1eT`v*l@0 zrO2RE$}(7?zyi&Skp7@aK@MQUvv8qNuz6duYS%zuPO{q8J3bT=)D~d-cJ9mwHhMXX zQNjN$(ot5}{jj@&FD#G{S>tz?qMo(5~Tp8aeOhQA8L?@goOH^XPrB zH$fid=hhu-Wg^W-$DvNPl3hxf3KVJ_^MkWZ{GfrRO=j_l!J-;`&_SXkjIgJ7bh?K- zIG~d1gb`HW>Z&ok(t^aWI`r_Xvcwuet+gUzqz@qCs-uv(?9$7W978e0{|f>YOzM?n zWU>GS!;Eb2im1#0uE;5roJlF1(I@*{l zEG#GEfP^@}I-1TkhrE0)P3i=gAVWT>k);`*M8R-Q%%TG?yh9BYGtn@^qz{)`>f<3l z&hYzS1tZW{LcK*U$&#c&Af(VJQcc|{NdB1P1db-qC@d}xWc{#BvueemLtE9@YD5;5 z`ZcXT?nuMd8tJ-`6df_C!%3JZf)Q3Ci4Y{k(|1K+&g3^Hs3)({Ee9xh?aGdiUU&ptX724yl` z?m^{WJtvO*2#J=C-udcIBm)|5O+`hIc&L#-x|Vq%Xymd@{)brzo8zQBh$F|EXHg_~Wr6`Uuhw@m@#o zk_Au3qm0?qyng#+GH&akJ>vZUj-~kX7=hx1%$xVaxaNeDgzfWMAg`j*`z)6nG`Qj# zj8GUP6ag7l)WTmD``qL%M<*8mV1kv?7%oCmiz=)jPp9b|w@M}|)1^*jtD_JvULmkY zxB+$-uu9Q%cDt@1fdoA33hum86)NNfh(Qb@@`{HeWz8jd*P1}K&JzUzT1pO4jM^Kb zIFshhKwo%^(r4CSA)*9|bmj}4+InFw1n6#iO(<9Zlprvo$U==J$OJl6f{3Z$&o|T3 z){h2q0rZ(*jTFRScNRzq;MKrdi#ieUXKXFYH zdB8#pIG`e2SyT&9%NbdAR!fh5RF@$=q7rgO$EHQ*M@E6c6P^{~(9}%PrJwNhaaxEC>T5MGRAZVk_W2 zcLN!=ctRH(nb=62=!Y|i^*ZQ*iXuDL)(XCrIt9h4O1%)iPBs*;4gC{E+`uFfRE44& z<>=0uN(P&uC>DveWiCakOCDLV2~ANVBHi<;BVD!!g&~l*c=HIVN}`1l@&+8Wlqt@% zA|gIT3Rundfui1{8Ana(G-Kq{r@jeJj^Q5vdTKX!ZV!==eII zHba&u;GXO0)2)@17j7;oSAcEgN?%QJyR6Eeew8kvLw4v-v}_Ch0RnHiQqi3~qy{@z zWp{?9)eq%6<@qA9WRooQPj31o|Nq7^Ujsa86UfYn)O>NYy8u*dmko+IsHD5iFeD;8 zZR&4$@i9K|BamcaXQ<+N2FQ$(AS)&%PRLkr(+ct(W!^81lB%N$F*_z71~vh8aP@1RGUD9jMVVyV=Xe?>A!0%x0^mrg667i!%O7 zryiVq{k$$$!)JyF_(0_;X3oE0=L7F}W(hA;D|P^#W9W21$Cg48Qq<$<8VqNuKKc;^ z%N{H$15Qf zsjL--eFhUH{tYPvUI)*)vH27#^x8#fh0q-@%R9Eo)pnmcq`RJ+6qwj=jJxN%3gvb$ zv9O!_=Ed!+aY3bXuHLqhYY$qx=%|QnuI52Q%v^$=<EM78{aMR_4W$`p-a$r- zrasRbImuIJjhx49zkq3$%ENsw~Z) zEGGCs#y46@(ef?%|Ktk)DveWKaQkME2D8rwZ!qq-FQB+%NM=oEH139mr47C;y;|)F z){p(%Z)qL~EpUNwr#r_PS5guX0Km%SD zVHp>N4+w@2{~7^=kfRacMbd184d`e7Hf8>>(NHSwtO)TC5zz${krCUm5!t{%Br321 zj|yz7@7Bw`s6yj1F$tBhFP0C1B1H-%o7vG@2kJYJifUi4X4KfUZFp zqa#wf$Q8onYdGi;>g}hhQ_TSEt$kpk_HQo=+ET(dr^kq6k3D$Vi{^zU9wkK)L$;xGzbYdz<(73#7U#epJUY!~K|*_=sSx>MYa={@{SJ$}gq#u0+XCF@FQDb&b& zq9EzqGfG^OV<3JI!@V{Onan6;$DkFNujW1XKmj zgP8KwJ^0mMeTrAFuR>wK37*0VmrN^f$Ox5;D~(kwb14mk4+CK63E(0GTB%U~3^ExL zFq|c|FvUkwM!9Bm)NIsbxzzy8aA73FBc)|r#WfWd^<9;q?)a<}wdNrYB*D7lJ1$RC zqKQ0;Ns64vie^?z5(&r{jYw0qq|_Da1|d52HD4d^U#XUAGsOne%2iOUEC&|*4p!EN z^DP!uSaW7ijdN*6E@A<-0mQ(a#&%kggo1`+79wg7mcjucf$JLW2OwcO|2Gd?v(;n; z1v^tVBP+qJ%^5eHXW79T!VKY_cK`#ZLEf#RmSYa4(t~Vha-ym2Wg{#Ye+|RrS^d3)te51 zde4>N5*P$&)^kMqtmnmI*)r1wueW|MPMD+6!ST@d!hX zPuVXr*!Fo7!4OoyV&h^4=x1Uaw|zhNdt;GdC{l23G%qfa3|IDO5H?(AJ7a+rEykLr!C&@|?IGU!FA9&#SEOs)hmTx&p z^1y-L@(NRHYOe4Pwm70bmqj&!j1d@W)mZOXs*Qc=va-gG{~@@1ICz(pSy40C?0jQJ z96^wMH?w}TLqde9?$ijw_Cr6^7uK1blh^PT3k(*ZlHsoq{J5f6fQJnS$IHke z3PG7lKsmG812J4Vc9i7buG!=Uj-`?AdSW_^hBljNT7%Vq5CWNme|O^8SJn1UVbu#^ zfuWrvS$R+4M%Fe^n}7)%0TFgbhs8jUOf!l!N1RU2q8mD(^HLXNL43)#lq=zsF;&{e zPkL3&eW4(&pE#X?+J~Cht=n2!82Uui`l8<#uYWqP|0x<+e;A}OOQZ*TrDNLoS{bo1 zaEemerg6tezta1)yPA}1yS5RU z-S9;*b|8fR*rAy_5A2zoPlx{o9CM)+oS(3E1a8baV0K@!N30-3D-}+ zOF*7!cu>1swRhD**?X(+$WaSgTPK{t-GLH>xw%JtxrIem+?=|LCc~%uuZHEwGyKEz zg$hKR0X}(Ga@@{s;Ke&6gpJ$1r@+q<{gvok&qw+VFg(L`8NgILNklr%3BWLhJku{- zNz@?E;lk7Xf_J^vJX$WlwGzrz-I4p7vkT9iz|D>JMCmmg#v_iiT`)J0;fm(%ITt(d4)vG+#0o*Q@g?)Qq%av8&6P%Mh zRd9lsw$I$xQ%)kPXxNEB+rI$L4;ffPd{(ag4u*x|=(3YAb(o1rj3tsi1hAT|2Xpo9j7R`lL50IQa%Hx{V%bu``Vd#Z{^7&oLE+6gtda{2S6^R^$7ci~a-*0(h^eJ-c4cGpK z3WwKP`WfA>0pi4wEl8jgEP@b)Kq?Fe9yAzYp+kc-jwqy<@Zk&_W(a1iXhR{Lh7Y~m z5Lt4D7n3d2jKo-^49hri|2|H6c~gzekt=a#l>1G>-`lSUHOk-^k$ZT0a<)Ko-{Z<_pyP2pViZHoI z9B|LMm)vtfEoVoLJn2YB9(Vi*B#(Uv*kh3)qSm2zS?Tc|UmSk86_ZVNCtfgbk!PM< z>1|m_dpc1eQEg#TC}wD70^=X zsG(waP>^0txm|a1hV_t?U`kWvm0414aK7PU3NE=RGjJAng^GnK%7Cs4t-oWW+B2XP>q#vaoz`6N$psgxlL~D_1g@9> zH!UeunJUZX&-1p_7^<@t^kaTeicKrPRbgyp#u{&|9(!2mRwB?J8Vt0DbuOx?$o7qU zufBLI8}FJ+D=MMR+AaiW-%sDX1qq4%`6|qJ6x6q+_+oc7;X*-ip`TeBIys$K&GvAp z2|YfwY)aWmTL0Ff5lfbw;318++G_`!mm#gLOPzJavWBbP#v;q-w510gJX&~WD(&Up zx}0*!f=;iqxFHxM^WkRmy`Q&>Ll3yJ3Mtx$Nlpi`X3wHE9~XvK`yDq`TgA?o;nso%+!SnaNRo;p$C z5N5WR)CXSo5{>he=Qrwk&R#cD-Us)VLH9%~a_3vwx16Lms__qf@C)FH=;NsWb%k!G zb0Go8GmIqO?tpGfQO6u8yyAiEXVbz+_U2|Y{H(BqECe5XfUv;v5wUpaLyQ_8#)zGv zE?y^e6#r;cmccSIPjMK07s`)b|SR)#YMB);0 zcb6{B>0|C%U0E8EJ}LI8ikmWI0|l3^EP||0^qSfDCfPJ)8E%hC`iF;ZG! zpDiH>!|imgjbf}C2hAt8TZXWRR&!krzo<1AN{^7?JE9T|2rMQh5PR4%73qvgO@#?B zXu*3JoSp_ulHu}?iNuqgiUx$@X>FagDO@^v*}~;rkeeJ7CI8+TG~x}9g<9~@ImLKS zHd*D9J%nHnH#w!vSqOexp;aTF2~9}m?roif5!U3EtU_W`X5f?<^57KEL1yzq@;c-} z&Ho2EO4<{0%H)(8^R~)N_HmHCl-J_I5F#v+uU?-CR6i@a(ql1dk)G>jj}j)y;@PSm1;>!Y9~8q zdRYPDaa46u5lj&Z!*GW6YW0Je;ih`j>wJ`rIJ*@N`{=T$x%GsGOr2jod%v#MR+Lu^ zP~g6pywom952u`A?b6m%wza24XNnupHY*X-Tr(u{TWxNqs5!>TN1mfRqYCTUO8=u> z@Mm|rVqQn7GrxrHrRKzKTQ!=k;Qp;Qxy>wdEDf#Efi^p4Hoh0 zwMjt)G|1e6bh9c_@p8z%Ar+`~cbmx((RE%|5v@5Z{86-?k4)_vo+*=NFmq{gg_ksN zeTNBNY+e+!faGsY^eey-WCOs774TDs6(37pH-6QTF*!HMTw`UGsB=3NJvS-Gfu%K; zgA8I^L0r{pelI(HP2)YS$5#^~&P;-wa_nYYOKm}ys!Selb`I-btg`NF1RWrc6}wo8 zVxqv?)n@pl`a7_NEQdL+=BO00qL*Fywa^mn6K{y3;QfrkufArBSRyOXS_@j6Q)+*mBJ&a+d`4U0)E zSBq|T>sX2k*ix@3s?vsSp&6Zp(6#c0`K|J9GhJz=njRs}@-Gv-liGHY z(HLUPX(!vy)?RcspH1=+Jg#tLTizW_iAl)9n5FZkI5D}P)*!}-+5Z-7DN{dgJLJY? z? zjF(*SI6TWz507W98*t>;y0<;;cGbwg$kuW8`)_}q&NCC8;nxECw(YI$4`Z3rw~4l& zCWlK6cC+v(_v>#M=)1Xhm&z&#gsNLl^~yeacIFBL;*k+x#hcr)-9Fdd4_wR}ZU}hF zuXB^BD039o*vT{By=JQ$ZiSb4`^|?o_hp}J4y!qNg|x7Rd+1rPhxtjQ|80YjDXS6@ z^!lob!;bG9eBm!P5>wYrrEh*MOILmK18fS{dN8LUk0)+jH~)I>79n<5Z^YIG&c|db z133P)VKx*%f@3@|24*NXP!&RR97itO<38|Z5nnPiyk>oWhBR1pe=!JrSEqmE!GC5a zd0&?#l*BnWh)xsXa59&EqZB=r$DOGvrAiMn3KK zPZY)#jss^xNH@1New>wkPRDdu2z^A@I)n!|E9h%DsA9?leTS$AshD{B*DI#LOdnx- zfaVYiCWZ?rg7q_nn^k}s7d0z36!=Da6v2m*CwE#1i=oFpr+9_sc#I_$Dg@Ao83B#{ zWqGY4kb4DhG}4I~H-GXr2j2LN6jjzIavdH+8fEPG%t|howjV<%X)Lim-r! z$A@h)W+E!}ihX8KpJr$+w_$8IdCfJCMfi1bmH$=yrBk&ccsq%ZIf#`d<4=@_c6Ug9 z1~`&hnTu^mkxI!_O<5ltxQ<6?a_Z=oefDM5Ct;pPU0^_DCf07F z2r!kG8+{;l_kow6vW8u`a%6^M(YR}rd5_NrI7vB!r)ihJU?n~od3>0OFBzKX7O+B4=rlGanxK_T%Ly^BNtz}}Zl;Nv z1$3RKIS0kLeqpzIoT;4a!|e2%f`M zRt89pkGYUtNt1!NpBnOhE6GD_xOONhLpi#QwW)) zr-W*`jno-oP1$*z=YDnRsA4LII2wfQ>3936kzy83bc#`qmyUYcpW$PTcKD`+YNi0% zVwpILx1xdVw}=C{gG8Bd|LLSB;QyjO2&T7sT%Ujmk-)3F%By>^7sFCuX#iF0R+9|q zh#I(j>)C)zCWg=WTvCW{E2%4&3UkfpV$R7)hWV+-if@^A77PiLFL+?dxq#?52Z(^H z>-vr8)>x%L3h^2Wx=OFVO02^?CE$zgK>N5om6mR@41=b%B~*^ha$Hz?<%jRa0;q$vL;)yD_gI>+N+;X z2!)`lF}qx3O01vOufxi(IIFChI-k7Rvse~)Xf~-f+E>b^q~JG~iRpeKx~l2=YO*74 zQVSjhYPBHCu2?6sj7732>;JVX3$`pPwqaYZWlOJP+p_sevueAxZELgsnzL`al0rMG zItQ@Iv~&K6w1D`kKHHor8ahqFRae^$glo8uimO`du3X!-U+cJN`?!!Bxsxlom3y{i z`?4~-uWswI_Ug7ei=IICwtS|yQre`*N{G{HibOl8bLWw{Sh%r!xLE6P(4e@p#<-2E zyOgVEyz9Ha3%r&ayu53+F&-=X4d$T}$wAm|! zvrD^(YrD6LyNjElEOG`6EC?K?-b=s* z9KIZ^z$HAs5In&soWkpC!VKKPFKohXvsfA|!!xWNS+|NF{J$U^!m;*-SXa9V48tny z!j4PCM@+#;Y{Wy%#4)_JHEhFCj3qg&npI52sL8|Okj07B!{U3w@7l!b8^&Wy#${~A z==Q`?tT8xz#o(xT2fD*qYz|v2!auCTcZ|T~tH)=|$9?R_e~iXy48?;y#pGedh5W`J ztjKXp$68#+BMixQjK^O5#gz=mmyF4od^>8)$%uT&pKQoX#mI|1%1P&AlARvYgG>tj%of&A>d&iCoOaOwOqM$mgug(EQ8YyvxRG&-o0`2rbW+EMN)k&Y=!A;%8tv1#j);&Gj(QVAkebk}7+{=x) z&i&ldz1`Sd-LQSx>CN7P?b)Uc-{cM6czq7NZOZa3+sw?{=l$B;jo$4I;Mon_tZm$I zZU3P7ZQJra-*%nOsfgeWp5KJq-~a9310LWOj^WFY;g)UR=KbIaUe_V+*aaTpC0^kd zF5pn@-Z{PDDBj_vec&S~VVMo#H&e(GR->7ninnZD|*KL6^U ze&>k}>P*hXre5l(PU?}a>oacaudeCAF6_!c?8Of2#(wOwp6tDH<&r+msov{?{^-9> z?ZD3I*1qi4&DB)S*W}&Zxeo2o?(M^#?bSZ(cm`1;@aNzSS-<3HbL1jF^}=oRU{C8#?(YBo?q|RB&jj^eANCiY?x3C} zj4l^(AN9>H+8clOW^d_w-}jAQ^M3#Lkw5Q*U-*vC=ZSyyN)OPF&+gg2+Hw#1obUFQ zkMFpi`AM(EYQOoUAN6t``mg`>v0wRBPy2}9@jh?&w_obAFZ#XT>AcSj!O!}(U;DYg z>&EZhl<)gh0`$Ny^vjO?#Q*NPkNJNN_{#tMu)p`kAN{7^{pAh9t}oEo&-dBi`LnpMU<4p8wIV(>QJJ zp)CIb$@5?w|hr59vGa|I7dOY@hKyF7_6_^75blRv*(oo-y)Y{_>yx%S_8+ z|NqTT;BU|W?f>xBPyWMS{>xwg!$17<@BT&J=Hy@g!@vCWKmX28{`_zFrcM5!5C7#q z{PVy3tX$+p{_J!e5C9?h1O*fT{{Soi0000$0YCu&2>$@}3EW3;puvL$`6*0@FrUMR z4I?g`$gmsxqQ-joOiDz_Qsv5&Fk{N3H?!tTn>bs#%(;`N zPntk)3LQ$6WKobMkt$7UwCPf(J)`RUDV3_#s#miH)jG8+)T3arhW+}qtXZ;5$BJFc zwXNH?UgOI3nHH_uyLiv)wOf}h+`oPS&lSA2aNxXr5%*oZH?iZQh6hI`Oj+{e%knV8 z!>qXTVb6?1i~el6GHK4GO}AUkdb4ZLuwy@NJi7L2)wogL&WyS>P~WnH3%5e#PGpS~SB_wT#4i~kRw+WL5|;AwYX|582s?B>sthj03Q zdiVD6d!HX)K7aps=_b{E1L{X$ehB^tAY|hKw;*5#?iZni3RXy8c^0OY;Dj6*=pls} zCdc4|A!fxPh#R6s3VRy2C1BbLbmu;j65Rg z%rw2FYZSQ|{;FlTl{r7L{BoStXQO21ewSB7RBZnPxtQ<(gr($!3~x#u*@) zZua-)ojt1PWtVf}StoCRj_D_z+2twcafA+vm7t0?x}%|oqN%2%lm==kqkmEgDW7{X zd1<09+6ii%n?`DCrVolbDygNO%4wyIV*d*3tpBy@DwwG1BX^Jv&-uQ%?> z>#@Ej3oNl18jI|+$}XGir>bgJZ9LRo3vHRv;@Yi{+LGyPw#@D(?YC)y8*X*$vU?r5 z*t*+pqUUy6?_hSRJMX^1#>?-&06Ti`t7Q2*>`?HItFXKaznd_N1EX88!3X=*aKH~^ zoUx_^M?5LT6|<=^!@gZSGQ=LIeDB6Aw`?7zW138_${o`TSIIKRhjYv)b1bpV{RS=c z&_oO8a?d;Sob%B>+w1htKo&aN6q!uzl(%b?In(L$vLt<4*eSm%r}& z@4)N4bnvmu-Z|pB`|^V__SN$ZfBf?&4L|w#*RMbK)b~IBF02S(C_J$W0qR14m>6II7s$W{7Epl$ zJYWPR$O`l=@PNPA1^NJZy7YyweIN{>2=hn6^6k%gB}}0TiTA-3p74b*q+RWFQHli` zq7G`9gARAd!!-C1hdxAv9{+AgL^}+zhd(sp4ky7wM5LjJLX1QxFo6gP0uX{yM4%Qg zs6hiRFntFEqZo~$LHuQKgk^ML3)jfTHmXr|ZhWH&{mMa{UNTLqG5C$c-A%}bH zBOvv7haMiWkVuSTAo(cBMlzBJg%BbTC#gq3-jI@tgrXu0u}MzSa1IHqL=i&~Mk*#S zfTu*IBBnUOFNX1zusowIgF(hIY9o!cyrnL8*$rFvQjByArX7blDD*iIiBBNIB9F*O zLpHGweq`n!$1nzIX5pI2RHh?+2!v!ZGn?SlCN+=Q%paC>nb5RmH-os!O`4OBP@Lo; z6=_N$QW1#t^n@v=i2n#IvNDvUbYKBtAIXcdfdUKjF?Lta1 zD$^Vi;Ru3^K@oDQRkHH5tvuZV6uw}|Kro`EC57e|PS64>+;pXRAj4RnS=VXCb+9|d zD>}nU(-9=rrxQ&iJUwa$$ZB$mk&NO|D`8Zq#=^6orDbRdN?L`wkbm03DjhbuRzE6M zq<rnAv!8QnOhMs0S-&j$i1ZnH#=?J9pq$oWeD}BX(UIsdY_^(r^JiBu}C_jIec@tFgx+aeR!$WL}IV$nL}uAa5fR5*cLk#Oh&LpiTl z2C-f>+66)P`O6eutRgVlUo_u#%yDis7gRuDm7+AtHQlq45e-rhr@6L=el?O!+Gsfc zG`khXH8%Ctyga`&$SjR?UWqGg5eNCyesu7B{~H2F;}_DpB>`{uTwnLT!MK7|wy_mV zQWbF9u7h?fs0l3H`bJl|+Me`~)qQF}ANa^*#_m~jZR%r}JH*!Z?}^zgXNt~&e*i?a zk8i5iAi&npW?uBTdkbX#(p9hu_brvl{o{9UdB|fWK@q$Q-!pR{0=89jsK3qNHh3GQ z>;F7CGdE3YemlG0gdRh>PbzCZv%0U9{x5fP8}oYuHqd%5wT7=uXYP?N6sz^~fZ>W# zo3}f^wf?adG8}W9zx>z3U3SW^{suEUI!m$Q&s?8yS@4PW+#wGHu>A9nFa zusmN|i*h-i9PX6|ImEYlc9}C--7LV;J%Rk>)mUX_TZ?jf^LZ^Kp zmv)m!f7?`Hd?kCZ22MxgLePh00%(ASmj$IKVJ0VP|HpG0r+&<}eyiqyx#oZVMN^HH zR(D5JfX8_-XnCUddcdc8Za0IKwq&GMa3sZlIRtLUWqBPJfOCg{;>L5q)^gMcgD;3+YBzb67lfRahOia^smF$s zW`vVyA+>kQ_>RVS zdmqVFw)c#yScQA`d1`ffP{(vAr+qhufgS~sZxnD)*Ln2Cd>n{-+9-mL25jIcZ>PwN z+sBMC(0{Agc2U@BS;>nEXI#iwb<&2BcW{FlS(0d3TjMB=ePwMSDU#$DU3I5~Liv-0 zg#&(BhSO$+^(TK8)`ZB{gHlO>(ARZH`CD)3O%6AR61Iw5IBpXO2_Q8B_jiJPI8wk! zWC*8`ikO-ISAGzubbt17AsCW;G4KM^={nIDEaw1;Z(j z2Dg<#2?V$ahAZ`+Y87QSX@csgkH$uwT!n03HTk#o`sYx(Rn_bMx+^ z@OMq(X{7LZnQxhs^+uW|*>@7EcxMTLYDk77X^hAyatK;eqM4Wg=94DJYCmRnf4fsGAiMo-#^h5yNB@#%>s*^c;EkcjD@FIkzP z$(Z2@1uIC4ICY(2iJreUr4{O__lBWnw~~jbn|V2CXqcQrnx1DllnLg29wns>=xT5n zTk`2?4%MIznnwYsce{0_Fco*?xt3cxe<{e4gi2YB=$v9Tp)I#csp^(>XQzSSqPnL{ z%{i#!C!h-Yfpb-x(1fZsT9<%XeK^{!w`Y-Jh^c(%lLgnSW%_onDXlN}tH9bqp9GZm zr--8or+pb`kD08)$d6`stDO3j4qJn6m6UDDqtd0UEvl|ts&V)EqIK$X9e9x`)tv45 zW{cUOxA}GXX`Onhm*knT_sE3lHI-HMp!i6u2mg7ZrdO~Zdt|w2p95%^beC!nxUC=f zm0nm}us4vk3Wa~LV2UPi4NI2{8;_+oo0T_iMwyVT7ilqRvMrl;1aOQhhqJ>9seLM= zZfJzd#em*vf@&q1=;cQLT7xW_ozXgmP3f&e_jH2yu}Mmu(}c1^>5yHDuPNDVMdomH zN{@GXcH*h2(-gKCR-L(3Y$%0;dpUbvb-7mOs9NTl%F2f3NusFMn73AY{2I6G(_WS2 zfu`!8QtNcYscbv8w{-}J>WQ&#T8mS+mV_&zkR*&&iMYZmR|M&!Wj3MEdbk`Zu0fZg zmT7LCt9ID>TcSI*uX?Hos%J3iwi3IONdM-ww)K42cvm*NWq~?xc89)+)>kU_sPG1J zI4g9rOT3^4ii)>}B^YtvS#~{&dM5z2F$rzk_ozh)rl4DuU2B!uC$A0WzTFpo0C;l0 zNtxpMy=c3q2il)yTfyw>V~0w<6gZta$&F*z!GOn??Y4&^ICIy>fyBtBl80&4=eQDl zhVdG4V@7apDR*(Io#;x1Lz+@X%eTX5wj~_2!mEZ}yQ98WwP{Ah(Wwg^7NhT%b1oPJatydw zS)_G~fI&K~vh}z#>3Mnki!C6zIoEW*=*f26zfCE}J>0M$s)75Kv;)hW87O}-8I<$; z1?<{m1@_Bmh=O}-U!99lCitP5=aa-Zn2R=}V@d=sGe>Ysv%46mP-vz%DyKRdrbQ6N z?FY+XOUN;0!1LIUL~C!gteT)&eB|0%EVot~=zZMAm7;5!(94yE476ixzwoN3&$^MU zx~8azz1n-Me8p0!+pm*Zh?@IhO+0D}+sE#Th=_c|hg*sz8qh5m!Dvi^wamL`$C?}Z zf*-x9TKS&aoU`d{tPtGF@&DSHeP^BbTBUQ?ydv6wjcmlG)}&K~3D2gB`0R=)*U*tj zp&;9ms!GUz_qR{WuyIh*(yNNjMb40zt^^>t4Xp+tfV-k6pgiZvdpxo|YrNnJ!KZA8 z=}FVBY{9*oaaw)L;6%;5?6&!tj+;vY*bLDhG`Cf(v-hBpt z&uW_&s7(3I#SZG?E#22$F5`6Tz)@YS&l;(G>U1M|O^+Sj6)k3(TE^N}t;PG^QqI)J zy-}ZC*m1s_>i@k1GYHWAC~2|n#$XPE9WBh4?BTZglXO?+5bbL-OxPPP#caCg6Zf)y z?yNl)$xH5FOI@#@80izs2zAWFq#Wv(PTVRRea=IyzQgzJRh-&B8h)b}vgnS> zPASiOX};4e;PJ`FzvzZTSlOJ6;o5Bn`-u_WLweL6xaMbl=VkfhXI7ha?Vpo9#&p_;+iq}j8Myr{q|bcEw;9`*3)Kic z(`t>_i2pv83QXeJ3ed*+@EaTh;O4O-PQz`G@T(ixoG#TwhoJtPzWrFW<4SBQ%I!%G zo1tWy(EseFH!o%Pe_tARfg6 zPN^w)=XvVm(>&-sKb8_q*N*K>lr}!Su ze9y{qmnzev>HCGhr|7O%nopKLdR*o>su6I(=C1BkuETGBt?WtH7p?aYAGB$Yk;0D8 z#Q*+-3Tb4X%KEAs(chc41X!uWs_5d~p;-!;RqWMMef&P`xIO2ovR3FX8OL4ymz)a_ zEyxVq5+{$MKqMFlJYfS$3l2xR1d$lx2A4Jk$50#rgu_CI5wm3E;_! zT*{HJ)EEpT+oa+ZW?>sRLp~X!X0p@Fp|9>80kWvf%#VY{jIr9a*wZA4Y82>mp#Mp* z3QOvZsI)We+ZV0k42-mA-q6L7&|%bk3+c|dr=I*tl?-a?UPngt+O&occ`0*|mFuhURC29!KOrM|+8Evh>WCI2x@$7n2X zyAlNyvQdc4BWS*)h{F%RfhxpvLoh}9Q47RI?eaI>ByBNACfCFA)%aqK&n{Cl(hVzH zH5=|ePVGbPPW{gG&O1eYT&!0=P3uWh7u#fXjL$makVmqR2%%G?dwtSil$^bl&eIls4Nh3!|9QCvzp&NN`1+cGSxTB}27aEvAuNI#1dYSYKIsiPJ&> z{;|RiSIjrlkuA!tUx0&D)2f{h9U80rNaglaL9ImsN1Lcm6_c4J;N!*un+b z%TQf{1QLbE0@BOmqrRKyulw~hr_=WKo#A3lXR@^&Qa898{8!f>8HAxW9eMZxXl8w) zo0!JLnd}+HeUD;VM|`5JN^OQvfYMmv;-wpKl`Us8*g>_h$N#`~?Z|Ms!P8)1SG1;~ z5P<$_4)~_l5_0isco;ihestA>?#vHjI0zg8Eoi>KvBZA)+g}llL5%+Wi%?$h0taze z77-TEJWhk6g>(cwNI8cN4g8PMf~P#}~y%1?kyq+$e}h`v7YkdA;nPk^RmESib!X*?Vv5sjD)OdcbN zWk5qGIVp!vMnMUp^g}6Ih!9k!@|3Im9wl5k2~W84a-gKa6a=Q;>e32{`TOPUTRGYT#lPLnZ1g4)KGX=D-9qRjN;K%F`T(as^F=s!Um+ zgQpsSr#;1LR%^=DHJ~)CJsoRVJ4#Zsrj?)}-6~tx;7NDt&~_oRAg0=-z)D^cpixz5 zU;BE7NAyCY7M*Be4V#X`4mK6D$OIf@UF72K!D>Ot7R=~A7NV5dbFmprEP6(TiV-}wyRF%ttWfyTeSvPxS&icTHRn) zk#f>sN21=v&LbkC#AJa_xu0IGo5@g|*00+=EgPEg4#Fl@yyN8tFTnc>*fyX8>Sb?x zPutZ)u#~vyVH!P`dtWBv*I(?U5-$C#-2CcNpzNgQJO^yxcOqE9h3sV+T@ue`>Nmi} zVquq-qTJ|aIFucotRIw6;CXQiFSP@b(js`Z>iSc~DrN-?xQhXWs#gr&9qdKJYvUW+ zgBuy;#Cm<~UiZQQw2R2-Jx~!k`UP>E7XOH=Fr6GsL!y(I7Ewx*PYf_WCV0tpLm!pv zmS2^j0Ko0^GLy#)9Ww)%hhC<^e-DXHIe>Jsl4U_S54cBE9fwI%7FCK@{Abmn!V?O0 z10VQ zmBMw&OK3z{=PcO)zZsvld)O4cZI(Rb70|2Wgc}gb1UC>u-JB7^aWk-Jf;zOuk;ZX& z1zl-N|5ytA!v;3|VUN`&5g|`7Q2z-$Tmr_JDUB<@!-v70O*HHQo4i|fNsC4Y9?^&o zRtPoS_;Q6txB!JK6L!e+md0cInrs=@L9?d{wuQUA*l7eT9Ed%tT8BWhOIU8sci!`l zD52RSRq3_sJmsfgYBV`?ZX)~X=heLyp<;LhArMg-S-)HBOpwK+=S}Y#djqkW(1Z&% zknaSnH>ZUtgev?E-g{t!?O*^47}HRLw<0_XB?IFJ8m{-tL^GuL1*4rTzO$GVz6TrU zAQ?R)cs__cU&tds$#q~j#Uq=2qb5Mh`H%u$72XOmr%oCpAY_{txbPfcgUgkxBg4CX zi_-)n8X7V5Ka^4uMz>|PCjWIE6jDmAPan57*l_McTOu+A7ef-@O=GavTl$i2M)hq$ ziL;|U?NT7=E&Ba-fMeqmbw~pu{1)uLlTFl$Pd|{*$N=sO_}P$uC&n|+@ySC=^o*y7 z{lRGY#%rhhTLG6a17Pbs&1;hP8@<%y8_0`1)e}6~6F@uhg3KZ*e;Kt4kb~PByfz?& zSaO5Pf&g3_u=Wurycn@N!=vUQKBzlB2;e%a!@9?4zUQ-rubVy{oHy&+22seqvi#%ME!U&+Y)XOz6 zoIsB|0R|*M7^1RhoVkYhxpc}LY$P+E!99$dgGxZOL!2LWSwvwelk-ynsJpHrqBC@x z!GfCsA;78yXn-bg1Hs}%Pb4g{tAwdCyHOlM$T|a9Dh2S1J3$~n%t^9R%fJ6PfeC;@ zocoU-XumO{pZ}2J#s5eFV=TjQvOUQIGmPxWGQfgy5&;avylX+Lo8v|tLdo`fGr#-0 z3F@F3@Bq!rMkmsNSV?aj=KE=rk-i2oTv#z~oKd zbTCtb3IDe22{)_1#Qdek{LBApCoSBs!aPjJlrZURObpv1IrvTM%+C4JPU7;UXF>#P znlnV40R(BA(NPt8X&$ujOs5P@N_2xc7=o*ssd|JrLRifk^9Hq0gsyY|?F&M)!>T`c zg=FY7sk+4k%Ql2tPz4pJ1TC=K{GQ;EbGv+WvMUzYnFiB~_x{0J8D=ZyulMwc7PwiSXY_mZOsLyxfI(frSi`vir z6aXPr(tuREg379?GJyk_fD52fE1gmUz)~x<00ZEFF8$I4<IcDw}#(IjcEAB-a(VRsRuC zDrlWjDZNENeO4N91){o!pduQZ!vw zL*-aT#aVFuRsr)?p%vPHgxK7oEpr`DLbRvSOr*t$vJ==*b_4`Dm^yTuvD1Vx(|T15 zxW|6gS4$8(e%-Y4YK47e1?&6Qe=}HBrGSmiR;i*-R_y?q0*A*DO{LEzKsGou%;g9TR$MXW|h-Hh`2SN*%DyX z(2CYU_**Wh1_ds0-K%EOc*LSxY7zBT$aU!J;+iY04kPE-75fKM(qUtO@(XN zgrI8K3&;fUiro2)0+9tOUA=?^PT*)oTnN6>C757W_yJrU1fi;iYOC0Uwcw$;ghx%b zC?KlF@+~940|2gt(8}G%I;fAbEo^hRqxhEJBar>57tt93H85UDJT5jEg?xNqLYPOY zGh21r*LGupA&3J(0Krkn3P)%KR6<^Gqy*uNB_WAZ)=jorkdwV(1pkb}ShscEg9Cs9 z$X~aeDl+}M3)+8mfd5Tl4_m1SSSp!i zWcL*)Or}{!edJ0`Hbb`9I1b;!U1txN+$-R~Y;{|Rrcx7LVgH6s*w_W(8kp!PrRc7V z<#PS#$7(&5+9F4&Q4MKC(uhYmc^au5w=5O{m^NF9YGTn8O*g1YoVHiyeW>Qc!NR>} zO)!H|0Ebx+>S~VGr7|VvI*CANI9H8ho2me=g=ndE+gTv$_>JHZ#s!1ERX(6${`F&^ zQemNctICa2M%aYU)mWR=;3$Y-teyo;zTo~n=v#H`wi-nFZR0_+4nbmg)d))&Es(*ry7Uz(IWNJHP|HV>CJ_Gl)Yc|lT>(=Ooi@4)HZ+#A5dK7H- zrf9Fy-H0`)S7tz3c90lojFq;@0Gb`auIW}SI@sv*_Mf>FU&!KZ!(Q)*&eE>-+cuQ~*^c9qJ7uvps%x-aw=_Q#KskOsZ=V%z z(5>Y2-feoOZlpeM3a{u57p=m^Da4-d8{UI*Q=oIK7pG~uZs~%S2DDKSgd%IL)&c}l zO{%J_Pa?l*&%LUVMrE_kJte5m-sVX@p02b8CI1bNYFM=@4*zgHu3rrPa0Je6dM4m# zYbQA3i5H*oN~YPawsAMd?(fcH)}`uERe;D=^YeBa5gs<^jrSWS2?SYR{LyvJp5A0Cx z_7#5baDQ|;L)wVt1FALhrDQq-qC<5Jn|A*vYrqC|18rp9h0gA5&j#&;8UpDRgHKB6 zf=_b~U{cwXRj4k}VFGiyeSp^t!f_VDA5(gUul7IxEUvHLwVrl(j&`vJcCugQgXZ`- zwrD{QdGprxcN%%!lY~nk>$t^(O90uz_3^I)tT{*Al=_1vSfr;&vSHGpusM(>*DFJV zX_yA>&j#(#M$OZt>M;+{uZHa=h}EUv05vBvNjOg?GA}ex`t>6HrLX#5*Y`mm}^e5HPTGN?%7{2nL(iD?(@jCvJSu9{(xM=>?$u4cP57Km}QdT+=7TvnN8h zGV_Ff|INqxr-ps~AA7zAhz}J64x~`a;1qA zB~-5D)c7%lldX>`&6p8|ijta;A@78I>GCG3oU%Mk>giLa%13)1&EjQr)+M2wK;{X9 zqy-5Tar9^*qQV9lHBVJU=z3yA*C%b_IHh70hqtt9*CHz=C{nC!+T_x$YwQ;wmMky` z>FXC|yIs8o<8mVg5XdM$2>%*UY%zz38D7?i5MVjMg3AV*7^w)r;Tk$yuzV>^nkD58 z12K1C?fSLr&lE4xPN?xV#@dV}>b9-%A;XKFdq+fYF-d4NDV8(uu}Ib{ThUW7l2i^I zIcn^-vun1EySwY*$CC#i{@%Oo_grhq$|I0y}F}p5h7i(Ry}LuYY;Jvp%oeW znbnM12r9S>HiEn*NG^6+Ga-d|5#k0Q8D=O3AwvN2%`T%1_6<0rh?rg?jQsIYbngIs zK!d-)VS)#o`GE(F0c7StYc@Vd(h5=lx5zb;KyjLBZM*^wkFvdxBWw+5W}^a8s`l7y zRc6E+mctRJ&~OcTDI9QJS{eW25+9t=*_TzKp;&W9Ofd*3fzV;cA9fVMNOpOCx2K-i ziB}$=fU>tqGTew$%`5WBAtw%vwy@P${uxN%6Oxv7se+jz$e@FTz*UTdpdRy!F_L^p zB6{|K=$9C+mf`BD-w-mDCWR*RP9rd$p&gPpww9wGsRWtAbFbt!B#}hI=)yuxK6#^( z5===EaMNz70k*42i@~-ya%*k3su}ksxz|>UtrUfsfJ?4vx}emWGx|^xy(nza2fliU z;%A@z{>fc3HmQj3k$+y(Nhcy1il|V2T+ttYX4wbC#7E?3F|wDEHK0}gZWCC8X^0Hr zT@{Xca*74plU|A3vM~Pv%(#gm;!d;uvJj(A4Y?b+%+%$!xlNtic|0SZg4xkah%og+Pmy zfP%v5jOQ`m+{wb|XjvFd=bVc|l6N_>ETVQ3Ge~9!r4(m8)*XU#K`F#Ql=T(Mou}81 zgI3j1NGT?ZDr(B#fT()nLS!HWhNd2poEwRmK?4>^=TxTK+A{gs6jv-l_q=~h^vEAb z1iw6KB8lMcvrdBx_PE6T}b@ zNID9cBP}H()rDHoix9D{7egS#i7F@w$lR@k7l7IPXyYua^fIcI?JZ{Beh0KC|=%f`0UhXVmAtVP>&`%2rq7H)ShCf!OR`MXU ztt|f=OfAcy3E@d6V6`*~I_~j^9IPP|C|T1f&qS2rnTaw!kWX<2aJ?bA%@w$Uf_2=% z2JH#pZnE!Th+Bt z4t1s(5+R+P)X#oOTk2eo36fyBrwm_uoD#~{JIsKsN7TV#QCwMHziOhRN=!v3dQc@Q zE})@O#Fj&h3K0>wGg+)XS_8ZK;LnRk2C|A-eDA#dJMoi5*vB5;udZz;wc=M49R+Isxx7$>BNZh{8>I z&_tZxB#KSWX|E<&!$(PznsISfdP4NCD>)gq%JSF04K0CawY0rP&?vOAEXV;2a7yD& zZo(0!TyY_c5-zJ5Tt{8ZYfIq17KM1Xz4dLXI)OYXh$dq7j2wVwY&ikZxW;Y8@e|%6 zLUBgb7LEXgBtRjOeRbLMZExt3%x?x zg=^#v(NTmKYSrW`RA&=%IK*BlOk6&j4M;h;l{TjpJ+RSMj004HP{usoE62>77gSSH={zwje;8uLCV;QiGe*hnAx>2Z3>zSAn>s zkukjGJu%3=o2xW_gTaQ3ZyjIve$t`Dr)%UU4}* zl*1Vt1vuiTZ5<4SsMpYx8reLNC4CPwQ9z#&Vv7d!apE2@?_Bl3LBap_Rj>xNpkOBk z03@8La$XbuY0&x8^MF%IFG%%XLN;swJRok|;&i&W;|RY+1=k8?RNlt=6IK_)IQS6^ zWK@n|Z#ZI_x|KkLB0Rxw_ZWHY-1Y8sQf-0lpEL)D_`Eag-r-r`3=K(v5g0earji7X z4s!)!vH3v3kLyAfX-Q_m+#9Z5YhGA|L7!|P03HP18t7G&6d*24fX9K+=UqVqLS6t4 zlSfEc`^g&xj>aG18G+E!%r!zLNYr9wOVTM_X@QtiQG=$4SnhQf`tSkc+>WCmUm!Wq zx?xNas2f);-}T`egs=kifdVh|n{ExE8uXc>tI32rf5Vs*kjM2G+IqTeVRLO5RF$r+N| zSSDCr66lpiW*z`0!RY1JY%ybOkzeR_4M~CH&ut9ch2w+G$wN4nY^jfvz(!-lTvDy$ zHuTTOrNcXh+aqdW!^DelHCS=U;5=Q5q*2UOb(KB>C=A@pEDb{ax~? zz*Bg=AI$LDE;ZW#;f6A%gs8#KBGd%_0HG29kvDN!4t$^hHi8ad+Y)3`61aj4B<4`C z2Ya}{uCzz!d7ub%gh9AMQISg-a3gE!(Ge_02R#1(Rr#D#B+6n)-J)rWC?$XqD4Hg0 zSt8Z|ll6q1P>e1l&cdLeQ$FPz$<8BO0@amK&uM|W4PS)uV^&rgAdzAz4&)pl0uUPH zkC`RgnM%bmA@|{9C5F{o!etaf%MYASHeF^)B!vgE86S|^8*tn3ybvzlBuZ7}0eq5T zH69plTj@Q}qhz1F&U3PFiio8*?7TSCURE5ag##XIU0SbxxV~K}D@qRRM8l zE_!F!6eAn3Tm}RTda?nLvRRUm0VnZ77dZcy<48;(=oN0*=hpzK&rIo{M4w;XmS+hh zYH3xE{3n2>o9Y=~VweeLUc$^J!d^*_2+RaZrO|jz!kf&&PTXdO?j~=d(|RZxRic*H z{VCQ(KsRbmih82RL_&)i0(4qmjOy0^)X7J{6LN(>2z2LnUaHtkRmc^E3pxRjdMcY0 ziW+oM@fnaHEl0EM-`C7g032pBc!D9^W`7|>X|+MdoR%K&fD3>rWQJqX^g#lphyntj zVlF{YNY4^PN1MfF4xHUj(dHuFX;bEDGen&vWI~1vTM`f_S?uF-P1^Bs6_e^mq}V{A z9tFOkkPp77bT%r%wVfkW#~)B?bJhPVSSZ|%f=|cQ+ankUr+(_EhAKI%XBwFww9!Wh zP{AOe3H-U)s#YllNCBD#4`Bt{W`e@4-ph0P2(RjZ$k>#|a^2-jlS>j~ib<0h_{wRv z6iSg)wR%EM^Z~2}fh;hkw8G;fri8ro-tXm_RZ!apJ zb^_lVcoinPloiTX6cS!3Q6>n?0=5AxsK(+pw1LbCnk^ix=%kOOa?}@ALyM2bSRWnkg-keJ3( z>zyAop3ixSgcrI77Aa`bu_M1+=*>3nQ;wmztpJXiZHdmH@xgB?8Y+vng6P7iyE^KW zRc~e~ZTyr0G#RcR0Oudf2(}pn${G&O$cA{f*}vv)?nWOqWWjA*z+5G4@Lo(&=unb% zPhBGD^0MmVeFrNjY9SPdGIIG+D4+tJe8h)n3B5TnW{LgH28v{a*DK@$Q63wfCFFh*i>f=@l9 z4kVM_S;rzY&0)@@HW`J~73q)S0isUua+Fr`Ty7wpA43!uNoa&?!L5I>DjYzfFhHqN z3=wu$(uPEOq;jJwq)p5Xa}*8kdJ05CaVuw`D?ePQz1<{Mtq~yB z#c7t5>T8(*^CX|7sez(b^Up7)nSBnxAH;|J)Jlfor@pTB0MtO1mGs~!ayPG@ibU8_ zP*dhj)=P&A)M%+qx3fBB@>8kr$wjrKSuQE}wASHSQ5$s{2nbw60w(&HbV}1c@3a5# zLfhT){5=0zOXY$8SwSwzr`0}5Wb%)RmGF4+0wid4NV}8#bg}|2S^?(_k7i9P@Ke2$dSI?P8@8<;vzv0}Nmb_9h$LCJ?FBF&8Dq;EO5e*V&9ieO%dsnBD#pf z9%8ltCtM>-0q<=FPAK1g!O*P00N=2-8m_Nl8Jp;^3jTEu@17hW$5gtBis>U*l=8P; zZst~6@;yOzOH;It-7rfq=}y^q3!MH2H065IqSy-n`@l!H_xII8U+NHP^D!l`Hq17| z8kGMQpRwxl9;F|vIKauUwz2j2z;WQbW(M!JJr?*EfKj^Co31_wIbIYh@f|xUhdN(4 zhUcu?8K$-x4RZdpbJ_DIG8g2k#EOF@qK2Z2m)cZdslg#`-sH_J05EDoLX$2EVg*x0 z{D2@Y?Cf5um_d$4Yk(6tJ~EQ5wj>JP?rc9VW>G& zQ-7CT;sngj<9S26_2v`hE;=7NI(>EWCS5fSD>+Dxb`Yzrl-wrq-BdM>ugf49ELT>AGUgI{3|On%MXJJy43C9E zDA+l*bG_GRHk2ibiEKS9AkM7%3g6|5d)_Kax#7=KPr+=$7_Gd@`|*6i88F+ueL`%F z1HHUT!lcV=OuRvQ4SuB`xPTc)VrH!veMBvALeD3eGl98VS^yaoq3s2>p4R^nz;_4W zS2br07fpq@Sgf1Dm%d_aF6zri)gMmOnle}flq!(@U3UFbGyK9owUX5-?UNfPfF~<9 zjziKiDpkZuMwOHZ70a-9djp#D$I#4`&1DRNmoI4Dr~7TF4Lr{p7;i#L#DOo_yZ5ix z8LQFpA_e4C4>6-0HAEkJ0DTUjS$kl9p8mT$7K~6-ozf>4zyZV)fdj)3i~;1}K!gjy z2)y(VqNGKPBq2J%Ai{_`kyLeDrE66vk&_Sw@`LKyt3MR0%~CT(O&^mpUnMJN3)D!F zsCaUsQ(!@W1xgypl2l`Y2oZx)qL9>#jKa;js>-CVoVo zjP`)#0gVb5PLd?9=QWxZ@l5q+(X3ROd|XI@N7FWPv(c5SeVcY}-MHDg^ZmOw@8G}h z@dCVNP1+F|1(KQb(%{17L=q~Tp5_VZh@~qAq8=elp--hmo+@vO6sAaS-&X6H3=#_Y zQ)tk)Uj~iLwe{(kDbs@n#sK^;DG|`vLog-Ks7!;P_7DZ7YOvA)6-ZF(N~{buZ0tTN z?xRDKnot^KF|6Kt>x{Gb>I;A}T*>4=xO9kd02HZl*E^Uxd55SUgvk&!&GD8Yh66W?TSeGQRAxb=L<$aw%{IwOwd)GiXyCw)WnObcyhfd0UhmBE%;^8xDC&0tpB2wMCzfDEK z<&RlA$)PMd%ne%RWDCeOm0jgvg%U+PVWk-yW2JW4tbxRuYn%DNr53ArI9BT&U62gZ zD!KLc+i<@e_wAeRL~h+2X81zCcH6Ai-ZAv`m*9r@ed1qM49^MisRIqKPd@)dSiPD2 z!*B93PTWG0ZHi+`B?xOecNW$H;iVS3AUlxcsb-PQru_Dx?DRMoS=p;3MCpSPTci*n ztfExo1XI#^@os0GJ=_8{?Eo|k=^h@|ky*wrF>L<|UT?2tYF1#pnn+@|o)Fuv$v%LO zcMcQ3_Os1~Y_id?Y};+P<$oKoJq}R@$68K%EIR|*n$*-% zg-+BTG;Nbj-h_?{JcLSmA%TGXvqS$Z#@SSJmh(^bft@ZT2+uR3L8@8RY6sPcfF3;J zceB7}E0o{^2K~S-pQ1!S!{o_+f(nNTo#aB@426hlz?RYr|4f(RvwD zLV|!Ny1+yzxUmS=K#>)Ttp?K2HzAj<$g(U=o2zar&7A5Kv?v6`BiwMrRi1!w2SM$3 z)>NIq?JrgGA_G(X=v1gaFbXN{MM_L*n#r2=5@z_JeTaKnNofL~keq94U`H8AOrn#F z5ZNdw0*b84#WzN&Ph#oXu7)1*p^zw@TS!*4`&9~;u{s2&c-cz>Q1PpwD2aBH;)y~< z;xuKs)=HDbHNK2x1Da*#2t)rA7Xu5A8nOWnFUAE)2S1pNwcu&c^7qrh+3Pw50pQ<+ z3QpSEwzhx7?YTH}lLVi5E9_dt8{#0`w%~1*Ua*v;Fj^rmd4syo)iEV@%u|7ZViY+J zvQ4sU-5&Jzl*9dQN>@mwVb(LSKAK+B+G}6K-IAm4Y-U*>iS8jANIvkKk?grgJOqUK(`F3X;9Yo8rwznQ>YoX62U;BZ!yp#%153by zlb9IAth=RI{aoq0wx)0Ud;@P>H^1#$J1dV#}X%|j%lDMJo9k^>(O9ydLUyiaB z2f#E$o#dmQsG6k$+s^-3rI&lf40D)kz?oEB>nWW8;Y{PHAY7B~VkQZZxynl3LzD&& zl*`Yy&&BPd0r`a_W|y0$bAmp9Gtl)?Em8T#zgXTP(TZL)pwiEq?Btlp~RB5G7S6OpQ704jA*Kg7iVmQ+L*8bEl-qd&p`QL^qeY;zc-< z0=k&K^raty)7@Il%IAaB)!xfGtMylhVasPd#QUjXe;~bOXC^73b3hTC42+8VYLn3O z4AxX`I_UllZ$$sxjh}U6Gw{B5NwxwfR&`?4F>b+4K=cybd-BX#lx+H3BD$0aMX;{e zk}&aNg=o}uL}g7(n-hDG&4$UzNY891nLf9vZ_?_ma0b^`;q|bOz0-4!GdS4qNM6vh z-rfXTeD(R8o}~70-AxEO3NCJ|rOR+_7I}Ga@EY3t>$NgBD7Wfp`vN)6tyPK2I#dXL4R6 zW3GUO(x+z5hV>ed_GAwNBTxco4+2N)57Zzb{)ce(#rLRShTMdI>d5#)Ecude6{Zds zj-)dTBL)A9j`~0d`=Fq>Rw6*Wir_+n`*y+mz_0Fl&>vFG(HN^o0|Nn* zN(ah{r9s9f0DrBjzN!aK23Ib{lC0{m6t9X{>hnavcd`uynJvKf3)vWu_R#PFDX{h0 z&<)=Z4(-jS(xfD~4g~jRfbb9rz)tK+5Mi_@5mMvMR`7Q^?qnKm@EV2iV$B5H2d=0uJ=z$q*z5aE5)%Vh_N9GQfe? zfayh|fKx8J!Uhr}6gg z3=se5!0J5k4qt8Eq9emj<@o$?>~w1d%Mlg3ik=SOy$pa&9>uw$E(gVL62os5<{|uW zETk}zof7eGsxDFnX)l(5Q$%s4ID`7a$O#){AOTCt=He0%az~cNGr%Des>Lqg0A|1l z4i1Y2lYm*qf-rVx3k~F~kO|qeLfNhb4P9UjKye9H@-tEpL1=~=Yw{Vb@d7Q3_kgNz z*liozaozZ^BgAp+3?T~KzzuItDOs@zeJ#^8Zd>39vVw3O)Zrr}F@F4^6)Yhhc%%Eu zkC2S;QPR;VfvJRCp(S6ZM-G56vW*~_#VVA*FzN{xDDoO8@>#6H50EaI2!j(ek|X~O zV|Rv4E6mU$B@!*mhylT75BQSw{IV_0tPGJ3Nn9ZsZE^x(&)$MdC(G?Sy3r0jGa-bM zD2tM28emCgttrv*EMo5QT%`z|Yg}+_7v_;3F=70KaN@`^k;bwt!xBj~Pe(ilGYf;R^!h1D;!{4=rAz=se?rh+ z+U>$Xv%>brKmQX@2=psXhXuMco!ZZf7KElGr6nWe`_7>h9<-XgazZ{LV3kuva<^j1_e4k;`dsTE_T+5VB>#_JG6?@<4=QCT#%%8W@fQv%h+ zOpp*t^=0=4$PVY^O0m=s0BlLbK=%gZOE(G^lVd?2Q^ImEO@DAreGq%P(n8;K7a)Yf zdXr6BD+KD4LlsG?fOX@T>z!ID+w>G!4Mgq6bWazQPfcK*mUW%H;PC%aM_LJ03z;=r z0aXK*peAiE_Lu+$4#fH_tbe{y>+JT)AV4|39@5P~8=F@OmiK?^XVSF7+;lch0B z2M3Vb3R2VrRP_8*(-P3OZ6ga2 z+5oi_ac=3hV(s=H5u*e;?jJ`MxR{m9WLB$=%{85s2Aarw)sAC%j%>&GAuu;)DXmdG z7wt@DB9>1FrjvrFL!l!24vKc^i*QCDhkZ@OU!|4d{AK-mhb;Aw0R@8ANw(SrMHC%gdgP( z3pl}Juj{L@H>);=yl%MBnuhu^rdaQ63dFWpI|@chPKM=oeP{M&R_utS%33p*=i(Q2 zEZ2QG7JvWO=k{q=2!H@4N+JvxTI<&9=HRthcW~BCUe1*Sd;yLl_)0;Hb_LXS?JW(~ zggKzFK((svipxyDuY}X~HciydV%kIVU|9gS8i9t9WN6#)s{9lO=g%doGJ#8Ix0Li%Z6q(W=^72~7r=5=__X zk_tRt_YQ+fKa(~n;~0Yd6U02BYUi~H@R$g6muvs~D`fZbC}c1Q2#;C~c96ZYRtuT2 zmiKKz0r=>nN_p~wlPaAhHhS5aV%HgU1rDBDn4aqyRkSLV{uots`Hx==lDSRWye%pN zx?#Evpby$;^nm^xfepx5XJ8m84`P@jnrW*8nUh(WZ7W^}vpH-Yl*Mx=H;S= zGNbFj2?O+w387wuKr&wnZBiqJY~Z9*I*|WWEjL@*LJg;FFI2A>VP`IIsF{Nfy+dhr z548wpS25zLwNZ~4C7WDZy;Z5b*Bc7r zdj|BH#X`Wobq1R<8@6TJr~MfP8oNUhTYvmnwArG?dU{Ph!~#uMh%?oJX}ZGLn5ckZ zt&f{DYY+qAT1$rwS?o9&mC>;uy2Af0h$^icy8-*t0FNC;**Wq+zri@T^@p)(++2?c z2kg7N4J9LRyrBedt64$T08kDQg5e7@TbBWOIr=R~~`JjxZk2!0&A zb$P+3+yXe!-oVFac+f0-SM%hoF+9WNSi?Ia60!+0rUk@7d{LG9y{r4LSDeKY zu0n_lv3Xo}$biRhKqj)u$-AJx7d^I#ywSV-p#g5c&l>}?oWHk#12Da{qnXn$oyxI1 z)U}+;P1UyJWz>xo!nIkrrz5r5+=2DT&EFi(*}4#>BS2E~!)p$+k_B1wHpLs(&;L9w zHil%-jts_@qT}TQ6MMa78Yust{hiJG)QQ=>P0PQLT#g!>ug_YZWw3faowM5;V(jS3 z3p~pwXUoZ*k`PUQ@(#yQq{Pxj@`3)+iDdALn#V0Ng{kY@>Wmn8irnO+f&t=K&NKky5<45nYgAd%L z8wwB`(H(icSCD;%^d=f0sD@hRs}6`!KN zT;Gu%%`cxA_?_1OJ>Uaqp4}PrQ=6yf771)>un#WMQvbwo;t8Cc_x66ek3ROVZuYbJ z?%5erIlJ~JUE`-4PI6!FRXfJY9`*%1@PmKJ3?JlOzIfFA`j}eu_Y>c}ktA|O4>Pza%4(6 zD@}^5*utPpG6w%+kg%zONEk3cM)=iX)g(U%8q(r4~^8qw1E{i=9>1)A1aU;+BnLVqQbC75i1_J6=_Y&H!63f9C1ufE)o^eB;7FN>)13u z(k|>0cW&9G6Hna9C@YT9Kz|ZV6DCH`<3W)U8wQN&(_2!jRkz;Ss}LtT}iw;IrQ!#*dGre%twIivYgCAAdpM=bu0V+Mu5re~~~FfjjI6 zUVw}(M%n*h_n{YogZI%Ff_{)i0GoIaat2;mgt_J6i83id*G}vucN|hDr5A=HeGE0i zPR~In9d$nbI319oM6#Y~p0P*ZP3?h3;gSeV*aZ|6qUaNdU-)KIh8tRmB~CedNy3E@ zZAm7VCv4Fnac3d;AeEzOC|`Z>Rd{BW1mxErf)`e4A&er9sbm~P6`%fX^dX3XXR|7d8FyF<#`%iy`YLZDv_yi%I~Axb%|}c0^2$3iMkH_ ztFLS2S}=M62L|D<&T_~F#2`i{F;ZBT`C5u9I+k#=w#i_m z{7MR@gXFn;bIvaZeAK7*-iz;4`u>*ZxYt(Zt*finY1zRq2U}jWPcX{o!>Cz&Fl(1B z7~p|`9uYB5#ilo3u1OL+Y}euSO7+y=DgqOgQVaBwcf&CqTE`7T`sk%cH0yhI%0v8rObED!=sHp!{5$fC)ErrDL3wbgKp;Q?jsMY z&s3sd?S6F=f%&3jQpRvJ#=Imt;&ZW-wV>6o{_t*PcK2ZY?fIAyoSjqf3s z`cfEZM!Y99(1D6u;-MB&IRySIdjg!@@C3y{AO;a(7!%*>W;eujHaWCWZ%c&>7ZKytJKY5^ic35KN+uXDx&UPeJegvT9+)q^4U&u{(w?!1 zL%1DAOOn#kp^KD6F#~0;ejOBya$+||H{yp@N!(V3c4EIbzS28hq~B%4b~|5jz?FDh z+M)OqMu#P^d$W|I);6d?D{6`mfq;>+9Qnwg2~Cm|lvk~KLo_B@@|H<-RjEc)Bvld$ zM!eM3-}00;U|q151C${kKKGAEK{i%!8L~GR4<~YBxTA?qIR;Quv-i<pY1zReFxA zUWaz3z6^SXw!*aDBninXw!lev@Ju95p#)l>I`OMf`{euXSI+-ZesQxo1!f+co1l;) zQjm28q_WTgMxY2vxBYQuuE`P z$25vgp8W&sx-zR)5OQgZI$4&R0GY+tey~e%45UzBbjj#m)p!mC?qRPJJt-mdj~k0^ zporSCtl}27PAchY_Sj>n`Lt-9y{Z^9JYM5KRQO8noJ81?wBB6?;6pIWS;R;D{1ZmUbQG5Bab#h2A)WikB7lJ zf!Cbk!lz^-Mk6O1x&=^dnw}AO(c1o)Utv~cfJfXZXh#3@u!#ky(x~;Pk1hH|33i+> zf!IF!l82*~?aWjs_f!=vPtr0?qZpBrq`3kT(b{WH>7BVow2U<1T z0B3owXscm`Gq^F9z)Pf=0m;eVfJa+8o$lfVyA3iL5}BusL()NTJmb}Zd*ba0+FZ_}#|FW@-r-eV%ZQr0)@3%9mho0^Cw`3l; zk3KPltU=G9iY(A5%D z3GdmKWN7pWYsgAivG4I>Pax#KIb}%0+M6=wu1*-yOsW2+as@V5=qq>Lv|p2Ux*zO8LzdIro|b`v=zNALT@o4D@} zZ=wx+2z7(xL0R7<_tSjy*b}Sk2-#RY-Cgf`Qj8ywPtP;MM~`JWdpf{jO+7+^-*V=( zxm@DE_42HVYWf!gECOA#H)-eOU5j;7*yR6soChC8qFzyQNT}ggrM6pQW^osI60Bi< z=T~?9WeT`ObY0?f@wYFu<`o~MLa5hg-X~)MCUq7_1Uc72yw_Rn#yno8Y*->VmGm7s zS8cZTg4ec99Hv}SvSU4@O&5rPG$&sk!E+smejiwOGKE{MvwvaNd2Y63dB=Bnr*TY& zOvuAxMJR)3w|>sjO|y1*nPhaJ246tPQdu~Bogso>#xr2pOQ?5(9tDBR$AooJf>Jm| z9=JGqLvgsbZIs1Pr^ZD|6n$>EW?P7GiWpT}=s$pnSzsq(OEWrQmTd+Wgqo;?X2?3r z6=6vSRTVT}g(qhvBZ);(eo^RwhKTYQjevid0>ZXoGadJ}mcK97cp#)_&-=YhW^qwAgdt_zO#g2)D>&&*gP!h;nEK zdl82of0%~i^nbARdr)VGthR~vh>OyMv}CsyUtYJ46xan)$3i(1co3*i z$(Do2c0AuGj$)7<&K%i}ARJ307#t7m9{7KCMVwn+K0#wOQtfRSq$R`pAdg zc5&Sp86C-vGRcuUV|%Y6e*5QIm^Voo7#-h8sFKWfN~V^J!)QpvsC!zs ze;Lt^R2hTmhEO>f9+rlRqlW*Ei|CTQ*o3Xd6Le^Q{D)qM*ioPei)R9aJV6v*DSsnC zl*Ukzd1+X28IuPomCi?%W4A@`v2ZEHj~R$2*&=;j_)j`hff%WbWF&DfIfi^WPi1+Q zY*?4URFFd?Sx&ij2i1zZl|tlHhKQM8MM-9QIW&-IjdhkKC9{J%|B1wvjCw2w}n(>&K{+A;*c>^k#hN$^TOle(ASu%ljOcJS@dMS>HWsu0( zn=mPxSofE%h-H|SmY;c?z=@7}cV5ufU7T2qOGzj_*eOOmHgL$ zRCSlysh3c}lHS>reL4Sz`FMun8B`ewj7tZewy8cZ7=$>JDnGVxJ9$ffS)V{~nl(3R z*H~)z)Rb~4k=b~F<5zLScY4|YqXR0Vd08EwKnf6aqeM~&k-(!nS_e962#1)WV^{|? zDSO@4XWes?mA8&z*>s3BX!>bq1~;OY37CF&bpd&w*A;-nXCKsg82-qiN4lK-R~~9` zddFd$Gs>oT>89H$3_ud6Fa`^!fTO06qj+kgKFX&yiV1%Tq(NE;KYEBPi5x`=qUJc8 zWEzo-ntGy^Wq62xyxE|m2rD6Hsa=|&<;kC@Ws}SWH}2AS2njx;*QTdRpmQ2rh*hU| z>Z*2{r>`2TeoFtVJo*WJI;cNNsCBTbJvyj(nya~*sR&A)6Y8jtNurclmyJ4!GKg=* zdZhG;Uw}p_Vft^SYA#$5qua`@sG6OvH&L=GuCPk3wMwhwI;-h=uIcKdd+M&{8mR4h ztM0m^zACKJifp&2ugDsy#)^X6xNCAbtO6^Y4k~!-W{+YDh~4U~Z~Co17p@W8s--Zo z=xVVSi?QmeryHxW8T+p03bORdr@eZsB&(zN+M|bxuk;zLmV&5NimxiWSkg1HC_AkO zN<#QXu=dEX4g0XF3bd?BrxKg7MQgN2i?kNIt{mI3OWU;b>a_Fpj4%vRvD<-}$-f~x^Anvn|8aM%emvoyPoU2Q8>D1 z8@!}zy2Y!us=K&xTf49exwA{V&fC1u>$;h{yVaY!zl*!r`#c1y3&QKYsk&6gi@J!b zxYE14=6kv5tG?;GzU;fc*9*V8tG)BvyWz;a-aEX8o4VpVzT}&{=i9yj9KZwYzRoMa z)SLgk14qC0%fJqMzxeCG-^;(Jd${}CdKK)y2)wx1z`h)Oz#pu>AS@LgY``KMzYQ$E z(t*MY9KkJ2pb(tD6Ks^q(7(@S!!@kIxr?JDT)uMpzb2f+BfP^U9Kj-?yvcf5$cBu_pe)LYY{e-&$CT{Id5p@cyvmZC%G;aCvAqAp zfec)0Jj-{PzulnAyDYezOv?7_#-EG4rR>VAT*<a?9Ipg%|Fx3*6hOMOUph3&28MX(j2_z;Lg3A&hd=R<~+{Myv*H< z&-F~t`rOI4Y|F+A!|nXe1O3cWD9`YG&iqWy`5ev+ZO_-d(EI!;2@TQye9-I6&IEnY z0L{xAttC_8&D6ma z%!573VI9|oE!cnU(p1gT7@gNuebODs)`@+#gpJNK9ga^y*ED?CmfYELt=JpQ*pEHh zKyBJ8ZPkDs*;5R()tt5}ZQkdN z!P<=t>doETP1($?n&KVR4m;kvec$i=-SF++^4;J64dC(Z-_=du)1Cj{_|2_)-O}Za z-1}|a?ET)s-QMUu;D=q|1YY1Wdf*#g-vLhH4u0Vf9^oMl;&7eXtJmSqo#7G8;iNs{ zAI{(wPU17p-|PM0B_8A8jo~X!%TUeZ7W1hH7e%D*>(=LwZd!FY=Ugn41=Tf}rY7Xd-F6n39um^7B zlWvibp6GV|=bXOjQZDFr4c$I&>18g{0{-cZ-sv!o=&kSjgx=+@PUoo}*jLW# zs{ZP`ZsN5*=qw)RT^{TTuIaRn>dXD;jPB{k&g;jn?92|*ln(4IKJ2|6?bi+cTls=me8t?2Vk@1|?-31990e(TH)@T-356aMTI|M0fX<`+-xkG=2h-s|!16cXR; z6~FPK?&T=1@X%fEZO-v4&+^=E)43k;Eg$g9F7n2X@doa#IDhgbKk_n9@hq?HG~e^t zPU<=jyc(|ZNWaqZF5v|4^AI2OA#d;uAKpY?^g3?KHP6BHQg88Gzw7C4@;v|2Nk8de zpY`tj@jt)yP(So-Z|YTV^=4nVX@Ap6?e;V8^=u#adoS(>U+{CU^-nMOd2jQ4-}i@) z_+3BhknHhz|MDF_`0#D`;U4uupZHJC_}_fzjt}{I-uP=@Sd~rngiraEujB~s@{ix} zk>B-NKcJ&e`g#xgT8{adullYZ`%N$VFhBKb-}$p&`k_7RxX=2<|M@Ym=DeT#mjCr) z5BJ@g`owSd$G`l)efP#c`3fHp03rDV1rz}P04x9i002M%Kmh;<{{Zy~9Jr65!Gi@8 zBFx8dp+kibBl4R_v7*0=7Bgzxh_R!`j~pe21S#@i!;>iMsl=DkB+HjDW73;Rv!zR! zICI+Esk5ignm2Lg3Zddt=qS7&&pkkwys&ScJJz?%QtRXwSEByo}0IDU&4kHEB?z>@ZiUeAyb7% zxiaF#m@{kMd--tV&yqby4h>psyVI6YcU~=evrD|JW0Rgens(~jHC4-Y&AT`3-K}lc zW;}eb?%}6>16RJhxp3sqp+_&yob&77)v-I5Eq!`v?|!Xc{}-=4dwKK0(}Q3Cn7sS< z@VCcj-(5ZZ@AC4?M}MCGe*50Nvg#EDCVuU54*qMGZ)@UP&$Ej%HjWCkfql+u@sN;~e z;YOrwKqmR(k4p9?865v>bc{Bfku@mo{SE9WufRb+U277Q7WjU zk3y>Hp@`C`sHK;B+NYzN8air`bc%}Us;6dJs;iu`|N7^we#+V^siwXfDy_WU>MO2K zZkg+Uzp831sJ0%fY_h=WDy)HU^_r@*%NA>Gvj7eYEVZcCifpwia*OS_u=-f3xafj= z?X}us7SBBIe%EeR-+orFx9O_OYPtOO3opI^U_hV0_tF@!zWEaT>%4IOtMJ19+BdLj z4;w7;#I!lvaCPxA9Bans7VI&^AA8)M#TS>%F`yu$8nLJ;uT1f52$LK$j-$04vdz=F z{4#Yj%k1-dINuz!(6{#dtkEQ%tTdHG+c|WoNFS|n)Kgc@?9NRCysybWTkZAIUlR*< z*6Wtdb=YW2o%YykJIc1!Siddz)n~Uox7~Q>eYf6w_x~L--CNhq_cwzVZusG3Ck{8@ zV{wD{(n&TzclC$IW=>sfX=N>#Vnay1Gw0 zJv-{EU;TOPulF9W@1z3{JMY8aZv64c*PcA?e-l5o@bCH_j`Y-5Z;tiYU!MK)#d9ya z_u!v?4r=3vU%vV1qaSS{ozx(uuf=MXw&#!;}@!RkJF7CshH|jS%`q8g< zy2ze33V6NkHSl~8OkVYV=fL!d&w>$rU3egLdQ{r}!~zaOq*i1}M!5DCb@1P;)NN`&J0 zq^Q6RYVe9!WMCG_r@bv&k$Ye4pz=QWwpK)_Z<=2!=kov4txHLK<#3Lq5WBkcHeKAWK*YLKZ@YK6D`w4e>)A9&vh+0Adm+`M)SO z(UYJI&0jekrgGV%CEWwLONJEUa^1zAf)G%}5O^d%;67|r|jFp`h-W)U;lKOr8moJRbG zHqOaTW0(?^@EoNQGpNN>uCkT6tDG#(X#a*-wvm{@jHL~hd4xE=!Jln-;Xj92$7Z(S zmk^}_7Zxf^S*8$k+$_SF$QKLbyDo4R8$B6c_p%ry2L{-YxwH~t%KzQm)Qz}EZhLo%) zRpU2Vfm3l>Gm@a_k5Aniz7U#3B}RsAJzPkT#-=JvH=H7;F6 z3QivO)r5x4q$YzAidb9{fYa-u7Q@#u?pfBe0S!bR!28$BmK3?LwdFy@tK83~7po|Z zFD=>I+SlGuv}4e!G!4p2-|qIfX&tUwZ;RhJ)Yq2A46i>S+EJ!q?10+(7=9 znR0b0N_AL>=oa>-(`6vaZ0CuVww4Rx%|j7Zd{Ea)mA)F@0!z&+UjuhRvo0nra$`*4 z9TwQeW|goaSu5NculBvYoo0LEn_LqDbif#9s*m%k$3LJq%KG)Hd!>nA*UlBgXjQ3} zZ<*mhS`^C>6)q)LfeEbi>i>yKHnJ9=fCSF=)2;%}agAAwQX913#uTn7a~%w1HQb;G zg$^>2Rr}S4(m1qUg)yOV>|`8En#pnn>4g7#Wb(3irGM4y6SV43w4R{I_|e^-^9vIuG*W^5=l+5gz-T6Koek<SlovK)~9qcgt_^#`& zx3%S}I`QdZ61;?Ow|D($ZZCPfxekQ9Dedb!Sb5IuhPa*?%GVtumDDHTz(~uATmszt z$-xD5RJYAlt>(3$#s7TjFj-7zVS`!ZBL_LFF)i*+mpQ-&e>W9KJ!sTYdgrP>^D&!d z434Xq;rq?CSR)?u3KLn}ZR~Q>3;yIVQ}o(NKk=|7o@MZU`_-Ox`gWTa@_pVY?ZAi0hFkm%EHCh1@cL$vpc`KJ?$3_Nh#|0t?SSR;^6{l1uV1HD_f6As> zG$wUu$9iF>Y{QpX1}Jth$Vdj}YdfHF$n=CZ7=CKNGGep}&i8Opc!a{oZj5JqpQTX% zcyb4*by?Mf=ayk3Fn-L|WR7=(sh4r*r*N&df~Xe(Qr2JQ2Z-J$e`PggT4;uE7;B5S zRcr`j0smG|iMVhVD02w+htT&`(S&=$c5Nd#TDF5eUg&TwD05M$S$gPoI9G#xm}X;^ zgE|LLaTo?{7-)D1Y8)7T!)JMOM}zBke1Dg9#59W4S9|t`gQa+W&8BUMIElHKfbUg& z2Uv&Lr(7X*N3`c|JvecJ)?N1mXUL3>l6x4cadjj zl?f?|WXXg8X;KuqV&n*Bj+vOThkiT>kxdt2Z@>uJSAAAinXDCu-Y0osxs$xCjpi4eN(YvK_>>Arl-8##-3UQu1eileY!kJP690#E zw@8iW7+Qr%m?oEntyy}-`HEeYntE`MS&5XRsC~P+Y;|dXWl4Fp*qbAFi9VQwoClGM z=YVlZk2pY)!AF{92%ONVlNhO;+c`z7@JOl%iMD8(*a)C&)QnF#kB&(K{t0X7$dV`d zgz2?$BDaVlDueE)ZBHqYcZikld1}a+mDpK|>eydXwus9}Oeb)MX_cVrh>{1li(fg7 zE(voQ`j!#Kh1z9L2y~%%HlBYuOL^6ab(xWMH;6~7CAx4=1qsgxk~5}>p4Vav zD35mco6P2)zxix$$BrghpI3L9;rR!OP@9d2k5DRijQ5$N*qp>?fn-RWB>(!C`q7b8 zYNfQNs51JFFUqESxO#$#i`+M+64{9)I;6BYg1+~q%9*LH6_`Qrc~Z%i{1=*exO>TG zc?asDPAXyKR+2KNcy>8{SXGJgcaJj|f557iadrfi0jc)XovjCR7D<^;+KfwPrly*r zw90dghGw0onzeGb)QSOiY$hEaS4WhC!eKtV2dW0>IbOo z_L)msjE@P9Ug~#Yz(vZMMef8;9|)n68J%+}txD;OE4iQ7`ef~>i8jiNJ*lF_N|@T( zZgeVR-`aRTD4=c{vc|TWPMUMLrl*)nZJ^1Xs%K-KNu+O@Yj+uW(f=l}U$k8aRHZaD zoz%Ie;tGvn8h-K#tX0=*q=&7wsf%sEvfT=kAsRz!r~(8KukwnC;Tovb3Vpsxr=q4+ ziZrCY*_n;^cB6-|{<@MHXN>UJnJ5dM;WoAJNwO!pvBiY5ni*6Cdu#mawojR2Bxr~5 ziKpmzV;?D?>}j!_Dze{buD%+y;M!;{Ah!L-qRE+7Xo|RVJGnV%f=!F3UU_!AdTCCz zxeE%nu34s&JAi-*n-4l_x_O-538l|D26&sdI=izCWK@ysw^@pXI=WkB3T>*IaCrE7 z0`;O*hjAxVnwYv7Ojjq5ryNy=rQp+n_JEnhKhe z9~-kNO06R~OlvZ*7K)(=R%Q>Iw4xT5K9JhYhBtKWkTDx`A_;n#;?!G`WY; zE49^11Z+yV`Z|OkTd^iw}co ztPYEs!gqa8SBCAX_N+>dD01$Eh@M% z>b|pUzxYa@cEzVsI;9;c3P)wD(uAC4`nyevVHy~Gxv0D!NyX)Brmv}k8OOnDb*hEC zy+Hh$T??VhD3DnSz!>aprhKB5yHJzrzgk;)uU5$ISxj!MZ@Z?aalFGL8k}^{sRL!K?0b~;cm!Gpn$9WGT>opTC(OVV$bdQcp=x*0*5|tXIkGcZ zd$9!3d%H+X#?L5c$CjFpiB@o!TDm;C1?v2gVQ8EmY^fYDEhp(!J<{ z3n-Lh<;boIySBW*hWx|JEVOGohmwHW8UI+RGaRGbt7t?@hDO`EQB8gWtpy?Qp19Sx z1?+n5xx=iSbAX-ASp*8ls@c7xEvv1(E&0XPTm(2Zpsz>+M>K(Dsz1}zNop9ZHz#Mw|3WM{<(?9(Nx{cpDE2&B< zaEYtZVTR$G9C*h!)COtYQi+7M$iyo>%;cjng;3cP3+;FfNa`>f=A7~26oeVF^B3Qf{m%$x1prq^l)Qa!Q3Ri^^& zn+0yV`n$@vPM7zZyaD;=af+q9?&O}TV9YDwDlWOJnVGno-}~Lid8d{n{+OOSn^7G{ ze9n{?xtC!L!clB>fm_HUHHN^9;W?SQTK=u+wSWlRFLdV8vi)lofWDV$IEa;n>zZc=F6xW4}NtX&Pi(rSIf%x3UNfQ zYY-o)*JPQ>UW3GK>ili!p^cH`N!`U+(ra|jQyufTmvqd{(N>93L5agFFVKy8T5w$+PO1QwW5|Jg(yWmR0@s30qf7pGra0Qrs%J-Yabp%$E9?%U-+Dv1x#A z9pF+u)N@#Xu>A7DuAe}vv|WDEk1yxUsmutqs+M;6u05WtzUSJ$kQR%Wo5+54-=j$W z_n}@uR~ytW+_$j5dkyWt_O{CVHqXb+&~IA9T91c;*{E`>?_Ec=w2ICQf9mc`&R4AE ziQW0|nx^f|hwO^S+W%j2oqzNqjM=WO$j_^XVOI1p8soY<`?PPuCVhQj}To$jBRH`uu^IvL;HFEyiTov2qKNHv=6~k%;sTQ<_?0 zcI2?74N$03XWk^~bH`MqB9s2qd1v4sMS(P)+%hHn1ASyz;6-&P}_n^z87k zL=rBcFhERH=>KZ4k8jYncy#1rv7pBg5Y4(^)S+3DU-hjxwp-CYpZ28KTskC1t{pQ~ zlZU%W-~zvgHCr<(&K(w~8)a-0yw*63){aNnkzmd^boK7f+rRIB`P$QtqUgFy?y}4p z3a==kdK>K_=Zc%iAjbj}C^0?M((SoBsv5&M2z8?(H1@c%syUTxGx0VC9pXr>K%`^w zCIiFUFuKiNTPiKzCOYxH2F=6pLxb>RYrzSN;?2J9cB*YY5eb6suqd&bgFYWm@-7CN zK!cKucK8$XKQe#Ou*Maak}@mIZbWa$0y#sIjOyIv>bx9%QiLcH!yvIpgqj>mypnoc zjJW}cQ2+6SKI3Si$iWPhZN{$VRFjD6RNKzb`ovpv(c>EGR8aUJ+s;HSnd@>(l@6tX z)Au|qPrx@?J&?ms{d^C)F?%KRFEdF!ti0{8S?{YkOABV0ZhJmJ1j&tQll*5#C0RYf&tWLsVk>u0;x71lS;9$7g$Jsi1bd~I^~x8E$1 zx$0U`D0Wc?&G0zeTwQ+o#KCWs@SyI}t!%(+4NJ9Fx~rTLw{|s#3$DnOw)!P-Yui=L zn!@(4Ol?{ZWsa|hW}8?n?2B1l&&yH+SK%_<(C?tv5G@Jp6B1X#zyC$if3)zP7o1kRzqM>PLj+e0N3%hwX~9A${21PZ5+g9|&11qt zf#@Ju1cg{mcEgH@i2?zM3Fhu!)#@S(=L5Q*mCIekL)r0|W}U(nE{yq;+Zq9h$QG?F zc8yfy4fc0QU}Z8JQUpu@y+DUDgc1#M7^NtU5Cu}oN0n(|C5&46N>x%~e3Ovnbd)g4 z9?(FSpuB<=eVI!OOkoC7Fas}hX~kh8QJCs$BNiD{r)Q!7i(XvjFwG)6IscC9AT$}D zl|n!q+<5_VA34D?$q1A-N>i7~M1kef*~~~0vzqgiL+ySUPiO9Op75-pSI$|VM@AEY z859e^sDMdLCUldWL`E)(Tein71eOZI@s}1bcEESA_XZ)OWIMC zvLh9wOac!gdIp%rbPsr-X-yY#Q<~!R5Hr>38cx}SB?MKd5)CQ~T6)Se#334VYEuP&9TKkX_S_7DZ7MzscMg{WA^8db1<6{&44 zWnAeB*P$M@2{pYdME&|#zy?;Zc?~R3OK{915>ay*q7{^`$s~j-l>ea%ZKyBedQpsK z7NZ$p!zgfJkCB>Gw4)_$GOpT)m%@Mvt91e#*zj6Dz?Kh0d)rLpHn&eLEKyVI zRNn&ErNec_J&b!?ot=SG;7fYY!<#ukOW6T8(RF9HoqvP1~rV!2OY>az5|AD20)+#x$^X> zyks7O?`cYWMhe0cPGyBHe9KVwQo|VDusu6$PI)5sm?HM@K9eb4X0G#r5B8;mT}&|t z7gLlkWUyaZJBU-76L=R(Z2D-JsQRc^vg`%$dkuVK0@K&N`~N*Dlb!sIdu#&^0qy{S ztz6|7EMm8)Fm6Zj$f?A#lPW7SahT27F+9x-iy$QP&_J}fH(#KaZYHOPeioc4ID!y>;It45%vnu< z`UHBKJUxr;!CIVKU8Ky|}giIf7I1w7)nwa>SkcI(D zA%pmImqgGt115k6ETRzE4EVtw&OB|H>N8XZ2mu_{5MtvPfCi{d7_iCx$&D2I+|vGL z(Bv>vQ9d&Q2Y~k)+;C__AC{eAhGFiUX9pqwh0;+(h5slp4G2OQA`L?rB@b2|UnP&) z)DurfGdd~?Rkxbe4LC3*UV#megEq>Tm>9PJg@+ZiDLcq61IlR>BjeCC1@<$H2>?9v zF2H;t)NaHT4qIzvyB3%#vsG-@`KM%``$R!k`U)vj+ZTAaW<{c{3y@@xA5q z0L57~o&YQ-6?UJ%MwpDNk5UvO4bcK;%YU#fJM6scE+-)wcK&Q-Q(hq}xcO(yfNUw0 zzGy8my0&+&gr*ZDj%y%yr{ZV;JT%>g1bDXYZ~w3L(8bXRvy}paDMWxX5MRbK&^+;J z$_P%s)(NV`fTDY>taCZ7%~688z@^9uKUmZa2scFVlI;c-47dt?|2yCV|50ynqZzV5 z`IR4lc*IL|5208E$myYv$HNB6OM|qC0pXi6%+tIp;5?jRJqC1?IQjqtG`%=NKGh42 z+k35{YXh$1IVU)}bmKjVocLs7#Op@w2-+K(@p{zrCw3RBONYGqw1$xTG2|{L4ZII50dI zhj#Eke+a-&sDnCC3u2><2joB##5s+~K>uOei8J(o3lyg{bUg=T!@5yHHqbX|GeHup z7lk+g+GD|DdK4IZ!4b&1rt3f*h{1$tiR9x0u$v&mm@P?cAxn4!KsdV|>;X>+vo5(5 zF`}IK0FJK+I8Kr*PS86D7(Yu$116jSC*-&!U_U8zzx;|nQ3!;fvNA1nfel!J6_^CT zXuL4I2YL`d$tyr$OT*4EfDCg9I^c=Vdn!Kkjsq-%5)h4~**Pni!{7TrJA}s8i-``* zy+2&Kg~&bM3kl`p84gfHDFC;2^Ew)ou~un;NK`y_`-mG9rcK+#)#}7X%QGjzpr;v< z7J8yeOGP6@!d65$X8RVXIsk(Br;qxg!_>3LQx0CTRA6v0J@t}cg$nRSN@gu+BL4xqoy9HRa_4|e>#K?>U zt&Ie$D)Y!zi>XBLuQ2I`$n%pkX)ql#A6_e-5Zf6(5j1{Mkrengc(RfjScAqqu^GrQ zZL6l2#gEd*8OaBSbLc3(OC7^_Ns6yM6F|=q-C)uU!a!yh@xf+r|q1I8jD7G%FgiIPVp2?^E6KyBc&Nr&uK)O^}Mi_!$eKwK^X8k^MjL8Va+}w zf(8mpv81f6%B)NygfCzLhC8J#d^kejgx-wHij%)^=*8ijP+$B=EL*_-qJ&S%Fq{KH z5j#54oF@_+Pw8|{5-m;b*&J4`Y>@W&LCG(U7eCxzI zOBp)C3^Qz+yCD$&^gh`P&`-;>irhPkWY7k6GSXUyD)a{7Oi&8d(z^sK1VgY7pnwZF zfH9o_Gc8kYL(?)1f&VmZ(=zS4)8d4JE4Y;!(@Vfm?2tPP*wa1LH$Gic8U53V5L7oM z)NPDRLhDl(T_-=KQAaJzdkRsq`qCv>u$JRA)yh;LHHx4Sq_5PSD&iPMN>bVMpH=)c z2!K*m%>Xfw(kV>`D#fpOs5mV3fdgC1TXg{|xK&*3)m)`i4e$UdSOEk$Q!+hPWKC8= zEmUq>gRH_TIo*U*SOjjnQ!JQPNf-o8P*ZO8Q=V8>aP8JIZPZ|5)IS~9LrsCT8qcGtB{GO;$18MrLJJKcK2v;3$@@g-tNi zSuiVHkOXN>1#WdyWBmg(IM;9$S7eRYqQF8{THQ9rQVkd<1gr7xdDoX=UFaH+Me{0Oi8TFNR0R`><8Wr9xN&2p%!vTemo z)5Q83g@$!lxsBMkm4#NA+lgJ+E1kc3+Stdu(RV|u1jPef5VdPqf{%P$Gkt_w0EbTC z)+NX)#f98rt=U1i1`$}`E`(V#IKo4?hME;b1ipr6ZCxXf;A^-9ljQ^r#)H;9Rtu)8 zd_~p<#)V8+U}MdM4=!C-U|D6$-6(k3-pw&hfZ@7ArR@D*+BDwdMZba)RZ-Lpve-(I zk=};1JF{)o?Ty0jjW|INU$Rdx@r24?gk6-58VFryH5hy1 zvj1%@Tm>M#(@iQiE&%>r0cX7_+%;inT?77wTr@abKWNawwQg199FEiPEd%}rGJ22S__&)SCg z*xq7FhHLoZa+n04eO$RZ9U5vlHU`Fyjj7@rRtoTgP8ft)0O69g<0CxQVyytl_2iSC zDyqV+8XuT*aD!kI8N#TSOahv;gY?8MxZKipjJ74=L#lP+#BKd zP2qQ=)+x9t8ZdxL&gvJ&*<|emT=-vgp5wH(T#~J73T|0S9)KSp)2zA%l-pZ}hUkb^ ztBR)7Ql3;`#psNlC^i7!ig_FP$w#bhlg;py=#6YjQ{n-QV#^kUlaA>OFxXXF03mn; zhMnrUo$4DVZNJlmy!1PU*6S$9ULWiTNk~qkuIi%(XN$JzhFjc6xBzRcWdCPvXE6=r zu1?%MMZ;$1+YoL<5tu15)$2$2HsE${(ky7WPCc7j1ikLxZ+q^Bo@j~2ZDIWD@c!$( zdVz?-XxGX#5UPqab33^=jlvNi*LTsal$W2I*~bp^8uCX)jv zlMPdYe&>^I(|_J$+Wv65)@Fx}zeX!q(K)x~cl9<{aBVS6E3RVkmZU0mVRmwnZFa>9F zKJzwjasG8y30~`i{@+>%Q?kC(Ba{O<)6mLG-54MA7jI;Drt#{|?gE}|2WM^X=5bU4 zav%3wdJV22w*teiPCBFI$6gO4zZ&P2>|IV@DW7swLt7yb-(6Tx?$z=*=UZ6UaajNF zDx2Y1FfTp0Dx=<20H}bDy>Jg;bl>hdr3UfDb?80#U~OHgY~|oS*eEkNbUB~#LXUA5 zr*=iZc1MqO@Fwp_pLB4S^yqpnOwaTQtTSf%#FyK#eT8f%pR7@c;{2X+W=8e8!-iA` z<`mx2UW|?hhbq4VK_h8eZNu z=O~!=iO+6nkNB#_@M{yobND!c2vzB_rJPQMrMshM}5EK4|P zLhuWa69jA!1fq{=U+x7$5Q7Cwgqwr-!WAk3htg5CU?! zrta0P|N76DW4AB(q<;HuKY4DS{nrn6iMRc*XZ_ZXd;hw&=yOhmElzO1HvH<0&tu9~ zRRTYrIHu+_d{L5J=$F<_uwCN6S*^m>oHCmdVI3( zFal~v4Ay#E377!a#{}LQ5Mh zMBGLgal$SKB5LBWrSe3D3U7i36E^nClAuqq!vAG!?D%o9QG{4f7Gn9bm|fq#joRGH zl@K>s*Cg|D1Y3616mNmRRo28jj`UuAJf#C(H=*oD5DsmeYQ(0o`FM1FYDaXi-Ue_l2bLfG%=BQgUlwJ1PFNq zRCH3TCODu67 zb}Q9DS_&d~=K~r>^}z@uWpzc3fAl4~XgT`v_vnC~49N|py>K#GAY@567Y-iUu<3`K z#vs=X6T%hOh^aP%jfp1~QVNU6OmhsalHv4?fB=5u#t@jr5n7F%ts_ewnW9onk5w*^ zfh4sQ$wNiM)$j;>MnrM!A|vsJA(rG_cbsjoHAE)4;BF8tatNfGCvxbX>+ZYpDwigl z+Px&H2zwaG(iw~RtDBM?nWx(fd)?8-dk8ALFvE=6^3zg^h*FKN0J*23pzd9|G5qf(3mb%QEAQQ$xucSDGw&9D~Z?2j&x3 z%F81%HZg_Ac9@zM8AotIh8}&C?BVHLeA?Gr0m`l`%%dt6tIann0!WXXc~)Z+V#FvA zX-x=K9t~#vfzu;7$lbu0=OKOg6lL(KHNjqYkw-0(#H4q)Xop|sopxI*x49IAw@4Bq z%`GZT|!wWAz|p8MI{pI;6s7<51KA9p*S$K&*j5af&1~SE)v9 zB?;x(P#10xGHzw6EPJ^STQ=yH5$39dxoU?XMD?m}tjHYaFih@n=7?V4$TPzdf*^XA zt7`ZmXsQ7k5cSujF^Q>9%Ij0XjDRO3(x8b>j6*6~1Gj_7>uf_LS0lLL2`j*aYB&L( z_+rxt2EqX*U<_lET;n`EfM-u`i~?{_!xKJr#ENG)gAz%?wygCjH3RfYr4r~kMl}jj zhI~{rIu{;Ik$?msL|I?%&_R(^=r1EIoe532ixlEzg^dwWVK&o_&DdjSo@qsAlCg## zvT_1|%HtKqw2&xVO#hWeP){e8Se*kH-~xll0K8ZYG>-28IZ%%Y z-1k1{sqG=gyaLm%0SO*fqb5<=1h!_eO-L|}ODRY~88y%dtepXvYwFThk|!B!xccL7jBI~lA>E(5hC}xkKKg`nJnQZi7*5dsp1we)XaWnm(Tc^ zu#~$!#J6TV*=w_X7l>`rLT!Bq>P#bC*!~|jJLp*`z zg<+=eIH;5+`qJh~^5kJPY6ue23|g$ILbXtkE6+|UfQw6*AaK|F4m`u^N;%vrkoVLl zS{sLo@ca`C8vj(GKv6dbOOotLA!ER(OsAoTp72DlDrK$!tI@#@He=`}GqolQ zc)l%eyj-eTb^uAq@wT_x`vNR4u_bW0BCKF_6VL!D$g`$39r#q>SkSheetY6!amy$fFj)QU9JAu%{w#VBN9zypbcy@f>JjS%%P^@KD%C5432|lK=I}_XtU~FMj)@?Enb~U*8hc)IyAiFB`+pC zfe1oWLW1mlihpeZ;AAwA%Ji+T(q!k8pO|M54Fc#@l9-h90I+(03ntp4SAp`~ZEEX* zXC}m1VHb~W&7)b{{5;ScUi=}qz9n&q0UBr?zO#zQJrnhcvRnl&y2dpY5Q{Cy)VLsY zu0qZ$TvbrAA`^BL^}5gn|Gd)ec_$6AWO z6F^HZ6P|Dy45ArdW}qa>Zn<$pO^M#e2NA1Nsh*Go0@-Bql00hp zj1riAn+nQdOC*|49|n-)*B<+Q2WZ4S(2deXBpw&w$U!G^mCMNE4;La3Eu@vO!~Z+g zBJ;ycL5Zn=FiqXupi4pV`MOhEc&P;FHBc<0iQZ_VAKgEjfCOnSm-0JN?m~chdXQBX z=~q2K0EyYx7#WT zukDr!NaLi2-jL5W@12}+Q2_DIO8g#v^bS5yfJ#m9znvxCzrfk+wh2?8r9SpZT#s3W&=vwtC zo42*e8BE!Iu*2VtNHREw#}xqO0mvip0iYMb9N zkb3Y$cp?6P!y`Zh{=kXjEy&Q|&wxyT97cg1T0~3zna%AX zAEpkaRp1|7;16t-g!Nn@P9e}R!s65(Bequ~F4s5lpg-}6Zf(e2)c?g>xP|X!;^e?i zUTh*skxbVuA61NEDf-|H9tePZne_otX7JgqsfHZDNAs9V7e*5hRYzx0z)aADF4EVP zDatCPMjClZ|2^IWC}aK=BL+|j9Ss_CDV0_?=ZsY+A*qX&mf0fH5rq&sA0}VgF5mI9?gy0v>JE;cdwt zxXnaRi|RlU>3Iv;Jb(v94xmxNdoW#C;8EHvgfU@-dFaUmsG}$(S)qW8Az&d^n$@`J zQ>!VG%i)|q^4JGi-6fSJk=Yv{(O_GGqQp>)sU758irrKI$@OsndJ*KFoG^1!otXx7G~u;o*UBv)LGi&SYBrYCKMQq zr5cE*S8N@Uy=4x{C61P&1NB)%a@Tq`Qby`sw}sZF?Ek^J_`z0b1sg=EWeL}Q_R$}t zLI~)UU9|vC4k&7QoX-r~foUdJf)i&|*f<8luJvANNT`H@!6UgSioL=rfo3XTpuZeK z_~_If07Va2$5e{wRCxm?_-H$7C0em4(^b+7`WTN;vn>zUJe|w7?07M}nryIgU$7s?#EjXid#rZ;=JEN@xdI0SHxS9=KZ` z(CI1`NFyjw+=a<}vRe9a2XRUTBCw^2CeXKnD?Dw?6*Yrgiud5-E&QCfPAUV9qYG`T>W$y^n1&&&nRawxNKKOae$qfhPQmnhYmZ zRwXOQUW&S-&Wfvj2(&VK3dud?5|f8kd6-IreZNG+sD^+QwcbafBMB2|M79q5m{ZB~`!*SQ)U7*r@A$gz|kD^>0Fxy_CN9RqmBMIz{664SX5E+i;e@FEOu!mv1ZNXbsf z$=*p=tZbKqvzPiKc&TA~wBrEfv zDHbsVrA%asL@Vv8Hh%!U2-V%f4;#ec4Jw}@Y-;2x%`wL9Hh*583Jq|2iLWNFdrrU} ztnQ!Prst0AZ&1S8UQGTJ;w-@!C)0C1ZzXqNkMv8o-BQqg+25SD`HRobaMqr`UWko zI_h1ms~+s*!K5WCum8s@FCWx(@j=F;Q-9>yeX|gKHRNm)IhbQM-?ckeQ?q`rq0UJf4)Y)U!UK&$}ijS=~I8BPD2_$Gw zbmj7B^a$*n#Pe3u1>P?eUnW3A!wg`AZoHdy)%t8~%ErZ83O#zOi;Gl3{ zv~e$+Gi_rRhu$ME_Ypy!8)9H5P^6uj)D;x(cw9G&YxjyS=AEY763}>ArssFH;K}eK z@dY8OeYU37_w#)-Q`GYQ8gZ62&m#OmnI#WmT>=}Y^a?ESh0<=S>e3G&jUul=fOmOn zksn!~xd69W#R}#a32Ei5xpLp;Ax!uRkYk*=z<|S1e)@zhNE}f%5uWF{o+E13Jz_qq zXQ{TUcrW$)vV{X-^~4}wrgk)@D)iZrT~&`Zk|&-dB%RlZ!Y)$GnHrwmk#hQ3x~1z> zhb75oVgGZgqI!dW`D;vos=Gupm3ah!UadnitxrgB4&s(9t1DE?)8rQ_pg~CmP&!VT z2)a13w+y`@u~v(>jK-Ue!|0$$JD+3q!*ELS6#~g88A#r?dh@7u@!#I&bkSEw$zC-X;(q|7To z-xOB{drWRcm>Hu~i#b3zI_7E$eb7U>uZ2s{9DZ1*^aY5=7DU{_Pd-vS6;)-*YGZ*U zXaAd0PF5!vd;yV$+i%}}*=l0)Q^YgQA?@rvqj=_`J;q;c2w|31(I4$(-bUb0zrkUiT|Ul zNuDZU@`MVlEl4^j1Q75ciN--C+V+_Gr^F?!3N*l-O6?w>wh}g7qqYrKj<<^wG-eE- z0D=lvD0{Jdd5WA(2Wxqd47q@ucN@V)c?=qKBRq>JoT>FXRi{ChTrC(788_|~B6j!w zkfVlg6T#Uo5ph7c4qUF5H`4Lw^|eJ&B;_=?Npoy8bkTr~i19o4isLVeFL&==HQ6Dj zkRbYmOq)6LX|Aj}VrG8*5p3ezxzi^*NW#x3SWYp}6sItG>W$f^(!@JDLbza{9<<7= zj0~ajVulX6%5b2;>dNeb+iGAhitP%(;5P_md8d)j61ilOWD|Oh65%*1IRoX(ihJ?{Us+x{jtk6`l%qtpZ46(4^`r(-`9+i8bGK0J)a2P!{ViQua=xdwZ5FuK=mF z_FCxX%1h(1P)YAwa=E-M-YfxdL2h%?9l(W7qLq%zsokb~1-4ejmm_}l{1;8^Q-gH zO52uXQn{HIMsTv-H$Na*BLy`6N`{-;_W-`gAM%JJSYOecSr1e?Nn!tiw|IEa7NrkJ z#IRScJ|b(aSq>FD3l>R66Twr=BZaa3?vgo4v?YuFSLE*!aB<0Y0Py_#t{zK!n}yWi zRkY%?sV)Z~oZBur>|jFKG5xKNVKSFsZ7E;t~YjA`sYdFDCEWkY6fK&fLXe#fb)aJe z3u=|XPJVJ#_7Wv;+E+?8%54clqKFX9fGeWWj#i;Tk|&zDC09goigBog5H|uoC}7b- zO28K$P%s76iLxi4YTuX6MXM5^5t_14g8Q&^8@O3XTym`2{@`W>bdfQ6;jC4@5E97p z5K@qRs+33wwJ;GXs&a~qRDL2Ai%Z^fh56LfAIQMTQ1bAfpk!5W>J`v{zR-`j^hQ=V z*_(vQWo&B$7;E}chG^Nt9l7j<+XVmPzURUBu*3%QGzrOOBA)>@ zN(FPUCqDIw$xL?2g(hf%Gr-)@zw zUG*x1me2&fY-1y{=(^bD&?qg6%ZfB(7j<b-KBp?j6Ge)6*tUtwt;(EwxrxQZP>K32>m&w~`XHB*mXz^wQMbG`z=2l~ z(eNxPgUHNb#&yZM;W#bq9fhJ|W;7FU{gRZiQ8f9<6@jk>0--o7zN8CVpszfa7|S{F zur18IQWTiOi20>)5GrfqK+8%OvtBbyK0Q~l=)A7!z-3K$jo})MXce>^m<)jkolj<0 zXpRPHBN$Hfp$q*88V%|_S)EwL@yX@&i;oN6t*3eFHF+cgbvK1iII z5lyMmn!hzlaODP^`itkO4k}0oKR8E$>xtQJk_{QvZW~?o zUZZBxr)NE{1}!aHw@EV*r{=DqOa^hq@jxOpDOg=-_Op0S_TW!Qy;G4lDQ?89ifJnJ zfzvZqZa!_u$rK6m%^?y2)OX8ei1GhZRV&a$E#^vFUxWPbvkm;PeWuoduy@N@i%O_w z(W~EJljW(AjsY~YS;JiO5X)(WD5$Zd3ZswnqU||2fo?zwur&pCrQT~Pa3!T+g;F|j-o2rDQDH2NLpEGFZfr#sycz0$GUO&+EXfT&34WrG9;9W2n4>6L7BoV=1}< z-zoNxcuv^{uvo+mGvokaRu8E3NCz6w_8f5b9uW6(j{+;u0zcvccMlNUY6kcta)3ZS zIH+Ri!&2_U-|9*E){gm5=VJdj0T-Y{fSM!rmVjLLh4J{(jurLZ}j-+^jJk;NQrDrsR`Ze%dW(-;;-~%DJOtNHq^p57!U#@ za7`o-4b#vH){qU`&<)v;>JsS$?7-594`Mdx1BDC)Q*Z_Quql$xY@#DtV&f8=r#D!| zg%YttPz8os%}%`U2gPsvv;q7uf|t;5IFQhz5Gzi44h|B-6*^(r_#m^WuS*2u#&iZZ zUeEx_#6*h3HGCiwt|7Cu&=vLOn~tP5tR+XRpcA}lXY6g52B8-7Xah*G49`#vZ4VAq z18Uw78llnmm@0%Q$5H>tPGNXUst82W_|Okm(1u`eIbx6&tqloPV}B1vXlVnyl)jM zv8Z6J6jp%EMLXiRuLR{;zsxaZ!V8a;BOX#rZiR#46mdQFR%&_t`HB) zseTK^J~N9NDJVmTD2q~LjsjmGfJIcQDXR(0Zf3pu;9|;ZDgw_RcHt_`?;gjG8@945 z!OI9OAQR7VEXgto;jam%PH%9b7zZFAr9&dIZ4eC;H^}4#aZyB?10_Y|IcG5~!NiXO z^D*|oAor+_NHG-6)AuHm4AbOjWDo++b24M{GBNWb_zC!g671k9CEm$1Pp~viQ)C)Y zHFKmjArIRO3<~Px5PBei%AhuP^EPu+Pci~iepBQG%2kTg~iSMN-jLpu|LZY6%+QOiZL8!m$6u$1scJv3QPrlDO8f9ivos0BAglz%OJDIsqM)FT&J{?*HaG7~ z(NsK|;X-BgA>Q;0>XazfaaZRQPt$Lp;4Wy$5=8I?SdaA|NvVhQlurQ_SsO6`2`Wsd zRnPyZm0ERR0<)D;?QPOb@KCJEC!q@9EXPw#lV}Q*5CS0v7~nm&v`bx+Hc)C7t3t^N z@5YQwLR+B}V%1jLbWORkU#&=Y7;zERF;#jM^$ahD>J6<1>gIBbqDrYpXG29OYwTNtiwbf!~ttdo9InZHm2SP_{=n~UG;L4~wuR!{$?UVmR z)RZ1}RsbwzOY{g0k0;DaS`Bwm9n9WJ-~`@d18KGiPA>#7wKFw{XEUuReo_r$jNpWJ zD-)!0RdcKy0cj0|D7Lh2sEg!+gU>Puavzjx2aiJi6?gyj9!Je{A|Z1S4+X@wSD7?; zpY`Up;%pFcGefqN_?9vncCEe;aGTd{?{=VaqB~f_%~mF01nM~MObIqN^=4%=5!bD5 z?)AtkW=__;>P>JPw@+6s0@yX!D1i+ql>)-HXE%sqaA0%)Ge2=a-~v>1F`>Ck*9}lN z-wcIysZ9}!w*^pdYWKuU{jOh$Y8Nbl6}*yzHKq`{wP3?`c-8MhfK@Xji*Wy0g?-CM zVJXXZ;5PLftq00i3ai(=xc3pfV5#EtVN>{@sFGv@cM63V>A)0r?N+~rE_)sACgHbQ z-Gcz0fDjZ5e>>1ngs%j}?o!wFw}R3azIcGWkq?cdKob}dh=5%gxN@m&D}+;FQO)7< z;2raIgJ<=EwU&3)3lUT;gFfWsvX2v+7|7hMSO>X~dn}HJ>}4>-y431~Z4QQ&0B0K1 z@RqoFr4=?ySoLNllHpFbSk{x1xH5)`sX*D3N7*MGVG`J2W|4FS0vM|#?Ps}|8+S6& zVgMMxSd1s>p4!B8nLw56V1X}SjsNBgXPC!4sXraxQ?yab1+wg!3TpzIGj~i9Xomsg*TRAx}6}{3?es{y}1im+T>(fo!Kd; zwQHT*nW(jlkLOv32sx>7Qin!rMkseqBE~&T5R#;ti}x)LpP->r@S%IT1h5o=DOwDY zRHdTkwaCPKc7UDLdIn_E2V*TwFF4jh;p1ctH$g#2F6Ra^Xn6ng6YtVkR)9~{T>6(5 zrVXGtp$(f+65B(9LUqqMg3+(94^L~vS>K8PsCD+8gJ7~B$FfbuvuoOoF(ak*(6Lkc zM-&SX1lkEqcdfU0i$$oGBgHd6cdNPD8`%e}%XqAZh6}6Z48{~$*x9Y^SgwzIYZdLu z4x{AgkPOoJa{19xh9Zg~Td%1bvga^is5`ogH)ZB`DBySpQYH?Bf_^D`p|RV#C0o7U znZ3h%y+t_4wt!^7dv$?qK)w~fAA6M3`@Gd{weOb+c);+qSOw18xCk47a~3~x`L;n9 zw=+c#Vn8x>8?1j@R6F2a5wJsOTCI!wH{%+)lUup%#C!kF`3@R}yR&<`;rp92r@b+V zR93vOk)Xa~9ATh<#<_XMah$%3V8^%2WoX>J^R2pjq{rbKo|Bvd;+dn{JE_rFy+fI1 zUD>)yZo$DhtGpF}3&p`7Ji;eQ#e$=HE<8)?41m)7PuIE1baTW<+^(YwC$mb$alFNk zJb1^T&Rv|hI3~z3TMf*c0kW%@Zk*5edJF_znIjv>heD{I9LfP*(Vbk!ryMiz&C;(M zyZf87r(8qu0ICzICoj9Nk8d^niC)k2H_|4&b5(EZJ zaUd$bfWHlU&lg>M+KJFpT*tAB#kc#i3;nPGyHx+io7p2^*+(VPGaJt(4%jpO$J029 zL8REdozf8-Vf?t!!@J$e03@oIeX@MZaW=tqlFPeM)x((8+m$kGMb=ky)?c%IvZL2R zZWro0PvHE-FWDsU)3je25;(^07DjgzFL<@v*}X+u^Zn zzFkLh)N1&a^wcy}g^Y6!Am7*ahK zRz1v{!aj!EL4P$1kE3kkl!P5F$I`*BM;f^a@36gFSw;m1Z5$1zI^s>t4?>=O;JA$y zJJlGzr!hX|bDShFzOhw)0xZ?ZG3;p*_;p7RBqX)nY8MG6Yz3l>eu zv^?ky{J4n0`s-c!22Sa(ahsd!^F`RDhj-9(+Ji=4NM6Lv^JEfE{lKN&Qs4jjHZi;A z8<=K4`J0sQSKs#ce^CIU$c`BT2_At$*uAI4!NmyRoRh-r=xOaYDCmB3FSml zdg$wyE`p2enH>fkXl5`zsb7YDE{vFy<$|c~vJ`7cXL!)2^`gURxLO*5*lzn~OZag? zFt>C5nbRHGk|IE?B- zz#>{DP7?CxvBg$LVA#GOdmXE*4{Ho?$GHX=7+H)c{dKI+8YV1`<#iphgIN`4?srFr zEK}Y_FU@M;&keqFbv%bFt{DY+?NUw)eoLuEMqmA2jFXZ}W`1Yy3o~K=K1{cG{~4^- zvwu2>tfE<#De?d28Xv9fK_9}-Y>ca-uA-mnPI>QHl#kr9%TBKaIMRt4M>tYS!OJst z0{x8mbEP}sjyc)x>*9xlO{X3D1Zi&7S)zxEwPOchbcsq#_r=lzW^tJ zB&?d4b||R^;;M5e>=epEGcLXjFlWz*VBvUH!EQ~9P}Y0d6jf)ZM0t@^v8&Su8EB=w z#m#Y_Qx^Z@7)S;YzR-sWqgL*aRlX4wk#&K&-203pyeqj7BK2t&-n?@_^B6KziM-wu zLlUXR&B1*{7=lc!c*W-VO^ksQSkacQhhfridjavW}GP zI~TPOR><=;$c<6lAPtujNOa*~hKCa3-HH=NWXkPTu&bN#@@2*q@u+l@lm;dxcqAi@ zM0Q-Xk}7%Sm`92-l+heRFZrk);uUZw(K_ECHS{`#x$$BBLrd5g#JQmml9jnkWx~8< z%1Gfcjf0G$J6%Z7F~!n1KNQgjCJDw*D&{;|aNo5E5yd$`@S29}CrKTp<1XA*_sVV14RF2~$SKXi~n=V98nQUZNOq-NL zVj2@p_=;#8a%`@9 zl->^3BCImT^jT+YtcHv`r@YZ`i>P%Z5F^Xlsd8(wkJ{E4hr`6!dQO?0^(<&Jd7b|X z%5to1irjO1xz#tqk*MF~mn~~cH4w_xw=M~UB98~tr@pngz&sTnA(l-1%BZ*dY9V_g zv`pggFoerGqaG+JS;T4av>&vYH7w;-j(YSIqctrwUD*)!NG8A>eC$N23tK2wm6pn^ z>q3#6)K==qcjk29?Jxx6js*=ieBu_@$m1r-_=2qwYdO z!yC?GzeX0JMHW=V9u_etpHwroeE4+c8}EP>OX8&37@zi|TZ|X1mBN}?hsE;~fZeON z=^iyp-{hnPWjW#3&TYVek@8nVTckPeIiy!kalT?x)>W!kwGka}RqrgamCFB8TAuVP zPq!;*$ZU@ayan-DK-xA_fey|4#H?LM8tOp@POC=7VF!sq zMcZGds~a>kf0~seMu@Qk3S8O}6vom8uyj!!DIGyA*6bCsq!T=vR`0IZ4J>M~l@?Jo zY&64%{_f5UA&^Ov7p{3$t-Phil}IxQvnJM8osDbW5jn`B))A&JlVxqclpM+y!pq$!;cc4JuN?tV9EVe8P@p)%O9tTYBco)~_2S?y-dXHXWUvCG#%O2LDM>FPx3Yb6t6qFZojtkp{)pd-3-T_drZzS->}uku?7@ zTe-Pk9W$5DO64nN+Awrk1}pF0d~nBV%htN%j}!v0s1q$jrXxxy+=gb z)*9GIzig{dI?gc=e+z-+JtF0c8t653Z+8>UH*(ihd6Wll6Sx0TOV>MtW*cosUuYL4 ztH*iD)^tB7GGcaPfo3|fCVqs5ZKvmd^fq1b=QkJEc~19q)JGArk_Fgz7*jWWp*J}# z1#^j~1(0`u2$+CMp$Jg7Bnp9p3U+oBNMSGLfRAxQnMOS8=Pz$0Bl_c0{4#&#B|;Ub za8lAa_?CT*K}k*MG~p3QzL$lJ=XuM=O*UA6_V;IUgLNynI5^=WJxFq0S9u*)e1vC) zL8ozUcX~x5as9PyjYf!_q=tQ1goT(=x5YU7HBuYbK?bI90Y@&l*BQSzhhB(g>eN2R zHg*$MW9StFH5h|%lTSRTiV<{&$`cB8u!rmecO3F;>!JU4w}^P4GI!-xfBo@q%m<3U zXo!g@PZOmrOXy6H$c0GQf_B8kj+LV9LMRriNpaXlBPS*B!wc8DpH_Y2PP zis;5j$%J}cHjPDQj!8#v{B?n9hG%KlZ%?>^U$}brM|Pg{hTOPs=;vwlM_oPQFJ(6x z2X|KbW?3EAX?}M(-`I-cIF9642u=o#o8go)fFh_Y6XECVN} z^?jB|f%b@au?KL6$cJ{rS^*_wVkQ2|W%fLN7Q^%IEkw^^$wkrC-tuy~NWXj(=m zQL%_}KIwwYWNR@sJ3nr!HgBA8?X*#a+Qg@7?*x~UU3BaeMKO2LFm!}ybncaKkCVB&*4k^-8dNtdK~ znl*=pux69j$(R~gZ$jCSzsPcIlAWFSm|ZrWO{SJ0C~&06E$BCs{dRJ7(t7_!NP_oX z=LtyDh(LMsW34fr)MyX_89mVbR!F_^F-4cz77cb8jbi z@yQ&W`Ggoaf^g|+fa!q|$Ca0vSvgi30V<%A7YPL_ht=nh-3e{;IgeARqUjlrrNM&j zHy}2Zdnht=7`lS)hk+V-ceg2=#+i}436hi)m#$@>__<}o6NEUS4Oz;fEh-E}LJFUt z3Ss)Cg+Qi;fC(>Jrmz?ZhJchXTBDgaoSjxL5UPwb3YiVcXy!+F4Tq#?aRSQcmA^)z zOuBpsg@hmlma>+Tox*$Osg+XHjReV#C$*)O`g{@@R@g*Artqbl3Z`hfrlI<&X9}um zil%p2VQpHGHTtN1T8oLArwj;V6>9&RNQ9s?I-;yfo8f6|ZP=Fd_eFRTgacSnS_COd zbQPAWtkBn`&1!j7CZ^FUt({7(o;s@6imj1QL8i*BWV)^1`mJ?9rf!O?lX;|es)%^n zm`O*dc_m<^g?Ae1p1K;MrF5a9f^9XTrQNWsbIGjEnrjm@tpm$p1zWAxO0Wm}sR^sF zg-}7)Dyp9#s@>|a5zDQl0H-lppyZ0GcN&Oxm8+&^kiuwnt;&x@x`^$_vHQxh{ko-i zD6lcxGc(JuHLI{Td$TyZusNHs6^pH7nysap2Bs>s5{stnk)|07oG?1H~Vy0LIN zr=Yf~K#;WMCbh2mF|hfxjoJUQU+c1%1ha^isWOYRXG^natF~*)vpV~;Zws_Wo3n7+ zut5vA;JT)zAQjpwF9&;hcdMplDz_DTB;$ChLu<5P8@B!mumNkfX&YV%3$2pNwv}tS zm+QBgOSzc4xsfZaWc#el)41Msso!W0i~FzZHoB?{x*O)Yp4+*xE4#DHxwT8X16#Hp z)w)qpxZu^hze^+&S#s?*a*At!;efh~OS}=eyUmlkuY0@Ed%4gnz0?c29p$^tyE8us zyxc3i-uqd1XuM#1y2+cm%&@$ptG(CDzPM|>?>oKlE5GhLzt3yGy9>VC8@&13qT|c2 z{tJD|%ev{SzUy1RpG*J02duydyub|Hz`AR|y0*We%fI~_ngIO2!8JU-I9x^HIAsJpz+)T1h6BV$oW(}W#aiseUp&Dqti<8# z#0f~lPn^b39K~16!ET&9QjEkZ{JdM-#ZY0#Vob++Jja?s#_o2;W~|0*Jjlh%#u!$H!>W1Pd7 zyvm~M%DkJ!tPKAztvn>2tjeqy$Y>nO;5f>KY{Q6b%E2tfscg%%+{wkv$(C%)vy9Bk ze9OAr%+M^u(oD#?EXuun&7~~Nj7-ekY{k*+%-|f(&n(X0{K1@H}OwX-+&iQQ5)U3}`tPAQ4(7x=>8cfg{%+2%cuL$kW_sr1!yw46@%v`$A z6FtrRT+N+HATT&CxG?&@s)X;U9MVj^$WPtWP#x7&jm~JT);hh_UH!F5{nAK%4yl{fVqMp}Y}RM( z9NSFBFz1xIc+ogQia^2Aeo!nsE+i<+PL2bp9J9yk-Fa=<-EGq2eA%X5-QGOd%Dvq8ZQk#k-Yh!b>+Rphjn)8;-0}V2 z!(IR2?M>haPTl(*-uUgmw*A@Xt=`@}m+XDu7XHxgt=bo!;0nV=kv?cmNw-s4^3 z5YFHW-rpSl;q=|%FRtPk?%^!{-{@W632omde&RYF;cngHGQQzJ9^)om=)np#z%b*NtuI51fkp5<`v6tCvmX7F|uIVgV<~=Uqo^JoC zNuKJn{^gUt=#Wn9kB#G^z3XD$>c6hv*&XVzp6j)K>4RSE$&TyGuItD9&h4As>zkg~$8PP9KJMZU>fL_s*q-g^{_Pm;?3~T%@4n~duI}}2 z?B-7GHa_R%zU?b6?Er1&&mQIRtnK}d>b_m@=|1R8aq9{%-wyxq`hLUmPUeX{@B}aM zckS;6AMsOu?H-@+AHVMqkMF~-(MNsp`K{vKF6km~@173w3{UDY&*!cDOe>fjBy zjoa-0=K^l%G{40TFY_>O;4XjU|8DZ&?dt+x!#p4COW*P&PxUk3@c%CK_wN7mQh)PK zALcJV^;w_vR*&^yzxBKB<2*R;@b2_vU-e=i@v_N2aUaot5BP%b_6JYz-~PW8{`E!4_llqS>@4gZ5BX<5_v=3S z3{BeS6}m^@BK?};?&>y34i?DfBlgE z+q>`j&=2MUFaFq1{ObSw9H0G3|FY{}>dbHI_+QsJ@6N^#|K%_Lu3t0%_mBV8U;n)y z{CRKu8J_>h@A&eM{NE4%lfUNk`Fh{>0RaFZ`2+b%Kwr_G-|fubBrvZzU;NRu8_8nmg?r#_)d zjY_quRiQ+uYQ@TQsn@Pxvu>qFcIDWsXVI!%TedA%v2WADjT`ps+`Dw$>Xo~ft>3?Z z^WqguxUS*Bh!gV-%((AjvyBTYmQ4Au-O7(4M|Lcf^XAWwCu1IMnKWh3pfjhg+72FS z)39Tgo-O*c=GUuntA5?OZSUW*gA1o!d-(3$$Xx?Ry}Y>d=f;UcKYm-e?&R35U!NXb zdUx*BgEs%)9zJ_y@8~asU$4Hr`Of6yBWF*)eR})%-H(^AzdC>S`2{H8cLcs?;DH6! zw-J5}CfMJ92to*;g9=ti;e!&wcb#?_t~TI@4Tg9jh8+4w*oGaZ2I7hqwn*Ym`$d@I zge;CIqKhxe=vav}YDnIKHR32Fdp(+HWRW}acw~+?-Uy_VPY%hXjwm8IUyx8nnWdCe zb}7%7Rz`>=ms=9KBSPPwLdd^RfSq?EQv*P@D^Sm~ymMrvrEnPOU|rJq*mDVV2rs;a4olIkj> ztLp!Ws(+7`%4({w=K7|ixb|ABt+K|d>#m}5d26t<9{a1Xh1xnSv7s`%XR^LZ>ujsf zW(yd#+;*$%wbfdyZM3d>8}7O2&Y141R!jkK;SuesmS3vRpaz6)=>^VZw1z5j-W zFSM{8w_L!;BCIfD12CLuxU@Eon^iz&2Z)^Ul)#Y&66xBVAz7M87O<%o{5$_0&{X zZS~a;W1aNVTc_MK*8-oc^~__J-L=_hr+xO?Uo#DM)IYN=x7u{iZTH=6=l!;(+|>U~ zw%>pUF8JVj<8Aoi=)z6-;)pZexZ~8!Wz`skt$ZTjrApPu^FuqVBG?z->(`tQK=F8uJs7q6u8!PlN+@xGe_yYs4t zvpe+7Pk#>b)io_ZFBy20E~T5LBM+#%4eDoevR(n1colQHKq7@Plap zAqMqe2oi3PgD3pp2~%3jjoJi8)d1+R+h1r zXM92)cNt4p#sCO##3L)`pi5rna*lQ^rZGX71`@(>mB933C8mkNIs%f7&&*~aN4QNx z)X;s148Bl=K{KH-frPs_WCR(9LZOTv)k3{GFX!=861S6+Boo50M1-=ssuhajf#qB`_%}+sArzw!y@&dCmG=sQ#C?U;VE_ zH)~8~4zdt3ZDCA#*vOt@@-xxh!g{Ss+8t=OwBbeQdKb)I5QkQkoh5HZAq&ty{?fJh zE%6`k7}gWtlD*wMp^DWYVBm&QR~_Ek~pIY2J35WD^Vd^ zIfN7qt28}PEzvAiJezj9TL>~zfg4q+PTHRgrm{%YOX!JunypV@ ztoKBLh&*^9$wfAGfwu|;Ig**o1Xc5fC2i_dr+A=;zOgZyx7E?UM)g9++G)*3+S;b|@Pa}8Mn#t!vUNtZvp)^$ zIW+5dQv^k+>7Zd$mKn$3R`sqmed}49o5|5;G-dA{1XzDqpOrrJ9XgC+0sxr`B#1Mz z@BPszBNePq7^{^3{p60~fa9o*b;k2+21>tJ-z5K+uf}!jZOaao)%||BiEAf0ZOEGr z)c$svHJoLh*L&GPMzokG-DEPfV7&HTDv?1h=^9hjzbPI#wBgKwN(WTgE?2p=hnjF% zLpu)I?lq)yEp`_7eADDkHJ?%W^~kau>pO0?+ow(Je(qCb-ZKW54KBvemfhnWw>4$0 zZgpI<-Q(iscLa_-TM|^g;nzKJ#PxfGi_h4X{}!=#yW8r6_9552eln(cyl=3>J=%;wz1@94=7+ViTHc-w*MmUUefbt&*mrS(H(LWpJi4HM4wwZ}#&=EUe@ORUCKrSO7jFAS zTOc@eBY=Llhk7eke*Z^MDj-bemTw8DefyVyxWs~HX9q~8e9Y!=_T_p-5QceJcGyN_ zb|!=EhkKVMa_Gfu*cM(dh-OOYe3Adv7h3cS)rE+2_)u{82RJr@r^k7ED29!9eFJs^ zHJEdh$8+Mxa&QO)=J$ofS7v~CfhZSzN2ZEgFnxH2ake)DiS|%KsAY=whW!R(eT-;*SN3oGLVyJ%i7S+2Z)J$yb!`l{iDq|jbhn5W*ly3qQ2qCbnO9cw1%s=Y zjXF1oqsWWnh(~u=i&f`!cKBoFc64D^a8{^-!-k2$7zdl|_h^NPj6vT-*N^dW9E(-Ug75C5|!|keio~&h>j_)P>wOOuxoTNC=2^ z$asTTda=fd5;=ZCNCl>+ecl#^j|YBF^?6}eTE|F+{S=QoiEN{YhU}MtgO_?xC0qsh z20B?~5chrI2#z3kYkjwfU&WU4h<1nvkaVe$8`g}@=s-rfR%Y0OMrDsfs74W4V4DYw z9m!%a6o6-_jlO7fZ>fwFCwXb8h>uBQJ9v|A(2_OSa#EOCSOGJ$c{VqeWe&skYFxFsYHp{nRv*6XBSJe`EbI>nPvHvQ)h-ohkv&R zfB}eZ0g0WI=3BfV6F34VNqTvZ zc+wb<2qt?-keLd&nT+Xf#u-oMBVFUAnwh7L+r^r#`IzPx1YmWF1X_7~=wrh9hkVGM zX@!$Ymr8$7mG>xhRXC38c#nU#WJ^buFxgaf7G4#qpYPaKXDCNznWCF#j+4oEtGQ^y z_?@+6aSwW+2&zPwFqsv1q16bV-U)#;mV_cHm?XJ^x-@wVs-ED}nB8fA?@6UJN|8kBhzA*qUm&HMHY>+jMdYM& z{z#U4_>rYKZh!wMmT=~EOWK8Psg%z~sqVK05m%dW&|%4kkJ`qZbSa=1>6k@@i_jEo zTS%c3N@8Cac~aSVTGw$viH(ygj4!Hi-?g3PsU3p~MY4cT;mN18sHtcwt8MpYS_!C` z=wX?uc8!W`14e>lnWK0(dw&Loq_~HAT7}$Lh)M>8t%{B##g*R{neJ(*b9Mxw2L-rD zptY)oZx>cs8iXSEslRcrPn1}UlzTjys>_(AHi>}8N{tYjo-J65Yl?z*R*sEmf-?$y z6bEq+#eNc5uWD(q{Pu@KTA<%JulRVUYnH7RYH6!FV5cgAera-KxO~3&Rrs2(`g&o7 z%4V(fkOKd!t>czt3<_R>2%suxYV&BZV#j_rIAT3$p%}-4T6M1TDU6)@UL)G9S%$Q( zCZ=R}jswMAI5@IyN&w2~ogsyy7wM4_iJC!qogq1lQTDQ-6thVxr1VIVSrwIpSf*pp zm5M4`>UM0&nsP+wMsoS5vbLUhnu=-IOV1>N^>(fqdU05oTlwjCnznM|$#y4rjORzV zeS53=sbZ1}fEns?AEuS!T1;#Uv&}_BZ(E7W7qCJ*nESbvtM{}g+MBU^d>zKM%c-Pv z`>2xlbMCjYoK}LS$$DtJROtzG!HZqlifb4fp12C6vU`^rI)tGcQU)uvXSs!BYohFl zMm+y=x>zJVm5`3L3!zGBXy#av9GJ6F`lckeykD75u&TEX$hY4XX_1+Skp`>m%9#s` zejy;b$9qfl_-xI)fv`EJjhUt%7MQTfrV)CbcuTtC8NqQ0rhz%SR>iBmIzYe*w`aD7 zbqb<9_Ot>@zY~dQHTu9*cv^Dmc1rt4O&h6gD5D9Rr>Yu*IV`v7T9)f;wC8$y0BgSV z$E;Y0qah%#s)(hr*|-Vk#FhKHdHIr8NR@bIq=%WHCj4}bdXB>RsB`MXDwdiDYmM`p znOsY4psSkN*~9J?#Be;DGMS!J{DT+j#McUTy@a<}nZv}ZwYFQT1lyaFikZELaG3uV z##Jhx!1j;b^1%z~SU+rL{3piMYkctdmoUh_n>ltn?7|RCbn(al!gr91yGA$Iu0NWi zWmTnhSAJ4=xfYAUBc`;c$8h4PZvo1{Tx!L=ERPim#fYq^YWl;pn0E-Ocb9j*S!Zca zOsli(x2XEPW~qWRnawC`tq}Xa9~KCsDV_R-dXHzURP4p6dXG(Lod;~g7zoG1>Y54Nh9&oMg!yo}sJUGnm}~z8%e6ZK zRJ6=cWDEi9zJ{EPG&_a(X1`&}#C{2?sLHwXY`1keeFeFW9@MWrYpAX4aNTDcFn3*N@Mt&7EAPBWt!RjFLDhyRrGp|y*kRBgGj z^~lccs!DnWU>^F`hP|hiEK)8z(1^Lyr3T9uxxtpY$uX#-vFg90dCgWio%~!&L{B-PHY>0r%j8 zOo<=u;QiQ2f;ztXnn(n>qtJ|@@Qttpp5;3Fqn&HHwkNsUn4Op1-HBSoTd=g4I@uvT zpJl6-mG`RtmY+dt;k5^nSADfpInD|Uyb>*N4)xp%Ua+bCwNw92FS{-K9 zh=#RoSj}SXm^cTFxVDqqtYIF`L|iQ9qF&S{F2?P=mj>>AZTyQHU3peL#k{@Rh?lCF zN}qRY=M_upzOAIh9&CTs)%eV=*X-2yeFB7@?VobpXGW+8fcUTJW8I%ZrDf!yKB-evB;G+#B}B%dX|Ax#!;8zs)zI zCFiMOy~N}d1Gvme9~h-YUc4PobtnMn{GI0LNWBS<<_P~h)c@@6;2H=0p09~{#4ta8 z58JU(uJTihdxw~~=174^zt_vR-~MXq6i=b-ZeEcato@zYGrrANUh*3|+JM=ln+vZA zJ)y(h%K+`rBc9DUPpG73rsiCDkom{gMPG2=vJ8&LW@^X6e4n7$oSq!Wl-HFeP0?o7 z>WfLOYPqo)YoEA^<49buE~wmiIrapY*y+ z(~WQv2#WJ*edYYT2u|vmx{;N8trs7nu%FcekC`ob?75WDDG7&A`ufgH_Q{*wSiR~W zNA5!|#|*o@u62#|`JnvC`5^8;998%X;KvsZs1^SRh7mfc zpZ%;}`0e;-lD*C#-~8)e#OX`pE^3PMAK>x~5DZ$_$+Klg5Go0ekZD7vhzclnigYoE zB;uBaS{@RyVR0kFj1~3tkx21NoI4~(GE%7qc=XCBg*VE_Ea6oqj z)#b?$9gquEFo@8UCdM}%U6KG9N8<>HQ@I==ae~X6ky_f&1dsgef1_9St~e*C%5Fn>`4TWk?*HO%6Z# z=4MWw(S;JFPIOXOsCHw>y;vG^O|YmZL+)PubMLuM{R%{B)w80MS?GSXR>m8^#cbv_N2UjlKQafJ8pa#%n8!>=c_2xWsC6DUb*m ztBWO#JR$-)5J3WNGz8~^tc}Xt%7efZrPFc8>b$Y;DBBVYjztw494oQO&N8kbOlFHJCBjABw|lVzicbMF`Ev<#s$$P@%H$nJzx3$@44E6u(C z%>3}p?pW)P&-_Hfv!e)8qUpskWAOhl&Gs6djkg&qoQtE34$=ZTOnK~4sOv@~s0$h6 z3slO6q>?f{`uL*~Mu+w~%~RUo)Q$rapA-+HK3=pTN(TR`)u9JL>??#f9b@e*{8Ti-CejsEBnBhXjN?iY&akYq+)4#$74sIc6(2RWIZ>M|&|S{({4_-l zC45R#d^go47)0jyl*!mOgBLyjOii`Ff{3f{S~pjGs1L4W8sRdowk6irEP|z#vvB=1 z)v*l4YfRbl@(35udI6ocO#?v_^+3-ZZb1V~`R4yt4eM=45 z@N&cMMhQd8_i5^UX=zYOX_o)2HzWr}?nc}gtu(D?3APsLYo~;iKa*kJ!oH4!mbt^U z|ImVAHYZAW+GXnvjkZ23mP+g`Kk62(cOQh(&{bnnGg@&qRJvHP7K0RTuvPTzzN!_1 zdOB@7_uLp{r1sa0WRrVb!R8jtEsU7J*v_;^)lkV}OVwp)yIy0Q6mz3V{uVTY5Wjk3 zftv=Ebh}G79(jU9rcGu|)YD1J{Op=J zWLove6e0jtK&iicy3IfTykFfVZ#X$v!K1fDv$4)dN%GoPYP7OP4T(Z?QcCR7MnG9z zL_GroP=AonhYMxrDRIc)7ku{rrwB@KRboP3?ixZOopEeiH5ne?EDGx6E}+62N0P2qbq!5~tC^qVsU@HB?& zot!qfI<|mMU^ko7VpJk5FQ!h7YP+Ee;k7@)sY+R8vyIlWV?`pCv38E58vJ-CB@wm{ zW7eXdzN!Pn&?y28Qj`N4jL?KNL^6_)TtXxVAqi1rQWK!yWG6lONlj`J6{8#_Qlj8U zC8W{^Wk978S(!>281V|E!($nypu|{WA&IzDq6KDIf?Ea?1xqXg3M`Ps>&235w5%n> zd^v>;2~%x;(1b2`nYUp7(sBm9GUg1f;7u#WGM2-XB`%Lyf^J>_nT;%>55(!tT(WbP zt9+#^VL8ul(g2w8#HT%NSUx3*1xL)AiKCKINJAnr8$rZU4J!Ep7(hUx6Qw8|Hc^Xr zxI?4t;AlrT>d}veG^CQS0uM|I2PdFZrH$aj9$5N_Mq~n}GrfdNYiiS$o^+BXO@b77 zs#Bl#RH#68K@(g74y0Zqqe-pB5_I6yr}i|eK^&yfk-T6z@!u<>_nLWh*{WU z9g9UrV;%cg$VL|bq{*W%6tZMq3|W=?ekC1yLzsYZGpE_K~4FujDC&GtPwCf9^;5~-I*#_iI!dJ*31@b-E zVdMQKS;;*AxI`^TX=IH`;#^`Jr#I&)BMZX}o3}(w9HpscUV2b7dC4be#ORnWtGF<8 z)fNX(2`(}EDUa(}^DMU*0y6u-me<2`Q(lRZ0O7d=$~AFHtIL9PCg)b2@fJYIvaLYK zc*c_sB4#Bj1||@~5t;^s8#2J-^h)8!MILgHiA>*(c0vbAX7zqG@eX|i1<0KmU0%AR z1WN^CShUhJSDd+?E64$tXX?Hs*z-DPGd!}M3bi%76QZ$}Lt6WgwZU!xlZfFaQ0Te&FEQPA;KhuaSNhkD& ztcWQ81P(h2Lev3}OkhEJc|7V;NBq=4R;(v7@q<=(K*=6ZgeFvh4Q${8%B3*G5aAO8 zZo)waI}4*~Q}8gk^|=IIeK{2>(-s|wJ9$iDiWT^v&mc4}j5KJ36@Xq4ZptCs>f`wV zT)Kks>|p5SF6Ua<;&Yv=tLH^G!noVQ2d-(T*vnP% zu@juzI36DgTowd3c%~Dc>@|5B)E$7h#0?+PX3*E-E68{PG|q!{8{!iorw5d=Q3#Xk znqpp1In3>lYz7zs4{BWneA}L=U@MoM1?m%-VS}pD0;ZOI!FT&A%dan zJiCbl(5t72h&i^H0Ge|dI$*sO$U0{_zMs=4p{u^xYdSPox|86F2n4N_i=W|ZspQi- z8>oV?Gl{WlzOV8+uaLp6D~Y=(gXn`d^}vJddjm%R1vW4%&0>LD>LZI$AcH}a4I@8F zJHPa^1Vd0WhAO!3qV-IhEFKK15CjG6Z8U| zi9Hdpk>8U*G?)N{xH-0Py?x5StCKwrWWW?OI&cD&*pmWsf|wK(!P!_uZ*vjsQ$QB9 zsLp%7HQ7O0qP|NEJEQ=-w4*jZz?De=1U7JjBFrVR$)VS=mH)_t?O_floHSH$geWWo z(8G$~0yX*L!iiHXACa=B3PUk`ycOsJRyae;69hKY11_NfJb1$tI+HV~Ii>-h3H&x( z+(1HsCZ{mLf~vr0E4n@emJyIZ+uKHTgQ+(pBH*y8Vr(aHOSi0Z8{{*v|G>o3DX1T` zrtR=SmBRy=KmY15T1e&%%4T1Bk@(Zt5phXA}gFW~=F(EGhUCcr)G_}L~ zMIR`t#@nwFyCh2}1#eKSdl*2>Gr(s-0Y`F+3Oc50$rL5o{n=~-yp(kTPFnbKT13SW`%%%g|naqrn%sisd6iw3nGdmM8 z(=0#2*0zMGq&a~2O{orKJ;fTOX_Vav9h1BfU`D_j8oUF3wjbpkJPNCY(mW=WR2*OPG*u$&XlF5tj^NhPVEHGe)`TglQYR2&jX82)>O|w zgDzH*8#36wzB9rxV;{Kao|GXV70JzqTqxe$u}l*KAt;036wdeqgl{NLyzD413`G!l z(7#;HC4;0*&;&g*uDE--ebP`3RVNSSC3H$B5S;-z*d!A@txFIeV3e{ykh043$to*E zE=x40Jf{!sr46+uDe3_l_0TetFd*%{3KP#)YOUpJr$ms?h@ww3aSDIwGMf_-+LWKy z=}#;5od4`GsRA_v{j?HIPz7DklPp6Ah0reDO9$A1G+k5wHQlYSV!RbN06CpgI;~R= zwEzQ1wmh}bJ(Yu0-~>Po)Qge>Jl%vI_=8lSCIG|JJPpwcXw*CHHAeLdNUc;$HGn&{ z(R|9(8|@|>1v(L(2q7KPY;q{5I@P!0EX^`Se8W?rX}2T{1GLV6-X)sDNg5R>nJ3-9mr~uvR<0Ry(~^OnuZo zodsI}*FP24LBP~h@Bu_UQA>5y4n=`-Z}CwAS?Fzcs&RN&Ehs!u(zPn017|=AJ~K7%2wpc)@x0HZe4_G5La-etZKLf zIc=&`aD_bJglmw5Ze3S-b=N!eR;Ybis6HAC_w9SB;uq>k6-`tKX#noS6h4&MJgX=WFT`!FEE%mcl0Q`obNYKQ! zSdm5C#1jXO<*1G&({SAwZCV2WU72dy zEHqk7)qz>CEG58N>P^}oh+u8)U7&3QBA!!MaI)5Y-Jr#VbB)^Ct=`L3*%T1r5k{`# z{nK@Ws@@Ia)mq=RO<%Lp(PMkD?|5JTW>gScs}}C4-~K#5za0Wj5MU^C2SD&&Hy8y0 z4&ZO-g#q=gxTIK4kX1fj30WWrN=W1{Sma3zI1wZpA37UOCjtv#y|Gy)u3qcZcjRlVg~0j*<= zGO~N<--&bsC>Dfq7Ca2F***SaA%KKxxQ9dtgkG@cILKkLV+UIRRaCa(?Np7PqwmQI30@ZLK-X=WY-AG|nv?b&PXEoL6#WqxL#7Tbb0=xVlR&KfOF00bkt8!izT zlnXyPs)>f}&s7l6As7Nds949X>PzD>y7W~cc!YBJhSqfietrc^rURx;18$nu-)mOK5d{*ZEEEzqVB8`YpF*-gR5{f zG~%D~=`ieorYvS7WfjH) z3Pvz|Hq=eFYiKR0gZ3#1{p`C&>|thU(;B_Ywk+|@-H!W%1&UtX#as_K0Lzu{uchT& z=4mOo>r~bO?j75n1@K^Y-qAJz(nf7r)@cP-ZLv~cqQ>S|J(zz2Aa>ImkigZvjRitT z(ET-MbAAL2@LwJi&J2EM3}6D^ZqZ&C?pI6HM1W!dbFxHO@K1ik2D_7JomuX-?7+_D zECg&2Hr`H1@1liiLLCD|s>6|vXj{%@p5<>Q?{WIJZyu-e1J`5&U+~qYW(IF?*LK;0 zEoyaq-v~Q5TD@)mjnM6f`~^Ya@TKx_BO`>Z<}~TzYB}(<;zFfSjwIC!E(n!a9tH;; z;?6$cg>vw343^fH)v8an)+&dCy++}{Hs7U8VX!`7JzGjf5C!+X0!Y~CC=c}~r*cOx z^}vqPDNl7%xANku;uT=WYln>; zf=d&FUBCwaLV);fXsQ6?(rN_r+@qFMp&^-D zu%QcsN2F4qYC;eMt>5~pcK96=0*ZXURr)FoHmNgpsYSTLl3)9^*Ep2tKkTZCKP`K= zmrya}`2Nm#Xm9jVhO23$?wfaan#Xc?m)ISbd1fdbQjIP#DlTu4+N(dads25jS;=f2tNgbP~UN zj+a*d$c7j;DfDY-_6LAgyVt*t2j#<0{(0y5#8-aqE?vZrcfOZ?=cnrf2>HPlg>h)w z?G<0`xrFL0l-01#Y*tNI1k_MMQMU5~_Fwl3eSbiJsnB1O5?RFrIWr5NZ6Zj^QUHij z#ky(BR`8%fW7h^_keiHvcwk_$gaw6=04jhGsQ^S7Kax0YkdQ#Y z%a{RHd@RtC0rYHk}{Vv#aZ$xRHz3AOr07vC)TVUv}(B8 zl_&zRRJRh%Y4U7XoN39L9gB5Qra_YWFj_R!$dpUAP`&D_DG#J%$xwOXMXi}FMu@ln zG3M5o9Aw0NC-XHd_?oLnuBve^H*3=)xSSm2q0-`n3LGPgF+QWGH$Cq^9A>)Wnx>#Ds21nBxKF=(QoIioUD z&7D_AnM88zS>LXsaoz5<|o0>01z1yZ0=%^Ee}@yHn;tZ--*3$mz$=7$-p?a-SRu8q+eZr-*w zTeqsUF~_&8;cytc?KXoAHdq#tw>?B%n(Ns=RVo6=pd9(^@Yd#q!Lp zrtE60IhT|KE_l?i99_bs|QF5_%x@9Nn8?!IME65PJ6(W$m&@=y2na|%JncwgfjW)XD(a(Ip{U73{wFykL6aM{;iRy%rv|HvgbH^ zQYkisjMJqh?>|TJPJi(w8`0MJ~R>k9~M1WpdPQ35V}>t zY$x{-0SZunmynD{Y{SAr(f)8c(D}(^C?j1GQ1PLcf$b0$9NFVEp$XQ#E>kDz&{^7& zAQeJo4Lf?o0#RtFU&$wUzZ%}4oYuAE@o+}Ri<=HdXua{^z&5ZUgWAByh&S+Nd_kB^ zZIq+}^r5dEbQ=K{*|(f{1cy1Lu*fbD_BSwU2S@C91Pvyj#tRtWe`^dC0WathJk$jZ zDNr05yYj|c4bKayu%m=TXF`>A!fT2GKpD!E3_&t7028Q>Eo8;2Nv7^*d{~4W?DNM> zW-?EnjLX+9vAV7Bp+6g-SfJ8Vx>a(8LLniZV5qml9{w=@ZnkU|^}e<`ZP^W7K==vS zruHqnJ&|ou%#ClTc)o9BDT{R|1iTa?MmSLuPX00@b{tVWyC{KwA4d=1_5ROC&wRi^| ziXe-7;6hhMv1dOq%FZ4fVlE$zN+m@jU9M6xS7rHM|L*Fxyoy(^x_iVmz)=f~%I>Wk zdMCpYE8mp@^j1xL7x-MAW{KFKZb_K>XCpDfj*+l}K^RD~g|wlqG6J)SJ*yVzyfRz^OG z@!QhoOJ0VnHjF3uZ2$Sjj|{RcDD_m~l$I@_i{$s1l%I zUqDd=B^Z&w6|YCcOB|L*WhY4D@~L8tJcC|>EPxciMG0x`s!c{Fs`k2oDgkBUl2WbY zpde0LkMaY-z``jFiHD9BI z^`M$afN{=xPRyNGVkQq;INO`3a57;3t-tl8vD}MZafy;!7)UIs(TS`^wva$>(*@FE zn9LI&uyV3W8_2t|AoC7*mR;n8IaU?XipTfa9xo!K?75u1+1 zN@in-GLMQStG4?{F$}^aFl)T0tY@>_Gi_FIB|>rMeqDQsy9op%K*4Rp_=V3t3fmrC z%EqA+dZ3B8LPQ~f+fC>y-dpHrCyd}?S?3BD?qC-<A`+>RKS)KXK{%OkXUA&r4vRzw!heXFFHg&8CUt6a$f~`>x zSifcCX<2dqM;t^9ad8@jR;|ZP7M#i zXhw}3oXBv;_mNO=QPp3_Tec+Nr-_9NU|5eNA7gk}PRLQF&|Uc5!zJw?hHOR-Yz7Zd z1!e?~)m;Mit-(F0#1d}*!aH!<5+X>U!5?fWT_CWHPppmJ1rw8L*x7~Odgbr_(P!#(VU zMHFC=7!9Twq9pX+pmfJSS&Y>U34WA7Kn+}9d4{!h9M+*A^hsaZ`IY$r$oAO?_BntM zP6ZfVLpNR1&paU>L}6Sp${jWwl5JsT@;{*Gyx@;$rKEo z5-bj^aov9a-J-4k)haE)hpArEbeSRIK|$%CDC!DeS)ceJz-~$6^hrUYV1^uY&>?bP zCUl1rM$W6ykr|8&UOn2)#b4jmAr)W^dEkx{=1x;IfZ^GY3UM0`rJ*t|BRY)O<=_Ei zfmvorqec=K2?*GL*`ej>VK*|BVfa!wj^iK}qB(lm5`_R$wh_|(5s?_s?HmGojLyxN z8iRP%eB91t43y8Y2;)c{F`5wFg$3Cmm(}Tj5IW?uF;@=k;1VjK9F$$5>|iR61XW<< zE#`-=@P`7H7tUeGU#%1f^jjkoApX5l4IlF#`WGVG2|{~BosZ#YLHE8 zY~Exk+xi6mWrx)PIKr7BT;NeAWm2MoA}kCiESlvtWq*A^h|rs*RLX;>l5XV?w#1WI)@E&Tg$%gIa%RAvU_ceH*b<-;4_IbPIDv9*R6DWXEs|VyJ^*+= z0C}mwOnr+;^30Fr+ty)9R`J3ftdnmQV>38rX(&P+JeV3vX4Ejv+p$y&rd(}UqcNcz zG8vn))gKp_qfqt&x)ef#wZdr%A|a|KguPynv|0jj9_M|aZQ6|f1sv^QM$Z%g8WdzB z(8`GlCppLhA2F1Mrbd$mWGH4x8FZ*jBmqW7UW)}w6CII#3PpH=rwcrgqxfAGLXV;# z;oH&w#3HDtJ(+;cS!8^692T(KeBx(h>SrzWRIo*sUEL&2`B#DEAs&vw9q{3TexsJG zNraLkg;r=+vKs^$Nt)fjO%7elz!mI}rO!xNEo1^8H~||BDxn(Itr4k4s=!LTSA_K2 zS2$|+VTK~G0%w5Y0Yt$oU|y2Y#YY5b$uLp)aHMRENk#1zCN05eYyut(*XaS0tx1OS zAOa|0LYBUyuQdy}al)WN&K+JBnU-9*AX9^qi)i+tn-wK$rY4$}V^`vwVH6Ql#^#2C zpq_qcdu%`^pq(oKNQtJXpcX10qyvkt5q_Lmxge*ETIG!P7#!r5Oy$;Xz7xsaq7>-= zK#*P(s1BRbc~9->W8CQz9ry*ShMVq8sp>6}I6eWE*6PN=mCghw9PH4w=n_oN{3Vfh6Zq{a%8>q#rwVuRojcB(9>bDN+iE_iZ z+7VMlYcZXMB)DQ$Dgg047o@7vR`u8f=!M2+f&Bfzz1mUyy}`U{0jVy*d){86w16VY z45nEZ3mwMk9c(BtUNYLMuc0HptA6!aO0 zl>R`xk_W{y+sRocq891N=|EEW_$+jcYtp|Bh0`2xu<)Dx^MIHm`=>8y-A`M~9g6pnM z?oJF{a%tna_UkK+@7#F?(l{zp=%DEtM*gi*wqj=Xe54)B)ZbC<-rYgsWTC|L;e~qN3lHtY?VA`t}h| zOb>e~&h9xyW-w3CR+TTB*gKdK&!injC6WjcYN0X|D-0c5T>?&Suq3|!5?uJx0QA#X zEybi@P)tnAEZi^Uc9jtvnH>J8naZse&#mjqUUNl(+-30^yqH-nCK~dt(S&8ou*^d5eQm=|yG4%RzmqnP;jnF5@ zC$%bO(z22-4;r2f8g;?vZCZhi3{=Rr*=b^e&MgKw?~y3CQC2eln$}&a8CYb$4bp^s znjGh3s%>U8w{HCv@V6w+Befz4Ek!-0qyzVm1h?@3svaj)LzhZy@;1>C-|-zUpoMkx zi6w*HN;92NGtYTt0meaw&hsvbj0uo(DYut!k&`(O#_P$PO;;E#YT|?>o&;U0@4)Q3 zs%u#Qp~$Hku?4jK;awNl@-6}6i&dHm4bJ~@S;kTDGX4u3Cg5o#+cZki%Kqx{0$4St z-X4dEmSFEllk{&k>+h+gh^4|3ZH&|f^`olJ)REKJmWVE}0$=YnwRqSXq^nRfN?yW+ z9VJR%ZWnF>wX67~29K9f6Pw(cT))O0IIuxshHeE#wHiDB^h@?GF7fA|0&hPrN#~&) z<~mW;;8gfXZ=ZqmFP-&Szk&%5Mr&g4%)x12SlG8kN{SUA6^s-)WS!BzG+wIJ?u4T5 zc_rUi@$7UDGHHk*hHR zkyS%58%DGu`hk8qmQVR{+IV;4{jxLSE0vq1A5ld>I0 z=tY@KXOrG|N|Xsz`f__O9%$ccIaKng?pF>}EKjQcY){%TGJ~cxnn-Ww&>Yy!g0<;U z-nNMlVh#nlY`p`D`3ilXn{<9SIpyOywHS&sh)Uo#K(!yyK8BE}hhatzCGvy_;`AIQZdn9@x6 z0U{{Ed(w4KLosr@%;4ZNg&Q}Kt@xfBxdX168lAz8*bbo^IwWNAmUB662|TR_6fkQ$ zkW2KR|A_S{fj=V?=as1)N70fi>XV;4*4P0ZR^UxLxz77sZ!->S{$_#mwPk$YDLWRq z-r-sVMv!k;8H@>jnSjAhV${b(JB7GG1&D$0AFYQs84&W*eF5HyH!xMM8tn=km$$?) z9d0GVIpbdbq_6*J+1b)kYCRq?7xUyn!A~}z-KKWg02mn1vVWO3aGLv_u_ns`!QH4r zD4arpmpjn+Hbl+*R}x0h>%!2RBw)$sXp*zcI4$e=seodg7}A>O5%%Q{@f+r}$#y+FWo_#yFHh1+4C;Dzz7^*NLd?_WQ>7&g_n$#;ZIHEJ8a&^i>~^iNJAW z{iNYmP0aKCxsoXv5Psn+f0~~FGrJk%%bZ__iOdCZpe;Bf2)`8ua(b!WO?TOvZ(0nV zxGyilH9X>9EfTQMw!8yGBMK5WY`XW39IhV%7$$J|upxi~6MZPT;^iVnEIrUH>U7D~ zr3DfLl$3+;9k?738Iokk=ggV`E;7|zgl7pJJ_;t195B#`Of)ry;_nWQ9nm1AqG^)E~5FesLgOP3K((6PN`N|7LTUGy3)i9 z+$`8q%d?*Ql1neal+VVY2uwmb5!75zIoi1M4iXT>Yl6J;N;+>NAN8aH3q~YK#zYqp zIIw^oYFSGl{SchsKj*HH1r^qGVdWi32u$!&2RGQS!A?2F?m`(sD9O`Kb>Q%a6b?A5 z#6VR{v91+mr3;nNeA@#KCd#0ZPst{8OtaV;<03IKvIy4LCyY%tSFxSTQ#nj;Vg~P^N`$(O4t=$IZlPHIZg&S;Ji~)^={NNr~{>(M0u!&EL&6> zsZLD?wJ6Uckl1to+fACO)fP2nJkLFu%-hd^Mv|e!02*qY#6Df_(PY(67Zq9HPa%Y` z04Xe<(27!%8&v}rt~}iWZF7TiREi+Nx7D5=I9XaY;;qRBnzBNw8g#`^YFu#P1I=hjOaZ0 zEfaJ-x=O1k(5x4d=E%_(xZ0F>c{=TOTJQ>2_|0=l8i;6q&WLL4>>*_P$u+cO;0`v?}!9^t6bH{z%HAz%!oSb7_p81|z zOy-_W0Ks+tR~3-ZLl4G)lx#{al2&#|VbLMX!#e7IGx)<;oC1^_m}L+sNyozSw}*jIplXviobm8SiAy|ZB)tRBB2<+F zK-FqTmfKU&NN9mYB!d$s0A2l}RyvZQp(YqB0VpJ~8dk7^Mwud+opz@H;tdZflDme5 zYNxvm$m~FBicTlQ<0|Ftta;8`2a5Xg2m>g0)zPqHN+gsH&oeOs^{|N8fr;?2 z1AyW|M+64IgC?B#swhg)XL_j{XaEt18jX)K$y^NhC|Sv6_KFtKM58LCNlme((KWdM zg(571OK-LjoSqz03NkrPai(b?sW^x|{D_J#m;gEOtPP|dx3{)<;t?0XfC7{-3rD6d z4exbZ}o z#x#mECCeic0m@_n^_W~7j4~NvHD-pE1WOa@8I6j~u|a7xQP`^f+~o(`#_6tW;6xc_ zFhZ!BAw=hVL`t}`jk0<}6W~%RM^GsX-nI!9twVtdBUjc$Y?2p|T5CZK`USUUR}=3* z=;wN{2yW(+l_cqeLRMALRJzfYgJ6aFSfLf0i9&sK`w0>q8-Na+6tWMItOr*Aii##$ z7C;JA@PZlaGnxjinNGtg17e_56~U zR2Zg}vd6^|m1x|XM1t_PziEXe%$A6@*k#9Ookc4QObLT=7h!YDu?WgZp#0GCOcM&k zDVsahj0V7FMQe`nc%?2@J~m+GIaN#wgsL>g_pyu3RUN6-sD*K62siSpMFU4#B9Re84JRFI| z0_TU1Y&4VUvN6AkbGAVSXIE$@SB-C5?Ptdg61$|H7bH^}%>0=#+Y;YsvB%T91;WsW z_V9;6jIAcP;Sq7T(K2~d^1Vxl*xCuxlrt#qkn>?Qcjo%n+ z=A;C6v!B_{BBq@Q$z|mcF-gLRb5eApy9ZFV`3%uokS9U=6j)lU8u=vbXBryXh|DEy>;nBv~dOe_KMH^xEFD}J$Wx}dtBh=+}=L_hqp(rTqOr6*IL|? zL3))4>JE#_ioQvAn{Y*9snMMS{0A%s9+1mT2lI%aiiGpZc!AH7gxT% zZ3Y&9$rRw1U0(G4{WE|+e2!t-`{jspC9lBYDY=WK$h2+3{&D?v4T@`XO}S|c)_mi$ z4j9TbPfq0rlE7nLf&K?q75^Lk|Exgu29N+(&j1Iorg-d_tWNi~=x!t|YD6gNn1%Ox zuiZ?6W8CHEe&-q}q!#j~^&XDWDsK7AE{_r+E0V7I*l8F4)FGHuqy<;--xz3|?oI;v zBoO`tg}h=3u54t)@5^dnjw+&PcnpvLW~rELr7&(@X6=;%Y+V|Gm_knHR0@e?1`A0q z0Jl&8{jUqZ5DW|O3)#RAh^A=Dq_n_jIJ79j4#V8murey}0_X5rY-pH1aJV{9Ex_XB zazaF+3J}pwEUtp>bRiyI(Cw;E8+Jhk-wwI3?~Tsl2KB@Xy2UqqP$KL_13ICeNTMn# zfQaNH2LP#~WI*}##fDhJ1+q`b+(&UKvFAKSl z3x~!9!q6BGa1X4Avo7w!OiM^`P>j3?wSkudx!iZUfaq4{f8@;KfVA#6}E& z96jm9)=mb0!5a`U5vRx)7_l84u@&S9oNN#fb0ea{?GD&~XJ>1CB={lR4)N9A~!~hCLMEp z@a`Igh<=9RsnKCX|BOcqy?jjK!r!pL= z>dKbDB(Y+MGV1Zbp+W@GbUIKwz@g<3K!_mER3-#B`*QGrlPi6304P#o$m2gkasOb? zVmb;0LZv59ab_|E@;p*zR1Prz;xG|YF^g#{C z2WA=HK+Pl#oh|`2ms0NB?loGo;pQ)5Wp%p z<1gvtJcjQGCz9uIhI+8dNsNWHzf(Owg_d>oJf1O4cwEPXFpgdu?9k^nt=|sd%(P;#3LiPb2>{BY%MC z_HXEbbPTJ2>2j|D6}1i5P)X|$Qm+Lmn$$kUK(@BZL#32K=0!}#+>_LkK%) z2$R+Gl$BY@$rn=+^%7HBd4Nj)cIrK|hykT0!yxccuW@XMhEm@x4!ms!Fg5U^U?+nD zc+`bkY$ii$BUD3SRJXJlP}NkqA=E+v61=Abh>}cWHCE{dkB(LJE)kAO;8qTSI-tsm z4q@iNB-&^IS3fD*kOHG*i(Q9RIwn>Svv1!3=pvA{gXmFYkF{ib)nrk{3;Qtma%Eb9 zPlQYhnaFipb#GA&ps?D1f&h)OO9V%J7v(4bQl7NPF1<8C4o&JI~^N-Ta==jg^c z-pJU*)=notEUc6Y%g{RiP*zo7$Wco6kVsa`K%|>8Qs{n!FdSieoIqN$FMMW_&|oq$ zD`V=WF5M7qXLlA{>yrt9HU@-t9L;rj?B#zL3fkIq3ie1Zpw^d6)m~NA#E49>G@~)K zWNXXRRpXXQ&Xg=@falUQt~er}1h%#oc7706E9@tS0ylY8@olxk-hB0Mb=4v|;(E0V zC$txPa}03_);HD<*%a3-Y=puP;c+AP0w~O8Z`KZI4|j(&X*Rc7IJa{r6?C(4bQ!>Z zL*QJBc9Hh)9=qgipEfmPcLu4}OHu8H$|mY$ZyaHjX1z8Xo!2*jS9I8>d5M`~3EHch`?I(PX_4N?VN;2`)olJ7|Ojy>?lT&grgFip#p&$DQFT#QfzGcSN3Eb}m!gQQA@ zM;PKbqk8T3kO9#{?vDxW$B+Mbd#f~TsZ@xQjbPA4rvSkbkQj0qGe{t_YMgj7Ethg9 zql&Fq4)3=Vs%5I0G<4x-FuculMe*y-N|E*Ma$I+fSz#S>=3W`tjT0*yGb4^CaR+>v zne8!mp}9-{@KKtTPa~kY1bqt-A$bIeF!PpqT-c+Vl}!%Prd*uu>&Q8s&6!=>8J!D( z5Y{&_w`wx^tTf~oGM{mdTiKOgnfH8le}Og(ZW#pzht_D(M~{{fn)yBB04d!LxtgI7 z)wo`T`IuMdhNy0W9fL498VX=_ngLiaHu?rUdUvtt!YG(Bv4Eq;F`7wQnnQYfgmk1E zvjj`p9xq9b)#IQ>XPkLM+040nggOpb(s9=}1I%TWXYxqDO~Pcd8u_`OVOb8p#Xce6 z-3TE9Ss)KKGnP~@RG4k1=@^(5(GU$(jaQ?Ay$TczY)G2Um|(ikxH(FYx%jXVs*D)zku9L5nT^jjZTBOB=vBj7- z7`IER)Dcz(5FnQcFyM~Hh<+n@iKm8(iUk;6TdH%G_wLs}IkQRK0JlS#XvZ%?qGA>v zd;0_%Ug?#hQSG9`V;kPUn(@bfg_ao&yAK3|bnRLVB4E0GM6ZoWXbE7t7Em~D#JLMw zfRx*dpSuVS1F?eu1u z8@~7Zzci7gfj;83;aH?o&nv`T{*7`(x=F6)pAg9w6Boz%ij3W-Wc zO9VrJhWo>%7Q`?5Jp!o2x#`HUJGE4dko%t2cG=8Gh)%hyOUiOaxq|6a<6g;+NqbMo>3djr&`Qo znYPEr7ty@J)7*4Wn2fC45qdCrV)2jy-4O- z%SUTkB;(a#o!`~14n~V3DE!s~3JX))VJWjS+0JFaefG#-utZ8DP!vB*J-|d-|hFog=!8i_j$hwm!R}P*>qsTrJS{; zrQBi{LT&DT{g@euUyJ^%T2fCuM5H6W3!=Lc3;O}hSWAkO`?6%uhumQQj@_5{jJYlT z@9VqoRbB#docsD-(_>b2G<)xO9;czZn``>!VK!AZyt3anm1nQQh~(&<{^_A!-B=?v z>Hyp%*z~*B=25>)B^a~S6V4Y#&Z>{$-_ERBni;dbnX+8(bpOaaI=C@D->lu4VgC0^ z`itCqnUP=Z^z#@>t2(=;7 z4sFW;CtI^UsYrTdOBN$Vd}ZbRix;rqzdYB5IkA?3*^pV($P7#NVnnw?g)UW*QDC4q z3(;0YI5dvN%3VgA7MrlNz|0n#e%9!*@kWlcMKXp-@#M;vyH_40&3H2vpGm#$G+Y$y zlj6)rEGJUzsnn{ss$18JeNOU_Z6p6)OlI!eFmq6yWUYkFKp1R=s5mRxkw`2r+$*U>^`ky#m%22Coi z21Q*5^#>YIiML5ue1;$Aw zVrq&hRDsBa7^aJS`UD^Yd5ZU;VVn`#LYW|z*T$1-rFo~8I9Vprj+1J)BBt7zIVo{I zLKUQtLmEk)j3$1kWT9m(mRf~Vo);+>h1E$}cuO62A$|0%YM7jEo~i4jAm)mYpbYu? z-&b%(y5^3*s_-ndv?hxt83USRgB_@ic&MTj3G15wp0n+M=e5Jx7^R^+I;W|5=4or? zlFm(Y%#WdtirtZQ*u-R}C@|_|xZ8R+t9i=~C2kfZ1SPDZ3=%snuNs0I>{%eScjuavBoi>+~w87yDS;3Wq%W<1uo z)2Gwjn{U3Bl1MN_RC?$0iIgt+C8R`b$X%9ax?6Ri&Q0v4){8mp^{a`341|L(lASEB z72BEX!rtKrakYV@JFK^6BGqZDHfKAW+-SF8BD<2s+8)3-N?EkMM~j+tD5fA16l_gD zUAWk<#S5U#nu{%_nYmtw)5jRUy|C#U#w~dNS1ePz-+~M3cQxr{(~YtXp|hRXvwbwR zJBEwmU@_*d`HN-lFOJa_-iyN;v8zl)1FGZrMji^1l3|f*imf^7BwaL~7qi}&_e@{Z z!jGjfxG}5W@Z#w6w6SWC+|X>FtBby2!Suu4vh`U_VIz7Q#Ofq2t0hlf1#k7LwsGxXyZT`Au%UE z!k7Y9nlSooc_vg9<3S#(|kCAO>%CxTM&oQCm|3A<&h@>OJmyN=wqpUe&}|8F7?9 z`(pHfb3;WIMU||q66+SVu4parSbs#MfWQ(LT;|e%!OCGT<#?Xa=`tn#x)PZRRK!zO zNS6tOR+0lG~ZoK5EP^Ns1vNDQ76Z|3XW-flzi;6Ik$YFo!YXGjI=t;~$ww)39+a zMC(+}FnRc}Ksv{KZ3Nk+W*I+`qO5-=6A>{%15qfVi>B!zN=7xh(Ic(!J1*KDSIxLN z@zsWBtr2Uv99S~q$qcPa?2@he7$F7eFCh?eDDed8Oq>EOfI_Y59PYSFB&d*1?R;Po zFQ!-MCGv7LgW{$~r+R)+Q?T2v&5f7{Phc!HV*VirFfn#N#%20#AX+ ztfwhISsp&Ulqhj990AwXSLki3ccOEdwoWsx!1gJp(45u~k3!Yd^pui1dl&nJY7NO6 zO`DwTsA(w!R?cpgxdR~)AiIh}|I(5&yhQ=v4Rpy+_ARfXgw3t<9GONW^{p{1l5R;n=p6?>in5*t^&0N;jL-&CUhCYQpnL&9j>| z6J{}7C9wvWRtoHI@$gt&84k3*iyf%ozA3$yh7Pta78v|a`B}tpn5-H$Zz3B-G_Rg0 zuqZiBQYK8<>TYFn4Ew5g4H-#^hLA$-g=lA=*2*16^Rx)W@nwCiOo_HOb!v4LQuSol zu?cjv#nEY%=f>aewlcGd^(TWJY~#+h*{YLGa%E33;-8&0tEfAQ|E;|2R2+F9$ke8q>!VIYtQ}JLB{~8+V6RIpS87nksNfTa7 znX{|Rlk!P7tOn{OnZj759IdSlP1qif%ZIx&c%#}nNx5#$xV`juAM!bp<$KQdPIe&4 zO5>o)N?eG1r(l8GS~Swuq&|FCPY=6DM9Ujv21>MJr`zYKXdKqpF?TU1{k+%8>a}G2 zjc20_INhpT!;V_lA!io^Zz}AqEGqF=32skV8539~H+aovzFT9F5a+_KwtWihX=UE| zeM@7_p=I;XsZI|W{}RslxRFM=rzci-tr{@5uYIarpYULNUL|+A2I-YPo6Y-8rMDTr zbJf=PPy+OIod3BjxT_Z5sxG>}BPer{c0ncHM&-G!)UK*i-Rk5nwdSv;FU58CGa_t`c*sNk+9z^&dN?Lie zG1@%9vH^T!D1PHtua2pJEoyXkN;aC`XLcVs^VH;5e%z{{K|c?0w8x;xY{z-2y@IVP z_ucQzH~n{6PB_6o2@1d4_VHbkd#{MR@p9uH2qByz`g6Ya5C z31Vz!#Z>5*|5=@fV<@M2ImTi)gm)m;X{>}_y@zLJHG!&ENvP&ZB^7|@bZEcCw4LSTyCdypH@g?C~cgVOpz2{k!NSLw|pr@cVzY@#l}ue_I*Ehc3)(1 z@;4rQ^KQoTg59QQkBEsbCndvohBSzA`_f>@78aIPC2!X>L%4xoxGs|SD!)~UAohGB zM|vb!|4?_>Y-53Ns?#qQ=!@AGE@a?Q$oGAO7(+LjxPPCRafQGWQz z2OJ`$eqZ;CVYPCUc7eWEcqP|x#o~2h2#6;}MS9qXAM;2HMtwZ7iFyZ4^(JRoSbw-z zg&&Cjwtkc+sBVG=#P#Q6_H?pkf(y>1YcyOW#C9}^f+kr zw{4zghY&b-%GeoEf^@5RlSs*97FmQn_lQpz9-Tr->T-m8)htzYV3nhkE$Nal*_DeI z|B$o@lm?@d9B5;WMpvffgcw9l@u-%$rja1W8u#^ED>;zk#gQc_RA2#%QrL|k$vA!4 zkVL6U?%|O`bBPlLEnNAPjcGJtX=|zYYyLxwzzC8@_$M3LDkVvQMtEU4IE^{TfM}&y zk|%(EsT2^#bv^}eDQSs_$$f44Y3g@6;{qctG-B3L2b;o}j3<1KxeC(wmwqFfuK1aG zI8sL0gNn$Fvj{;@Sbah!U@IqkwHJaO_H;J}esvi`%eZEYn26n}JNP(d!ewtc$#8-x zRU(*?G~$f!37pg6ftFX3KPiZw337&(Vdl4tI#!MGH$DP0Tr&w=7j}D}MO@3a|2dqw zUKK%f_LzW735j{PGOsa}U*lkD=WqupG>#dczZr@E*nH?{efddm{HBFT8EK~oiPwfd zUO1YCHhN}oEI3k?_L-axx=IB(q9%%Pu(4Yx`GVkhjOx^i+wmN|38EkBBTI$}p8%z# z0HuXMrBwQVyNM)eK&4tqn5kx&!Uw~g_mZU zWitXs|M@p{`KC|69t)>!N(!h;N`p!!Ij2ylQTn8jAd^*UrHHDij0&l{c|4LTqAO~t zC|aT;T8;>IgLk=_fa#6B<$KFEn04BzY)T${1BNFUQxAzLfeNck%75Gx|2e8)tG8+j zxhkc*il~n2tBQ)KjH;!=8mYTErC2Jd#(8PS<)3+~m|&`u&R0Q%`AaJIouWFetqDC+ zw3K@}ZEbmslBx~kO0KagsIoe%9kgWY%C5NTuJ7uryehBr+Np@ z?JBbNO0p$uvL}nO^lGyDdao+$um1Y4{8|T(%CA`p1fn>zk2uS ztFruhuU6}_0!y!}pbKw1IWWtwk`uL33!hZWaaXIiUYoTei?w_UxMUl+gZsCBTegL} z3Wlo{i3>S{iZsGN6&~7><`52cd$&_7Zp9ZlRg1WN+qr=2xr95fpG&%>8@i_(YmA$^ zkE^$;y(x3L?$zInN?hPRxnx2kKpr@Onn+qJ&iyTIGIm9@Hy%et@Iy2uNPE-Aa+ zP`S?wy_RdckP8i%OTD=3x568|+pE3Y+r8i0x12=0npv}d%M^R zzTO+Z^Gm4`#6@htTdc*mYQ$lD#A5u#AFRYnoWvR2#7;cJ%nQXfJi8gy!Bs59nv2ByOTrUe z##~Isd)&u*e8-}4#)GWJYh1`tjKgzm#Z+8;U2MjH48wlR$C4b$kX*_9i^+tn$!KiI zaoom;EXRtx$aS2?nY_uS?8uXx%9o7Ftz62SyvjQK|0A$09ir^XpA5>09LkKm%eu_N zvi!@jEX=3u%EjDx{dhFMe8`=g%e3pf>r1(B?90>K%hasQ#EiwpjLqAe&D@O4TI=)?Cre9D@%{(j~pm z5&h934bUMi&`O%h9{tT7jnFX-!gXxXO^nir%+EW0($Ngk6TQ;m+>ai6&_&J5GyTyS zebfbv)I;5|Ds9p{-P1n((?DI-EFIHC&D1cx|I|v|)m~lDO^wwft*LZE!Wc}D?9odo{*ptoHcAeDJE6SP#(yZ*+ zovql34cbra*rQF_r7hJto!XzR+P}@&u?^S54cxwM+^Q_xHC^2BY1_9A*|@FSRNdRe zz1h?)+p`_q%01WGo!qBQ-OSzC&i&lcJ=M!?-rjxQ>Al+OUERm+-rfD)@D1L7&D+qO z+vF|X(;eU3tqjh<-*Ua(@=eG4E#UR+|J3tM-}Q~(_r2RF9pLPp-RS+@4_@E|F5wWq z)S;c=#uGl+8(ab{XOIVP2xOG+ZS%+D30Sve%XS(*Ki!>^f9^-e8=N7)`Zw}~i?&*0R>4MJZ zt}W_uUg>6N=26?={5D3JCk6!AM-k6tu<_&)9u%6~}?d!GP>cLLz zW4-I6?G3A*-?(1v!|vo;{^yx(>$2YL!Vc_gJ>{nT*nYj+y$;x~P2|)5?9d+V;co5Z z9xCM?>*ucFsGjQPoYwKq;nhCw^}g=X-s$YF@8(YJ?XKwW9=-A|@T~6bZJqD-&hP!M z@Q;q~3BT{ouGRpr?ajOD(=qD?|L`h)?hSwN3(xG>F7Xqu>d7wh7q9X6uI}ib@g2|d zAg}B5{^KKG@#}u_C~wUx-|`>t@^fD0mY(g0{_?+W^RkZJEAH^%j`NnlC5okFV-N9h zuk&Q@^K)cVGB*Kk;6i z9xb3 z&y#3;_3qoR&yQI?djItON8En==~tj~_X!vvfC?ITpo0xAC?SIr)|McG_&o@rdKY#` zVTB%Mh~bDDZYUye_Jyb-h%26GqKhSx=o^bP*7%`}Fygq=ZZGzDmyI{37$lH7=2#?x zJRYfJl0FJXpc&MRr{;8&;)L}ZP zrFj+_sgsp%ny8MNa%$?MgnCM8m8hO7YO1Qn|C*kwqtE=*{f%_(i*6) zyF&Wwr^vFZ9InAG%WSd69xLsD!al2;war4SWU|w8t8BK|%1A7@)keE&mEHnN?zrJD z%I><_rdwz{^VDmvy!V#dF1-5Uwy&x9b{lY;@BZs+q3!m%o z#dk(bFSu)JEb)65W1O$Z-)1#2lY&xQF3BvnJo3u~dmL<%D9gOE%Q(aAtIjHZta4zx z(JYnC=H_fPdK;JgbInT^-7wEkM=kZ#M>#zsE`t8VXT>J9OH}CxO%11B!?$l3j{q?GQe$MvXcaQz|;1gdwH`4E- z%lYW1kNzs^v){h^=fmHP`sjOOzWw3c5B>i9_wRrB01ROG%;7x&8qj#+>zDb~hX_MB z@PQ4~Lj&tT!8t%Mf*4f91ugi%N^}qt6!ZiLHHZlhh9VTLSOp3fXg^l`!h9=qA^qxy zKlRlRe>lwH|Fl;?0v=C?KJ1|o|AY3!As$d@zRO+$bHs{5)M0{Q07Du$XhAhpFp5(o zgB2ynMJjd?ic<9A9KvWtEK;$8CETDGIT#8=q#+U*6k!9?XowS5LXI+wL>^1%zWnh~ z8~JhhvXTf=1Z69&2m~8sL6o3;LKZ!l23^i_jJqu56dmY6Tn-b3A{1i= zFNjAG-cf~V^aM2jxWW^@P<}JSrXY=x4M|S&o0SA7CdaACO*Re`6+|T#eL061bYYdk zRA(FJiAp`AmUljE$Xv^3st2-c^ScA z8grHbZKy@pxWQs#keJTI1RUWgLu=ZRklW;DFjC4=Z$i?M;e=#4hiANGrc$1_)FLc# z>dt=BQ=LWF=|kZ;PhaA*r)Es5QsZe(W|~vs$g^|6-x+0CnCa*B-O=9p{YWYuJDE~?Itqv8dGyxYI~D-pjfe*Q->Co zuwTss8LZ0GsQPrNKgDTbq2N=kGS;hL{iPnjKmw;q5T8WNEG=yrT8x4LtNbjjA`p54 zuBsK6g9T=45vx&I|5{dz!zAG|+sIbl&M~fZ)uvuyI#cRtqnJGX!c8SR(9MDso|F}B zbeW4&H9YmT{-kYUi+WkqPWP}Q!0dA)dQq>kvWh?$19_cW+k+w%y8DzURUKPew91yf zTxINL&+1wH611#ged0(3JlB%OZ-pv7A8}o03`X=)z3@fCD*vE{BhdG-@4T*QMO##^ zD%QGj!0=a(DqRap^aev+uZP7-gu#NgzFjr(XA|mK!!A|7Bo-x#^Z8T@7eT}Mb?k-Z zIb-n#nXK}i?2*&EVT%saq9BDKCT?RtH~P54UwE-=aheApBNnvx9Px#NFxp!-wYuKj z@`qKNuq(A1;>doNp(?&_laXBJ=SKC%+QsjCY5dTtDwWSY9dt&8Y2?N( znXJKdRh8S^g)ny+%{=A;6ui9Lhmta=ST3}3x7^%;YW9@$Q~`HkeBI7wd8m(Na)qP& zUPj!3%%XnnQR~TKMbjA9p1w7(g`H18S30M3F7ae-=}Ja}+N+TM^o>t#V`(>aou%EOqoy{SXDv}K2b&78Ci6mRU1v@o``H}+X$$;|Ai1WD(iSGEL zTTbtPTbsF12*R*gj_F(%d*YA9E|?{bV~jrzu${K_7GSLEdZWDI5pHdtE39^lA0gnM z4lm6A8H0mgJjqx7_!w+3)wNUi?+|{%j;(9!9REC(S{|)-=ic(d=UnMX$avNnulSk+ z9O96F=XOD!bH7EB1Xy4OcF+Uv6mD+raK`mM-jf8qvD%;OWeD^}Ejw z^r$C%>M;gB$Bmu(vU_>&B38Sb_g!ap=UvRo7JJ;uC{#_`p4wMGdZACCdjJpgYDU~e z{~H5AVLMCz;)L%v)-#Ru$cvWyLG5|(^$mC;z+dNZcK@-vm}8#0eF6t&Dj+fABn#u$ zY3sHHD0h4aCw1p{fRs0H*(Y^ycWI;7a?yr&r51sYrwFeWZCb!j_jd!GwSqR6aR;|t zmW69TNK0}@cSK-og|~y}Ms^8^grTQyyvBODmUlZvYoNwvNycw~rb}%nW=e!5m?T$p z@POfVPcUe8GUj6NW`{3mRI8_exaWJeM`@52g=08t4rgY0*ah52ZZl|anul(P|A=Za z7zQHfUgTGRMVMji28bxeb@k?gji`vXgoI<}g)_)&>=jW}$aOZTY;O>CCRjw;gMu1% zYv)I7=m%do=!aTYgFTpHNSAMXhk0}Nb_EB5FNbV6H+*UMc%~?Yn%Io96?Hy%1f{2g zPFHk3c6)NziNSY{?>^`D1)m+0-i zdHqO-S!R5Fm}Xo!TS$e5q4jJ4`C*Zle}adA3wdXH@QUHrM53mAiwAVI|CfoWxQzGc zayRLAzP5S^$#w|{S|V7DdUt`mXLmw}0tKdD*2w z1KEx9WRTnFh+P?$GIxuqCO7;hKsd3u1W0YBe5lM`Q2WkT-f2B!_typeI$pGe8meECz zHKtW}>fI+Vny1qw%rfoPIc*nAQeYU?(S0=QVp7gS(qi#95huxW_Y$W#6&k>MzL zlqH-yiH@xpj564IJ(YGTX{BaRbBWl8COM%P7J*M{RVk{Xs+n8Y)Ehof|VTd7wy;tB~4}{z-5!2$&pdY;9JoGdH1ys!x}wq=ibOpXQ~c zT9av*fy+v8#yG3j_@4Ecl8~8^<4TCN%9`6cL<<&9og{DMx_p5thQ`^d*oT{tr=@uM zfAi#cy-5ZenyUV$2(n49YG9qhC$JxRrm#1inM!c<|0knux{YQ>e2WTsw?>hOXsIKJ zcU9+?uIHyYOS5`faN#$xedVoRda(gnul%K`wA!eT8kgs$e>CW*9*4D1d6&baab zc74lwQhAv-Yl_dexjeahg?pZfd!_?ujaxcn^ryJ3`+{pr1>MJu$F_#tszZ<~q8$jG z=m(eTI;ZZsyEaOM1W9QF>#vpefwuUWX~weQ|H*wkNVg(cgu}UMvE`J$`LfUXvH0kJ z6&k%bN~F5W1vDGI@(HAw>$5b5yecXR>2#tZ>9M-%uyF~eoT`qKDNrp4j=R}vXFT8n9~ZF8Oph#hQMZ5r_o1>S_`|uIGI{1i0{^+C^@qzY@r8>e7#3lo;ru=jkqvb=yRN4)$y?oFsjd}ekov`S*F*T~c49Lv&SlWo)%fWizl+DZGhKV+C56O>1cZJYtZPMp&uQ( zdpws>+r`^_drNtCXSuO?42;xyk}wLI<=e~*2X53|-G~%eknE{lnXlE{(cS3HP}B%AEWFw=3xyPo+C%uFXpY7ZUWxM=d1Wl#jtHs@E~@sN%Ff$*eIDlX zOo=L4yBuq~<|U+4|7B9oz*M$SGdoJ^SP& zoUG=ps!B}0R{7ubCE~OVt|rdkI{@S-*x-??u%sE;C@aj)+E3|y)QE@av`d^EZPfN? z>vkHS+`Z`mYt%}8wg!CZly!DIJ?ToD)2d9fhUJ$;pkrGF;^t zj){`lxjCA5<)_ihE|?sAHe>8|eX%;RVttkWmi%ZA}$SdThg zcTQ}`47zS%|2*kg5X-H~uhv`0jhfgz-EI~?ls=trxvlY2tIye(b@~p>{JYH79Mj;e zcU4Z2C1jNdHHkBdlZXxP+YRhNJA1h-IyUM>fa$ULXwegwi!@TwgF z<=x+Y|B9h+(DO`*>-CC&j7{udE76aQ`5&F*GY|AQ+S-2aeB4dSz32NnU1H}?2Ep&N zUJVdI1Tr8v;sz8dTHXjGgX9DmJ8#q=B%$I6mxdTsgh-etB9|Bp<8%R1l7o#b7h4oW z2qj`4C>Dtn*;rEG6Nm&(&a`pj;1-%SBSfG{vPIE>6)h6IgEHkqo%0v~PQ2{esktB1nH4t|akAEJP#;*Jd0Uqz&Ph z2}3H@0ypc9#bE1l1>BKQVyRqP2FAe^o;uLyM2{wI&h+VUsZndwwW`nS%rSl1tjxKy z|H9X^7c;DRc(QHHi+!3c+u}`F-%o7BLH;zPo*Phs3{8o#>vFe1MQZOYQ>D`rOm~J2 zB$$rKq8=&xF0NcRU8-qAjD4J5EYpeY3$NeXenf;e@~}>g*Pts5iRS#`4nC!33eG$B zjB1d%_}WshK==kVBnFwRdG53%X?vAD$1n#Z;sw0l4z7EWgLL%}r z&OHJ)!!W=WlY=m`_rSB`2)L-@ipA`Haw@Cen(RwIqh#A5Hlu<(sKNJuRIfTDm2@&b z7ImbmMcp3UNkOHu!mO}0_ZkDXG2OHgi48gYP&GbLbIlA2t1_z#Jr08mLW|m<|Lwck z(6p-~obLQ9(7>1^aKNa11ZqM$V)Jfx!H7E4X+w zX|}Nrjcd-PE|u_7x`wQ+OhM~3b1H>QjL$C=?Mv>#WE0gQ&vx{*)^Nu3{4bSr4@`$YAGu$vT9593gTi{(>mGkUDuGpv91^>A-D=a_a#H62$W;Qw}BaOkteQg7t#mQ*pk z+1hc*RZWiFW6`rW7Gj~OQYb%X} z5qZ%|zMR#$uE?cpIs;zBzP2&UEhuh|+n-v1hbjp^FK`>=VDvDN|2iyq&I|pU0^RU8 zx~MoTU1SsCQSxw^#60YL6@u5pOjkSAv5O1^SHO2jKVKX?> z)aI)oCF{H_(w_vQjmiLh3?d1#~#pt28c}L70Td6gDp}-A%t29Dd|Hc zT2hEgY>WJ6fVcT5a)g@1q#bVf1`b@sfkWBk3(G@Fcq#IN729OpfQZQ+)>4&1WFaL- zdC5cy(F=00B%mI79!$29l*xRgEPu$lXVq*jviphln#CCn|E@85Z45;SmvDwEyg&xx zBxgDI(1>2RL!InkXFJ{bPI#_Uo_knB66^_1eBuEPd*Ek37xB+S092p>eZ(LR0nTsM zQK34xr$WK`18{zzo;iSMJtsPdTFgSDso(`iHA+V#q@V_CsDTMgkkEzJk)rf0DMeda zN0O%Wr4(Jl9WN?VIM$Q}KQIDLd#X>6rW6fp@Iy>*dQ^uR6^}#BX)~Qkw+KQ@GSR#m z=aQApZDti2t)VCcy!utJHsGsOc!Vw9u@1DNL#=6DYg^+PPmkhZoJ??n73z9dHsqBL zjo2$+{rZT%UV;;wpr=KpaM*?>Hk^uGEDs#Z15VuI|E!XwtYqc-Sj?6)vzz5?XB*2{ z(1zBqqqQheOM6<>rnac2Wi3!$yHAyRb9!7c2$_I|GGjvNs#w(~^g@w=C4^PD8Sv^i zWii>evNgGIJ!>i+JKPRLSGo*sLJ|N0&^fNon8kGDcDs8_@P_xh+hPte%?n;fHuAeZ zkS%-F>&q*M626UOU@`6DR8ra(fR*%apvEgA00$Vn*(LB$=}K39{?dTvOP9(hrK^s) zthZX_U@Kq%UE*d~tkzAZJeB)l5HA-TsF*H>Az)lh1XP}z;MP%g!$}vvSc?z5WWSU- zLKD_Rz9GzU7{}GGUrr2H7l;xSC0R)pst}n#{~jfrLJ|TZ2RMbBo8*kyTipZ0fUfZk zV=AQxO>a3Q$4i=UnYRgw3wu=zfFOh;*vteWh;_qk=x~TdY-et)(-w}EbD#aJ13uUT z9p2bQWkpdaV0k$N_|1c&nZjiO@sKEf*fA^T(^L6cdMNdb;!v)F2PNoC(^nWo4HBkl zmR5>v|CzudnkkXtq!Q3@REwjf(1`CojyQ0@^)YjNlEV7BUwm+FmYJ{wlg=1jBbwdv zCgYv5y^yyO<{&2$;Z?e}nGk89!Vrqu=O)nE!{c7ApzVAQRFHTBe`fa?jKLag*n_gG zC^Gy{x~@A+9N1POlT|3;`o?7S7k2)GWFrw>5z9GPJ!1n>Y~O+7D# z_gd5i;We%M!%Du^)PkPALdd(jLsrVc&52E6DRCjjR{yL3YFGZfQwZwDvKx%PG%AY74=raK>W@FXbUJ=Lw#383SSGlCSp z5hO+Y+u5LT919-T>h-d}R}RU(6WiC4_xt7*zxbl+5fFEU0DWWpEcls6=Pwr0MTBl< zZ4?~_Y~FUiaUg~yI32ib7=_fU|NeV(U)>gxSa;VGaC~qsV(ep=x1h1{32q}t@ITP^ zR~kM2XR6`t9H@IG`Tl7sY^UE#@VoeR^6>H#p5luafk+b104wM{3`28#=hqS*c;z*u zV9?R$F}=3)Q@PG-CWSjED`2XDffw=vJz{dR(W9#qySO_U9onLih+!VV*n!tGGd6fE zLKp=@_zz2qgJat;COEFCOF?o2zGfh_YS_BsW4GgDC|6iMvOBBit2ab|3GA~Qx??-N z*gkM7B_7N_7qE}k%_BIWWn*JEEkeySz`z zKbZ3Z{EL7n;HxtzpdB2A|166wXiJWn5(>*vq z!4+gdcxu5`__i1{zLB!Q=DUX+OanS$H6UyXFZ(xNAhrs?9WTnbPb9??*ek}XwMn}?Gt5LTq%D~fLtb(@UzE9j8bHh&fm0;5qKbhz zXfrktgStYtFDelXsh%f6oyC~83EM*m7zIKIK}$n{ozlH*|4H00xRU9G`$V zfrJ8yio#D?NF~HP{}f<4`zw}Dy0uV4GNkZG{)({_fGuG=h!Ri&&8speSp&aI#xQUM zH$1R9f-zz7xjVF_Cp(Ivqp;r6MmJ!yoeTpOz&&wQQBN8dH6rT$^vc%yo{+chgluNo) zuwSAz^vcV<@QtC~wYQO*8>Gy$U!+MJ5M zyv-5HO~33-|KKdJ)f}+gG)~rJ!|M`EPYase!IbZyo}Pp~=+VX#Ycnwj1IUcb3>$?y zt4tO|$7X2C&FoC$`^*(!gpFdPlB&M&;HDhVAW2ZX=~)FtXD6le<&=&Tp03XJKb&Ua%2 z767a2!o5M{PNyS;Z@@F9Y(dP-%mt8u@|;l{?Ygf^C^(?5%$xupJ%As*03bDhAr(?1 zH2@AEQYBTS5CoNMmJyS3x zvibDV|1b@vFJ03Gbx`fhPC>UFi-ciy_;o!68)c}btAI<@02-}SO9cT?b%7*608$N7Q!P?9MbjgF(zu$^ zE#*=yrGY3-F-<^$G))0E#Z?O+rc_ncUTsofebXn^)n6n~H-%FnozF^=!1rva8o*S0 z64iUcCwrPJEvv~YIuwIUk+{SOCsK&rFw{eBqilGD>hh{ZO;;8Ggl}kvcKwE37)@D- zSMiir7gWS~3O;c1g#cI%~ z|BAiXWaxw@MOZ(0K~2!8h%M6rFx5HwRbQRhA|=*g4Ot^?Pa$nsh~30yy|VOjR`+_T z&XQKn+AsAi2)HUGj1d7+agwDNU_WuN#DS*wH@ofKu&*D3ya#MFUrWTRwQ$QgyWpz%?SJ z+x5h#jS>L^K-eg#20xI5icQIpg(JNsS%X>xT}@d^E47q;*`b=-MX)xPHPSKwD6;iY zo`T1J`ofoTHK%gcEKS&)^(+Kqubi|dH3>8Mz?-7&$zR~crA;SrD2H#rh0%lB|30*~ zsq6)-t(L281?0t7S+IrVRbEAGK}jgv^LS7>IH*t+Ra}@XwiT-d7+D`pgjv|#8d%Z; z@PSOI*fqdgQmtD#xP~YIQol_F^F7ULNZi6j*o^Y1md)O4$b?nh+xoqRMMzUzuvnNS z)s78Z_3eZJrZiKvSmz2U3J9t8%~<(;R!qBujlxo!x+4fCw$;_y%~IM&0!*4QlC1F> zF{<6(VlyEK%I_?PQAk>-gI5;)hF%~9Az*rk!J;&i!dto(tUXZQ@;%=H2HgTqg`qm!|5-J~Ot^$K z$OK}FQpuf#@x@yW3r`Sz-nVe`YGKR z-42E*xaEX1s8h|df=u3IPP0R9{he^Rm2X@<7S^U2e&MV&G#YME9Ug^IcmyH9Viur- zvigQHV9|F-ggi(mU4DgJ{sK|pU7A=`CjQ7IN}x2>;|Ye=d{S5)NCk|t(jP_OP9U{L z5M#dO+XFy@`Yq&3P=b{u(#R!WvVA#+Wv|BlfQNOjZx&;9POyIkU6st^Y2HNY%gl8& zsU`l`vYja&g;v(JR%^Ykg3{0Hpg-+# zu-xVbP60H4{ec2b+y!3hK@jJM&8PxaW-P{orT*j3tz_RdWPO%kfqmM4 zu2Zod>vbGNfJUrM7HBEZD{Gt0Uo#)jIg7?Qj8fJrN6=wFkmz`=M?nw-h`zAMJTbxU zR6?kQ-hJhCvbMwqU-Z@8qxN2TUIbi#iDO2Dad2O^1%R2}fM}Ig%|rl{np^+vTrpmP zHcnqN{;0z>U9E#MNbqQ`9)OwiV{9fVIELym?kTJ;)wQmH|8oM}hc(^Z=IXBQ>aRX- zfeq`k4rncF?q-C?!35>Xfgr0Ik-C;JZfsHH^1VjbPN&1-8+C)l-URd&>0W?l-bL?O zaO2OuVw9dOu6R}|W?SW!QBEyY++N(eRpe3~*+$snALZu!oovXZflfTX{6=ZW&SHmk zXY~c|1=oR*rD_K+ZhodyvR3ZU4r{b#ZhQPtpAI%Xk8gBX-n%@*PIhV0Cq@bd+SYliSM z2wy3N-)>d{H|9@D2n715V=?~n1)o?2S7!(>^D(dR|1)=Q3)k?mwo?u7><<4dwdSWk zNTv@>k;70(#B(5FToccO+olr-vzGgtyHkS@JHY57!>mP8X?3j z1DAGzBZukx4pmL$k94m#?hMaxAYX0|A83FCvVEJsoM|G5G|Wm7SL$pt-DPp-dvORD0z_8?U#^A} z-Bd~E%;*XLaj(%Ch1hKsq6z=@a946o@AOZ{;#;S6EUsXicHd+ks#h4*G@tb_M{_fe zXEh&lUKe-}i1p0n}6<~RrH&1fM^>^oYCa>g7h*DQj=5y|M zp7&ly-*=hz^nhP_fG4&0g>>m%?Z<8N%*BN)=Wxz)$)C?(gr!B6{MLmAiO9I~og^D* z2XwT5x4RZZh#rE7c6+yvGZhU-hV|^}3bbv&Y@Ap5ABSl`0|!djO+82kaPV=oJ^Z~- za-~0d`(AvgXL?9y{LTJ%f^YIr&vn_BaL709%msz0>*LT}gg;Q}Qfnlw(<4+cq?VF; z^)RR^eSPeL+l|U7?7AW1`Q$K&Al|U1|KXUQ(qvH7K}iTvy+SxiAjE|RAMIH*%NE9l6gh25 z)aMJ59V1lS(WB+ai5OklPywQX1I&~#wXi(dQUwtt-kR;~+4GkrL6J%o)0XWh(xky0 zA!5No2m~QgrFQ!EZC5X=-~hP+)QeM-GA33SS+eGg4iGCS%;>R1ga8L|=Qg{`AHFW4~rDJED8uH`rmNRd@t=#kI@9veO z?yA$ZWaOZy>Zs|JN0u^kW+6X@4EiTG#?Mq!(>xHK^5=WQ7E~xvrPlWIALR%TM*(s* z1Rz+Af>kfzaP-Y|10^EKMZCPjhh2wAvWE`=EI`*`Mp#taY;Sq6$Tg5a5g1^^q0u2? zJVfT$iyCf-muxPiC7T8y#P%AEqHT~PkgSb1WRFB9=3`o|;W(KTADAV=1Uphd7H%m- zshb=f5yl83=V>>NaM5iy|0bEnIX4}e)lK)@JGf+$2{&;L=iMK5hJ+J*H9 z9jN-FAF0$bh!2;{vPqJ;VR0*pQ{-0bkumxz$s)Tfji<#{q_H1# zl8Fg>;KHj~XDqTU7IX|!m~K;CQA81+nly=+WvUw{yS;%T5J9M@S&elIj@cboW_S_O zpg7#AC!l$@w9*sk^_f!)&?#JIqJoe#6n_yPBTZ5Nak>t3lR9-llpJ%MrBzJ*0U?{C zvTBeKGXh(IMX4Op|HC8QC7DT=1P2DmB3Jfm3v4$lI;w93Tl zD5Pkob$HzxB#KxW9NFE0g(lm88?FU|e5u^JWw%>NdgYzyULSJQX(2;kl7lZ=B_Lt2 zO)ec_(_nU}xAu8l^m}(0ds27!;Z`L|ULm9u<>)WKK$I0$R*iD07yt@bSXXX%F)j^! zq5Ofer03}KXtvd6w_BvAz(*jV+okZE|&dV}e_sXsv3v|_s$icLXLK|=7WO${v z{L{%V$;LI{Tth7d#ZKnOC4WqTvk_K)P-Uxyrfas@vY5{hDiS61uDo*6jVTT!_^Y-` zfzw2g{b>iP|9AW`_>VXMzoDp+0HF?oh?qLeVT=VNc zA)yXN4)`1-#DbeYpvHT*p@D5Qu?LPpOLs1e3nZ%1k<$4F4IXq_0Spa zI9HY^oP~%(6e1oBv%xA1>kQzk9u%V(t|&msbx{(V_~b@D@|~%R&H+<*;NV0#fTTTS z5LCek^M*0(kA5c#)Cl^OFaRb_5NRM_9U6xepFpsW5KNOjAlQln$&gf}VqFP#^F$vq zW>s>L*Q-(>D_ii2Gix!x0m_hyq#X}2rcs{MkOi_$&M1A?B3|(ZP=rlbu#>+clGxNx z4mrqB|Aw==<$}5C7B)1?VH(kh1K^<(mjoqiuysK+ykdM>bS4+cR-9=*Q%>m! z;~3dkzqHMvB=6zZ5&T9j!L0{Rc%mcXEHMNF=Eo?F@`iEHX^%9aQy|0{1rg&`3+?n~ zIdCwY?+#O>EM+fN4OvSGqo_0?!J(jd5T0iu!#Xn@2?Zgj5;8lv%2qORX~^6S9m=II zGX!F!YNUg=yzr5gz5qf%y}u!Zx(Sw>Y9O{G1e9k9=+=ZPrtmjJ*SQ7CYeQIYBEC`>2c!3H( z&@vg>YODbb&Wp$bR;5s-f>2yP_OkVj7#m&C+b z*#CAX7ueuvSHiv_PIM*o#nIitAm>7ee^VN(fr#sB34I}ut61J6Y0_ozoy7|2n^77) zCU`z3uW4%FGH{Yp!2dPa8Y?T9@0M`BOs1n zyJ64*=03NsInt$sltCdh<1Vi9Kxj_hn#nH6#5Kz5j(=|2yo^!v0K_;O?v5%6YF>nk zv{jKNo#mlG9BPvgqb*K0uT{t{58G-6;5?|&)pv*_Wg6iM7J1n+hco~hCQ0UBx7Mew zRfs~~TGZM`^}>8&Du;paY&ZTbsYKv*uUTA;IaC`F$7My_xZ^HRppO^JO-8ukHQp=H z8Z6_wlCekjiYU}oAq0Jdl@2gYJ8WV?MYFXhU9hdfr1laMcqm#-{Q_hW`5q&0(;ECL zArGbL10({i2>QCF6r|~&g1-j6ql{tnlr+|i_omeQgi>WTcC;o(c_w7B^8cNRqnf}D z_PfMe#G=~f;<+Vyobeb3{nZ?gL1!_wM;r>Hqiao0FY}kcojxx}6_dHqp>=4WYr&@% zqQ1k@_l%m^Pj%1Isrzh$?y2wZxWg3`73i<ANO%b^8Ls~3CCcIUUf!PJIg$1mbuNeyGeco!l zSocvx6^PeL@L#~l1hX}pOQgrL6^zh*;5HB)JL!VAEywMh;L}xyEl5Oo%v-g9pWU@Z zZs1$^p;C^3pY{+A3ppPEpg~`mRha18sgTf({EoFmkAmc#^vvBODE~-b(bNlogDW5a z0vsNOv`N`*i=@#&4SLc6`0(2+JrcFF zg$AL` z7<$}aoT2?NLWvl`|HaBFb=0X%i511+x+n(&(xC%xN7F%v1X3WmbQlTrUI%U?|5VvX zpobvdo(MurBAVbLGUC{PhZ%5~xz$xNNP=x7nmeEZO%achLH|((6$$frVz_-`Jl&N- z{m?zUg)3Z(U%jHiO-S@ei*;Pf55|;VHQ^P`BJY^SzVMv6WPvx07X{*vs|k*HRm&Qx zMFY4{5h>&Mumy=c*EBvOG)Ch)7Mk107EzH=8SUZx*w3>4)(NzlOMD|4grn0H;s_#w z(P^I2p<@cxj7Q9X0pw%|R9LsvjF7n!C=t!_pjQ-~K{c!(Lf93AASKh8P_9{z$q1bH zRY6)BlNESh%w)oKD8O6d0)@DO;ZcZM(WR*Qoi1)xI3b&jJir6Q78$mh|AELQBqjh3 zS+XSNg@jr4pa6+@0++;&OJG9vxnu*vB*u81g~(hsS^r@F*Z`LGTFBPEVR)$g#abI!z9MWC5p%;E`TZ8laVzK zuJz?*HJ++~=V0mpvf{@Bn~#R)8gx^)wN(AJ z!6S$TF2ttW9p@{U%&JU*tY`w1Dr3yN<`|5OA$UR%Vku_^p@c~wZI#|`T__-W7@1<$ z{|F_60!2Z%>6`M?gnFk`HLJ76M22Q(Z`NBuvZ8Qy0-!nppbFJRY{3Lz$tz$ep9Db& z#6fytW#t{uj3Piojf>mtiWh#!m-dYwxc@+=c4U^ZR0S5P-@PRtP!Xf$j2q&?b^%mm z%1V{SX($i@t;&t9-s)Fy*XViaZuJjSfux(Y(as%}fhs5v7HgZDW3oEkgg&Pxmefa_ z9wd;^4(4f|!hu~1m6P~swrVT4+JbU+51|?AN%W-kebO6>7VfMoy56H(h|?~1n{uu} zc)Eb;WhCD~6dRra^#p7R6i@KHjKP*b*BFWrK5UjkSh&54%$;a9A{E+-;Kc@MH(_K; zccp74eiXdaR1kmN=B>X;H9>WIO&+=)r8+1j3Rhn4ZI@vSs%d$ z6{$c5&k&evz>Jn&Dei41XVO!bD&HTn-UepaQfU)vqG{Yd0deg@$R_2;QlrKhR<|&n zEvA4HK*$*g*=%SDmlPui;Gw*EUyA_ewerEA0;(dIk4yL-Dy&DcmFx82BSZKSF*#il zxz+*)tlMhK_C?I3EGH@#>I*;thjQ6TV0 z5_CsONkWNC$*Z801EY!qB>yDUAV3mq19)r;9;trz?zy~j4qY=$pJf_mGQ{~pC$G9jt6b6WOA?uZ?GN@C+I@l1cHub z<*Nw;Yp^0MPRM3(fGpiAG7aZm`aB}y+GIT1%GC9a_k0U(tbhx8LH68fs$j>X&K^s_ zro0RS(PR+1`JP2yRJk>eV{jUG>1ava}K&zpQM$)$?VAXJoY-b-?3 z=*=5Vs@e@)nKr$dAvXee2xj@bQ@e+aN}u810MAN#3DCD-q;oRQcS^>CGa#igqwE$w03)NAfqL z)FwJf=Dmy#!OEiI(2#7e(!59%;G0V$U3><$`QV}rt}%FwvkowX;G!L=byA1&h3OTr zMw;+^qU`-?LFT?}lwO)=Sy$z-`sfNU!xFi}2gZ@W&RAwTYM` ztQbnG7_xe$b9F>WCSPK{K^~l#CHixRiZKkax7&5?b}Pr(_W}jXY6fI|f<2D??N%tx7^q(t&Pp?mw$@ zJ5vWDN-E;TGKVhrkEEOh#`Z~g>K5=aLLVE!`~kn}K+F)J+ASPt3><-J#wGM|1DdwW zy`02OEP#UPPvT^ov9;Qsa7VuwE3r~a7ab;a(@LKewrvAKs4#UkQ=4EErU-Ih8+Nmt zpZ#9ibONp3VjC5u^FK?}J?lk-8?Yu+!QjfW7iLm=t&LdCs}7=fvb{I+}kAm`k?JyK^a;RcdQJ1(8&A1Q152`ULehmW649r0D4WaiqfNAd! z{9loo#mY)HC_G2))N&4nost`~fB!dsqR9uJwSlwsm8YhmBC?imIU<#eMDPZ@s;_fU36ZFf@_)(~m8K8|;dXE8w=x-;j> zGQlo-N&!YhQp;2yt>6nqJ!RRYmI<cxg zDG#r5=apG}p=TBpDV)GDx+aHp@S*ObW(a&2PVV;nS-4c*}3_7kQPX5Vq@m|{Q{J?025q#L?sLM(m^YzG zc9zr?8jNBT&}B8yrCDqVrqKO#2sEZldbN}FrR%-Rnbq__9L=ewW%kR#EIWdafn$ch z3XS37!@?;v{vn2fDwsk%drMdc>~qX%c=0mM{KmBr1SPyQ(%2P70R*I4mrDGZmC1(! z4LL3)vjvV<00$r7IDE*XLIRB&1#pyDK!gr2SVod8Nk>zvSINLdLGTfQj&i(Ey|Yrl zBTfJlFD>FzV-2oq*M_FNwqn3eg&C82cuC9VVP7cV+@d-?XYs~7O!a)a-=N}EK^$Pf`uJQ*_wG8M;@ zfvB)CqOs*LlrdB8`B~FvMuF0t7*VItCsn6bvo`e_QzS2PAf-ert1SxMZ)wcn-Q!B| z-n*v&ukrTj1m$j(L&ft0Ds&SIMU<_XYQe-Ms2IJnsk38Fm?4Xo-%>n}L0juvD$R}n z)$Od|VJaPTutXV^x@2(!#SWLZHdxBNP@(OI65&b$*FcN8sZ@o?)fujkxT38G-4s0ZXG9a>$?Ei@ve4K2vF>;hF zG|-Baq>ni!OAVG%nuHCuLeTqz5lu$3gCq+sAa1zcdiV{@-l#Ic3`o{!?y3|-hz^7( zGO6;KSF&U16-vB=BRl%^94`kX{QOh0K)Q2C6ju@gVIrv5lM^cXno`J*Y8KIQ6Iocf zR2Ob)xd1_s9tH0`0y7lV!3vhx!^I3FSe3yI=39tDBh-XzEfWW`6~$at6lIV4cm*j6>yyd_07*~~~U$vemeE;KSw zLTjy5tfY394tt^!two;NBfdwL^T{livf}VfffzNS6IljjqW@7ygLROsOJoGpPiH0R zg{A;~Fl<&jd^O|KP6ajq1xY}u(i2wP2@oa@v1kLq(#L42AQscCG4%uy)FOHETX4hGLlL)x zJI;qP?$cz8iQkd*AVutqPi=R12xL7vJdGkTpv1bB8hO{OAcYPYacFNy?6C9ICR`b+ zmrb0y0FRkoaTel;yP&c*IpC6%x!Dry7(tZ?vS1LdOQzOdP!)6?L6}|ssO6g1h#7ZR zU%tu@xqJpXMWKTa-bJ6xm?4BPrsfPsESCmS+AyRw7XJ!m)mR1l?6nUC%CNhSP4QU| zKr?OO22qP|4^85&472+-Fs4WDd3&8^4DNNRxDJ<`x+_tcZdIC?pF1LFg3sXw6=Wzv z^)k1cEsf6(Xqf^8$(67L*~ej8Is~tB#9Iq>!$JhW|XF%yg0jN|MS{lw1=H{2-zl+7O31 z1RP*CQKUck@J5=c8q>gV1T7$ON7iE{7Z%utB{mU?--{w9go6Mt{?dyY3KQDISH(2t zzzF&o;5Y22gtYxjnav2zz3fsgIC@8nwgANi^kc^aLcwb1xmq4k&<2TrkZ7GjLgGQH6XXKqMC zp`j-uqJxL~iNb)>;l%(gRYyx%>RGQ8r~jEyQ4L;D;vK!B8$wU1i6poak}LQMLaVj9 zMkc2Ug}P-kqF?}^?1Fo`Fw|CD^(hB+pk?(*S<7&W#`UBFFAUY|C`lQihtgq>L&QO6 zn8p~(Xl8jy?Ga|wb<2)EHj0K@1mtMQi^_sy5I|82137CRJw9-7oPdZ$mRZz~rj2NS zn9ddaAc{(iVw=Y41yLz6gDZIRo8naJ7RaU3co{BZ7eWK!LMV|`Tw_x4^ePBHQl_pc zLKdz`3yBlmOh%RG zw2N}#dXD8NWW^Sn;JD!tl(pYwP5-Kd&0-G0$BS4>eYqVLK_W#ph3$i_<%m%r@LJUp z?>fe4Fm4`In!2rJ9sxVjGdhw~pIS+B@97CjxRF;A;1g&cRZNz-HLl{>E~_9kTpc{8 zt{pO*px9-IEcD^UNgm{pmTaxtV3@*PJ<=u;GgunMAbRj^-trEsIqIoFX-pLJes|g5 zNZ1w+0T%FF)`N%NkR*?B&XSg7e1<2FVw5!nF>b^$3b|1XfxS@OfOpX!^G@O|ZJ019 zIxAuaf)>7+;xL*#V2=+}ucax!FyZR%nxbl*q%ecRMbrmQqck5_Jk; zGBHVjOsAI~ho()%!$4eu6H7QiP^{Ei7v5N|0?o-rVr6TTEFw6o*0{qP-aF9zY7J@5 zM8Ps~w1rs+5jYD1Xy%in2#Z(K1FI!TGSCjBs? z(Q;O>7R6nY;dz4_3)NwHISiP0BR!~FqTT{W90yt7X+V&MD+ zeRm!iW&P-Ro9VgFf8O7g0=R_*o1sc@@dytH{!MW1_Uf^~U=uVQMf2E2)TLf^sDESX zwon9^9@xOm4eG9@*SFXUls;O;0kBh>=B8tk7>#B2(`Gt&>r7bpySKHZeLJOx>dIY9 zdY|y%FZ}osUw-qS-wNrskop^+h~;rWd9ZYz%`Iw&X4f2#n~!$qUnBRllwQpsr}TJD zr1UOxrUW@sK-`8Zcx0~`?17UJ;dlw+kxEa28kt`NyWAc=I619e==ge>XP z%nI+Mi*Jal`2R{pg%sraKwv8jLvkL$U0lJWRtEe6&yvW`2a|vY){p&$kOg*y@Ah7oQ zq0sz6hwR4!H>vjy(6d{msD%um69}#Vg2GKOYeBlO zgQVfpKrBdc6jxCd%cA5uA&4NH>h&Ja^55I5owi?n1hiV*{WDHtEeMb2>s zTfzpPLIDn-5uyqW81X&2Q44jl7bhzdzzQ5%Xc8ZV54^&Jw(Hco>mF;s0lW_{p~>vkT5ENAFdwxTKVWT1kQ1Dzr)G9f715vQa;GmnM`t&N5Lus74}am*4U z(GvZb0P!AbE!i>&lPDGuk}m5~BJna@VnbC(2s`%@o>*o(Fm5&=q3E^)86^-gSMnP$ zAr#upF{#D%YH}DQGbZEn0)vVGbC69Wiv^5H4zCCS{H`@KF`51oE`aPnNM#QQ)HyQY zGGSy)z=9O(s;AIrO!x&Ajq^B-W(nHs@-`F}Z_Y>P>xiCaI;pcdvy+7SE9JHC<5U~lIpx+MpM#850x>|a~|+Q!a_ktq|vZsG6alt zn9AqP~U8;!@5?EIr_qIxC?NTm}Q&0O=SYQpF%YhwnFqGZlWVb^j{lo=_wl zLV-MghB5Ub9x5S654A_jW>Uj2mCn^SjN`O?0K8b$Hb+%KM{-NkNU9i&o6bjYx=1a0 z<+|dHU)>B$5bjm?C8Zunf3Cn`WVHuCL}3d@x?q+2N;NluW%0fQ%>bcHkrSk*KurnD zMvM(j*YpnP&z6W)Y&yyivSUu0;GuwkS<96MOi)xhsZK(!$pC8%e?X2R>@c|%Q592M z$Mp+fHb^ZXQYDpAKco{9<8<_uAbRIhLquNOCqGKb;fn4)I@Ml7gv#*k&El0m2NYn@ zhBh)wLc^0)xkz2*peN48Qv1|P`wm!!W1?CrV=+LbHdGd|OmAf|^Z$~{F7Fbu5CJ7vz0d+RL%F7&j)@K3ZTf>z{+wQrTb2khNbkQ{$mm}d2miO+7 zlah;lN;iY_4t0A%@3_e}*|wMN)nUArHyq9nQndipl?HS%Vb%2z_BD80t#>?*iqM8a z+g4)vt_Ns@0m6U~A|zMwu{U0pqT;WJ`ZhWt^1X5ad~aby$xIQh({K}k5S}1$wd6YW ztw_1#qjokh^vpaFwOfIfN8zw*qEJIW*MH4}*z}Heq1Jctr+AM;Kh)+Ug_j^ucWv8( zc0sk2j!vaIiONQoEJCM(3rv8K*Tjt1<7};b{*~CgRO6fp1^)r7VBYpf!RU~Q>+nld&v&`bi@D8hxh-9dBa0o^OPmi=>#$SH1BKaDZ9bUWN;u<6 z3V(DBkMmg9`iYH+j->k74zSD;9;$jXlx`mqGNKT1Pb0qCvU_=0e0z8bsSAi%))AD3 zF&G1+BF(Q5wj3mZ>n>wPlBJ1njrkfEpM2XudG~e*_I`>2+lQ^ zXQY=$*?<4Hl!uu(;1}|MIbD<4vyzUC$HkeSxulX0ME^#LT$uKcs~J-@2Cy=j5Ly^x zH8wa7i)?`^z!3;RlU1OT8P_aBPAHHLnO!LkoLEQY zu70C+Sub@~mu;IF}$uC)X`=31|V zqovFuV?UsOH?(_s`n`e#l4~)C*_o*IZwlG9a8;!V#9(oqIy56f91ptfI{p^F>x2HPQ3II=q$xqAa= zbGxOBdzt$>X?UAi2>=C}`D4?XkB(=t_ti~58kui-9KPebsKMsUS^$pl+sK{TQ}R}>0lKxXn!sz;JVrysGkms>Ah^vs4qn{0 zPXPHUI<9rYt}A?|30usMd%{Kf!fi{iQE9~2T+7e23&6Y0y?V~gKwY4~MjD)&*&M^~ z*bg#x=*W4bCtH!9u+VjG7=9ecgS-&j5;*ExPj(E+Suj;aEB79-$?dlps3CcY3BmC% zqLt0VmvaiZtiOw`%ZFJpZacMAkE4;rH0Dgh`P!ZxTybt>rIq`uNj$olwbYYlm$7%G zbQ-PixFou}eA(GHlq%ndv)ALV6ilkDBIFSlQeN9t{+6U9mj8}k(1RJ54|`HI`j(+Uey#eG4j=K&&A`=Lmbae2mA>&K z|M4fE>?>dLxBT);8SamH)LXs79Y5(Do{R6Cnaw!Pg&wBy%jL)1+qj;lGmys}o#rqq zsB=69=pNe_9q;oVrRTcwBm1|P+oGX2t;d<(KWVEUBJr0W@%!Mb89$@C9E-!LrKdio zwI2J21dn)i?>o!$MLzUFUzx?6D^F^YZ_LLWo7KOX?eAQ+FZ`b;@?u}H+S{jb z-|lzcu>c}c5GZEUs6laJ2ND%fiYP>QFan7X91zA}c%x#Ai6p3K;izJS#y)pS0(pWm z5Tlif^r(DEk|fJOVE-7{w1HBh#*GZSoZz67r%;_iivqow(5AtkR49f_7?OZPEnAe5 zIht_GRHO@wV8!b3NY))7h1w_^qs>{d7(-4mtME%hf?Dg^6-)Kuz`J$}@?{G$B^gF> z!VpPo_@v^tbDD;I9QI#r zi;{z&mA#g>NQsAQf4xXx_t~AoI|7~mMk_{~(b}g{w^o_H{*BjWbKOm~KLzPs zH_`84jXPa>ePj6Itg*Y_iIPgoHi;qU9bvff>M(@#(61jxF_!sq*=7cIHri;IyusiN zdS!JLL}Z<3UjJ<0x#n9^5{+@3NH~y}*lR?r#}!w-O|+1Q?=f%~a~v&an?fvdksgLD z0%sgxWTZ%3Q2zz@(T3%nh*WXaS*RgZ(zW(tdgpNgMUzP4HCJt?`PbVMjfF_Wj)?t) z-+ft1BVaHE9*Aawpe;Dt3`f-Epj?bmxL=huE=FWpCo!2}ePMz*9*{=a2A+_H5=kPC z+u14Tq0_yQ(QQV~(MOOG@c3b-Cn|d5pdIzmlao8pM<`HT7*-yX&W&}0mp6GSpQc(q z#vQ3ME=8(+t8U6cfCHY1pqj#x#^GC9%^7EGk-;acm~BR7o=&btOB=1b5d|a_$w@j~ zsk{Q}UH_+zGJEI~yRPSvq(> z#K8``ra+1%Ob4S?e|v14Ft5eoQ6*+5QrZ_D9WPjrCJcATkvf|4j3+Si^K()u>U2j! z2DujrFz$?KpjmooF1FX^tU_U;f=juIQ)hXq$k$=ZFp3^En49HGbE|m_S!>O8f`!b5 z#s6JK^R(~eqoW--M+3X{Esk`zIQYx3s%6~G+t$5#-Z?=Mp};-fyO4{~dP;8?Up1;< z?Ri66dG0im4piyRsfw)K>|Wk5hO*~~D&y)uj5>i^PYm!|PEmL&u1Ss!Tj=fFu>a2k#>RjnC_bd+)CNcdH7@49mjP}{F zYnWtKBkgoG){W9ih%B3~KJ>lufiaAELr?b3Rvh#343bI|OA`f%P4)F@Q17CSC38re znsl;8ZbamE4yL_y{vh3cBzG(~pDP z&z*pPC6S=@cYMvuDZ9kSY{b;9aOK+SCL7qv0n>(imD6qb*EFaWZY(_%(-f;VPR9Il ziVJ+B+jwM$3Uv{wVf0&Jt!f??TGTCh7}#TtSX#sO2#1EDNio&Bu>Y;`BS$#K>h8O4;s_P-PmtZ`+odKG$4|v8zI7 zT8YYC-xB1f@8wr;A(X;E<`#Xqw;B#RSmOkUD*O)`?@ZZ zwSsPeDIC%1?on))yPW7q8Y%1Ek7&y`(@eD5I14=D*^H6<;vcvTwOLKcMkYU?IN~;kgZ@OpD1IUqLPpRjw1qZBkTh zrlJHJjIR4?6{MFI^k?gIN-4AVh_VJ>c=w5GLz{`Fm{77&r`%@JHEp&Y^Xb<%#nR&T zsG#3eOR-xyqG*nlZG--CuxYmHz=46gn=9s^GdA0yI$5_#gVihp`)usOOnKo2(1)$O z>v-!Dk7cf~Uc*W0Wp+8ku;sLLL0#5kS5;|lljF161j=xxk!eG&bA`{WXm&rh-E-n7 z#MOf!;vAm*G(>O z)Zw+ndK~J^CF5)(>ao*w&AGj2hh|X^JeEEmxPu31+{8xEPY~jpzS>J_THl?IMi#f= zd#v(c&WA2AUo;f6(!tl|yT)HHxvs-*3P;2JLLx0TP+5B7rt4hcEU$4Y{b#nbw+T|o zO=ehC&QLK)_Vj-OFWOu6X}aT>kEWF1-?fL;9dM^0jfcZr0iSo+Zgtr&0WoK?S_Ug#UICY4yOT)||QH^I&Z^U-im=HDL=6 zOK0(&w*M(8jWwHpB@`o_hrXrS9wNY>}PMDHyjeEW}Y=r zOQd_2GcV-gX>)UNH4|DWMnPFbXz_P%A@G2i1AA>}8n}gHSE6BDkX#f8V_U)@1$cSW z7j`=6Z@gD_v2}j_$7iyqBZ5_T3?^um!Y~RHcPm#&RFX~>lYxMBNhQ@w`1fY7)Pe~S zXizt4LUv*_sB^S76)E&JAeROA6oEW=fCiWgSXB@=)^dB+YR0C3TOo!X=W9&#hsbho zQ@C(c*gyONbV7)SPeCkWikGP3rvu%DQfA|51GX;I} zMu$TpNCb|KSct24 zaIMC4jy7DKS9+PIdcJsw<1=m0D2QpuFHVS$V>cV}G=Or)M(LJh);NmTSaP!mgJ+0t zJG4#2#E!1PeZKT6u1Ho~XK7KGi=y{xCvqCKg^IMeaC&GkF}Q)Nn00nHi3o%n$QWam z)o~-(eDQb!25FNBd5zk*fQ+`2o5+m-h>o&`J)B3BYZzY+Iag9Rf6W*pz2$;KsF9F( zNRFs`Kqz!}=x4Gbf=TI(`;rqX1^-Qy5?;iok~f)?XURS!7KkKSl79G&QR!$EM2$BJkA5K|}`C)rT$q;6FkY~vz z>quK`sg&G!MtM|$6XG^=>6RjinJrd+WL8wAIV!|>SELt8_Glr4A$aHng>|Ql24#9d zIFoxAiWwDLD_;6T>dJXwBpLs5`vygD9lCann2B&ntc5}D2 zViK5|p4E}034gjcH*I!I#nfAmMw#nLoSabzhHz8wSqGD`n}wK_%JqzPIEEg0pHiu4 zNa>8^n4Ju{nzQ&rcj$>SxBrk9_$Q#Kpnw>D=;%u&s6hV~fUX9awW0_vqz&w848!>= zTtf;YdJ3O_36U_OD0-su37_=oqIt=p^I4xWx+ilPinSx2;Hij@ws*FXHXWEivH}!9pb8~w3MLAsVfv+FDyAqJ zqi4z{GD@Rh@SbrcY!oJ=XQNSh#rYafLKbL z?HPR`b|A3erCmy*kou^RN~x7frk7d>CYq@fL#CSwrkgrL^Enx#DypQafTjwgIJl>2 z_LibqhU7`vi>jro;HbMKsmGeE%WA33 z>a5QSt(O|6o+_==nyID`qmz-X@~N%dTAyypb#fX&zK5ZJn1Zi3f8j}s%~^VFDxAPN ztilSSSB9)xTCdECuh5#W`^vBS>aW!bu%JqOvu-nS5bzr93sxYQHp9|Z3 zp(?Q58nLehvGmD=2#Z)V`K$06uO3RQ^m?xnQ?km6vi+*EceJuCtFr$JvoR~PlM1k7 z3aQr$rp7v=I?Jp)OS7d=q7=ih0PBNuXs}*ttRM@bi24jY$f&|%vMGDAQX92Y>#|pi zwKJQwTidf!>;JWkTC_$H8Vw|Tp`b?dmx+PHQbxsw~Wj61e5tG^el!5fSX9t^PyF%)uZW!@<(QGK|Bv3o$3m!*Mvc28_ZftinIs!Y)j~FPyk= zb!8Y#vKh?8HLSz83&lIE#64`q)f>b>oW(+X!1Ig5M*PLryTnh7CRN#REdezPrGXOuD2j%5bd5iweq| z9RJIm{L07cw5=S8 zjLg;C$iR%s!u-fD%*w}X&D8wO&792249(*V&cvL((frNcyvo-c%)h(M!>hvWjLqGA z$l*-ONqo-ge9QYx&i;(dr!3I%%+2;}&$|7yjKi$2Dd(#v>#4jDwNj;oJZOk;?(@&k!Asy8uExHms)l>~)OU=)m|;vE&bIno!0Vf*3%5vJ1xp+?bCm4 z$PNwF#T$Jn9oJor*mVuoVQturUB{1I*kF7h6TH?8UD=ck)Mb6xES=bkt=OLJ(whCt zcTL)AebrX&)@2;Qr7g}1-P)+_*do2zolVr*su-Qm(AO{ZQ8Dl*~*}{xSiQT zz0tMp&d5F286DeM%GSjF+-?2ZyDi+%9o@bS+}VBF-K^Z*{n?Oh-QkVWn%vvC?bpS< z+JrsaQZ3#d?cFxL+_eqbq3sO8jor=N-t4X2`CZ-ojo$wK-w<8V@9o|5&Hvb{eckx2 z;Km)``mNrP-QW-&;e$Ql);Qn> z^}XIJ?%;VX+a=D~7k=VK&Eo03-|U&-IS$}HuHp>-Ffu;dHg4iL{^KzY<9z+%NuJ_B zPUJLR4(1fzd<{oUqp-`30JV&3Ip-s5l{ z(=Wb5tWA5Zo4(NeS;z$174UW)$KIeTd=ZVhfiZ0f84(V3z=zPB8 zjsE9P{^gdg={X+hoj%`_e(77j-H5*Cp`P2Nj_Q=2({Gi!Y6Om|p42 zuI<}i?U(NB&;IL2jnul{?UT&y=YH&qp6)Ha?%7W4y$;vmUgfq8&)2T%?2hhnUhc-; z?&PlT{%+{a?&)g?&W2z_NET@ncdPu5J& z_X}V25s%;&@APSp_J@z}9Uk}`ANKK$@Y$~QjsN(N|Mu1{`G&vw)1LHZ9{8EB`NMA6 zPk;774Elo4_8X4#u|L|Szxs@S`m`VVgzx&n9rrsQ=>Y)%A^8La6afDKEC2ui06+mi z0RRa90QKz~NHCwkg9sDyTd2?=K8FwsGHmEiqQ!m~GgjQlv7^V2ASa3pIkF_llNTSV zTuCva%a$)=vYaUsCe52TbJpCcv!~6UJb{`_sS)K+qezb`T}o7FP^V9QMwLo+YE^nz z>t)r-wJFrEV8bRI%e1UnuU6Bly&85a+qG}IibXrO?peBb$>y~Sx362je**&+99OPi zynGYyU96Y!V8f3KOP)-b@L|M`6=&XznKMw!l|e@iO&XqP&8RPTUhSE6+tZ~-$1dE? zwC&rtUF&YWTeH2!y;IAc4V$>U+{cmEUd}wY^XJf`OBX)8_-pINK{x-)oBDLVzrzFn z|E?Ij`Rm!wpC9kNsC)M9$;bbxTz&QQ_PNW4&wqb^|M%q=pnwC;*WG^y8d%_h3=-I2 zf(S+kp@I)Oh@W=#RY;+P9CoPThZbHKB4q$hXyS<*LIxsw7>-CHiXO%&ZqWOy6LH?7TV&fn3CFRszx6Fnq#J= zhI;9%uGZ=*teo~rC97TCimQ3H4$JGYkLud0u*Nov?6a}@n(MI4I_smf&_;VKVbyL~ z?6=B>8;`YtW?Sf!(}L@wy5tIJ?z{V?D=xdh(mU_H_P&TMyfDg}Z@2!|i|@aN?z@w= z{6;yjzzYKjaKjM0i*UjQ!y56$7(bk`u@!eLYrq?atTDm3dTjE^^MH)o$dIkfGRZ2N ziSo=di%jy%Fkega&U=naYs)#uEbqq@^PF<0LE9|!v_>E8G|^9^tn}1LFI{faQTv?r z)?aGvHP?uJ%{A3kTWvPhW2de5+HAKi_S;1S%We040nQ`Rc5< z?z-se;tc!jrBj~w)_c>gJKU=G?)&e%^A7y$#1|j@@w=bHob1RqUwe|nYaWjDyXc~g zH`QZrJ@wOP-@W(XUk`jY(I;R2*Uo#LzVXYi@3Qjn$M639&!a9l_pF3c2_}d{;{W~t z?4Kb7hzR}h&wvK(-~I%szg6@ve+`7-Dk8`V>s3&D`1{57wuiy*anBqcEMExCN5T@y z4}>R7p$So_!WPnRbjG{gxO8zqPZ%N%W5A&u=Aec;^pJ;O$YB`&fOv;K=z)l2;2{x# z7{oac(1$vV;sBk92p>*xfS}Od|LRac1$u%N{{vtJC3rv!LJ)iU%U%Rg_=`}8FN9_^ z#s}x~t zHpcOnz`R~B?a0Ss8uOTa1f(GA_c~Uba+E(L!WbZV$u_*rAra)M(voTupWW;qHi1gYl>$MLGqvYAJDE;KszHEpC@L4} zxz2tv@QbvJ1Xlf*%Ue=#jJW*ZF_A%5i6%3K0m~>ibIMYD+7zVhEagU<%F{c z<~$kEhjilf2r=+zRDG&ZQ65#D-&|`nJG#@nzBP&PRKyrma96e}v7SxrDI{-b%Ejul z5>_oNRVx9(8XB~WzJ$dpAllGu5ObMmo#880I?_P@3>K~=9jZ*bfYi$3HKmRvY;O@8 z2Afhru`fLVM}Pa&&9<|gkv%0zO?g!1TGqKSl__$si`#gP06;*$zjL}Yy)131`&cT{ zvJhDO>S}e7&^UJRggp7yE?${KxNgCSw$;KH7<=31{?wDfB`R`{TQ%y28|H3ama8ujqTM!%fh&5iHEcl0G zOy+pV6ZU2vu*+gB_t?i#{V|fePy|KO)ww_ID-v*AW}tR+&QMmfPJ4)HMkrddm<_6= z`TOBU8~Mq3Fz}z1Ow~lMIK)o~?22#T=nJcQ(a$Bcm%ZCqUPF_f-jXk$w{Yu9OBlf| zIQCGxoMdT|naBI>@3UboV9aWn+CQ}}2P*Ahly2M4BM9@KMM`12Rv5*M{&uEKE#^S; zR>Z}wHm6_RZWXKg(JyxMByB6@92eWp<+a1q!qJLWf4Z9mHnh8eOk8qHlHG{iLBGja z;CC4#%}3Lq6(9W`P76|37K04{ir?`#au@^*E*}O<8@P`sEEL z@Q!~=awdm5<_LG{q&sOd>IpT~#9nqHdtK;7uUg~~PxqNAuIX=w*wrDmdWpAdVw-dL z4J)^Jmf6{Hly>{wMD}{0(X4EsZ~E+LFRsmpPWLa@o7wZ;HPhqnTc0+azwAEj#!e`T z{{3OdgT7>%A6avlW0~pN2D^1tu1Q)?U7RDZE|JZR@}{CZ=poKLwLkykBR_oe^ zy*SUcE&A++yTFadv`8(lhEJP0>!(Ne|J1?!a8tWF*;LSa*a2&PkS>11IyUqRi zC*A(Y7Qeikk7k$4e%|l~X*#6>Hehn>S5w0WbJlfyyQ2-ML{dlxeR(HXlcjmWr+aNs zYX@k7NS1h{W&@FTVpk`1B}Hwi*L|30Qr}ht+m>yU7J+i7c#GEtRD}fO=4p19b|`3D ze05SJcVCFLY2$W-r1pJ!KxK@!b%up|zQ=MLmxTDGSZ)@4|JQ2_*g}&qMVa<^7zlPE z$c0Z9dwh3*sE2c27=9<`VBbf1nn!`n26Z|nbwR*^Qx}1uhH^-_b0v5J8#f0yfNm;) zUmN&mhj&~rsCwvEe{>gx3m125|D}W#hjW!yhvp_>f2Cg_NQItdg$}bqAe4L_2yM)z ziBZ;r#P)kmCy8gMO&}Q!qG? zA(xCMxRfbZjm4IDc`z^&^$R%Fea7f_l=pNFxo@M!Ww_{QVA+t?HitJbitopLl}Cqc zrh{*oVLQ2eKxb6D=y6e{e&pATIGJeK7lo=AcvATV9H5hA2z6XnmGj7y=k|tC=asN{ zcZU=%$`n6flyUVJj{%pCQnq$Y*J0)tiwC%bqnUZFSAdwbZZeg5(JanP2KZTnrWDjaR!)7sgG+(fV9_t&M28< zc7KA#mJ(-=6sT_OnUk()NP%F3PgjT0HkokflViw@d&hISl9~7;UKs~~j0cZ2>5c$; zg2>p0yH%d6Nt`kT3GOFurdWvydXZ%}nPm8#XIPx1n1p_LjS4rT?HGpsC6qvzmgo1H z+eeW)DuMjwnT=?bvxlF-D0vlXp=>mEb!M3NMhC9;c4*g**Qjy(*pe|woV-bMkp!W^ z#iau{Z@`$P{$`sWN}^lPhmWV2(U)VZsi6YeVJQlC)ETEy37_9-RK2B_#%5n{d8Kpc ziPuRzY6Yai|0t;bcbu%4o{{O8*~OrI$e|2(foXV{;g@D8YM|mdfnN{^O=gs88h$~_ zd2G6n3rd@X)>49Ko81YTxRs`twx+Z;p&$8!yc&tpDTcrb1no1dnVFqPDwDdppM^Q8 zD_MkD%BDT^b+Os2m5Nh1S$*E*QGRNiCfS#{2$#DViS{+CnHps0H;a#YkY;eC=XRWM zs-`8np!GSSvSy;nI;7K?W5f!jY*a`{YJ}p7S=YypR|cB;cZh#Tg5dc{H~6lfI-(Hw zmYKM51{ZOn_>umpj`J3I_=>8|nVhMZb%edVr&I+B!%tGx-VjYy)2Ik00#wJ>U;H;c15>#sElw&|pnFRF=g zijF^fux*)YPm8IkCy!DZofe0jXcz+8lz4j?NjP}7WLvjvnN4~qc^pV)*ov1c8=|2q zpV>&HTl$V|ii%5_a6d z|4OLR37MViv+#$s(%$k$> zS)jp}8oqISh<1vUHHm;3`K*f@#N%7LU0N&rs-^xyYmXrj7=#8%v-@X}s84#|CM&rRT$yiHRaR zSBezK{5!|))_@7kCax!Y8l=^e2*u`%g0RhR&Tj_uv zDPS5*r**5rAc>9kIF|UV%**V!APRySJEHLHzOf3%gSOBBe2R1ml)l=xH*mlGcxva$ z%3}(78yLgz8_=rPyCPbfK8cdb3Vh9aoOL3>A+!}5KjHfp&ygS;T=tqet&5*VHn_S$L zyxY*<{J323i1gWLw3D+fO+(52mB*}qBz?*OIi`+iyVU5Ya(2-D3&s&m*V#9R2MW?b z42+E&$v-9uwYRnJ8nq$#wHVpYu{Vr~Y{owPuQ_<14*OqYD zshRAua~s!ioXdfIe|gx=yI7QEJ;(%CRloeDpe@iQTd(n%-fiWo%|_w1hsbHE$3_j_ z%H?ucDV|`AwLHvv?0uL!O}T=Z!u0s8(Z=C*x2x!S0vtBtoco9AeWq26iNjU|K_I4yTQds2H||m6G?r0 z{nPF(srHTD=Tv7J1Kfon+GawImO>1bq8i=_ZP$N3ro;@Q%|)et z>f|RL>upf)KYEQJPQD`U=%}3L$bPE8*WgEeL1^fFRazKtm1CWup+3G!CbS%|69Lo{kf63z<S?zAkegbN!?&Wc+;~aNf~$?_h3M6BD73YG&IhfM$~%0op1|Nuz3rV?cYDEt zd9hC)-F1WrR$A$;Xk|M;nu#9Vk*n1f$kP=lv@5BLMhvwEI!&Tizc~+(X5Q;NOp?Km9)K$Ct|beUI;C|LV|er+n>6`Z`bQqCNTPp2~9{!)gzW zwp;jwtDiQhtz0eGJerlnzm@0?ZD&mP!3_|4;>3s$qs5jnM6^IraRN#kh(**CF?eKR zLK_W9a5yo7_*vg*gEUyw{ys}UqfJ82;< zU5ix6qD21;5F|D%L+h!6my$EQ`Qf9`lD86srgH&cPP3~JKh zSubOP*Y-(Wc5=_kEmoJ=yzStcD@{8+jRPdl>0#S)B#l~q`s-4E0~S5gGu*htu@6VO zxZ(ce^vaDkhPrSJGRhLXEVIm5F^n(8Sh_1h!ib20Dhn0jDzuKID^J7s?nCc5JBDlO zBJEBBktwv)z>S2aR9KI@znEjM!-6t0uCC^m+6uM`XR{+d=stw$4e!htia@(m#F2r6 zn$(Xj_(F&%r`WQaP`uJyI!nIr7Bew0&;&H&$}!#3>aGnBL{LEn7sSjMPv*!^KkIBG zPa*Nh+{sBEpJR-|`D#=x!!SRy|H{5X3smDML%pQyM<{h{;nEv(>@h$Qi8>QVB&q7O zMbse7VnRMWRFB&I|Zma{)xu)`V@UPFz+$ zJdrFV8}@Tdkd?fYOa_W3|8u`w#riQX=Zf304@vM8^}92FzPUJvAGWP#HMMlPsKbr)^lLs{H&1Qz%4@C@C^BTh|B7JCO1(RS;h`i z_6GOV)!I~^H4sQS|%$&N$r7VGumE8 z=c{`OELb(sTlD<)|G{I_2@7XJLm8AH2Naet3MMRJBK$B3+rdhPGpqy+V@L@YY66F# z2-^s+PzE_frw2U%q6w$)LldF^1=q8n^&a>{CLWQAO2h&cn>0cz9q!T{>8@Y=97Dpr}I)fG7nSWby@@rwjl&gFP zDQ9sAOeUcUXMjU3X}Jen;vtuhSVS*($xB}Xa}7(7B{7R>Ok@r-nMU|Q6NZV0B&?tb zuBb*eMLCOW|6Zb*P56NcxM|51B9ocn)MXCFX-;!0lbh<)0SlJk&SbJvp6@*8GIa?} zbv{#{`{d^u@afEds<59W{38?rsJP@LW^DhH2>Av@l1F+_lEk1S2r>WxjAm4$8hwN) zSGkT;hE${@O$SL`DGOJmqLwgVsTG{y(ps`XrZeROFl`FVP2|)M13hL>eHzQ2;^H3Q z7-~^l*@<}m6sSsFDpQ;K({(OWpi|{3Pj_0vs$MmP{@mw1ZTU?Fx(!|-3Xekvb=HVV z(u10?!$#e@Q9z`^9v?L+NbOqJk5c2P#Oy#{{n~)P{xzI7C1~yzC`B(Gc8^G8?8+ci zwmb&0|FJ?uEM@IjS;Q7Jkeh`9XZzU41`^VI1~n{Z9lJotK326wWbLy`$OPEJb`6$I zQ|A_9F8`rzi`*kqS|u7iPC(+V7{$OvIV#t=-c`BFWv(ho*@z|#7qEX_L@F*(U89z6 zfNqJRR<>tZ98C+69kD>}tb<6zZV!&?wNe!;)*m>YE`CI$Eb4CNMfi?&W%MKOZ1ZT4 z?tt^Z-^FTdW2=M`#D-K6;nqQKrC*Kt7Lw3=#2PZn1VR9U5TnpAag7^Sx;9tD$zAR} z#(~tpPFKZ!Ma4VjQ5h4DQoFYSLP2sAM2dI~EjZpnAL3^L9tX_6ct~&6Y%GN}Oz>>-eJ*uj$zu!q@B&MgrT0UwMgT1f^p;)C6UM`|}B-DsA zV3mOd@v0v%{L+@10JMz~+lu_tKi*}oM9!PA2ifpp4HE)zLm*voKkU&EJ9oq(7DXnE zu;K)ycmp7q1S+ucX?+ZpAED6A9ANe20XNLZ7l7uJ4b~AONSOve$lR)H30fWJj}KCq zAW)9%^pO_NSigOF0vS z&TEb{BzS3H8DPK%SX7^}z(YK>`7UE`AatSq)=~^RLeUW*qZZsC1{J#@6C|E=|G`b! zjDDnphd;n{rXBvt+1B9`*q}!~L|uwP0PfTY!^5%lH3kBNeA{T4K(Kcc@+H9G%OnXo z8E@lX2ioM|S|k1dp_&o9hjP zH$%#r4y->}p$ky+L)6&FcpXN<5)Q;#1Ppl<7nV+Pefb#YQhS3+mVO3Y+Y9Hs{<+A@ zX)P6Oe(_J>3%`+`WiJ6;rqrfdzp-5t4urgqNr!^7*$igV_tFYZH;9;b|Gl~lTOGDo z*K^>m0wX_DO1x{Mh}g%T4YOmE2^vPk$&c)YO&53WbmzXo=ZbfP@4ev;*MJVzzzuZ- ze(-EB#QktS`o_0>0*i=3f_p4^loMHr+X8yeXWh`yJH7uKmj2@d54gFI^8la25~;(y z&eOHqlNymLgV#F?GLi$@vpv?kfv2k@q<|7o`n{$}gi(kstYd+siJaBwy4yk*SOGg^ zkuZshJ_ry4Hn4>1b2}EmK1bWWy7N9%!aG#RgLdM(hm$5n06dBlJQpLpIxxIf12X-K zJddC=$SZ=?Nw!$Kg8g$pI&(mM!HR;2GS0&SFT64Xyg8510b`@N|8YyQ0I5PrpuK9L z!`84v6(|8bz(bj9x&dj1V|`_QfP;{x`%df!Y4dF8i2z%Y>g|7mGMHo61hb@ zpd;!iFO}oPIl4BT3q$OofX~yiVl+Ar`I?jiFH7;i^m-y%ipHb4wf%x3eA=AP;VM5^ zqcGSobVI@M;*Oay9?E${K8Zw0G#MI%fPAwMn9zb9yfg{WKIG!WPPD5}^Z^&hzES)~ z7eKNXvITdL42;t_XHg0{n4s}-ty*+QGW(Sm)B#6uF}xEUle|bG9F`PXxj?wejC4rs7(kH($*=UG)zZk*O03Dc zf`%l^uPjN2)XKJ;s;N9ll|%$U5WYMMBUGD7MS6`h(jTtS$r_qTGN`CZ5rd%->P=Z4+yFoAmu+lmE`cvyMM+4_Lr~4s zG@@?2oj=ITJ&VTKd=@GyuPiD}-P}#e8Wz!fqR^Be|Iae5;rz(S3Mb?Yw>zjyH)tv1 z3nMoxn#6H87KFvoP{(zY1;N~?CSZb>b0{}fFEt*)Hngao|=OEZ8exP%#C&_B?G zRPX`;?N1EN&~BW83++$>^-vGx%{lte61blJ45ZvN%?Tn+<5U9kv!`dOOXs{x11d8} zVV#>i$G^l*wIVxKa4WPtv zNo@vCFd25iCwIWT}IaD^+Vg;e+hY$CZ|byF=p1kjvQHw95HMX)x#Q$St3&-~Wn1SBrx z)o~RtX$nO`^&~@WDQ-fE`@yYyQ5@kpzN?5-N&UAPoId8XK^E{Zcyfac>r{ai)lD5( zZYZf%Fja3jIJ}F8Zj6Fdz0Yq&&WM%L|Mv_?rUM6vRj~@#&t$c(HMrI*ODSdj&jVP3 zPGGf0D9~ViRt&Tzl1tVS5L0KZ256nXUTV-f&DooE)9ccvElsCI_*VWjf=mF}Ud2$R zv#!tVCRP1e`kUAk7=)Z6s6bse%cR=tiZgZXBy{~+LuCVd!2#DmA4io4Kg%%zl8$8o z%)VhxHs}SRYNd7P1R>f0mxh>XxU7lQ#`1JYS7utJ=tsMs8nFtYC@q*n1WPDT?*)g(&gN1 zxY=MfF&$#RSL-9 z4938R)wBP^DI?I&2}Xc5V9**s-D;q}7lK?VyE1(817saqT@_ss_OFmN-u>*xG!5b7 zT>&`ILm9?lnY%6%!cQaKL#`45##Lgka#8BVvJ+CFdJ?8~dR4Ez;#qP7Kwu$+;VzT# z9>lA{M;xmiu-CC`UojYh|MD~{yT#jH=mo;uFhbz9TXlm`klVa{hp_2|d!Pg}kSn&@ z1az>4(yZZkYSBx=;40?YTZ*#}7=&36WEtMi49=n+Ug3csK6!(#$Z={s))r^lQp);MQC5r(uTHVQ%tfL${$A9 zMTxE9`)py7Hoq7a*|e77e`ee$*aOYY&n13dN;cwOmRP`c>6X4ynRcqKRpulXzjfi=t1z^)Q#j4N<2ltOD{E9l0M$Srhrq1?&$vK>0SZg zwr<2;X_k)ZNzt5oUVeIzju-Z?KMTVn%RLiaX9~NUp)Y^4)eTL@jF-SeP-+!e{6B`^O>gUD;8UE{4Sp^ zoYg=J9@p&6em+9L)S(RRd{}aC#sDD@13_qqM{lvqeNQTt@+yVT0O)7a?%2>&R7&=A zRrhjaUiIr{aad<)+nsf(w&fg0gV>a1JKuHgu5%RU?G_JqVQ*|>hFGi(?>}DzMouaS z2dbE|B~aSpE7+h!R1u^p5=G}PA@@;hD288f-yy(sq#k!cz~;M^J~5DAC+1oQchY;x z0LfiSrA*a9&CG6LXZmySrR?&59APl`^*Wyb-@B<@)^!U%_~-`qgfDg4-r!(Q&g;VM z|6t}L0n;G{eWmc`+yU>ZMy92dvR&d8CJfpjc*ThKWzHIZb#!wzw~T2_i!)m z(l&uj)^{FQ?Q(Dj@a*>lD0MP-fG7yo1Q!z9RrS8wcb^A!fd6#gesxui`hRbF&82!4 zcls$O?6PK7TQFR9sL*t>t{A%HK0+RxX-&#%NUKy+i*I`!dTU-1Q??g*FW7N-#g8#w zK6-6iIr)WHK*b*Q(U;usmpAtf69oPiXBPPDddyrY#r4&m_kMq;S#H(ZuE>=kb%%R^ zR$cm^*L=rjdg<1B#+`c8pL$(Qan)CN)35p}b_EH2+@WPvOnwE!)hRkag|0^g|551r zSAZdESa9Lzh2f_LQ8<3&Pfgnwepjfe;oXL0NCpLsdov2O*YOkkvX7a8R8oCRyeoyI z7KB@n)KVx+79jkWzhi;zRCR7`{H3Z2bJ7wu1AmWqi%KEKC-u*5;Mo8OO_FrD0ASEX zK>-OmEL`}YVM7HAG)OE^fx<# z5KWB|Iz{Bjk>?fAppqCRBWD!d(WFY*xohh59n`1yp5p3Mi`zJ(sV)K9Brs4H6=I3t z;1c!-*)3|aPn0sEYx3w|V#%gjYig`nB&Eg*KB?%a{o?y~ zs#h@`983uDc!1=}Eh230oVCiB1VOAeJ{>xBiW9MG@7_`+c$=!luf|9{37^-EeoR0> z9ycELK63t8QD#t|-gQN1-PYeffJ>Fblv4plMb1>NWD*-zwzS_~8egUlO;7Ca>%%Nw>J^P|Y7uOtGaI zTZLB~a@@7K=9?(3D*;2%EfL+A`KGz=cJsoS?Y~LZbmt|RTyus9FL{u}EVJZSN&(F%+l1n$itK&^Le;MPLW>81{^eDR!o8UWDws!&b)$?gy-R&mu!{m9Vfchf1 zS%fy`JEdEO++LO!c5+I%vW+b1B>;dN)m+6s`z2(u|0%!?_^s{V`0%xB;o)SLI1K_f zkw7$LqH-`1Aao=LIe9^jCmSiB<~GN++jcP(Wh)c-a0y497ELLbS z3x_f^f-tlycRc$BHiYIfULB8k%ljd*>`@~j5^EHifuGl!fI1~G5o?dw!*J$TxGxzj z8Sw#1C&WiV#eJlJ*IVD>nA037o?%f{a1Z^ib%{9P&j6=U1W2Z+Ex@_&4vKn3w~D04 zSwP|%MEL_B{m93jSV1*+qCgZ5mav&zr-Cx+8ly%BIu3@gk^%yh=}?D+7I3n4uA3Bv z;KITR5n&866blO1Vnd#dLNJ2Bl{tpzL#C{<|CT*N1&wwyqiGE&I86!P3Z$@v+k8Nn zHaUZ8Y7?Qr;c{&xNnhBcus<{~FcrvDPD7^W1(sN#n>wioG~GxIN-$1+<{aNgdh#}O zy^dvCGQ)2SWtG|->^k-wkL3(?g#}G=gOs#n2a}=+(^a8_TaXLuWVn`4+F*95WIzNu zG_;9U%9fTG%qqM=L?RBcqXYUVNbQHq$=pXMMKV%q@USUYK#LO(N<}S@gh)f-Ct@ps z=Ks6|himw83e>m`(F=$DFab%2A zcIekII<%>gVw~FuXYn`Fo^t{VVy9L|nN_S>b%f?YW}TSoBz#(qf&DydTG?tzTZ|MO zrAt{S8R||NqB1XRQ7CpLI)c8|K@gsZkzo-_S+_2#TfK~wD8QB2Tq3n$;&p2eAn6H9 z1%hL$t7_`jkcvQMiEw`ltvQW()6c@erFDCf^oX;?*J|K7%vqdN-I#~oY>2m8l~6l< zb3jH0SGYFOBD6qpT;wKKxoc%B|IeN~A^O6KhG}4EbzPhmQ_i@?L*(gz9Pwg9ghEr{ zJytS2K?zVqD^g}H3LYtoOT`}IT-21S%FNdaR7^Npl9(M*(eOUrtSr8W1LKzp>O}d~ zZK?ShW16B61!sPF!DuX}NuDIL3g>pKm-+)JVR#q(q!th0NWz;Bcfku#Gm>bPB#BLY z;zc9H$Gj-V2s7!2Idh?QV4yZnE?{K4izWV3{T$-s)6a-n^zsHNQ9hojtZdwAA-6BJw@9O z`_qBrI+27GyzhO;>~>I9|Jn&{ujEc(^$vbq<*G?BV!Bo>Dy3F<7eWP>4`#k+o!BEz zMB56{{8qH0zaoc$Cjq!sohw`-xAg}kwhYd2V z6Ct5ZX3nKnR08!R>y?3rO9^T$sUL3L8%-$EC5!O(AFi>vK+W@**;8L9j%2Vs1bq|K zO7@Kqa3`KZF+q5$qK%#eRG@A#-E`(cek|ODK5)GauFm?M<pP6JO_K|MS>0` z0{upQ4Fz$eOllK{|1JyaaxQtaR{9cNTew3wQIz7lf{tUNf0a78s6`HVngRDX7XfQ< z?#XQXj(y%k%eb&W(ATWs6f8m!b%sROTdbDY(M(I~qWm=;ac_a$^VS|`s|FLTR?giB zDlugP<*-bq2>=Dm)9ta|2|^LlO-TiATN6Z2Z3IsDHNph&R2g`c+T0EC z@fYV6fgVU<3@%^YWmO#fNe)=WA=ID;_>U)^VTT#Y1W-)I6vz(pV2N$etmTnt85Hd> zUW`>5-pPu@-31YTh!*lk;Kd)|!CDsD-?Z4m;;l|MwHGAZR}UB$F!76;e1!EJ8|^ul zlI35wkls;P&K5n}`=~?(2Hyo{U{D=NAo9$5)JPQ6pd#);RLBoHP1@CY-Mb~xlfh$# zk-@;l|3noXf|-%g6PckXUcexv1Cdb{Z_I(Al?*EW(^3pYQcAWr- z*j32nqWk#{753r+{-Q>1Oi^uua1f(Oa!nB+T)Y&{N@80Bjn*|xVC6(b>MuozOv86v53w1JDjdThhq|7@p91h@BURavTNL&)uo&C6% zE|$`Y@FHP;p+(FfL~d545`qkdRao>@z!|(x1LcNh)<>Ao z24!iHMl5Gpo~0k0TEDHOboyX2d`k2@5fhwEm$}Y&M&!diCI{%&)%hi>dWecET5@DrXWz<&1Ue*)+txC4QX;R(%Ru3@P;p`Np0lXBb% zCCmX~4#JyIlldj#3v@t-y1*KE47;_PmQ0%N@f1t_K#{2^2)MwLpqA&oC^raMpcGXa z5CwAGD2@u;wLIj(+1*`Q3dDJ;m9_w4U0fp?p)=xDSCk6bnuYhvuf8wv)VNrQAiq z89>nlyg`cs&f6K4i;^NM_!wmw!a-Up+tnx+zTLG}L43t(cjlFz9blb7|3O1Fp;(kE zN45ecMB=%cCx2W8)iBq4nt=(?0PM_PxpdQ@G#MMV86#DTw%NwQR7IFx>`amaDA32Y zdXmjN%LS1TM`Y(lw5hQchXnriRTpnOgj9U}joZ1{J0C6TheW4eO zY1-XrTPl~(Ws**ykcRA?3ohZO#!z|MpCmpfBG4besw!dvMS58oUH-u&hyrO)4;@5{ z90ZzAP!qw?$&(4&OBhVgY1w98EclJ7Cmf0a7{CmeLDF{8Z(@WQh!jjn%ek%TgLYey ztyM^nq=p^ig6M>SsveMr38>yT7SO9jLA$utDAL@Bp=-JZ{}d1=0jVUeM7ofS zh3CT6nZBMXlLDh?RITWgo*0&9RV4zdkqb|h6l<-9f=Gy~y@R&u$+HYfIaq;G;#f8C zfbHP|SHLa!DZ)5_LiMXl_aP{7h1{54sNV1S#2?q$I|16-q|ExSj{$& zfDR|NB5M9{5h)%TLav=c+6pQZ43&@&%X(TZo^M{>Yu`!NrMxa;{^hpm?O z-iWspo8HWaSxTNfR!1XT8R~QkAdV2~C<&eH!4t`k(WO`1!NH1E!Z>V<_mod#q5x@n znMcqq4Ku)kI;i&G27bkaN?cwPD2T@fK_$*C5r+)Z+QEog{{i1HNx(Q;YRNCqcCP2D z6+@bBY_`pAT_vjo=|d3!MWvsq!tSFV zL2=O9sQrHKO)42oo$$N9BX@E!UQCn|acJrv3_&1zYp&j1jG;SOLT=Z}`5LWVl4r#9cr<{8!{ara>0 zCuZ(g&F|XzTPLUMb@JzCJ?7QHYF<<+EzUvdc3Q)T|IU$-F%|mjNBSZ?%i4H3tgq@9 zAJGAoGyyc1#J#-nYq|iUZL~)}`adiS*Vl{=$y5c6ebFg}vUP%{Lrm?gD>R&~|NG@Qzb**|0 z?5Ft0k}czF_?lY$F`Y*9Mkn%VjKZvy<##bxAdJ_5LG|$QF!lg|fU+>@Nnd3~U?Nhp zvpO;;f%Jzyu1G%)(Je^dFqLvBs;_Qtx|Xvi_nR8C>$P5iy;c{}e({jPb1DOHP$YIw z=Y^>a*^wcc+gaI3VWBVL!Zy*dA1t4@L^KYk|AaJo;riq-eRx5`(CC!18CS37O{v7Q z8O+A0a4jr?Mr$;ofhwT34kb#m*4bnKISw*0Ok8r0XtApg4Bgn`btmifUJGMQ4T3QJ zHObYE5u)-I$MfkDHa}M*0%Ke#7{b&ZWHr13;dL!mpYcH!<~5Ltnz>^!e(--Oas}|7 z>AA#erl4a>3YUa-YJZ{d#R-OX&E*+RvDOyz{zk3gw!BtzZpp?S#`gPc8=ftb$>r!Q z2#N|KH~lI%j>hheLid@<=XLonh~s76$)sf=FdypzP%FZ}a!fD6+Ao0~P)tj)mDki3 zwE)7mM5t3%MULf3K$=XBpuR*`!i&k)|F^Y-(IU3wY`4j1*XwTM7M1{$x8Y%M6zU;( z0_pKj9$mHtaPBy3_)D{0irJec!E3x~*QKp_`c-KqzRDfx-#DodQ2SNkbsQ&Cwqjc& zxmXsFjqB(I?wDvqsocdzaILG&$$ivU9jEq@Kih&qdXh`Dj$3FtsT;|qH7P?SmMqfJ z#gs7B*8w4we^lOue|fs*`E*_?YE%u%q|S#w903P$EP`sosR3%}pB_=SikN~hmsfTJ zTd>R8mN#sS1oMwoiE7K;3~%bWNT!?=N$GVZ+I8!(Qo4dgvooU6_rA-uU&%VfTM@hU zBXb!Oby29G$1n*+S@C%v={bZOykD4 z+p)^Vk%Mv%I)8NpLbJSF0Q+{KEGK4G4@w`{0v+%?FQf~dI+No#SEwg?8El0f{FH1U zI&To#BFsCz_aKf26ewHM2Onc$Yy<{|1I}OmJjqF@iEbotAwX4z6-DcyYCMrlxIs7U&$Tj(3+y` z6K#Ts3YT37sNu9=r*hu9g=YT#SE#{s_qxKDy|4TFV4l>(|9pGH(*VixS~EJh-th3! zz{=u*wtde~tgTCByl9?O4m|pxV{sM_3NGo~cT@~vD*N|Nbfk%Zz~b*Qvpkv%IFmQd z1(tNdEeJ3Hgb_4r2L2dy=@Oa=69&rK#_b_Fi4!A^vq%vgJB=INb@a#)q%B^yI?l=j zXGxhWH#kISsUij!Dl=(jd2;gvOdvJE5Q#BjC(aQFH4!aJDbSptg+84M1!pEI9|)y_Fv8mHr(#*!uM|p8vY7jkz~)K7rciPFV41t4(K%&%5jYmd-jamL4NGM zeXAs_nv$bj*+PrZmXq!9l*v8}lrd`J+kC(CXnswe7`0DRhbgG0B1NiC5V0zcK>q0O z4l~S<42zX$P)Q-{ETpi6GA^{vE()%C<&X}4Na+Xa{K5;4SMt!LoW3yNp%zpI1IIAa z7US))5hJUt|E50vKqVPeUU}t!ANVlBu`4(c1(hWZuyMD>Qki9n)?T9o6jqv{EjIw3 zgRV<2PZKUU=NwB>HCGy7?n^U)^T5gKu7l_z?d{aY2Hl*y9WdlOiL&nBr@Y zP#f%PgU~jrf^R-V=R+knN@2^=5CGF?YCr-##l(=S5~M>^3R6J^iyzCmhLKsQkwlbc zUSZWWR0A1lG{5}VWRnWDIyDsvp<7~*tI%=L1y~hvND0#p$%I!pPAtP!PVSTM2+F+3 z1K3#>u^^LbLL6X12RnpPOf+z*rVIm=gyga;Y3z~*GNWvcO?%sHY~5OFSp(lTr5ji} z5z&~H|IRzP(=*S8H4+INag$2|iy3x!qYp-fdcx5;677j&k2Tf_+xj?OB}TGc7=CA@q9poydqZAq{Qh=5aEz}-YH4NM{-@-jup3g zgfsxmm>J6w4l^MWr`Ks_>W5^skH?aGSUWBhE(x-(3oJYr$0&ET)?HDsOxXvr>OM`( zfcF)A@Ngskjj`ukSe&hbJM_cfiWFXW)$bg({2h}3$z)~{V$ee=pMF|uWVB7rX)^a# zp=IUOUw*2<*Gq}rQ~u;wTdoe#Qpv=ccklgXzm?Wu)~CObmDC)ji+8K6=@4uZtZ{V3 z|9DwK#G=}*vfX}*FS_u;;lv`3!^5DG)hz^1gBtjVC@UhcDN(&VXLo5U&JJ@*;W>Qg^ z5?HXb$LKD1D3r_idS`$Gr0|4BAc;r3VXdVRY=G9w8YMyl3h8NpT4K}TAA;l%PZYsD z-6O+QVC5<(jOv9-8qy_FfIs`y%?Ns%M9S8;8xk1-4@q#06aaRdGvwup`MclX8plS* z(O_4Ss}to8h(Lz9<5iM)gkTdF#h@(&+23GJZPiVqxAdo|7?r}Un$gLNz z(cbrXDad#gu^c}VQrr%q2t{;)I&T8g6(y;~SArr;L(&B%Ji(1`F>Nc(Im_!yct^u6$ek9b-w6X=a*z%r8K1H~4bbTdZCh!u%IL0v#2#9OvXmz1!l z5_#DlX#%MXSs4T|eUvM}jY%~oa;A(RdYhQCA`+M&TXpmn)NMwmH{R^P|Ci8JoNhWN zX)i1%0SlPUJWX_+9Pt?-61pP{s^@_aj6=puc0r9P(x0?*q%jEsh<;TS9FovW^0akN z@L@}QIhxWGKVs2Q)D(R|<7grTyEjeLltm`B07{J*1ePLlTY)WF6p%!}-u=yP{<;GE zp6Lk_Vezhia2)Th0I>P#3raV_mkR=gRB^tRA(Fk!m5doHSquph<{ZgxsZmuq_7=Fo zoubfC5LTGj^<&4?!^X_os5c}Et^7>uKht1>H9VpgcnoetAVE@YHlk#KGyBMoGDmDpeNW{(#SJKiYQ+lo>C6~0flEqlS^4TVTX zS$9BU+5mMxioYOBAtfvrz`H5IlB!e&kYY&|T*6Tl05vrCovBzO+Joc1IXD~L$RIKj z=Z`YMJJ^<}iM-HRQG}$qf`HOhn)Oo>7`9bw+eq8~;r5sdQ=KbNZCg`#Ex}5(2KQMQXO3P% zqs{F;cb@^?@s>Bd*<4Pdl;;tDz$cfJD6-OVx~$F}HL2AKDpY@RJYjU_{(NRSA{%)H zvo7AQcYSM$UmW8Ww>2sAY8Jc(TMY?9FG;N6Kg{aJemz*kLKCv#^3AI`gP!dgsv{(n z-V)g-cQE2u2)-3`<+9^uoDLRObfPyJ>10%T6`0<1E6m&JOMg25qD6)fj3Buo^EZ9` z13E!c=F*UT!oX$sse;#z6+W8spMtl1rZO>9{tq13BqGXO{3!H*V2i$ZsvG zfCRfqG2HN}_Qw^#k8#3<5tor*_TcxhYr=k`5&zs!4f!t)^Kb#Fu2)!q5dVn9P=^B3 zV;qBmWDM~T&CZx)AOqf!@lx>wNijJhixWQYao#3@Y>*T0$OgINDgIF)b#SEO<}p&R z1uIFx@W2(+a9!x^7N;<42H*@LW)U>&H2TFEaRV32a9Bhz3AxD$J@Q6~5gJ{K{}>XP zqVXA@ko2xGO;Dofy3Y+`1nRgk>gutNjO&kR@2kEJCc^IZCP*FE(NmHF0@g$+B~c!k zGD%hr^1d;W^zq{Qu?7dy9-4uguo5CfvCIq-Ax{x3GwL`l4D(Keb9hD4WQ{0_FovRl z=1ePw-muN4L*5nvfCk_qORjJ_;VdVB5C4ME6`&A}?4t9ku`reA8pFtxjw2gULI!a1 z^q2qwXM$uTh^)j8>@Lv7e6lEw(m>20DVH)w48w#L?_T`m+x(2-4lf3s?JBu)E5(5} zYqRgtj%ETcEX7hGQLr~DPdCRh@F2rCDar8~lWr8uAS1EyqRQJGvJ%OXMDC(7;qnY2 zfb^KC~3me?CLX(GNBq^Gi}$|CfrDDZgY3Ks51$>K{#QA`D6ckV4%DhVOf zCqyNQ6@zC?bp}i00wM5JME`qr5q6|^lwjr1ZQQ1%%`E1DcnnZdR0rB-415xGMu#)e z^~Id)tj^P)W@b_yfe@bHQePrNG&MN>L})&2LA1mQA=VIO4Gp`W-~xpwN(q! zEoAjTWwoJt6Do@(LiB6J1hPWg&Se`{fZiZI{uuHvkt9m7Pw3TExw3u+_@y16( zR@M^nAZG6?-HK)94ogBMYZGX;I>+tHs+0^IfvW9!VeYF zWM?4X#-evVw$FA|uK#2fLijdhf%RG&u1gQXR{KV8#dKNg!dEkCZZ0nX@e)rb_n}nn zXBdcEm4|aFObKSdB?ur52tjIKAW+NoEAZ7&mZ+Wp?sQ!AP*31pbHTGL;pn6NG)k-USvG>~& zB(hkvdjZhr!nZ$K7F!V)E2j5HSB^`mHDj?=LeZ3Zhh`x3h^|(%W=ZQzlU0B1R7*Fh zDi5;z3pLvc!{4FmX+8?2>4IE7=d9|YNuAl7IkWcA`a4+c5^THBvnUwRD;!k1=_R0UeZ+* z;Re`MTmNR`27POgFA4{{QhAw0yzC$-5fwc;=Rl7@lN%%_dRHCD>*a`%6XkGmt2UnS#l#n5&DG+Sm<3dH{6}U#RY1^q705`R}foKM(eh(~BCA%Tjqacx6|d zX*x1;0#SIE0BYK3td6IF?G0E0r_*`7hPtRzc?W_zmjRZk{m^=!TB&q^)`MRM9lx&-7M ziWwb?-Hl>?EnUZx)6{i!%UE>8GbW-O!GW8~VF3}yxZ^mK%g-Wge}_e>8DRrikfFQx zVi=R?yvVHM>SEcp(LBUk_puoJ;qF}6eo)_rSGn4`GlQVm@k-9C8e49s#QzC3r3GEu z!E>SAoO^iOh9iB;TcfpvM80}&8EJs6}sxT9;`(ds}Pu7}fkxx;;wJsF2x zy|-k+czrBDyD|_mjVJqD;Nu;ok=@|CJ;_b^;LH8sxqIM)fW#3V}X0 zjR>9Oh!FxjC~53@pBMY0IkhXE&@cYu23_JG9_j~g&n5ndmA>d_*@?5hY8u3QWtr>8 z%IlN9&=DSj%@Km#R)U;-zD zOYr$5#%o^e>YeTnTAr4?k(1)M_*fi~p0_>6bPl@iefXY8!0a!7i2tza1O4hlU+kpU z(=t=+PyC)>SOqw}tq+~KAZYE`zU>F@$z6YY(i)u`8G>{l9S!MR+gbOYE2Y!^bQa&O zJ$b~vT6RBtz8krxyOySP-r-f-KnYab?VeDgUr{Rgn>$yhZA2qX56?@B14oNC5j|6GA7IsA^#6fREaVIfruuk-=#GM5Uk*L7b81h5PiX3N zxs~J*xh5yJxl7h6kvU|-)a4t)uiG$v^#;}}t=KSNz8dFx>=<$^T9Q{$j#bH!m$RJ5 znz6ZXWJS6dodz8pI4;#JWsg1#`!S2)m~NRu9E^kK>5xTDw|Lo_HdwW^L#xr+v+8Zs zHGMu2=+`##p3Z&tZW!p{@zf$mvK^5&;ad((=Y~}t9qJalrPau04!CS#!6)OhKT7yX z;>C9;?opA4 za)I6PUQQ8Y*w$fBT=^cB?$MMageDOKW`B_h_#c@7V#3!HTpXC7Se*3*mwF}{8QW_r zUR4!tS3$U;ZeSS(T#h)FM&wk)edSP&y-DX_c!9oUq=|lVU?`)v#grCLJF=)kAaB$q zoL1%8;iZ#wev0CCNGus4pcAUJq=(ghRNTUWnN|`ETIs{W?rso=PIE`j0&0- znC67lQ}o3trF9j4s>P3?%DLxHt&w%dqc&dn8vhqIA~fl%*Yzosg5FNbW0%=>m#Arh zT8SWoD!>^njAk)O7PDRs+ibugg@J3X{vA9Zl?Hw%Z>ylfHYc6OUMknVMR2Miy{~56 zBAl!er54Ddd4-c5jQptwbw&WNAyup+9v6NHef~>?6l=%0(*7o|Av23(sZ%8wV*#s zh+%W9j{kf_y5X$4@^-%5+3NI*mZbe`hpH;%eGOW5YaO;c*RYGVjC5GBl$T~!KbBa{ zd>&!dU>wn;dlAA5b4X3)bk(cM5a>5PGZtuMw5Be6tUieg7X!Ct!C5KLeGw}h`_jd( z-QA#oQ~`pYe5jtmkb!Ft8JnQuz_|d1Y$iZ#Tey6aI@saQUV-zS&W@NehsmgXotsb| zsK=nx?T33Hv>f;jh(`DjErkI?U;nF;r$ECft_zLQ)3r$8l+R)8N5RPF$O~z-7Dkw6vjnR zMsJP|V;?7DC_|+=sAl$h9hc@N$J{UyLoecCcL7y^TOsX6SFFArNs8E;iR%()K~mKa zLVU3XGQ#;m@0}?P@LUQshyNnYk!7ii6l4zM)_KtOZ8B$EIbqLoWV6ZLuO+L?mMeAg zLPrKCf?JXz&QNyCIxdq-8C6Y6b7|9Es%&a|$znh;`Z$+i%W>T#7)TW}PI8(_GbOd> zN!M7fFAUMFu6$`Go%f(mimyfHb0Yk1O3mjREl42PSXl!*URNu9ez~ zIBPpW9So%^1|z`n*{mldgH1md?m?vSvY>n0wh z$xgSrF{Y_3s}}3Tu>Z5RQHDQ@?oGu*)QA2yM%U#UGh-;!o-NTRc_pr5)5s@)X_Sjl z+V5O@i=XC(RafS$u4!sJ%gt$$d~rQsTfHd3T)vQ2=4CB;x%ne&MVAZ3W2c3e`4J@o zsfz0Ti@DxwvFcs;rEnc@TwV9!?_ik2)CDgQ#3|r$qRFLq{cbAnTjK@8leAFXOO@to z%$4p{wH<1nRFxb+51Us3`5SES;Z;HXrldN$`UiXSk&xnn+UF?ACSW)Z2?yN9s z9qQ3=vwD=|CgYq3659NtbHy<2b+q#=X~lZF$IY`Rfd8any3R|rKxFNb{ktcIl1)jF zps|)8>BIc$F}}a1HjNo%O)=Xb5NHN1Rx~OxSdO8SpvLoK#l7lQdjj1KZ}E~TqRn>- zvxxI{s)GS7*aTyE+E!W#d|l?{t6ndd>ou{1qUd86!>Xvn4Q^ZvB-&Js8H*l1P zet*Vjd^cr3XM&7}f+A7v`s5!Z5+i?Ee2#UXk$@WD_zlhH#lMCBZl{qVPa=rzjlZfnExNn zbA`E>f{Zvym@tGJNPVJMT66e`BsgCNcx!UiEYFmHrPo$W7g6+Pb8z*Fw03N9IBd-# zgr=B*+?F*JR8bPbgz%$_yl9U1mw!{}g$k01XT@(VWqN7#ZvNzhqv$_pSZgoWiA7Y8 z#5gS*CW2yPXt}q8&{jU%xL;cmi^BL+5h8@+2#a19F^af|=jaQhkY)#{ftM(6^u=A& zNJ3HvP;G}?q^4qa5pIR7P5=5xYT zfgj0`Zs(2}xi}?eY#~vOL}`wgQ3xbAHCzXhu=imcnU5x9mQy*5T}TpE)^+~plpR-4 z(nV{avy<6{EDUE|%2kDBNkJ9mU~E~8+Q?6HU`|@44W0RxMQN0(Aez2%3YZfKr}+t~ ziJGf9mub+NJ2*A3>5KrmjGR__w)t+{6^*{9Dr6Uu1F2`QSdW$1W{nn*%|?T_xqUzs zgFpv@CApP`#Bo`L6`yIHpE+Q>B37hXn%$|IpHK+jX_~6Znx|<9ty!1orWvkDRd`95 z%GsCLSUUZvH)`f1CMS(p*+6^4lrD#xQ6r|Qg#vat zqbZ>iN(z`$p%f~f8Ty^!IhP&Ep&eSDqj344_2*m>%AFW$q^WtKN2;Vr%A`$-ni`s+AG(@Rs+v+t zm+Hx$>6xWYWzKr+W&ee5#>y37#G*qJ)YGh03KLIuIm>oYzYMN^v@;+i9mndZ&82s(=csu_~yuD*vmr8mPD0r%sBi-Wi@1 z8VRjRt8`kZ197X3XbcybsZ44aq*|(`3JtBMozWUXstT;HYOU8=t+lGH+sdukimk1R zt@hY!BYqeL4wOOmRLR+##8~+(ci?m_8vPyfdOY5{x3$>@J zwreZ3RO_~HyR|h7w{c6iAse(^i??8_x4ICuW$U+QYYu@6vuP`5&)T+mJGFMZorycQ zircu4%eaWUx073ue`~py3%G*Yw+=hDh0D1!YdLs(w~HIOqzka7>$sPJ4XHb|*-5aJ zE4HwUwz-(On>)Cl%M3|-xL)hJ${>GyroOL#rrg``?a#Gye~VuXN$Sc8*cyUM$~%qzWSd%Lzfz3Lmj+55fK>%H;ozVO?= z$Vrnjg1`yPzzwXw$@{#8`aAXAH*`oOPs%#F{I{^f$v~tjArvzh?Z# zZ0yH@EXaWj#)Uk?f@{Zn%&@eZ$Lj0FeH_PS+{OqC$lL?RGE2Cn%Eg1*$c7xob&SY~ z%*16Z$4kt@Xl%okY{_f<$3V=-t-Oos3jfNUoXAdl4!4ZUcU;N=tjMQ4$Z%T4ne4`r z{L8TX%Cg+av^>kX%*&n86$wX|-Q=HADtjyi~$)OC+(7em3%xdFY z%*PzX9V{6Otj31>y^j3I$n4EWY|G(n&({3R#0<~Uyvnb<&eeP{+RV<|Jj}c-&T~A^ z3H{B@Y|#3=$PSIo)hxgP?F`pU&;kw67ah$CeaH*#(V`5|{T$K-tzmQ; z%+e*@&_TS>^32g6t;;R_(7>G1HoXivt`Yeb37b z()fJTMP1Q6?bBHd)H_Yo?Yzm;3;!8ZJ=9eV)nWb8cr4RSt<;Rn)Mj1NLaowb?bdJo z)l%Koah=szP1m_A*Le-sNL|)5oz{Q7&{J*KcU{+GZP;Dx)p|YGY+cxlz1Zi>&ub09 zXzkZ(-Pn-5*Ae~LnGM#5o!E-K*{B@VlzrAqUD<#w*^bTGqb=I14ceWJ)33eStliqA zov@~T+Nhn+uszbfjoGpt+Q6OJKf2YljnKJ`+~zFYz8&12-Q3F!-GV*a&kfO|+}EjX z+t*FguKnB5YutF<-Ml^B%&@QV9nKJ3+}@4f-+kYgZU58+zTc~s;Fj&w^}W6WUf>8m*z@h+;jP)mz2M0m*z_IY z6E5N7{ox@F;*pKs`fcKBjol({;TKNbge~7H4&%qH+wM)zG+xiQ+~E}t-YY)aUCrS- z{^Bl9;kC`pC!V>@Y~w%9tU(^*L4MMoZOuR~tl2BYxBTT;ErH z(Nj+5KyKx^o#jpa;5pvpW$xxa-r{fG=H;B;3U1_gZsYGg=LD|Ra&F&#zT;kA(g!Z(e-7w@zUOsr=-Hj;O3dRd{^*O2*h3EKg6`?%UCJp==-vG3oqp+T z?%}Ae8>v6KAP#le(I5a>m|PHu&(P% ze(cmM?8NTuHT>dxjO_W1>jU2G)jsUmp48ZW=F2|p;|$=8KI?t1?VKL%T;Aj{{pq_r zyyIT%;$H6P&h5>u?r*He&-~=%{_gRP>rjr(^1kn@F7UCe%!*FsZeHpM-{=GH<>s#N zy*%UPXz=>(@ceG^{(kXM?(Xm|?RY-%6z>=TkLepP>kpsmm{G^JumYsuiYme5C9?h1O*fT{{Soi0000$0YCu&2>$@} z3EW4p;6Q^1`6)!0u%W|;3nMa|D6yi&ix~ZB%*c_V$9^3@;v1Q-q{)-@=%ply5+ci& zFk{Me*;1v=n=^Cj+{qIsO_3pM{sc-isL`WHlMe0aQ>fFXOruJDI<+cPsztM6-MUmM zNUs~giseeS>{Y8~&!Sb!wry9fZ_~zwJGQP}xhwIy)w_4C-@bDL3m#0j@Lj`#0W1Ey zn6XuOkRwMXjJPso%ZD%Xb=-J!XU>w_;S^1pwC2;O^7y8k+(s+QK;s-~Zk z8LO6E9A9)mg_E(>ZXh1w!(5tueR@=3*4&lep_$5_~xsxWBe8zETo?XZ0f*8 z%4=}HTiwfHtP=~2F2opT?3u+CbNrXaAU_#>41OSI4xEzR`PP)F_b)KnWSbJaFyy|UC?ckMOS zUkh#Z*jKAVcG+5Q9roH_x9v9DXqzqf*>TrRx3a3ft@qx1_x}y|+<@C1xZrmeZfK=` z|DE{aj1NxuaWAD z`s|=*Ui)ve7JMHAhUjF##@9z8h>9gPd$?j|Zd&=P>-%2Eg%u$E_{P&;6|NGk?A_ka0 zJ5$K5mVgeBYB;W𝔄obuXV_4UIukRJo{-6gWAL1@QAmI5webhB}|?7 z-p9T3nGbw0WT6bzH^UnCE`IVekvSR|K>+$s4L}To5C6#kL<1VJe@8T+5W&Di1wMg^ zL-gMh5vYg)MzM!ru%aR+@xTK*aEq0|AN3-Ey!=ftdt}_;^s2YQHg@lgaEzlIV@Su> z&GChJv>^`nn8U_l0)R@)f*^lr2O@@{kV{139lV&tF--B0M}*=N4H-os#NdB{1fV1X zh{+*N5{jG*q5-#KAn0eHrGCk%uZ%pEq`V(X-ZD7bCHj$8R7$i0`xye*klai<;ghMuPK)-8^R{qj|+pIuMKdYakU1!T(EJuJU-lEFSV!=nD}dvzW%Lqe2s^ zN88P;e(sEeHXV6{F-X*qDnJ4eFS0FGXZWbvn`@20^6o%`cWRg0*}eEpbWJei}5G z2z6sHj-geCCUbob9hotzxYI7cl%?D}DNogEz#E`c3l&u=T2o3_QLc5U6K%s>&l=L7 z+SQswMJh-Gxk#h7lcX4>X;~8)1idcyv6p;8Wa}DHNWL|bN?hkj)v3j%4p5(FRA6YU zszG3GHH8cPPrk+q&$GI8t$a=CUh{g_!~b?6usfCMUv~>wjlNT*cr9ce9BW%D95$mA zt*vG;dfUgUfe6IKZC-)EL~3SXw(dkFPR~hJoBkH4l~t=LVN2a~#`CiDglZb8O2PH2 z(HQS*)j3$3%HeLowKDyvadB(d;KCHKaE(J+Y3tb7{IoAd|H@|>oYi$X9 z;Ig)MoR*d5bmKZ(Dv*-000!?ynW|k4U-pSHQ0ys1yjc;SSCg&81o7}UEwlY1q!KnO zhe6!mQ(!QsK=PZF7fzJYt?ImJDRR^c}Fg==(-ju5lg#g)zP3H&Ant;aqT? zVcO*idm5&ZC2tXqOj7+Gn8j9pF^28D$%pnV926WdWe1(;;7T{bkw#~tT^rmg>Nu_d z_Hvy&N^5E(mjoj~>uXyOL;`TTt|6H9hG$LGPoLAxG=**w6)WxugZa6hrZyM2{9k0l z`jOwIG!!fxW~R2f-1Sa&zm?iYbr5g3hHWmu1>R!uPMFO_rZ=F?Jz}>8+_swbt(Kqt zyyxxL+G|(6A-*_{e zxHcg3*q04)S)(1%a9?}JK~8F(ue!?b2K!HG4&3I3*t=*3vkK&AW8>xF-}a8HrVX3! zeFMJX7N@z9GrIL@H{R^oygbvJp5nKLvfmF+-!izXb~3g_>} zO@Hybe_idFpEciu4C!Ns{qaNy`R}`Vg2$oH@LiTU=9w!<##@~8k`BGKMIHRgBbEbM zHYtO#j`E?WnYen;`~SWH{c>CPwDOVmxYW5H0h$Y#{7=AffTyo+NU(nIHubV|jazzh zQ+%B^7kOI-UW3+iCx=u;kahZ2e1r#VUDRMuc5g^WUd#t#&sS0^aC`1&Z<`l!NPq(z zCTyA~eg~*{J_UQa$5JM^bY-`BhIMAw6-W%QeP#u3uQhKdH-53VgCK|lLj`+{CVeL* zcdwUs=Qn3IwP!ykQwgViIB0=%MFjeCJG$_Ia%XcKcv2DvS1H$pfM<2K=2j;V0e(k& zTu6P7ri0ygg4c&?0*Gk>28A1Uf8AAPEx3M^R|L9+g6(E%#Rhdb*n<`3dmLzAe|H4j zW`bn5Q4a=#o&T3{J|{DJRE2`Je1=$ephjgk7l(`n1$-EPw3T0VQ=gdwJTY({sr_lGIhf(&QS6==XcyPyY^5l%w_IMu`iz|nOX;*#*n0RSedOqh^Wm7}e z7+8B3X#AFQvnP$3SbA5e2OEiCk=17$hljiT_17>s?_PTE$Hnn!~y$AdSQkn6~a_UL+T5Hphq3RX0N<*0Ye7>bys zjqJ!>xHyt17=`=@j9pn=0-2LZH)J}P2ST`pBOrW%cwCZaeHEu&V>nqbcZItsXY<&W z{div-xs#jtYTXEqdP#3!Sb?q=2okeQ#t;foDU{&oe39p5bwyS?fOCTRgFm=?iU62M z33_`OaM*^1cd&&?hL-5al0r9W z38R?wwT%O(oeGGI2lU+*#G%$ zWyhv#n1eYNsns`g|LJ2sS&!1Eqfh2+P=ZhVdhd9Z7)`pM>xTGjLpa7;FbXun+ zsHRbhfp18acnWzMxQ2sjh*HLxWZ4F!%A1ELsAZT1vbm-m^@$;gk1kq+S?PUdTBEnO ze{naOjwg;_8KHd2iP)N^rJ8_jsHC)48$SA@zwk_tT6@VRSA)rgni_){YK+K=sPY<_ zaiD373ZH!0kF8mmloyj*q=|netVQTZ8yNyWcb&BwmY(>2?%A(BxQw=Sf>64nYnqo& znusb^Vg-w1ir94TfohOlBWj?U#tAi@3E#;ai3YrQFte-iSz5iL0wQ8&S z3V?mcmH_&&hl)t6=ZJg>r)#;TY(@ec3Xi_YW2%^sO{5oXQ|=Xo8g+a za4McCr;9C1U6q(-`Wb?WwRjrpff<`^x9Yf#3Yj|Qvr>6?T6?7w^|j@; zx!(DnCi}OjWV)0ZtC|a2JzB1wpiV!zWU3juO?$9kN|BB0u@y*+mH)P9zzJo!`L00O zgd9b_M=*o3Nmy12XcLEz%`2Gy$hX})tBNYQ%*vV8x2*n)lIT{H5j$*on`t_=EuEXd zBLrF!cDxrDe$6YP$r_`oTS%7cju+aX-3ppndx7EBhk6&Rg{7afnS0x5l8Xp^NV~1! zNTs^Uxasb;`WD~dTnuO+?))@Wn)!KYWjWowpD>YHW@ zxqAtHJxIcds=L>UiZO|+9oT2g*mZj&cNitjG()GF++sn50bVv#5NEY1yT0D8A*2M}K_3_v^2A zx4%xSbbMx-OL>My`Cokrf7W`%+eyL#M!J;Dk~VOm3|odQ`N_05#`@)XLFkg@H_V2d zbXJzeqj$Dc3#NaI%4Zu`5kNy>Ye!W`z2Zg4l03LKuueWUq=mY;7CFdujCx->b7H({ zsK&m6EOWf9d)~H_S!}F&2$#`F&$8%|db^oT8jFvM(7_8=NNSIesdcLCxJ+!f_1baA z+RLqs!sTqv=e#|u;8AF}!58?U-nq{-_`In5f&QGQ4*v>wAscxyFwmHFc}NV91|7({ zxPdq8&vRS96bHsV9V}RByiv=|)nU1WmaJ#QB>Dd)3 zu<%CCP+{iklT zYRFSmb2Ytv zX6(wgOt^2LzU%nqD#opE$N>Yb$#6N_@c$;k%7=~!8{W;O)UYO|j^3Ab_on#EZtM%@ zZp{GV%bL#y;@EX|Mt*(=A0Gp&^u{>o>*#-1*osGQkdEqG*p+WgnldR=ln+{$;~>&HGZC?>(t`f zOwjfIVU$X83m`6+zl%&+TN#HQM-3OkfB4T!&9Y)p-m zE=-$vD~x$=qq}NTtFF>uI@KcXwpM+`GJ^SfxYO!Du<``g`HUTfC?} z)4M5;6zkh2j-6nP?DCtxE$Z)gEuh7n)}{^U)lTbkulGytSBhS@>J7f9_w=$n%g8MD zAnBEM?12Q&#E|}Ig&uQs-RM?q-8G5=4Gz3BU(D^-ziAhmr8caU@7cS~->L1`FlXF_ z+rrX(-+!vRwwrrO+|J(Y%9=WK+Kl5R4*76}__7@J&xXa6Y*FBydBC0PMc8tKYUm6w z>_q&GdWz|M-=)4T`bdq$S^vz=Bj5H*%Y5=a`myQ!fG>EW#@I>fc^0X?a`w(BKB+)& zwq)tnGA`L);QP&<)I9x6SkRz3gbBkbT-Y#_9*A4Ws2Ou)L<}N3Y(Swnk;=y) zNVIqi!2zShA}CLMBzYra$}C+%ZZvtZCW9Mq0>$Y9J#rnoUirSxe++%dKR!NM#f>0!NBCjlOBx zcdpR15lG18tFupGqjmkxlo&On+^3Hrt881CE91iu4|5UevE4z?3>%JS_|P($mq(y- zPVG`7Uz4gqqD2C2l94r;vfkL2fTC%p!TKzgyGG6CT=E@u#1S|2% zEvGV9p}H(f?ai%gEkay8dCrApvvs|l`j*`^=&{x_wPu?5PE}B>%6(s|fY(EBKP)z_{^#q)jBGuqtUb4au+#2O)7fZ&6agT1-el!!waD z@ND9Z!%^R(V^B?3>h2F*nPb(Z%^YNKOP{8+6w5I#i&Vrk<%Du1<TV+23eGEO*rdY(n~L`WzQ`Tk40igjkZE-Q!L;$!>Z+q+-unu@)!f&)Ygn4 zzj}cKuuPtW?Nr@&S=uy^KN7r4Oufv2Qn3JkjqcL?-h)(E^*9b!#(vYxI8D)_l{U|8 zRxt$M;8t6gD@Gu(<20?wlsIfD$7T>}=#mW*TXeCW8RQnH`eO0koSRdzIAxmPrxQiN>HcbKTvZrLeE7trArs8UGXEl6JUS>QYcH=UgZgVV zzn+zf1@Da%bHqz}lBcdir`u?l*;K9Yasl7CRTH%}yxX5V@BQjj?KU1|y|c-i@3`qs z8&DA$#m__aL@!och7(nkY^yeNeF{_+>|=()OEkM-sV~RzY4WAdUMn4UC4W7^ANP&e z_ON)HZPu?7`bzMHjdzUOUjAOylDEmPF_Ym+^@fGElN=9uCrchOn1==oMgfCsXaWa2 zxc|XLfC3ewAYlnbXhIaCaD*!yVPQV#gC6*x3>l2!4C6I}Db(z1YJU2x$d4gi@4zAmtuN*$7moQkAH@gepBL1rNrukc3>q zENS_|TE4P_od8EKt8oon{1P0U$es+X zlhl-DHM6+{IdD^(-4y3I%~?)#da{_}45u~exE?`GB|cgLn{cW{5kCSlka`j%3l#a! zL^j}`XLtl8*MUiNBvhdbWoSbs*^5+E!xf!m0wXI5N+yg_qioRVA`THkkNRPQ?Zl-l zZ)wsG){+z8sD&_r`O;#-@}x7(Wi4$=)0^g0r|T@{GkNM$C6sieBpoVIi)vJoiZq>} zv?c*nRXV7AE2>P>XI1yPJWsfQ2LKJIK)nh9YBocY2PJD+&3e{NQX`U=$Uq0W`c=4k zbp=vr)=>Ny($r~XB;eH-)UIFdJ$+xgtK zYD3Iw^PYQ^I4yy6HDUbtYzsrCh-~qFwia=0j^|bFFwJ|I_NJpXRQWdCOYR0h?J zLy1k4)$6KGuwlO2MMERbaNiroAOZ(4wyX#?N^4rz8vs08ByFZ(zPvtn|0f2?Uz{@|VIn}U@olW9_0ddlhZ zgO|;yHB>Lc*k)6oUEzR1{AQYSHK)hnm^Cq-e53LDEDEhb0u7Qf1@o!PM@e8?1AODhJXafSJj{xA5;}p_2 z1AfB?B9oGu5lRi2^qoPz&nfuDY29&KUr*t=mNk2%Mgx+UiQ@2GHV-w;fN4t^HW|0_ z%uilQFf*CjxKhM}-Ed|^vH*?4OIzP670)=~VB800gCYqr3PgxmePV#-y8%t_LKFJm zw&=wlB%tpEgq;U?$V3bO;0=K9q}8roI3DPDVyS5yjSMTo$T{5w8g*O;9H+SmLS2%Z z-~HnikM*sk#!+KJ=iw=bm~pzS<;iNg26AZmym(D^$PYdYQBXF&KduqXe_3JnZL5|}Ze3_2!Vt(PA=KSms`t47FhI}00q`~`*0cY9$*p(&>tI)aR$ajh zRG1O%&6va%^reQwj39lCRx#XRE`^5MT{3L9_zL(=05Y%o`ycRlyg)AN@r~TvB?mlK z7{3Lb!!Ur!uKW%bzw84M9{^t9Jdxp?ERV^SNU@E3%m6%DD2&yfx-kTa%(03mP! z2fID>p@SsQy+7-{-^)5#U_K4n01y;Axe`G}lLV2XgAQaq2Z=t{nJY620s@f{h8rVK z!-|blnw_zUF7rMZticITHCBtZ`YOBtsR=6}!t`?vE88&|;Dnk(K>ez}gn2yq!y=3z zJ1f{Ss_=o%(?T5Zzsh+!po0Nu;+=c(o{+(o_kkV1;o${7b6!wWNRj zAu>!l7%qI9OilPpD{{0c>H|%X1ILs<#!N>i>mtz%qjVh2(IleJ6v#s~%|;BxAv!(C zip_xpDs1WuO1L)K3px{^FCxhhZZk$Tql*7!JTSCut*shKk}OFkAcVM_ORcj@ywppX z41m4#MjmiUL(+iZQ-H$kPQvs~@LWs-7*Fyv&G8I`RM-S3R8RKAgYmqC9vFmF_ydqC zPsOy)^E4yTGzkkBGttCP0QIrcqyh09Py+3uYtkumg0Mq6GiVaC zFNz{ZnFBUc5keWx3i_ksOimJgH|G2Xf3P=kNQV_|Q5R)V7=^TX&@kxK&I)L&;*$a$ zD1hSYM(K=Byu3~X7}D@8((*h~0QG}ec!l(2(k2~*_XHo|n#=$l&-+X<0IgE-{LT_M z%r5Ox^EA%{MbHJ^(sx>;*o-3+sDc0EY*Ve0sWwF)l7PWH%q7SH zE+tY>4XRPKQZ60OL6E3fVAWQwhDRvOT2O*V3dBpWfK)Y8@+`6c%u-)9QUgc<{nS-8 zRRY8mRwFuAWgW;Wtf4`Q(6Q50X_eCmix-@cn-QR!pJBb&FiQ%$D+tUeP5=aQ_0xCL zy<=jqb}&>FjnsBszGi3zg_74uJG9{Ag-N|s5!hG6{6GPq&b=(8M*xQo_}6bF)mF^} zKL}6xJU${D0Eiuc8t4RJrC9$}ZK+vMDpp+tS-pZ){ezW?)fzYei(7>G1W)#=O!!1n z6D+n<6;@JvSS}^I^hC|?RJN0irc$k|SnSu363AoSMU#C5rjph-oz{#31ca0ha6`re zyrYnO0^;CPavOzqa|2G$KwM}y2*ld1lLUy`g>Q&dvPINUINMf8TM|+&Pe9ufV%wE0 zRW1z3VzUAY0N5(CD+9xI+}$#O zfgP@4il|#HOf-NgRFzna#f35e&o#J)l``B^9mi{MPs9zTUkY9C9NoeVRZtay%B=3xY1F?;ydpKfq9n^2=1w!yhbM3SHt;1gM*j~bw$AzRRI>}84Bg9Q%jKqRw!qlUE zB{Igu^i*7wwX6R;NQE;1V9%}C6evu|-CfRuOeH|o0G&Kq-Gk1h#2K6@noc0zo*`NNr`~++O9x(LNgm*7Zm%KD&2N+R}yGWB>=(6=5AvPo0!F6RZIn zrQuGl0*+GSR_Tr0IJ&3un^dl# zCT32!BwPPMXazxFux4z_wmi{t0NW9(Oe~(`RSo7rNMtU>U%FjVeFZ^?jn?4x017bS zK3291re7j;Bn=)+kq#*u7zBLV3Q2I}cfO_tmV&~xU_MTP3jWku`oTRQWCY-#AN=2; z#^4-yYN%#veNADm4$Q9p>aWgZmo8{d)-oGzpEn|9&A}aj;j0yS=m~nd<}_jp1BB#! z<^||8HwY$9+Js)%%+fW%a!5}l&0m1t>SoQHGa_o?_0eUn03OBW?j+n=Fx`3X1Sf3V zmXd^8bx$www3F-OPb~n+hFBzRv&t5GzyOb za_j%YAp*Eo9&0d$ijK=8K5ipsf;k-GM4&nwW`Vp`CZ*M(LDJt7yhVY2(>HC^d-`l9 z4NMT!Dg~$j85VEVzUkOSZQ2doPX%09<-A=)0g$D%9*~dGR&V@1>LS(W^M>rz9_swo zYS@lsf0pgpHl|C?)Ze}BKja`Gdb72g4~wLo3FlCj2yWpXg;%C)HNXOGZFLI?lQQ~@a1yyV+x-}4kN_ZyF7PQUX=R(Y1Mb|3$Ca*x=Sk7+31YCdRE zP*AH=jHb#=1aYQM9^$#ffP~Ei+N;DBbiV`^TKbO)t1}kzyQ;GV46TN4lpR(S;6}TX z$%RrVh2a+Nt7^ACJ9va|WexXJ=yd~IPUJ$C`4Tj8N*#HswsT8db}vxi8GrVX?^`_& zZxp}u0B3A+AN-lGc9-{C!Z&xtFL%UW@18eux8r%_W1(4L(tumr1QP|W+x!kp!O1_N zwryM0l7+0zGQXw<)SptjN-!LL+GE^~9;WcD|2KGDR9ydrXt1hc2nPS#TT9m>dok#R zcMY_7Td6eft`AbSao4rQIb3M3coXoTyysE*?C#ESrg5k7Xpc$gC;3dTR3?XP@Gtqo zFaN`z_Qj`p54d>2^!D|SQ2kwZNO014?iz=R(UhFG4!_)A0tj*fIb8y@X_KQsT2!34 z;V=vpB1I;SArny|kQgHv05P4 zUB0lzvL(-~PF5=bsQaSs)w~Jz>Q#VOg9{yj2MY!v7y;eFIuidIR@@lyh82t-L#|xF z?*O`)HDA_%*h6Q~o1sb;yT&wy(Vj_%t&u`R3L`cCNJ?wQ&YK{L1dijqt@rQXzR|HG z?(Vp8f$#2>YYwgxL2lBw5j<2ok3}3Bk4S8Dkz@DoPZo~<=uslI^y<~0Oj$?JCtxt= z&orj=8@T29NBLwSgav>A3PFfYf#h*tb9{V&fmBVMCIN(jfu>1ZWhjx!EShLmf?992 z#TE*garGAfS`ijmgjoeZ;sPzY*w<&Ih4!LfD%wb6iZ`al*^D!;Fk)$s;ev+<(785a zBLvCe#UL8S<{Aw)&@l-mRSpH*a9JitoGqX*qKF>2-68)FcE z=W#)Tb`McBUU*tmRNhAzseqhw&3)5ecag+GUwyv>gCKtEfHMjPLVQ-j1eBT}2TuIq zCJ9r^Jp~~ehn%sXR((+g))behN|#r-ZSkR7oa~|6jVRiBD~mMZ)!6~P_KGW1Nzl3? zvBesD?4%&{$k}SjekNg>hqxjaY?U=q%_ea2@`@5q`cNrOS_+4yxXqygrbp(H5r>!u zQi95QnE(RE8F22f=c@~mfbSayB0+?@A;o~_MiDW0@WJ1-lM+TNB^pgJ5C;QHLaivg z%b?`|;*%R!66J;uYk+Jo4PLlQS&b-mfE8UcU~&JNrDb%62gx0pYzvXOo>=N&JmV!r zke8tm#i$vmEFp^pJ@R0#$Tl6?urW$~z+?)raShf?&lswOgA8)yg5WZ>9-HHmyJc}? zt8J3Uy~#`OZQ$v--CBCK@g+!H=>3G;bTfo7!V0Uzk;CS^q>kY+{X)nnKq=hlppCb) zoTHCcdie#JUoNB1e67saftOrSvK)+XX6b7&qxEamwdQCmPN^L7;uKlxe-AY7_kMgY;pGz4qDJqg@>LfZEf&qvXC_FB)CY z&2FAhtb2!d4P8XA-x2{HxFCMMCro{h>%afMew6#8Fyq#g2-gg)TS_oOsP@1#mi2`% z6>z~Ucu|X?eFYUY%1rbQ&?M;DXf&6>U;uiS7usa#FEz2QS}xFT@I zXQ7E`M{j*%S08c%1&l-_4rB}icnU^@`6VnR@G(Z>5ciFAu(6JS+sT@)F~CQ3DnDjo z&V!a9$jb~dF+}vks>(1!Cy9oGQPWY9hNh|oIxs|Vx*!J(u(>2M&@Xi0#U(D_G!`MI zSEIAnZQfwA_1Oki9!jNKd?1-M3`ze9({jww785k;O#o9+EZY>NNX05z5goNC$Pmdl z#(~+ueT2hJGnvUZhD3k^bIV9I|5nFu)Gt1B{9fP!)jx5P^A+&QNo_b~i&sE`Ol3+C z3)E?gGT_0S#*3in!~&Wth)NF2+6>Zm#y~t^4L8kT&Q^})7C&5}SO;2OKs))OC(1=H zBI@0S?q#!lakQfwbtM)^(+NN3j9C|ro(U@TM3R`TrNJB~Ob1evH6)=5Lp)^*^wu{9 z9HCFx{LTroc}=?6@f&qWLmezp3QHiu5|bF)J3;wNL6ymzqJW|~^O#Qc{BmrkNko^v zl&R7&1%(t?66tQoD<8Segbe?s0V2N^LpfBaSZgX;F4;OmT!dvWCyk*7Ghqhxc@(5| zD{Nb6Lp=}dfh@aJqGf`#g3M7+icx$iF}Ej+Q)OsgNeF2}#K@5L;cGh`0V)yH=%+tz zvwkbN>QJs^rXDiFOQO((JI#b!d7e{`Q+g+$Y@5{@kZ zBOv*Oh}M(nUMMOvzfuLFALQhSYWk^vLd}%5A?(}``yuRz5mt~8Y_lMXQOOcerHq)W zxPXb-bQH5UOGc^cZh;6zv6u7|+ez2h`;R)Ctr*`PNLm=9DVVwXGw^pSC<}x>f z5ch>q!K=;o9kBoA<}V`^sZN#iD4JrX*!Fg|QSCF{K8(nvyp=;uA`2ozoWMbNr68L7 z*IVw&NzClwD{@#v+DO7sR_9E{FqGjv3KH>0yh9Vrd9r?7Awyi`M=lvR9>N53FkCbe zF%}~5T6*14Fm4JG-eILIOMbzpHSu!l%rBc*ibY|rgTI=cDYTP$O7tzWU_*VTfxpdd zH(UDBAZe;6px%o*zdNc{r_NAtONDA`Zq!(FislCK^#IG}v9o+@i9ay#3Mlem5oT!~ zE85l-aU$c7pg703;tG=Qoz`a(`2>kX(9+$bp7)qd;bmlN8om$Q2U>Ez9?!m)dxT6w zEBG9=3=NQ9s19Z?vm6BGa4m&wWT-PZMbC4X2uyeSH4Qof|A5M>0`Ixo)sKI@fiDM)!RYUarhIn=n9623@+Rqnk3%%-GdX@+H=eW znuPz#5A4Cn9ns+}g8aG2L9kDekwKz4-jXF5%Ao>Z4IW)(f@OFPbtzu27@*G$)9CSB zF|8A9fsz6#)1XZtHQC`AIg|aISp<{^2l}1|7J}}L;P07WOP!0pgn%ZQ0fqFzpxK)Q znGO>@90q9;>ztvu))DO6@is)f;~nc3ebjGeg=&ARTC`FrV-2Z z45rNOQ_Ymo777MSvZP@AqC%ioYaHPvfJPzF$Q2;v0Kx`OdWbiK!a22Jv!VY@#>vZP z`9PWll{FELH$tVrkXGJI<=6xwCS0Wzou8c-?4efc`O4bqIN4-Uk$5GlU?Mh2Uonr2=Lam+>H!!A%dao=@o^?4{8N zJYAfG8*Jj9xrHDh-sZbCBPC==3*04YLM1(>7;vIx%8Xj7g{1Rc0&=EkBQR&S_==zb zTyO4W>p%$iX(wJ>f-A5>ZH_}Kyapz9$ejS(ZImZtmQK6qW@42<6Zrp8Efrid_+zOk zQZ+zA*o4g#%n(+lLDZ!|+W@E={+w4@BTXsO>Xn(%MP-5w%p8yw5y0ke$R=&dS*H}D zn_^gXY~c)K6FqK(9g3rGPML@z%aWpr5uIp?CMQQWrwGV~hF;rRFhmNb&Tlr2jjkBI za0n=BT55d790VymrbP!3>6lR-210@5k!WKf)m#L5f<5VI9YB?reks}< zmCMTjIs@Q1)6i5bK*#Vt2o(kldq@EQXLpQ|mxqFz4J`?qgK*sJ34D3aEhzO5(?T-<`5q;8fGsF8804Vj9kmCiA8Z9s<@H^xw?ip_RWU6 z%=Gx!3fvy^EQO|3Dh#C>PvBNa&K#$*CKce#%iUS78P}Lm!7JDf`(aI0++7Ts;g4t@ zRA4~_QR5m$?9Qp}0+!RIeweUs<7urX%JI$1fh@AdsX8+2o>dj#nu~5yR<`L`IUHN0 zR_PLC2v}NN6D;bK8tHmU04<4|E@4YthU*z%0yk&^30VK4UfzqHoB(qUg0Iv-R>6kT zVi&Q{>t0~UIIzMweal*H)SdWb$_?0Kjw-C=f#3}x6*`(3cnzuO2&^6;+ZJW$c_QX& zBc4p>ntE(ha%@Ow0pLc^I+Y`Y>4BUcg~?i|D7ec2k&c%*SG91>0Cw5{g$`eNE3cJ7 zAarfaNkL79;=CL&6L2*11Zte zE)zvS>z-sS?|#Tt)xf9)n?DB4iueo~>Sx<3TkxW&wigYruXgda_*P5y_I0Uz52Gt{@``<6M_YtlLu2 zK`yM&kr=>~mYVWrB35Af3``+X;DlBR6AME~Fns5U*h- z0#hQ6Nx_fJtCRwfe}=O40b7mc#RsI|@@W4Mh3G&+{$PkU%?hQ(8RsHoN?v+?0V5b} z6WpC73?VO*?ePXPmw+a&sx5Fqhk;6<9LV?_u9uexNm9^HwHv~#wDLb;qZJemZ_WHv(W!v+j+U=pVD##s z4r*^Cpqs4zZ4--COiMF1Pu(T}F+DNVZ}}ilhXz?xGEnEh&}|DSr$ZT7fN6paP&<$Z zdC;}=5a+Uk+(}eE!;<>6*K|5pEgSz<7YuY1T*?f=fnXHeL47sax^+nR?MR0&m9RC%X4PGP+4-WCVuwtQCB_%C z3ew{93!Uy0-j+@aaiIQn*$vAT!t}dNYE7phxBSZ^%QB1tsSC%-?~dP1vGmQ|l@Z`p zy(!~X$F_jFZClR*AGm={k&J@wwplwUu}-Jl1Z0Mt6P)dW2nJB^8MiZ1$nOPT(}hdL zeRFQM+$NL>!;*6kC_%ug-Csm^)gVx}u!a#|=NDgvkEu2Y34j(~>rXd!C;d*xs#lIf z9#x3I9)ux6n>N0s;8qU#7P-lJCk!5FXDRl`HufN z0%*`$>X8me#Z-tmhW8MrtSqG*LDRuDF)ukBW}ie?0VJ#0SdvHG@}@JXL9l8^hGn^y z+o8F!L@4ZZCjvLhHX1=-__Jy^Ps>&*3MyCp+lN&0A&f*|C%~ng^dICou*0);%Q$(r zuItuo?Lx_s0j!SO%xB=iF~>J3?16}@p{G1>TD%4ah+Tk2?2;=R&>0UCTw|2nWgaSX zA?Hb_ZiLdbdrgE`HD&+PN-Wi!Vfk)}y1gqnJ8d&hj|4mkCbGc{rrONown*EtL;LbN z-tF>>v^b0N2=`&*VmW+Zl9B<~=eh6#Tx>kI%u~nb!K;V!0WVhN&O(s0^f%X51Lj*` zOAlNmIhK9<+J?Jr-!{4*S%DL)MlU-=uzI8sBqHg-2vp!v(ZVU5y2#GiIZ{}ok-7gK zSC~@qJ#3cJn|Y^vxB#r8BnYwNUH~LC$6PeQEWlz2d*5-yxv{Lk(A6vfUF3L;Q2%)M zBugYNeb+5=&DZc-wuCJ|?KMLs&lNn6X&0v;HCC0>cZP@pa|F-(X7!`LRHS>)+nU~r z+Wf$zBs5cb{Fwh%o~VlF|9pBAy{W^((wlPHlg=F7kLI@w76OB^5(5M$htSCB5($bn!rw-VZ43B|6 z0$Iu0CF=j)BLfar>b!VXfyE7J3zT&`v#o-)b9J0qgQwA%3^`rsm|Fp`hZTZOOlb(3X@a>+^H9ym?BG>oBQL~0JFiL_CJ1GW~~v7ZDD zbDK4i7@sfkc8MrW@H&xTohoh$d2&pJrj!Zv9MW?$r6bO0c!j57?K~a&*lr{T+bE1S zt=Z`JBL$Hj{A5yu>RMC>OBX+5M}&J1M~(H;eZbIjGEd3DBmlcm$PK%tK=k z12?&(FdO|?#EzHbqfZp_w!vURDMWijuaQ0&Xp<-b{O?Ono;vW5_)LJ1PPIhTi9kOt z{B05r!x{@k8X<&>#$bE>Yk)n#!R4Ax&?AvV8*ju>$HaatXIdeNyjC+?9<7BFJiH~% zx;dc?BDXud4CAyfU0e5wDaXxn6%EOZmtHs93`LcE*G#j`NDv9cN&mbn3E)4=u`mC+ z4Kp27(IE1KR6P^G^JtNGEKY|TS<1NNjPz(gqf%E_0{M=CNYn%z%Ag8~2BQeF?bHB4 zmcuXuBuOR{j}D+C&V~Y;6@e;_LgQA2IRqHQUPml(Mi!+4_97izspVKHYHT_`XFcw% z$7+iV(pqdc3-X?FI(Y#AQ%Qs(S*l4?l9;J9 z4I+yEAP7Mb?dGke#VvPU#ufx7x zY_Ka6XrMf>IDv%M?k*PzE#Fq7?%LGHn{PG$c0X|C;`IZ&{vswE|NIZ1zyQPVkGKr3 zuv{}R*OT>lK~x@8tHlerp zB}d&;n9p>#tO~5igib@=^P2WFs1ZgzDI{R^yoSB(VNY9#dW82{0KO@aZ+%J3;M?wY zn^-&%e)@_Z-~<;57V6J`>OlkJB*I08WF$QQi^5$JGr$6R2R(AA209XjI}rFlRhR>X zp^|`(30lB{s-py|Y%`#vz=sY5;e^jTXOfQi%6Sl?L2nqb!V$IP6`22EgtoMpiA?2b z54l?r@@i@=Z5M^fCR77-pf43*kS#1PLVzHuG#G!a-0YpRqb^j&FgAIxPGciBsRMM80zpjIh&xJ5>F z;a#PX4Hg0vpo2tkbw>f`SFod`6-{SLC9gWufqq);*mnO|S_hqWgaibyf(N{gUD$$% zl_H2DWNXgqd@*@9G+yowU`XC>XjfHPf>oCgF{^3uM#UA@TJ*?#4NIcE9<)B67g&V zU9l1p7uVI#f9|AlUl^FjdBi#(ah50}6@^TLXVj}sg2Gc>g@!vk)pcn&UIe8ZS~F2V zF|DY%FmB!q_S)AcRnB(-K{cb8oIto()UnCTUo$7k+d4FuwW2-kYR}`r{$O*r!w8XZ z%VsD~IdT&KUFj1QAaMkb!kDcqg$dB4Z~;RM0BPo&6Q?_o$HlZ@k{m*c((hS*>C1@6 zzHqZM9PJQS``X!V1-G-Ex?c!ENMGquU2Fd-%7mrOOhGPkzxzF2z0kxXn5U|0)WYO= zAp2eTD-k_oxYe%r9W>MS#=QuX$RlfZ`$QY312;Jg(fbLFwlVPR=}qcYA*lIz!h}Q`26AwGGR29?*;9G2b%BscJNd9DuiZW z1hjzT7VDxg4x}~;q`oiw{_aeUDbmg+6*@r*rLgtb!hZ^ln{1`7h|QATf_L(5Q zEP)ktF3v`a=hm=)+VBeVCkg+MCDJqyEh@$pqF@n8rvveW7X=6xX5k%@f&=}q0pX%@ z6cHT*LlG(QxR&SY215p?CRFYbMlLXGvTv4tx?T4l?LVYQy6S%ONQe%Vg0IYS9+& zQZ6{<+prErnClM)MsoisU_4Gj8I7S7n!zMbk~1Qq8CUYnxGi&>sUlu-*(Smzu}EjW z19AW;>8kKDQS1#PCL{??xKJ>m-fGz1@GcyPDg`biX;1YehSd(p-b91!3X&|{fCU2Z zA;k^XBGThLiU{PgBJaga@X{88jWm?=Rf^7kxG0O3U~1&={tOc_71J>tj4{U$+o<8e z?nw!p$ui$>?x>M6U(zyfiaS2j^9U!5)KH9as2VYIKHC%a@X?F%(=|y`KlhU`R;>of z@*o|~l|agFkZ2Wu&V0^=IG2!LL{1GPz%DU>2MECoQgJzDnGXx05@)vlag!=)z)Wqc&<9AICCjEC*06bEt@QsAW}?y6YogsL1U4^ zrUC+3V9O-(G=$)diiby52Q2rbGc>_G?9C561pf{bJG0XpFrhKKlO;8T!YXqj$WtK@ z)o~c76nlf3IOlAX0GViP-F9S{1LIK@U`;Jh6ZW zG_*xb4b1MtQL3iy;H2^CNsa~A|PMdhdx4)rBpbWvM0JtZ}9vSLQNg>!x|=6TG;EwX*KX5bQuoc(XiL!)V`6nR);wU+Z1PbCI^lT`_E4I|typqgx6MUcF6T ziwVQfzz1JUJWH+h1U3pjRlibTV-CSExFc0up}W1jRoJhkfVu~x$aJ2Z*_MnDfL z;ne?l6$SJ*q_z+5%%_Q35f!P=7jl7Q&GZx|0TE6%5ttQBLm=30kwli(A&TI|yX zST+XCO1&;B2xsa`6*V^&mvQ^far-7hm34B{KyWPpTAPAdT$X-@ba%6$zfdKRMwI?` z)-glDXYmVY5o>iFGHFlc1|pb(UAKrnPq8Zafi;-3c+*K(xAD9nVzWbFM)-l5PBZ^E z4KaMGVXe37BG!7NgN31sg_Gn75FvcGv?I0v2;FowGHbn%@FC$cHsDu&=l5~b;AF)B z4=j`j^tJ@V0CP#iZ85bRT#Iurc!Fmj52%wf4me!9Re`;=GkS}2ZOU9pplI!8jn9&Y z(|CgC4hg(hjoaAb;5eD^*p2Pj2DG>#8h>4S= z)T9p7V~O1$W&L>s(XB12VBH`L2x7U5FOPI@xmy`nfvDl4#AR>yR)C+`wI(`)Ef<<8 z8VZcampOVNuNkAKfYv0Mj^CK>=on;u6?~9Zkdfe{BOrp2O;PWzj@3Akm)YgWz?NLP z<)T?;`e33GVOKeYrVd)3ks1~C4rU}0m+Tpo@fqZ-$OKL{7n1-ZKS@GezzU=XQYuWT z#ch@unygRPp{WIgXNn+j~WxaH2JExL$^ zS)<1Qs2M!L5xbBDTMYu?tuZ^9E_;Kwd#9)4xXqbuwjjE%d$Rv=;Je>|!&$hzIXl=$ z`r_R3hZ9$bzm${HpopP5s^^i`MC16Ys~zhhHLYa244@xT3BF?xoJia zi2MsR;y{=e%U%A;umkYQp`5`ln01Rt$^p5;Rh)vJwYzIgj;;Bc(Y&#LT#?_nWwoO% zf%>PRn1);YkcRYo(447*xXI{wsjZ|nhC#=79MR<)itsl!>ZdvFhkr#A(!UMz@Qj1B zHMi%Cq4h`+96eW&jeup15qOvki0H~gKwK(%uZuaNox82Unzx@hBDMTU4B6E!+|~y? zu=Z=Yw;Y&d=5?#P!)LaUU)sVKnOrW|MtgMOdUcQg7&re8)D^i;SlzM?aKq3KUC}`< zm#%#b{EKK2l~K>~S@ZZ@>l@RN{JtEMr8=XdYrJ!tdd*KUkS%Q9Tep}GxpiSV+`LF#XiMI`Z@Op;?J^21$>j{U6+;vV>sjmC*q!Q?Y`x|q65z8~oO%9Z!<=Ku+~*%2 z=QSALZT{d79*Dae*@@NZ7dh@KIG))Wao>B^5_G9GJ{YuJsvB#yt&iwOe&qAl{Y;+5 z*XE(ixVMl3za)?C;fxm)ECx=R;z3=-NxkO7o{<0R9qiAT*2`?@2l(dW9oEB|@59-t zioSUDUGRmy>DQ%^9T`p@*<31L^3D9H>mBe#8_?ey*QS01w4Up?9ppaVHP(FMQD3Dw z8RgN^h~^plK1g-%&hjCg?pwc(yP%F&ALZBDrm)$%;YrIUcoByW1PznpHnc z%URl?{f@6c_x;+htDmWXAi4o!#0d^WI4}?h#D>9y2nQAk6Jw!40ue&ouyG_}k%}Y^ zHuR{a3ki$_S~$#jP~^juV>mK&I5Gki6-fUgx-59};y{ooZRYd{RANjNH9JmegVH2T zlN1RKefedFz=|bR#QZsvNX9UI?zBOQg9@1^TC#tx$Z@@k@TiD{tic&{zjq!Nt!i^eV?j&oup~8xDpn};Mv?oZdPDyGE z8pP+u&>fp*h7uX3P@`H*w*dRpwH>%6*XpdQRP|KUk!vfq=s3jKkN(QidH|Cl`|VQ%mk27irX8 zRUKqNY^QxxAZWn36hZ%k!^sB(86#L|+*2yVL0*W{k=Ip(%!wxAYzTIj z)C~wW#X)xyX(yjdN!2zUOAx6c;|^marc*;|kfozs=Do$1br79ZOnU0A_g<8Jg;EGy zTQp}9eg5UAC4$*yL8Dotc@zU!C^EEPW(xN9;AD$UXq#y;s@NiI%x#90K|{`I6_Eq7 z_UD3TUA5ss&!J|RaCb6*9V56Q1RicnO4gf5Uu9QfV>p0zWTNJUHK~!GI{74fQU*5I zl>kbo-$_HJCK6{*5VUHaX&U<4Z8!l)p{;!0X`g{3Uif1|2d-J6MA&Iq+C&Tr>sYTJ z!I^BKMNKzmW5`x3p@EDfD6RiYwAy;alICveBe~PM>7T4N07>ktQB{VnbRsa?6M zhJ1pO>T+4yt3C7V<%)Vt5Oaxs`1~YWgQC9i1gBOuuhJ0(&Hg!f^HqEP| zo{cxQfEVj5xaGOI9d`eu+c7;G!gQk;uC_aWoVCpy5<$AcfzKCqNZX598c#%HSM^XzJ~&Bppm<+l!sm=Et&6DKLc25uI=tsG#AMj9}#<;oCUn!ZUejeSHhwuFMy# zysayMi2Iog*+iierVl(lvyZhLafjD^$8i|UV0);f#O^T-OF?MP#I#7C!*$Vct1?&- z1BgD`4X%V78O{IP?q;{zt?XFV8O;r4xWWT2F=eHbAMVn~zq9EMSaHJL5&smmKs<_O zE9o9W2(u~9AgMfIprZAz*B(s50*j5@VwM~jI_SY>Z38P|_qg!IvY;?}LOi5c+@&%v z)u>xG!`R&@Rk0q@4RDK!4o1$#lGgaqCSxN=^R}clJO)lqx--xKU-JdRb>VO<^484Q z*EsZ4(TY>D;P?15N+wPdn={kfIK}wM`+$LsjPo1!OqZ=BWlTW!^BFtq$UX9*XeNoe z<+P?oNaL~4W?l)7GUxZ2s`0U!Wnw7lMD;dpJ}z=rtiM?*M|XNHl9$nF zB4?V`AC8Dsaz)>x@VGeE;cb0rdR8HgSu3ESZja!M*$y|VNDv~Gtcp8k5TkkwOIB5b zMIcJ$F8j_h>L+YQLKE)O?4NgF|Asuay+(W->e@8wQ#pS98!e@j}Rb?^yI6>Wc# zgI536mi0P%b(2uIN`hiBc6Hjp?QOwnMhB6SQv(8K_WIK5y>WfVcY z;EhJNy$Kdunp~=51d0g7_S9i8i;6L`#cjsaS+8-?M#JrOw4qW7BCnJto#WZHGX?H3 z zWh_cTCwsyJMjNDQjPC2A)WoXe?Fl`N<+KDWCg*BnSE-{&LVelLkhXWIpORtQ{`4}- zq(Z65E3s&<8M&h7_L8!J@o%?!+>wifdL_+8g1^$jP#7${hfS$?1Cn6y{I_=DEAqP9 z5tYWdH+fTn@8LdKHoN^WU~$syotT^A(SC7&|LpL@I<&YPbu6d>dGU;gdz=3pSC7Xp zjczS-PL|j9)NylX!-~nfy-gJEg%jf6hyzo-kKT`IFV$!w4fLpI=jgg8X<n*>s;K zGDa~s%ZINjsaxRAjlX-GhQAoXtSpMgr|#;i$KvW;Y%^c|&UBT7GNdm&%m>8|Z(QGs z-gQN^N&ODLcLG;{C*QA6rMy7buDi;Iz9*J`&eZ_HxbLH7w34IMcp!FIf;D+O)g^xy zQmC5HCI09%X$|1cjxpxQMzS*p<@DR-eC+hXI!yb#TE%^8P@}&#QA?t;vtM4>)7QL) z+0D|=m9gvD=AEb9Ov$lGoz)nGJmgHVPSy5)Vg-Gmvw4@NdpF==`^Eou;O9xWcYW!W zdk?5)8b&6ZMtfEka|yC@B;-=7*K6ita=ce`akoBR0FOX$zbApZM_2F{9bB<;bFqJQ zwsAcM3*V7Wu_tq@v{{4ZRWwI{;>SC)M}jCRv9rESwb^|PLh69!(go5SfJ64uz_p~u-XN2c7h+RmCwMKd?H!fsYhLvcEGdO~w zg>aiyhu`FTL$-I2$XjFQOZru5kHIxFRv>!Dih(s=A?S1Lk|ENDfbEz6ZhXaAfrxE+ z_*Pxe2QY+mMv_95$bV-DIfpoDGUjqJ_h3WVWH+ZmD3fpI6a;AqTHytXAj5MQ_#wha zgtRDfc_@kAC{QSfVI3BM*m!Hc6;unzX>)jN#Au9{c#Nv(iS70|^HnE#7lH#eGAY-D zy{B;PB8Sq}B|Zo%^G03cvs>dUGH+=(h`QKUFo}A2*m4$GarUP-?6QOn*;dlW zgI%MDH71PCBuF&}763GdEE1B%=zp}3RklcoDi?1QIgUffm1JfAh3ofgu9$@>n1WtG zj+l`(sfd*n=Z0a(gB6K`wbzK)wn#KJi%V%FhWC;9VtiCtm3mPKG8vJ&mVyt-lQmNz@xt>zQlYzyR zhjnN~*qC#;mi3u^!FivD`JB!+i!&LCooP)^m6!MgjRj@@eoUpE+ZGYr38D139>{l9 z;#r{2Za?n(XOKQmASqnjt7^EC70hdzO;}dYJ_3m@+Df=Ot<` z*_%@nidA@Nn}(f~;(~y;c?(*g(-WcH37C<`FH5?i9O|T|@T46YrBj-rA}XclNu{ru z2JT6su(_pOs-h?grs|1ybsU z{YQ1*$)p!5s7*?xgj%RqYN&^*sE7)sjVh%e+NB@bp6tn`0V{_nxilOcruj4v7t%aR??u^nqPc!o(;x343stx^{bCrhve zi?Gz=tq{ww8T+sptFA74u{9gBH%qe%YhWDPs*+Px$Op7R`wJvnvPFxk-k`Ec8+i$< zurKShISaK>E3-H&wN)FnJBzhh+gw6Bv|TIzviU-^Mk}`Fkg_r;3`={qT5Gj1yS8o1 zwi@fUaoe*^tF|G_wO@<3c^kH4i?V%Nwn>_{${@Ib`>k_Jw{I)AiL1DaYq*EYO?G=d zAM3Rfs<&b5f0s*)d`q@}3%Gz!MC?5lq1tT)`jQ!7j|fGfclQ9K$kee49$;B$G)7$8VAh3e9g|R&A5Eb*7M1;Ov`>O&cqzeMk~pY+_&d& z&eojG+YH3+49Cjc&CD#w;oQQ@%*|eGvh57X7W4aUxl!63c=)hq1Oxtr2AE!Jbr(^PHMXYJAfUBn0t))_`5X zm3`QXeY={S%9-8S^X%Cl-P620(2*_Kr;XZ_ZP}~c)qxG#gzXHW&DgI^+e2O2XC1_o zZQAFs*s#soNIlx0o!GuD+p~?+#m&f~Ez^*_+o`SFSS{JZP29bW*)>hw!fnJ)?b^Tn z+N#Xl%?;itd)?bj-qmg1#=Y6nZQX%e-pifWs_M&+9p2pC-ga%@=bhg5ZQEvj+Yx=r z5;fn@OyByA-$`Bn;K+^O$}QFW-QTzU*DU?s&i&vAKHdbb-V~0~2|nN!&fui|-GBSs zFWuU~?aon6;v8<`>n))j?#dtD;4c2+5f0s|J>na#-6($H7@p%Ae%$%Z;s6fgKR(fF zed0D=;R?RsAbsD!t>Y?=<5S(yNlwZE9^&r3XzdN^@Y0nI>X(k`JEPYvU{33+?&*!r=&6p_uAb(nZs{^E-+QgvxZUf#-s*a8?8sj12YuJ{ z=;2gO?8{Eyo383@?&H_4?7BYdzFy4i9ODxn?A{LSsGaRwzU$R~>u#Rw<&Ns#F7B{C z?$0jiwodQZp6K`9?%S@-@b2#69`8|Z@Aba!>`w3n&+YfU@BYs33qR$=eeMR2?hv2v z{m$?e-|YWx=-7Ph10V3Uj_>*&@#)U-h%W6VKl0sv=o0VG7i{e{ zuktA`*fBrxAkXs@pV}yI^DH0neLnQb-tjx{@*m&xJ}>4Q-|{&B^g^%IH-GfFp79Ca z^iCf}^&YS7U0?OiuJkIb^;ggIf}U-VQD^+><+R{!Y-kM-Gn_5!c=UZ3_$ANEf# z?+S0#QQXON5BOWp>1*HibN}+E91s8@`2+cqLTXU(5CgO(IZbm&K;8;>eAiFBz`o=|~C^(nRLRH|67THVUEYtg1!zdjW^ zmMPe>WW#1%>y>C*ws7C7RGYT0-MeVb;yp`Op4_;A{{|LJxbI%Rh7admj5u-OxsL}& zo=h2@;KoofE7q)+vuDSULsynOnzZH9s8jphoH^;|*Q{g923_3F_(XP54LdvWZvuY>>RuH8HN^WoEvUoReg zdH3d~4=efRYz-+TNaXrO-teia~t3&!V~gc06V;DZ-p zm|uk)DoA028EPmZgdKipB8eeFI3S3*jhJGIC&tL4TQBCcB7rx$cv*rnPRL`ADAuUs zYAp^aB#a*Z=p&FvHaX*yPDUvulTFctwu6p_gBxNu!zU zX}KkcNM)$zoNKbFoSSb387G!taww*qc)k>9o^aBcrlEuiIcQspCd#LteIg3zq>tJO zsiag^ieaOgI(lhhR$>ZVrvY+GDwcnyDp{nWes^lAroR8G>a3Qc7%Ho-%1Z05ys8sx zuTqv->pW+6iEFN&7WQj(!n)Y%vBWXkEU?Q8SZ$}zLW}Hw(^A{5x8FK>ZL!&EyDeMe zh6}E=>dIQKxpLMiZnf$5R&TrRzI&^_d+Hjfz19MZ@4y-!3NNuRif8J*0ym5&!G-cG z@u!MHT(QF#TkPz_1|J-3R~p}!amedN>@mkD63nn}-Hc4~%e@zh?I z-L=_gr;WDSO^5xo+hUKs_S|&W&Gy}P=Z!bqzP|t6x8HG--S*yk6K?q7g(D8Q;*7&A zcjA!$-T35`S8jQ`|6a~Fu#ZPRxaN|F{yFHQlPTh4l4eG0he$MRPU`_k& zv8P>o?xy$7d)L3mj(hOLvu?bsyN9!j^4=`(yz;+9ug&wiFmHYJ(_?@A_U3S(yzJi# zAAa%Vmv8?0!>2E-`snL^e&A+b9||Iv&~JbJmDFE|A^RiJ|NV-HgeL%KKSNZ)6Y~=w zDe_ca_;J{LqI+grXpaXoo@?vWR!!0uptI$T>XWk&6T*4PyX=Bp$JeU}&QMBALlS(r}P4 z#Nj4)m_}VBFouOVVE(MgKm|?^jA1k*EKO0$Sg=ugxP+b>jgiY;%JG+Vq+=ZKn8Nq@ z&S>6{p&yB;NJthE2#4&XCmT6UW^TcdoKz$uw~0g|Nb{1AtmGx%P(&MIvYJIyrVhnP z$RL7qn^5eeB|W)BPx6nI_naaD8EF5?31$(D%Zp_%eVGk{4pW%KB<3;UrVT~}vYAJ; zff}9(QF2-|4j5ghGe2p~BSf+YK^P}D-x<#ydUK-_{i8I=dD1Za(V8wbgi4)BO?omi zoF=uYNIluob8<7E@g(370XV@{rcsKxJOx0{IM84g^r;G6=pK~`(~E+1q*slmA9wl& zPPRb{9yMo3;aO3glC+(Pu*U|n^wXzL@L~H6uA`p?b+7xYZ!JvZD zYBQQ~<*0Ny8i<|Z6{TiHYhg8c!^0|;pZ%1q^lk@7_I-_Om)&7zQ%c&qc6O$*oi1-< z`q}!<*R^8JuW^G&Nw*%dy2@ScaOZ1Z)4o@>6Xk6=-PyzCcJqRb3K!lClb zri4p+U7gxM1&8ghemmUOLXfh%LM-4t=Xb_%RPa<~T&_hm+Xc;DII9STX=)3M-3&iB z$2aIHjXjH7=~j2Q0M5fpd%M}Wa(2T9R7pINj0Gv{@U%px?Q!7>)-AMPksy}t ze)(KxZ@wYXQyw%qlf33d-}Vb%p!96%E8i+}6%tT(>sTw?-U$o%!Kbbyd@oJo&8m~p zX4dGa(cEfOm-W64hH!RiP3K;x;bP*OUsX*l=_Tu!wPMzSs8!8GVlNuE0Io5cS^Zg$ zmU^%_uyTKa5P?Nw*v3cbwzE0#;|B*hzzkM&ok>7$V~cazX|=G1Pg~|Em;0tbfH$vg zed~`pTDw`^w~$4whn$5@*mN*7pp!joXVY8Rt6g>x671Y#6It5$UHFFgjOlA;yqP2T z?-rgkbdS2uldQOKeY|IJD;(Qi=ees9+;LhLSJP4dc*<4I z>Wni&hPC#$wPo9Euzq>iS(tVZz-(Y&V|3tRXK%#M+SaVIJ=8(xG!HASGVn2-?__50 zRcFr3w0k`Brj; z1Zr$`1CsV!V1{j!hjh&*hsJwSH*sB6 zeiZn7WoK@>g;IA%QP1~)ueE9vcx|zVc8KiHda^Zo9z_C^)&?thgnAHzROWnQM|-4) zchLXkan(0}{1$rem4G`2ca>Iv4Hjox@GW=Lh3^N6Cs&5(_J|8eb`<3aDv0rg01wh#}PnH3x&T_HwC+Wnnmc>W6&y=Xy@KhIA-= zE|-Nlc8LgRipa-tyGUi=fa4naUE|1W&4*twh>y}1c5MF$ zjXHU0Rn}_5c#Ap7e9nhtcfbL%sCBQ1h(ow<9a($J7mBf`h>2*73AvQhxQ=kfTOesd z9W+2JrH=F`apVY<)D@JC#EW1RVY;@Al(c#9hlc#fgW8x@eTQ6Mmr|~njun}U{}={d zrh8(il@kS>357)nV9(HP`i>)(=$KC_+e&9 zR&6M4K!<`0hj+Mmkis~WHc3+1#BQ6XZ!yVwhG~Cf335gmm1Y@;xY+`8*+Y!kmUozs zSr}HJNt$*jigu@*Cy9ossf%Z5QdEFV%^8hrshP!ON9je9L1<$dd2XO7f=B1Fr1PJ+@mv@0Cc#Ye)oVewd!S{2_IAP;iclX$qNoaaFb)MOTVOA!V=(c4m zIG(AOmv#Vv?iHBUmzazQiD@W-Z%`(jsXT~)k^#DQ7ipQIHg1}Tk=K`>llOpNnUt>e zTMTw)LspBl^#?6Qo>$3bc8HODX=W}OqV0HmE+}=?S%H>VgjZ+waJ@1 z*nS;(qriB6LAF~p>7(FhpGWt5qj`87S(`vONpF~f4rrfSN{`C;Npb)8kBORdwn+vf zb5I90ktaB)A1aq#YMj5Bno~%VrFfNV*PbVuprrV9kg%!e7J29SoilfjA~mKu1bSaO zmEabpJ*sv}Mp`QviBCv(OlFRUYLF1xqgDr~LRny>%5alPsoZ0!ib{A!=uGw4s5S>% zzNVX$h;cYMh$RVb{P(9AMyp|nhh{L2gXxntikW}dj{>@sFzTKMnxxZNg78+QzG$nj zR;KyMr@z{crI)MG3Xl-kL-sO9PSru+h;#pDuF*Fing!N4v)U@K)|zI}%3HyyvTCVdKS-dz*@kb3r0U9F zCbxu_>Wm8qsr^>57kIJ6`g)7HS2#$nt$38h`m?!~ryq8(2s>FCqzgtng>j3Z@D-VX zmj{kUrRA!VKF5b{$g7=~hY7Z1M9Q{7HiDH&fBed#^I5P=R*2#xfXB#~aC@egda~Wx zfPvVwZ(C#ZxL}>`b$oQsE4DX||$o!vKuMoEk@d%bXHpojmKo<^FZ%x9mYm$Fh=x1$)Y zErO+4>Vy_bs$Yu(P*AAZ+X8Lco)OBZst0+j^^iSBYBHL*;dz=5_hgNGTS1wi97|#A zS+i;u0W!#)vZ{dEOK+Y_rJ}aI9g4VMsRs0$!SG0nmMfL>b)|Ycw3EPz!FQYWDq2-n zzs}oN7fgf?T%hEsyA}Izvs1lgAq7l*a?rPPRQf;y>w08ctt@%TaN5eM8o)HWdVh(-5>>$L`lBNk zupy<7J0{1~%WM@D$pyH;n%JVA7Y9A;qWHU6Vd;_#%ad^0y);XXleE1z3zx08qza08 zqpZTCgoCD;h_2k5hAPet*t}Hf%IMt7UdOdGtiF#1vWR)ibgGc0t9}4HkRaEewn&Ux ze6qvKl=K;n5NEM3S(a_v&v*K&Z|koQYmi{Z0A(CUk&wN9D8kQ4is5LN27P)2Os^Oz z#Z-*Ib@jy|Ft@upQ~bQrt2vAAO1^gc&?EnxWxUFs!RNyq482uH(&6}hh9=V)3(USO z#x$1Afe5}vnZgMhLZ5)a*&4z%JDLh9z8P)6D7v<$hPUpd#E%q!@JW3q(9ONNgVnlD z6KV#&n!4_Kq(?ogVrq*ucEe&_*Z4@!et4|KyRvq?!2=m&XiUFOXvOxNoFM&qx9hA< zjm(8jvqj3Va7?_y+s>`|*O}I&*es@^%e1tcdI2n4fxNzdx~)6A#%*kMZ?~~_n!adC z!q)k;o-C{dxsooI#yh;jKnuN4tB6dk*u$h}H0_ZrE!7tpoW1(UzKWWKmeYL)!{KDS zMp<%4EXnt|XQcYOC5*+X#h>yTl>+~$%*{NqHwMsCM}|kJ(B<2EvO2^lo5^>2$SpW~ zGpVc5nQHdc+ZpP&+f3h5m!c!;*`s=^u^ef4Dr;NSsG@s_nT?Kf*T;gXob{VzAY9CW zkktcw%lj>?okiD%yVetyo=vRA0PV=sjm3=^i66X?p(&PlowG%h*n1m7m*wLByrh^s z%@W$&a{88u%Y-%_tK=uQ%Uj^u=yItoW?!txt~T4p$FAdjYj-TpGhNR_?1aXxm}yDQ ztgDb7YNBSh-6}3?q0OZt1>@v}P`lunCC=U^e$Y5Tx1xu&J|3Z~TEoM{aSs zoyBZr=DzJ>mQ3VB_{hQh!%*qVH?ZKgEMyEo0A3526$sdKKA0U#r{QJ>vALsH36^lK zzFcm9vL5Ji+=?5lX8OD9O}5*Wo>4DVMJ?r`nsCAxpPp6>eQXx)s?6b#Ee z9)!rs&f?AD6+F&&!0j>afz3(cdrIZOtb15%@0J;;UJBp$zMk_L(nqH0MO?mjJ?4g8 zl(PPJ?!?Pl%%Mm8yo~=nhcIi7h@OlWoWGr&@Qn#lxmmT(6z0M`x)9Hmof&68ZluBM4ak`tAr>`aeIB)JRb*kU#V)_ zzs~LEjp?Uu{hSf21+8e>?@sHh4ELFqvW7Ibpl|#ghkkZ^cZmzSW9pm%ds!xkJsm*E#1~!{V!S8_k7b5OZmmU^5+=( zpU0BT{Ak4v5J;R+A*nK9%Y%IwDT;HGM8i8xL3R;H zg-p?v8F5ax#R!J7Epn3G)smyW4Ra=i}W2KxN4th+qk_gyEJRcF%Mt%U07t)9U~x94CMUk z^_MMNcBd}2VAqb`J&vx;oUC{ybE~d@`E3@h}OU|d6Skq#x z=2Ba1!KbDZ&$!6GU@SnVP_u&|)21SCzBXEGFSY_>=WQM$KyoZ1@OH{c zApQ394=^(S0+2NOYzz*&nihKO%gmdbSKrRa(f^Vcy0L&Ux%wR;lKSxW@cKwEhoA;?7=gmk__t)vMxE|IkL zMnNB~OvO}9+teU&9n~_@@j{ICzOSZywFquq!&HqFg`&_}W)Wlc!&zfoH7;GxB7rB` z7$u6<_bRkjIav>OkKkRk0CwUswL!)cLqb#UT}Azzbj|kuJo4b-puNquj#ot2)lDBe z%4E+(jBUw&_1(DFSJ@L3Kc0z%(lDOe#qFcS=;f_2cW=}al8BS7ZCu<$KG|c96UP5l zw0f;%4&k7Jra0J&jlq~OI<%nCux7tQS!;HsB&oZZuUq@+6QZ)2z_eAK;>x`fUK?4r ze{!ix&>Z?vTD1{Am!`M6`m;()r-l_INU8;#Q-nqAlWo4!`#0K>v}L<)oGvvsT0Y@w zQN57I0Q+@|vx)r}R_gG=X|;)75I;$GZ#-Wh_(al6esMh-@t$xJ@m8x*P{`ZVtDO(f zdzCfXK=1N6_dz>a6}PI1dLv4yEnkcpp@FMP3w^dIshngz$<9vmVwo+>!I$+`7&^y+ ze5rWjosUaj>qY?$c)-~Oqjr5Oozzqop*0;LYeljZ1v!wX=Bx#N&N7P2TJZlN=#eE4 zhr`|Nwh#!_2nkLTQOJYV!m-p%ZhKDK*R;?CE`M1_X?tsuy_Tn{kr|3=GSuD5eCI!8 zbudle0pLQeH!3;7E^G;uoiQ#}h`yCUi(5p~6}*TBIdmr~bLtX!$f$?_bq9@WSqk>T zNCW55;Eg6UAq$U&1tU(uc`Woo3nz!iJyL{@cr@3z!t=a1ZqQsA;^Paow#YB3&LMb{ zkRA~^g>u=^k6nb*>Q*PeA~EofWlNac{&lm=p#)UtYn>DkNX05%3=uuR!WwLu1R3P= z1-s-yFRK8EOeCWm=pZIJj5&^DCR3TkWTrdVQ4LKo-M2;(GY6qJyIHIUN>U^Ia_Q-IEQo|Bz$lmsX2SxDo=yv;Gr1>Cq}s_&wcU}q!VT5NYA;^l6vu? zE!>#^RYfe{nSo|ivyLlY3Cmxc=?4(tX-|FX(>(<99nO5_I*NK!q$V|~y~so+;DAde zL=_Hfm?|F{Vb!Zy8bMyCQ=5f1+*yIUWwZhG0-Wj2-o z$^b5Z`qKvR)D3(^9IHo!#n!+k@0;Mh(To0%9Y zIA$@&P{om@;erwKE@!_=Vpo3_7Eh|cM=1$kl3W(h@Q8n?$q@XLjvWT#3j)#dM%Z%p%~<7TGy`UUYLq z5Yy(2=y{8$5p+Lwg1$P7U=e!Uou37{hm}T%BG?pVXZC=H=w|A4e8998HmUR|0HO){T|$3Nn`HqX z+qLMhr=c6F;hXx-a2{-nBFcUBonizwy2ooT$Q^PK|}v1oG{31;h_;$KsZgVYRzmz@ZsW3 zA%CBWhC^K138}^a3-CB&B=rX5oopTW-bTP%&raQ<9R(o_p`C_^A<7-^EFh|k>||TH zsLgOkGadokXMcH|cua&ixUmgS@Hyg>Ndg?;xRUM=K{siCv|O~(IScXjuz_5>(3*~KFEORJqIo+*62%eH$b z==<%+4}PAh|8M9w;sLFrKEnYI{Jnb`=0xy1rw5&PhhSCIh?vBa8xMiU6H#>N$?y@) zc7n?zIUG!YfDoMEYnU^Q#lnF7vbp~vz0zX`oqMKckOkCRJ=NO)7r+BC(}XF2gxIqM z42ihByFG@ofx{!d>T3+Ty9hkcfpg=xu5-7rle)2ELF!;W0+>MOySk}syM1E_@T&nv zqn8zAgMF)%x+_1vyP%-*KGM*^82ACJLPAWEIwUAI^CJwlGdwZ~w#DPPf;v7@c@}Vi z7#zAIFr)%oOF12b048X;I?%Hj5QG0DpgA!htpi*@I?TDH8nZz-Ckc!|K8(N$r~$!~ zgbU0-L--^$!#bzyHZQQVs#^&1Kt2?toIoQ!acV&VctNl$3l(g>4}3oK!$A<#o*z`c z;y?|*b2`Gv0ytB+?Zco|96W#<0Vae+Q~fgzRm z=^)IDon+)cGz369lcQk^tt1#d1WZ8ILIpEe0UeNlKI}#vyFKN)1VT*0I=Z(Nz`gnj zfqBxzwL!;t;}DFa0O&Y@2}nLXdB?+W0qM&$qRBTN48=+50e_n?hA=fwBdE^t0a`?) zC|F33D1m~L0#1@Jg%dShnF0SP#5g@-zfs`09>W)M(TQRNond4W;7LaPi=8v%yhK>T z(_sNIpt+j6xoWgSrSgVM$VL}%00?9~EBHwYqy|=~FEr~djky9jh_S?xvm$Vq!l=QP z+NXEWZPqVI%D=BPaAd4gos@x(H>O~&w z0Tj@bV|fDWFrCjK0+dWiU4Q`sga9Wa1QSV!nB+!oYyvi*$vU*jI&3XYD1$v*fCKBs zE)xJ~YlM4v1zWI%<@y6~I<9>S&Eq1avUs1;9HjB-op~|JIvTID%r5Z)CE0|{tz$0N ztS8&;%!Kwe2F(fFdzJC>sl~06)XhE;Ujx#nSf- zQ!{m*E3ncem8ky@T&M%3Pv^=}YAQ;(w9w>4v=4KVG`o|s3(4I8(IlfU>Lj-9+)g)8 zQSfBZbTGOG)JYmG01dz;4R}<{tW+5#&z%fXP2E&Y71J~Ij3rHlO`y_Opi(q2fVe|} zK}ZF~0#mC*)iq7ZPaRWQRnrn6(tpg-J+jq*T2o&I(7t(7g3`v^I;%Z6&Rs)JFNB$8 zVa+387uxX4K-H{4EmWE8B{)olZ`jst-Gy#-K=C|1aRpa#eNhki!#|X!575SQ#ZkPf zroO^f1ejMIyZF`&6jyI`QC2{v20S{W+l5vz*A?K=_FP(aohBW1Hhshe{X)!n zJp)#qs7>{|ugU>UjkAARf*~zfehn+Kt%frwwSnbVGnj>I2v{FwCm$7oA60@y$ip$6 zE;5BxbK2A&K&UiT*nbK&y8HpJS(FkM%e|4~K z9Dx5G{ejVa1S~y(Rjmb1_<*?`fHk;=XmbTvCEG21gffs`4O*dML7x5d&Q z*w<8eeH$#*al7hgVKG8&XiSLUC*MF;Xu%*9OL0_{Mfw8)S&VM zE?!AF$WI4X7a&fiw-H<)zmY_&Q^*+g=r?Azs{Z zq5$?>SFlwl$Cc(9aAP@LWj-F+Z`P&MjgMlfKUZTzJq?aSo-auVT_F&JU6?}-?!TQC zf?lSO?E6x_bztGlSuO&I1}wUWGO0ZHvtYngSiz<_f@J?bOj#K4Skqcn5#Y z1R?F=v(NTT!E7Mx*!<5My#VrHug+wT`Q2>N8`v$?&vU&n+>TT>|_RrE8Y5*RSI2A%@h3>~0^;W8~$g2msA5m^wW|-}X*s)dpUa*6#zTWo1TUoYw4UZd%!< z?S@|PZZ>M9&TZY^?Of_@1V-Ql9uy|RY6dp$uz}~~R&KbmS^e@jr`_QUh{OJ+?0o{J zM6m2o!#L1vCv5fROAy5Cx(aUEsquDKo{UuToYWf@sBS(OyN~cSY0xhoH?(JP}s&Q)D1npT;1HW=AKXsC+a~&9S z+-7C9GV>dqDvz7$B4QXgFUD7!gUuoKInS(7*mFh(aX#_E*FIO^g=nj#^oSm8AZK-S-*UGtvukMgT>z?Xr=@tT@0H%q) zb`UsfAh%S@RAm=fT$4+)+s5H+uPxf5R4h;TR1c;zF56_zE_#1?boJ3!$MTTx??Ufr z{VwW$*C=ulEP0yKRXy!ie(=e)I4RC_G_R<=gmVQhjq5y}6ms|~;_9v@Yp?fuoRvv? z?s9UUKqPPU${cwKXKS7qfF{J64&a-Lx^x=J27lU#keQU?Jop z#*4ka?b@}mS0^EE0CAG@qXvZ!D|8rP$x_4&lPVJksA<#0lAHh>@Jy+s43?l@h6?pE zvw+PGNog`|dUL}|5K^a7T`}MQ)~r#ha_#CBX@#&@#Uef6fo#~cUsa|Y`;yNfJbkW| zk@-`UtyjGCqUwW+9J;*h08{&muH4|ninsq!YqnP~W5n<7;c8Uva^g#mNWNlpr-+Ks zMAUR45(Eep(o{^3I1$4&hZw2J&}RJvhvVD@u?Z4X=o7Go5F-*dNN6N-H)58L;@Ilj zMqGfPzw71b&!0`HI9+z3a!DRWOAst5UtVd2GMi6styRm+7NSJ`yvy{%`%d&rojTP3 z_H0!Dk%tz60v2fCfe9k0pjunCrPhEaO)%kinxI07BpG@p3S#LT7MX{Heb^z1lbPtw zVU)Q;StqH?G7dMFZKByCiu}j+$TW>9k?K;4w59}iVtXH zl8PT``B0R6bgcx{RZs1A(@p^-3KgS`Qk9ZRRuwR3q>fT*DW-%PIKWw*8u-$r3P75k zD{8v3$AgTX;m0Cq&}CN+ba-(Zk7&YptB51&%9t#@iUP_jS!!9Cg}8FEnH)SKNhA(M zzNVwIw;i!;kw@|v1Bf8D!=xan6_U`D(NHOoBX)jsA~=9lG~oqVO~8U1ABA@%V_H@* zlbw9zU?>Gk@b#06gM_7?k%9Jk)S{Qtxv-{-ifUe>0=mm7gcKXduc8@iyurf*TD%}j zp+=~{8Mv%rL@oA>x&X@6Nb>(h9?5Kil7tsbp;u$LW~+0qB!Z#`wMArMM<5zzc!^;# zQiJRoNNVxMXHO4FZ5wff(PL>`1UcG~I-+Z0JMHvF+{^v+ z2qzQ?#fT=e%Yrgr% zoiAuHdbo&w4IZibhsTUq(7~rqHM^K|&#}uM7$_LpJ_+8rL$W(1wkR$09%ob$lxwr? z07Pg%R;vh(v{k)zj?cDcnwHg9|Bcu~O#d51=SWGeBfWU!(Yd1xL5N4!O9vAT>|4+- z&F5j-pWt!?9{3eIQP}@}5B>MQ`UgOlF@;fz2MkLTjg(*$DOjix`-+po1muEOXeSEt zb6kHe$hX$92PfZY2?$q`gARU&P%hYv;trI8maI@hY%##iVD~$o%}!S+x*-m8*t13i zF9~@lL-dTtHLaz_Go)eGv!qo#8_i4tTEgNF2D zEe^4WS@^+ai6mXR9ytg{PO^a}_y8Z&qyq<1KxGeVi-SJ-E-cBAhNi6HVnSrfO!QDz zLV}vNto9>Wf`smP@8W_3KQ^~g+0 zoFy))H9gtr@_M}ijy+PkC6$)+nCJqD@2+7cQpQx7lR4QT_B2S;UBU=!pb<{+U<46l zr>7pM$HTeJvs4%%dE~s~pn84${>Bpjt>w z00TG>t&xVIr6{UH8p_6sEy`kscY+;>KKqzEp`x_CNS8kI@~fn!DGGVJ(CR42DyX)R zEmLsSz7BNP$U$xmEkbLszIsRi;7w%-7(gDSIg?5WaDsNS%5f>C)TsOkt!};UTeCK+ zK)uC1oMdNLE@>7tq>Zl|AXdZFCG!3`OmfoqN^+1eadzKoshNRjtZ%kBcRzwrbn z$_1!jQv8_>9guB?#x zqQUfIdBTH*6v{tkQ&r7~zM(p(F)}_|<4| zHwXAlp$2HUjMLh8)CFOzDpI&qp96r0GrX7re)xkNCsrP$auq+;)B#zkI<-987-ra^ zC3oXjflQXA(<*EuJh21hT#XvNLq;!)+8Y;`C0VU)of@+yeKjSMp+r8L^_wflLb^8CAV-nd&X?$G zsZX8W4|Q3#?)71^rezxyEN@6eTC$XJ&9ym%7BkgJ1IP_wiCu4T8`a6nmT{+G(SBsi zM`NDRM4=BpbHpB@BNIT#K)#O(#?m^yn zlq_A8`M@I{e`ximCOrMIGDIDj-0)Ry?Bou8HO^48xQC0t;aj;< zT?f&W!#muhgEeG53d~P^pE4o!R7{`hqTBIy4qt?{zd^g&ZTk=7aB0=>+xb6eFVm?x zaNB;74q0A@i1u#kAyaThdPCvKf`ajoax`ce*Kf6G-eo8gDgeoqg@TQ6$kDN!GKrSQ z9hkIvm`vy%)q!5tQQcNSNOBcL;K-g@m|i*H%`i z!95JZ0a(GX;KBq*Un9K(?ct#GIT}`+-O$)lK;2vTEm^DKTN7NM$&=qe@g(!iV8Gu=ORiXS* zVIs1~BNWy~HQBR7R&2aRv@{kOzS0@`PAdQj8|K-mnlL4_mP3VL7VCrQ7U!;cEHQ%G-0DK1*OED-L(-G3}6+)pDtP<7SfQz z#l>I&V++7W5+$R4QC4DM+V<2Mh!9{1a?Qvg0AfI$zo4%5+DAy^G*YAUV8Q!5jn!lo zRq@lyd=iXJ3^}fVzC?&9irE`AQ%vPpTd>i8@KC>$N+a}}D|A8!Ak%+p0tykQAP^G!MlsOGWfn;^s^K$Mp2^Msp-wX3dhL!abRG&o zRxzSvG45H%$kUFM$0cs%*ugSV+~MG=W(J8dufg z>>b%b8qxVZfOO8K5fscSO3m(wz#cpzU5N}GX}}3-6G7f$Gl;^C%mphLg6}9MNQR_E z;p8t~8+`2FYY+g%#o9{Zr)3rlM>^mr(Bv-Iq^*pme_9Bd{XyXcC#*@2qJh_IKIrrH z5GJ_{53zxTRwygD0}XX=Q3$ zf9mK=W>!rGo9ITR{-TA>DGzCahU(c* zZl=WTK#;ZzaVjS`CdD<>LV9LsC!kYL_ymHcRa;q`j7q1gfM12d znFQcHseHW>;l&ym0ANdgA(&>V$g#rYfhMmKsE&qdv1S!56(c?&06DHH$iAj;F%nmr zK%6?O6F7lx-YI)5(|w(lYvcvHxZ~!4qqvgQ;*2O)UD`piD;2ze3`nO7G!41NM?%`@ zSEXAUHG-P>K<%X8S)iFwjpya5Bd;YUz=q_`K|*durr_Z~WI;jlz$z=grDk%JmlB{e zT0($wDNb7K=7lM4zKs51NNM4Zg$0o8jnFhPr+`2!>Ws(9rtGtN0z0hPC4C-#iiXX| zor~NH1@Hmu@Rn5Z1fdbEABbByq8aq4qAc0}S^jw;5tXef`rV~MB2N8<=F-vMB~*2K z97R=Z*1D$_cJ0@mBw`)o^hk~WZ7dmbX4+PnmbNX$rme-U*DC%`v~@@6S%}`APK{tk zQB?@x5Y8od(*iV-d(kR_X70 zX|Q-^B@nLyisp+D=+PPGP)b^M;3c9>7~<06b~+{_1PeV;gPHxoojed8j1Ko^7#}z* zc7#Is9s#JW*_lr14sb0aUWJ{_MG!*&Tg^I5P3Xr{Rvs##%IswrtRjJDOv}33VeV-`Z=e>X)!Y9R}gg#k;o*Lrs3Lh@Y^^6WjFHuHYOZ!N0+b33%t`9_T_@DARgV1>{(WN;A5dP!!5NYo8ed`TZodw}MTA!X7s>m^R!#N!oDEtTwymgBqoHeKY1L$|Y>cuP6d0 zRI=ph0+KEwa~i8zNE=pgm^a4%)3&NG{fL#WXsyBh9MxqUnGu#`9@1T0L*;2c*s_lPfD8A+Rmv)9``=$>*JwPKe#| zis|xs&2C8E5Cr>>q%DzsO`IIZ*D=!9Xe}7T1{h}bLGQrfUeu08_i_E<%+4M12XzJ+ zl=UpwvT_=wJYChESna-URw8DCC}Yf0a0=n*ZMyXoBTw=ojPbE@&K=TaQ0H4ev(-O8 zLQ@T@n%p(P88kviwNx|zX8na=g?1s#a_lQ{RA!2ywY-iq{l!Mtq#N?8VcLjnr*)X| zv0)qxNk76|wD2{=9zEN0O+Urzd4>$v4zIz7XGjRQCdG1Ut0>w6Cal66ei!9dcTd3e zV?(4v9<4$;b{in|O6X;LmYex~%Dd%2;J}aoR#a!38ZbL%=9Z*J(dtN%cC0E(?|%1c zm)1;r%Rx|H2RC3jvvwe5v>)4)v{DEmYit+uk#hP=z@bH;n)00$TT7qla!-nKtA*t9 zYg7fd2U55;P76L)nu){AP$erkBsC=BNlxu0^bzJD0FlkCcPh2Fz{bV(vKzWxHLMY< zN@9&>?yfZ(C?1dh2HT$VO%j-O;FH~^bqEFvR7J->i5kYuOQtHyKx2-!V44^H=!(Y7n4ow#=0W)7B$>6sgNq@5BJOB!b5tI$Qe zZ1x%BII30kkqI#}*D3|~U$Yc>u@s<|u7V+8)&?&i=c%Kw#_b!TS^cd7$%&v$ZEL=Y z?kCOMo8po5+E2)0_WNE{P1i*RE;eJ2qv+bXiKBNWaHqLJqJ5MwRI$olke&h9fsH2y zqEB`5vC_4P^j1F^q*L@7DjtwXEw)QSuaIFObbC6dBW>^Zfvq)ytrEXos| z-B@iYz=qZT&Z-kt?x2AJjB+@iIn6qsnY%eDha-hs7o*BrK5y|Aw4W}BKdn=$I-GKf*cr4=?7AFtd8o>f*F)VDnwgL=XEqO{Mu`W9>@Y^%s3rH zq|K1}p}*h7|MF*Zf~w7HN(KUrcFq5pM*rbR-Aip;>{DyBMjZeXy6dVzFqE#UeloHY(E{^7X|>};*uXJ12*AuHLFuZS9}I^EbtNJj}InRq$n9hgb|V{S)vpa5XPMcDlDwUl(ty{BVh5D}l z+&!_G9-UcJhKh<29H^+d6Q@m>v|8vA8DaNs483#l;*jh2lBU3eH8C;_gu{qBg-D%( z_1IM8$chY^@%bv5U|Xrops+9lG!4-{yj-D!c1mb7t4o(?SYdRT10rbS!L!n(1_equ zE%oEj#3MGo=o%5zWi=aD{@(55 z>ko#k;9mF4_Uz?<+;g#L6%a0L!x1S{r5~b@2o0nLTLUi^RU@I)O03RHQ78Ct8ez7RN$qOtMcfsbh*h{`f=xFvC{+ z49FlaNUDS(ComF8+9)`PNIzE3@Kbb98E!|3%)$^fHI4|e!zcP6z4?RmZ<<<@wA$h30dWP%5j#MpYmK7867QW(_sdhO$LR%t!8~}Uyg&%&#V3(GW=^c+;el~DTGcB+|GM)GT`D1&?6pLt4N<4~M z#L}u6fY%<0f*!hsn~sG_aJyVOkE`bUuqhbj8ifI;)idV#*4N0>l--+P>XQPr5=Mz# z@d*V>f*AK+Bf5tus$%xZ#7#s;sm;Nrd^M;=GB&lK)~$|p5lRPE?va^C^d>HBdCP~? zvX`34>V<0Y5E14Ohgb|xX)&774w?2VJ{`pb%v*^Nm88G^A%#dqdYA&`fD5qL<7}SD zfl0{c1x{eC3}V|8BM=~yuvCZ+7jT29%JVlU+%Gn0a9N%hfd&I$id07#Sd@TsmLoy# zfTGYu0s}b5J|YTZfegtREZ0W>$z)_iN!P3!gp?C@5Mwi{ z83@~Patm!y69?ltCsh2jNCTP!kA~Sf%)P54c+BJf47`(x^4O>xo@ibwSb+&ojN=T} zS?9$R*VXH|^PLB1mOSaHx_f#`7Q!jZAK=mj9O$kH92${959)}V-Su{12xTex+E<3s zsA(7)N>mIsA$I+15onxC_EIyYHNcOsA@#;eGwOqYF+z{2A1z<(V?Dg!HUC=}jUe*%lk55wZmBmjtQ)Kf<*T_Y#Ca$D6gSEU>FF^Ay5U7BN6jBnsAc~79+3VN8-e`S? z0A_p*MpEl37Q_7IjgTd~3pXzrZ<9a-CGe(XIm7uIT^Ta0q)Fk@p4P%bTH@urYSg|x zDGv&>DnTaV;X!Y>#j_=}jFStVT3G8}uGqwlohP94u|v_SjokBErY1C$0oA8|*{a>< zHoE<^)NX?nK%WfSX6mJ9d4-`}ZMa_lFaH{_kw}(k?JO-N9jmRSVJs(4!{1A=iGvC{ za-9R*OL5FI$n&|d_D&=X-NJ0+hgC>E_GQps@Yz-HZc*H5*@J(#yOZSe#4p3lP zf8GFpzpJ4qffpC^)}>g!Ztu#4T_A`C$axqwK}W{`=re*lr!9RW1)}hXe1U4RBCMy- ziJs?q7Janqmgo&Wqgc17s36- zxA)&QAxkInGeJPnI1@D=_vUfFGiU$2=s_>Kr{b{-qhm$DbO|Do(D;Uc6?PloP(r#5 zDYiq11mStmy^fLrsEQdzwSJh58d)Zo;_iC@Xs>hzuU95J68w-!u%O*tumtK~zwdj` z{`R?V1-=U&;Yd*7u1RDExwJJ9xq~u6iQxFh1C~Zf@0-t^_$7vi^`0B7 zgG5O57)j6M3-u;T^^|BxLgVI~qYr{i;Pz!X+=IEMZaxTQh?Il|EDBuOgKc1|ijM6i z+R3QeCG9F86^P>0G|ig-LPzZ^rut&=?q-k%CC>Y95C?NG2lwDCP)S4DfDxDtuPo{0 z2CDpc$Nk_B*cxM3B7!KEqW*$HJxoUdSx-ZDu8S5SxB(5O|FmNNfhbOKB7l;WL&fo$Yp%5V` zAzYvn*x*0BIYUj_6ki0~0pzdOY zMkEQ95Eg?CdNjtPY+{4NgHX&6N#01%nr9852~>>X7p~0<6=W594LWhHa;g+hT!$oy(X7}Z9IT29_bf?_2qqV^44H!uCXa%e>nCeMjt=6bym2zW13%Km zKTOOj9S$Aa5I@{7`4EBveiH4L;QkKDLWsg1Pr@w2^2R!`Z@e)!*B~+-PAtdr?l6)M zxGOCZBNbZ(f3i$2=;}jWkuKqnG9aJS9Yb&gAycrTQtgbO;+TUnr|@GU6Bpfa zBP`@TM?wp0EJ1&fK}+X8E`bdQ$v15<6j@*)G06tJ%Pj2uMMOu`M%o}cO*8}0 zlE1`Yiu4kIT+j#rt~SW$D;@lt!Q8MrqW|45$i~XhRN_Lw=MHhk|fT8$e!SHG!G{1LPD9B<~lK(xh^O-UJgXipD!hFHpbJJmm~DWHJQ* zsx?uAgzd8RdgcZXJ{45es$U*zSe=jqjpW~I;GEP<$I8N$7)~j5kzc%ZO79R-@ePPV z71c=9K=d^?+>=uB4prt&Rh_^Itnct>Ws(k+{qBcmc;-wcQddos5V%WW)gp^7M+PA~rKqUXCRGp6luhb`i2<}SaR78U~0U;3x z79&TZTEF0+SjB7$FXT$@pvrX1ZdD>l0TSk5R};am5Vcw%>T(iJS)7M8xK>>MK{h;x z@lQR7moNc6uq|3ELuEaKWgB;FtuRT1V+PVy0l5a1$`y&|4QJbxGon^#ElOV3C#Fmd z1bg6Ukv7bpR+{8(Uv<-MST~ipW@DX%Zz1j{;e>aV)uYmgNOl$^I&fcc*+)S zH|Z84(rx=HZs)dcgZEJJ*2DZ3a#aIdI*L2})K5+J$adktcE;n-gmE3W>7e74O!xK% zk2Hd6-F#wwGglI%Noq%T1Ik1j(Xikv8zk1X8C*TVri$q1J$7S9YW4 z)XdC*NlGmUKn3QkEWj2+S(a7G)D*+OVhbumpDY4s$QN=Ud6gH~j)xNe)WCTo0WUVh z0;Kl>Q!HEeZ&(5_aJ5%Z4OgVPcUN#H+4>d@r$E`34cW3MYKXY9ig<|g3wyj^Z-rQh zcf^V1ZldO9T8?-MTIPV+ZW3ph)RdUiB1wr)?GQAffxnnqz9m4`I7>-QEqb>g3$`T2 z7leJ-A=`2x=}HYoxP)J^%l^iB384-EDtOl?SQlt07GX8Q1s84#2g_mrEF=~Q_%JmaPUCyC7$I+mD; zQ_X*NS;usF_oSHaSdG1pp=(6pmv~T&nN_- zAP(#nnz8JZ6V+z+`IJ?~pc@&LWqAf>jh@48Rp_|m%$MW(sR4)BmT{Sg{YRsf?XoBV znVEo67;7z8Ss@qlh1BmYskxdHDhY$|J52Ni=#~)lKnz~ykUxfW;!V@CCzIxCRU-L8 z%2S=!S(y+wT7&7d*y;olnyH~zh*S9vS^#=UV3UjBRjL}6=NYQ`nRXtU@hWzc_lux! znX1LwnFD&Fa}Emb8H(S)pFEj}IeEN1dS%CUSd7_gE%*cf8nAdDvN!>o6~ik-b4D0Q z7_m**@%E|`3_*d|GznCd1*Y>j-~b=P*P@|7s7aP^I~%Dn8E#07oNbB#x>|;U&~6Fu z1awdQN*k{5nhX-dwS79E;JUOWn6}^gw!ONxTRV6!>IP0jwi#fSgSxF}8@N*_dT%+l zyL#DV%5K3pf`41O6}Ecx8W6yixl913T1Bl*x|yx}30S(N6+59E$~7`zS5H*A*t`11 z#&1*EFR-_$J=>DIx0>7`etmjJ;tIKC`KfVxti9&8^BKU$Ai59Qw*Q;s-eR~#!=mG& zz>}M7C!D#pI=3YWxy{LzsCL^Z2Lpn7qw9 z2}2}%mQ4%vCPPMo3$6W$BjPS{LSK#sdOJ+j5-!;yt>?T&Tm-HT zTh9}l##QddCmI6+UB}tGz4MY&={xkoP{^Sbzk%5#qL{QD8K2!^s?qkzmE5bDx`QX3 ztOZ&%%ulZT*1XEPpIlE8IJq?-SGyUgF9!e?8TY5l>284tSDBsx9D ziSwjkT%`;9r}v!C%X_hv=Fi;@-KJATm8jSM2_p>O>?fWg?2UFCf3&QB|rhqOoe>lv#z50sVLyn#~?m(j0n z&$6I=KmDSi7|B;Yj9Q*}{n_Q~dE8$fs};+t_ub|-Io$G9)m6FK*nO*+8=^J3qUAlm zjvl;{{*gQGj9oOoO;qn*m0s@UySQJu>F1uIOAD6&G5hY1_t!yAoa?@l($eXQ9;3FP--*6MX?XEx z<*k#=Zdq+ z=%&K$-JZCAIHkLw^XeYs8K3tZzwTeX=o|U#b>FXpzu}F)=E0X%Iy#rl94=^P_nd#1 zr$7Djo!)1tnWf$ozPFa(Cs3FtF+g$!0`eoptzOAKFe_{cDUH;tG{2an z`jaFFGMS;^m;{TAKHc>)OR<+p zjIl^yWCSlLgSJlXv8u=1qcuOw93nPs;6yT`R#`K7XvDWamL^P7WlP?gPke2emFab~ ziMQ(U{e%#rIpiU4RSPVA+qZM+wqHfplOdsh39k-Jx+h@CVK=*kiO_4yY3A7+sa4jT zYN5$A84w6{_TWh=71SI5ajb!sScC{Y*C2QUdPLt~n0fXga=Qr#;7XxsXJLXX@34B6o!FTjZhL_)7dBudh4pS~+HHh#cv*$_1uN38X<8<0fRc(B zCy83nnGmCmVmZS9vt`-gC}n+qg=>)8?X)SFA#P`+Fh?dSs$HZC3F#3!&196URsIX` zgj=dO?7#;eI&FkhWXr2mayIMil*uBxY;@EWN-V}7LX6;qg_1d`!51To)sOwPdtGtL zW_*HE6``1!cOliO*vY7d74N6?*2`p~7@AS8nLjb8t$)Pcx@^PxPIgm?Dvw<0n-WvU z;>I*CWe}{^)f$AMVR+5uX?uPQ_QF|{tuX;){i$VW&WhRcs&J0gc8pWbtjDDucE=-? z2ltn=&Y|v{m(Ql==To!>8z!ZGEKA$;h}N=JG;hX!yBX9jm5J+TcLoy3W_gs!VcnH; z>9O9&sVi;&uGKaDw3l#SJhZ~l;V>j|w$e?ruz!PzS%#ek4l3dI?yHg;_5!&O(0(nedi^^=3uq^*^Y1Y zdJk#7^0=G2qyJIrYFr^+@pxvhLJ);B9l8MkgC?C+0V;pjnx12HG%lgGOh%}9baOUkQS=dX7?>UV&792M&qECqqlgw+xg(GFF^4!)r_ zAY@_xX})$QlTmAWWs5=Hx>7E)9gK1{+anVErm+ux@kvz#k0a9174i7Ufb&A2dnVCD zi}|EJ`ugJ_gDAczny^)op`4I9$V2*ltz>O7+?dE1#}x9hlqd|BL`VorKO*N2k9j4a zyayImQb|WAT!`0lQl0BDYcy>P*AV%pNj&Nju^SbM)u5AR2&)*9XB&dTC;In z=w>ZbCbx=J210+-7K}~=$}+C7P3**_DpmO`5Q#2sS+j|-l(t82jqyP21D1>^)=TSz zFJF&=AQ1>*&;cf9c+lJ#ULZM0U;^cEsvF5Q4cEUZO6Vo<6W`gM=gHlr@=HlWUp9>3(%q;TpR-yzZ!Q5%B`wirK`m`xP1ISBD4m4zae9SS9+E9lo0+Eb! zUQ<4ZPf4$S@Swx1#m(7ujFIcuU~*)4M(5D>f;+rK<+@Rl_AN6EC_`{NN8GNOR&;2?<~2 z-t354-JKpQwHz^0v$7tltzG47DLYb_b9(Hs?qmx)rlAX~+O=(1Zaq<~y&ozrndWg)z$8{ekVf>WkiD>JxCmE~g_5-lwi0MRI$BLdF335K zF4?YVMiARTw;LAiigheCqUDV~Fb<`eiEG#+74C=ZMKNRZD@V&hsbH9t?(=}SMl+XL zres}Hcmv2{K5vyqcJ*CI)9TpI(sOlnwP;yHN=PaP`NRZl8s?x&+LU?!`Nc&lESl9E zNy$o@UyDmHm@7SD>mF?-S~l?Bnyg6?3cF2*!{D3znWBDKGtwu=IPEXBzJdBM#aFmj3IGvvnC%_1T9qhHyD@2vM& zS@mj}d)3`w^LS-bRD*>}$t zxY2X293SW&iDE&L!UOogK>M@t0(YE-?*y}PEq91yTb8CCVtKPs~GAQk~GQ#5UeIqjnz){AZnw;d9;*5s8U;bM>m^=DMGUUf1-0v{-!qJr)N86PueC) z%|>u7u@`x?SBi2~ZFVv^avpy7ce(d>pI~tn_;RW8d6Kt0y{Cc4*K+;lc&T=A0jOZ_ z)^rx6SfXZU_Lp9)CU5#PfVNV9Q+I!F_7Sc{XbpxPHgRbKm2>F#Ot}+t$55v* z2G;c}gqMSIbTh~1RH){0KF33@$8lA~ac0F;8s>PVQ)g_|dUAC(iZ**9m0OASZstcl z$8vZbXHglZO%vyP(1nG8Q-Zb^3OMq6Z8j4T*H=PzXDSGFru2qSCUyKra+OFWWOax4 zL4ep-P`#yebypF`)Js~lZPmw3ZWxMl=!8AThX2<89}|>_i%5cn#f4yygBI9D+vZ_L zc!+99TcNc@Ql>bUNR0M3RViQPn zROoW+mT6U2dH}b8SYnE7)P+T+W9f)SSE!M?xQhx03p*%?O~^pQXpJH_b)0CByoQkG zkw1@kbx7EIR)TO9xHstNUS$|=b~JyeRFQk;LmDWOnC6UG1StVYk;+E|GpT_RC@UM8 ziycXVroe{?36m9NLTng~HJOB>r+`E`i%JOpVN8dAPGXcn*@MvNhsIczM8;O8c$NXR zU>uQxd5CJ{Hj&MQ7InBOPH7D72$jPF3rcxA76{^x;9Q9*koPje{PvO z`Nxb-_l^x{iFmY@sd$+l_>U%5gWxxsD4rO;ve{blHb6 z`GbWBWw&^Y1i6*3D3ByaanE%w2xw-QA(;+ih*4J{F~>t@2bx2+Z{KKl!1I&Yq-T8+ zntbV+xri5a5O}Ofo6~5Ln^>E>>1%rCgOiDLUcs2ul}Va6jRX0aW67MjX+>msa++Ci z#fg~yVlD+{ahv9aE$J1%byeD+oe9eSmrJAyr0@w33Zaou2oG8a6Dpy4!F&IRp?VRX z@z|eEgPu6qk6W3a^VtXUS(1v0aTqtElNVniNh=hyb0GtcC>d=afp=6jKnlvE3o1=N ziWjiJGZ9LpMJk~bdZZFsDjRxj6>4Ll^rWg8rDiFf$abC}Cu_&~lIyvdAR0X-W@MGI zLp-LX{P|k}_elUsdp)|Hf7x(AI;0LNr>8)tb84q|YNST$qadX8WV?k+v#pxP_~TO53)J3%Bg&xQ`pR(Ezz~`?qw$j@fx1L+Kftw^cgSe!7xTjmX?x(nn>$a^6wUdjmkt?^K%ekFfySIzGwwt@V z8@i&4w5j{M!0VSu%en+>4zDY@uUW0j%e>9oyjV-SyeqxaOTE?WyTL2CrYpRvOT5Di zwXTc2$qT#AOTOjXyXRZE&uhKw%em-ls@c1}@!P#H>%B1xzKuJ+<}16}YpUkUx!6Fp z>g&D(jJpI(w%CjR!1AlWii^6sFu!YCyo@Wp6Rf=X3k}Sm4Evi@{>#Db0l@C@!8_Bz z&}+Z~+`mc^!hlP;pt8U%+`td)z%Y!wF?_!jjK5iE!y_EHDLlF>TwFX{!q*GJ#bd&6 zO2quDy>9Bl^~=CA48ajht>JsY8l1yAY{ed&z&?z{S{zMTyv1I8#VowUjP$}ze7N*$ z#!Xzj6nw*M?8a3*yJ0NHM*PKdOvhjx$9aq-a$Cl!o5pJV$4?B!ZA{2;yuo^m$B1ml z9gM_xY{ZG%evr()N<7JdY{>}3$*pX|Xne`449cr4%D9}%rQFLn{L8`Y$AEmxue`X&3(KyH%rl(Ko7~K_ ztjoeo$$31yz%0$aOvQ$L&AeR9mn_TQ?99yk%+SouoxIKG{K@ES%@#b(>+H?nJI>F% z%(KkK@jT9NY|rFO%+Z|2`diNCtj_-oyQ135;!Mx649@dB&uyyD4c*O_?9cA3%@Q5Z z5k1B6V$OrC(GjfB%6twT{hAKF#se+V0BzBsJki+8&gJ{YCmqo6ywUj#(jAS=A8peg z9nSeo(j_g>J?+y!UD4R=(idIJv>ekijnp)K(>M*e>!;HuZOl*&(L+tq!o0`a9K0{h z)AoD+)ZeVsD=pP6?a8*R)hSKYXT8X1P1Gzc)&^a$2d$Aw{nc{~)?3}yVQtnWI2W;- z&1x;wRSno~-PMC_&vVVTOkK$L9N2vg)PDWfW&PNT&DTwxh=dK(mhII_z0h@y*K6(A zk)6SyE!q_P*?G;-n0?uUY}lC{4xD}0jP2T?{n}J~)uo-)vJKlQjN7)I+p!(hs14js z$=V)G*RyThwcXjCJ=&5@&AyGnyM5fwUDUxX+`|prs}0?}o!!a3+`Y}#-YwnA4cQYt zbWhEu)=k~SEzirW-R<jXl}m{oc$y-x&?pa-G`gP0!E$-SzF?{~g`i&D{XL(}`{W z-aU%majoCQ9pD4r-~=w;4=&!O?W6on-qdE-2fpC={nQp7;UDhc^bO(={@WRD;u^l; zM=jwMKH-!d;v){@@!jG;ec~aV;!b_z>b=<>KI0VbjD zz2ifU&qXfe@Ezna9_5l8;!2L?9IoRveZ2HdutdJrF)QFNF5+Zv=1Bg|Ilk2F4ap`> z;#3ajWq#%t?&4XV;9B0gO-|=h4(5G6=2H&nZ!YMAUfy`l(dez`bH3*o-Og3+-h>Y6 zRi594e&>h1=9-PzIZf7a&gp^f-E!gSkM86$-sg6X*_V#$TwdbS8tOo9B#Msz+?~!I ztuE>XF4$^b*p{B^OTCCF?&^LX=>k6Lwf^VCKGvFk>xQ1|2@UH%4%o(C*TW9&!H(v* zp5xBW=iM#mvJSvvJ?q?#?Teo5)m`bj&fwIJ)zdEF(ys34p5(}W-`39T?hfqXF6qsF z?|bg+F-_^?&hPC`>%$J^`R=^OdKHUz_@D)$- zLwraS-#f~-|-(G@FafnFn{nF z4)Oh^>`Yt7!94LlpYH6=yl!sqBhT|ofAL*C^hWRWbzbi_Kk!Ab=so{o?=>&=Q~&bC zUiDos_G4e|RIl*!uJR~N_F;eTNYCTP?)7XB_D`?zk4xBTul91k_Erz>SfBGU5BU4e z_kaKOZ}0KHPWXjC=?Jgkc)$0LfAl!t>ig{SeV_P&&-ff35C9?h1O*fT{{Soi0000$ z0YCu&2>$@}2^`3;Ai;wO_bE)6kD!>= zqCc-{-I_IMRjgpY5{(MAsadi{%ZlylH7r-RZ`a0UE4MCOyK2$q-K+QRT)cDv2mU*_ zZQ;Xy6AM<%m$74chzU!MJee|Nc$Ob)Ufj3ySxY+PXYO1ncj@1| zQ~#e%Rr_|k=P&b4UpTyY_t4>!AAg~eAAklH z=%8)_DyANS3+h*)f($k|U~B7jsNq%=UWj3c8Gg8-Xc3xN9f>0L$6|{r4w&MDCw>Uy zjX1hkB8@fn7-M@c=BQ(kJO1b+fk&b@p@l>a>7L{UX-j=9`fGQd(gLpy;+@hCSmD{9sa@r}Ak7~N!KTPx#@x%YqTrg3$JSR?mI8N01K-xzT)}| z;lGv!tZ=_r4*YDv#8LYz!dEIw@x>UY`mVze8$7YZ8G|ff$O{|H@yQ*}Qy0lBi)``B z$}v3J#t)~g^36DNSn{$k$LzDc`PS_2u_$x4GtWIIEwjl*H|=!J_DQR>(n;J8>+(h4< zx8Q^qK6u|``>im+(5iE|;gCNbx#W~1Zh7J)8;N!0oKN2Q=bmFOI_RWh&Uomkr(XK% ztdH*a>#)b}`t0SNuDa~4=dSzhyW{S=%jZx{`{g?itD!fCBd>fo!8gCW+s`{6{qNLE zK7H@lXJ7sH)Jrcs_rr%DWAnPOvPvlDo4-o=>SK~V`tY;Q{`>K3Q+_DStM85c_UkhJ z|I7oRIRxmv?iG-8-~-+P6Zo)?J!49Uu5FG@e2uFxP zM1;_U81zIWHi!vDT+oCqY@zuuu|NFPuzl;xVgC51Jo8cT7yb)i5dZfl#9avSbw?av z0hQ>$B|336O^h8Arx-;DTJee!44W=UmL?oh7xkn4jnur2fye=6k_lR zT^s`&#RvvNFoKJ9$m0h`Xu&w@0v}Pd{$xR^k&zp^` zBr?Rw$#QPeoKFO$D7ELOl5MdK@C+sq*!YB74)c`wjDs<4$^T1KLcy5-+-ENbs>@x{ z6OPV&WgwQx%s0-npQemuDj(`kMf4$=dZeR4398YJ)^eKCJa;earwvvn-wW>flI#w|DRG3(Gsyvk$Mm>g95!QU6HcN_49HR6W z=xb>vv%%NCqSKRE)Lu<*3d~{>7NSc{>>I>N1+ofMs$~VMKbaa;c-~Q$Hq`=E)ym9a zl0dRq#V2Ipc~BLkHJL{}tYM{E)w4zxjFUBND!N#_bkl1Ac?chzf5 zW$H1Jg(8gvwP+qf+t}3}7PQUHYEf?*M%=2FxoQyVDK(4QE@1bx(M_mn_371CPS&W^ z9c^r{Yfvw8m9$wPFHwQYM)hX4yY2-oc!yfd_v+EV8P%;^>&L(Jsg#@t#EmzO8{La) z_p+(QY(CZNSj8^3tQT%3K)t(N?n0EiP%Xko?MvP*AeN#`^{Rw7EX&&7)52|dZ$j^j z)cm3mvc8n%A7V?{ha!`xGIsHdD;!p?wl}~?4zL|{=)U9Bu)}2isdhP=S)20H#{>ni zc6ZQA({kYmQwB42$J|pA?^v;yT_tQ;fEVh$rAsZhSNrz@@A z6I?jF-WBnsH|t?N$C$-s&cmA{EMYza+0@S!^?t!jVOn#T&Ww%$ihJBtRug;CHKuQ- zOM6fo%WGbd2KH&4n_>CFxY}|A@u^2$uctPADN!0lpqC7*ydVW zTeEonHe;8nZHIrE)B0{`3{KoxF8eyWRsRJwjT_tQkyqKr97wqq;9J@Aeiy4=ZZybu z>SJ1OwZuV?`M-G{=pcvus9@gswSg{VqK9|A=hm>JKfKBa1`idHpo6q8{px`K8nq{{ z?8oukS~89x#lPgYi%psDZ@W0=?N)WD&x=f!|+Qt3C>>Q zqtji}Hbp(k!)?^n50v#*73(P@-{-w+9O2upy<&S#1Y6z#5;RZwzKj3zuKqmXTAnVO z8Q=7b6Tj$gAO4}4-gcu0^#pudqW=mju63p_{?Z|!J9Bsbal#6Fp514=*t74;8^Hgc zneF$bpTAfRpxXP{2Y1Q0dym%zGRFY-#6bp^UFDW-44`AkXMf5kYZmuit~F^l=YG;B zYKP`;L#IZQHG#c%e~#BwC>MTGqz&U&ewHP5UFCY)m2r7Sas&uy_vd?smw#X-cbS>6+^R{|v*8-h~YD71CAZUbthk#ADP=SYZy_Qx;hkZKOX*N)VtYvxQ zHgEQ(auvsGRLFnMq;`LH1Slv)D|m%s_*lO8dlA-z=rwi>C|w5UeKCN5VK{pj_j*hi zRY6BtRp@nIXMqj)OM#SuzyAk%ruSJ;g@jX=gIjh3bohvyM`S-(Y)DstCH7HYSBAA$ zf@_y;GNC{QR6d%Qfh+f5m=#F>REYJKdV-j2vnG3zcYu;eY<1X)M}>g%=ZSmQiT?M8 zph#+^Xk@mChj@s6N+)j^*ncd?gPbO9iZ_jSXo_U`gM^oMBsXk^Qboux3A06W6E!#(wOkSh|Q_LjM<&XD5V#2an4ZZXJer=~o0~cYFO;VgvbwsYZXz_-+$PeRODf zNYI1T*N^|`RS+XkYndGpJRd znNI)1U>v7~=>J82rx{(a*-SXloWcp6oB45w$(a8(Y@^kbC>Dwk=wTul0?WCE7pMk) zXowsMjPZGFyjgQP)(6JavcMtYHHo??oIb4YILiFyl}b4*F1^ZAZ-DxT9veXlr^`j(kunwQ+Sfz|hF zBB+4?+FYs#Se^gGiaoledAggQc8GR4Zs&HQ2Kbt9nuHUY2mGj^Qd&mIS({yHbANz{ zUtoWCTB^`Dgu)1_6UnBMN{q30sQQ*>P-m!mX`H_Hd3YJ84G5P**k4dotlsHKj@qPe z3XsG2qF(u|%0_u(dTNO|m3voJD@g`Bb_6um0+dOJJHVa?Y6jRAhDB+2f!UnU`Kn|J zq!3xD;dzV&n1k*rmEv}sOSqy|nN()jp!&#wr^lIH2$tU0K#clC$-146h=q*>jbO@C zx`?a!ik&eQu*_Pc@0gs0HlFF{fkOyONS|ldTM?p zccFS9mc~*9ypXAD{_cf&fd5N?5s*uQP=7pINP>`j|p3+y9X&aG2`ami?cZR5iwx*!uS#4)#1`}wV*K2Ou`lrta zj@$oBcDS~Ym#e0&C!s_6xb_CCecOjxz@)GXhz<(8&xw{QDrPdsuhB@97};gQn}SYc zM6c*!?acvH?m5 zm*~1g_hy`_nViL%;i{TFyS(2zm2kI>F*j&n`@%??p7JTJg$kE%5K5tWxFxH%UQ2Y1 zX`}cVYdYMyCak~Fnz}j~x&gP6NZ7>bE25;@ckafd6+6H5Nw(TIm&mJMnObl=C9k`R zxl!x5ty+&@oWn%8yUZ)L+AFn$ss})Pv;ZuAqiL?yS(yt8ymF_t_&9)}m#&zZ#xVa3 zy@QFn-ui%{MyYGun;`tf(o2L%X}bD$#5T*aAM37AyMuCPzyzwozDT-{sho59rSYf+ z40N#h1D@yzxX@e15e$8mI;nt$j&NCOn^)|Ua(m4e*Q_y%$m@HzFR8d+>{EMtf?byvsJ|$gD=Q(^hHWIH@r_ zw;K$+c8Hdq`b(`lzwQUNIOk>E+mHUtmSPOO%&NGm`OG@%k&Jg_8KhjTg7pSgpyMpTLK{e@UlmE4P*Fg^T}8y_<@b z?xqDaE3XbLg|s`l(kqu)35l*7)5#14*eJ6B2eVncMvYmuJlwE3dc2zzbA^kn9{s&V zr>C7Py;6#&m%FSuZLLI|zY^TKIsJ_Pc~SKn(wNxNQ_5OR49F~e!`Do!OZtak`>#Gc zzG5YWOlYaT*Lh&gl`;IY{fd9j+h0Np!%x|nX}XFX-9vq}zsY>D)p)~+Nd{fCCJLPpO;Nok*~9xo=a|a6tk+qLrc3;bLwtK=9d{AUy1f7AukNOPTpS35 zHg_Z{oCkNvS$V&RNu({t+cCP%UJ7+{I>;FPcx#%I&+4r4j40{Nz1-MtH`Jx+z)rP* zw#=9@S%&YY*(p1G6kfWGH`f$RzDfMRqPJXCYT%8E34ANYj!oJ`+Qu`$1-{$3uZW^4 zjAYBdZ zdKlBAyVXfY01vv?{_L2p9?9%Ur~Mb^mmYfbT;(MmUt?!loLb?DezOytT{(TXfcb-j zZUa22%mlrQbvdIhE_}v1W~F zuk4+X1msD&&D=fXHJjo7uCmg2bL-vZhE3KlI`G+Pq}l&$)LUL`sJrCLo4ECNh$L{k z%3jE#-J@Nq##brRFG$hsF1BaYhpRn}p%{me-0=sCb(}7u_>F0Oj)^$-)JPiNMW5(< z?Ud90o~K>rm7I|Jivs-2^Hi|)0jY=hJ-_R2doH`S^%+zFzjNP?=DfMRfrtc=>+sKh zknP*bD|^z%IkKarq71m|cTP!ewBmZMu#BGg0{y3My^mu*b90RNR28vxM~UQZ*YciI z)$P(<7=f?X);o>w3Fo)Ve(;Tqk*{0VZw%o$fYuRiuYgw6L%x*_tjTgp%TX)pLpSbN z-9K4(>M@7*^ZKyQ57cIQuKZ5Yn;-b{{miGX^nd?P_TawHMUaq!d8D9!s%Ss>?HTdl zKh;fs%;cZ_@#=@%ywU_e@c~=*e{6gQN@7_K5JirV>C#3XJaq}rEo5lUVZ(>vBu3O` zjFqB0Z@v`Fa)d(!5+_vL2tyD_5H(svrp%)z#>X8dRjzE4k&4EVTI`HD*@8ohAR&dq zRMRI=%a}MeCJnj8DS;4OfWpjqBg9prZHhF-NtH_knNXsVQ2{cn8Iv4r7PN7*rp%*n zhHe!Jl;hpIL3wsO!4@Tz7Da2m%&9d7;4w#!+NgVzh>AFRP7>5u*>GfmZ}jY>yH_CN zf|$#qv#$bqv0Mqt*gVb7_7p-R@(@x%ofy3v-dXZNWm@I zG4Vv&Y@5ioi*_sRyQe^Ftu3;Cif|?P%%g5Q%yt^-Lg8L(j*LL0>MAYoDEq_1ra(&) zuf{$zi>8srx=%(MLFJG~w*Sc~#EsopBCK@xkqbD1~bj`=U zEZed(F+%02|}i@KotiiAsx z+Y!$xM*Yt>yXN{*Rk3&^OD+qsgR#ajKwb0+^`Zkw#IGo=w7KV|8px*=$lF4}o#+>gjLmjP)@CKM$&`mxiO7xf+^MpNP0jc_ zXQ$mUs_6*0vEXxedlU$lzi9utRjOOPwxgj#+s{q>qIGG?H_rrXsj(p0&8%F@R+Gr3 zpq!G?10CkL(a;jjQsSOVGY#gLD-v9AalJ4c?KWA8*X_4^4%=9N8H{>tA`MQ8M1V~; zT5}H7oB=7PUYxOS6guXSG`QRyTT-FOUHM;aDYk=et7Sz=5@J>6w7PP^1YJkF(zWtT zxX(?Pa(0t`S2?0ZCfFdryODYD!97XiaO~@*SW<+YU)=lsl4bOD7vUuyDeB#;V+X3)M!Jm=VP3*8Ts_dCVevTuCSXPn6QNQ8AGw^^R~@w$wnnaLE5r~Ju}oW zekhzl4@qQ!1Nv|ZXc1x$@q?5;_#{{yDqW7ARGF6aOG0t#8qw0Xsk{A3UiT|if=b83 z_J#3=h-#lzN>{`(4l!;qQI^PBXquHJPgKkKPzOHm>jf;__` zIMEAkP(zWtXa*x2`AAzpa*>W);wWCI*3deKH?RHL2X zXeK=RQIK{N4}Ls@6iOOMLZbAfDqX1@aB&Zq#+0D%aK$E5dQ+U5)TTQnX-<3EgqDsp zs3+~IQ2RItqaIbMBlW{lliJj$ax$A!rD_hO3AV)bfXIYG(ec)Fz*Vksb!%O1LC{VJ@&*6gs|@=3SBeIes%R)7D8V_^D2$SnhBYK7 zHsM%QK=%I-i)}(>6KmPaW)`!W-7IHES;|y~HneLHZD~oU*utVVwW=L#YpE(!izZ=| z|MLr|EcV7b#gSQW8f$NTD^FDXA*~otYjI!ThECK~xyxPdJ@Pu-21J*idR>DnW^rB8 z29r%%#8npEh}9ry5scgYE_W*?LB04-ed~=THP3rXH0D;F~t5xJD`-3)6DkW3six z3RSM~OPK8HmGVQT8ll5XBIc0AAVGt6+b|TE)HpLAVX-$qw}Tl^zz?lt3yTfVgCP?z z$G-oJVv2=q%Gs5e0VY_)C|5P*J}MK(J=N!al~7IhW(Bub%iQ-iL*U=y@exh<=Yk1A z2t9}a5KLe}hW(IS2{X6C5uUIuF0A1W0QwNufT$!0O^$0UX^0zRF&=)v9R59r#*SuS ziAQ=(_{dKLe1I>7K4Q{jQo+c4kV1Fq&?y5%p%IV1*ia6zhdrzSRhn)DrPn7vE9BLx zo<;#5RvpUSK`Nt!6-{wcJ#1o2NyxQE15};xY&w6SjEB)u*Ct5b^iGjMXf7}wozUh{ z5TXu5aRZyx>V`+;S*jgiMe+n}C?}e)s9(lJ)L+TQuIZ^bpT(YeN>g`0JbJFanv2 zK@7!3?z-8{XRJ389g)q!yb}=UKwA{bdT=BsIwqcnBYERilS|r5K5&N%IlCZ&`pp-P zE3`k{qREu8-K|h*z5klyey_o-37@Hs_ge6*NjuDu6L&N?-t8I}B?(^6Z0k_s5pIAs z1X_uqi@VcypobUIyS?;%ih&K55Vw(8AY9^R0Nv`gI=dy=b+6kw6}2$C9U}kf4NTsb zI456wq#=KX6p&^~iGTa4JFbJfyBsPJ|0YNXk2>bBnt{*Px#2k=1?m^s?u%DB4az|I z!}VD5PKyE_n62f*tO40ICIm5k-h{$Srz#pm-eYDp9n(9#)LXqsK#<~iJ*sOf+N(XR zTc}4u1?0=UuX6wo@BmHlux$A~NobGqal7JkzMCVh@G2qZ)3M^=z7(RGyqi8N$iIvT z0q%1Fz$1WHi#XDlIFAFlbXhq#0J+LIreHz8rD#0zvos&P8JVNG?&~{+dp{sR1V>nX4m@C*s=8(TLu(GMJ#6^&Te$#{cV*}NL0sMm?fukG=$(hJ8 zjw0~FFGRh6@&rWiKczqdA&@sTWWY6yur~C8H;gVhbVjf{uGm5ah>`_O;Ga@KBDT=6 zM`RxJOTNIcu_Gvf!@-t{88TQB9cID7kNyiSd|#f-5{aGNSWKLC9H{WGlzX zs7G!}rnw^`-}nJXBd5_JrKn-5Q=5Y*ygwFj5;NPDlF*6Y8OHx?BLZGbGhc)NCUCkr zLO=WjBB#{? zBR#1+hQlv$QX!-~Ev5_))WWZ!gd+FI17eHH)M`bZ^SN@1A7nb7Kk2SXu{Mp|$ln45 z1%m@|I|&w;x--mxCP=q$Fv(;z$!3TLZ)gTb3p4>(NoWK#6KJbsNQKf$zQQ9igj6Fa zGad13zusUnYx7-D}oJ)neOFYcWk^D`*oHzeK5-JVY01!|B3P1pBI)H0B zG6Q(di+vH)umrq8oZOuMRB3biomOi|jY%(6-S^2l$aLSccrZY+>0YRy|Q4%mE9YmiOb zY)iEQs<_NeWOP8htO4U(01y4pD(KL?98nV0OO-TE=1fr)T~T_ZP8WSq4#3XsgsgZw zvQ6ldxr|v-mtRIWQs^h0hoL(HbI5_l!>;MbahRPyX!B_9{#r08m0YDmVD6 zZ#zvX2`^pDnv$q5b!-!|^eqXUP%|BcxJ*;tl%)SedZaa#q&0Q4hCD>pr9)QWd%)jIR=e~DU06Cft$^e_RcxgIRBgs++^a*x1w800 zXH-rj$hk5Y)D)FJr1}9CjRH-DPDD-AO{j&yOte&U1s~u&I@mo%O#v0%R2HRyDs!bv z6#?i>(Szkw=FFx}^-*X#F-_IfR`RjJJg5I5BGu8N(iiYkR`n|{JF%Lipay!FS;f^{ z?Y3V4geDLIHaH|)_y$C31Yj*j1`AI_`UZM{S=>}sn03jX-B~uZOKw<%9+=W=HB27} zClQ@MjuNOPDA55((M+J%Ola3cb=N`Ffot#qKfnd4rP^jggG60}1E?-6xWEgn1xcmH zfWod7{Q(c-y+ZBUc_rIS#e+jY)K(f)x>egIU4-k}gda#OO^t%My@VQ@*n-1Aimidj zo7Yr;wT_)!EEPLh!kgO)gUcC4`gxu(I9US|oK|2kKtQCKa)(a%RX0G>(Uk>ctv@jk zgg3Kw2t2<5%c3+zwmL_2oSeqh1lfUh4(d zOpO9uh+BY7S3-MKPTf6zH34lk0uvD1!2K*G_^9udDg&lh$xT%&@L0f_V9WBMM9Eyl zkPOi4lqot4H5gMemAWA)-M9p?)b&UbZe2v0IzfFs*R_K{c!hFs1T&yUwb6uapadv@ zRQFZU!5uNpv6{#|MCrX#Dm7XUkUu(DGwc=AM z!NsWwo?vuS+2f5AUCf!1=oHjEFVOu$5iZsehN%-40+th-5)%grns) zOGc+-v1##ys93>*y9700-b$P?KIleYG4#gE#)@MHu5Mw%$wAQ8Ne@XHL<) zHflDOwk_W2=>=Fomg37!0?P(x0#0M4W@;la;;2S#)jns9t!k@=MMHi^N}Ol@8<#&B z>*;9)LFk2&L+n?Q!~^ z*WQCnfXU7AN{HbZ)%S4Rpe>s<-PH(vhi%l$D&X6c5nG@?~Tq=6W{T~UV*Hg zZ_^&~(=PJr2GFM#?2Ucx2#(;2!art;l7un3IdCIwDV77rqc@vm-;OXsNIfyYhF#e5 zduV12?@<4yK2AVg+5kB7Snf&p5~2#+@ihl=SnhEiXK(nvfET~zTmJ8vOlq55&O9%4 z71wbezve)l(+>z|AQy5XXV*wK=OaFHB}b_VhD8ItB74qY0qgWR$Z~qxXCdfk+>{yUr&nCA z@1mtgJva77r|j%b^gOrq;cnqa|MB^~a7veBBfoEI_9N6*XKI$HX*4*0l#dO@V0t#d z6L`&B{RL9D#`A0kY(ocuIv?N(Fj}LAzLsW$?Bq~7M&QB>PnUnm#7$b7 z!>&%kTUfh!oM)>1?cWZ2;w%$cG-*wY^iuyo(C2{nVgLqXfK5#PReINVsHf2ARdibZ z(13pjDNulipZYbA?!G}(3MBZgkN7!Pbd5jzL!WlES9`XHL$fdY>}GV4&sO>6rdiDjjwC-z6qV5QS?*Dr&ag$f7y6%4!0_w*+{wfQl ztLwMMWpyM%YlPjiq?Cqc?vx78H50ACl`A_SZE;ij$HQGHg@7&uRapAb2MBA^bgd#J z1`t9(2Ze|k)oT~TZ{O~5a#-#is6TQNReW{rkp~b;qU4c7K|z280#t6O0AkRhH78xT zT+!&=AShpU^0YInnu{e4066IIVr7D)N*^dyO7v+>kx^YDpnCGERj66D7TAii>(&DU zT80FxW31V)tLIsW>F~)RxLga{+)KpZ59&tJ}Y1XHk(XMS9A~r#S3Vn*z zO<*?Q!dY?b)j0CHUPBlfH0VbmAmiUg*bbipcn>{JX*O8Hn|A`L>+>BG8~u%aWenWjW;TX0Si@4 zu%=iCmGg}wcUUKwV=Co!MjZDNasF8LuOd{gcOB25h{_`-q*>h~yDU5Pbm zeGuMvDNB%k^&kjp?THmljksb28rPs#m7Q=@lggls@Pb|rcg>YojVlu7BCaypGRYgU znbrp-e*KvVv9H)l4zYWzk>YB0lz1d)uSsc|ld4Ut#qOCdh?6T zcmkLrJ^S1T&_Ofemu2H*^5Yp-%m7=%(q5}b9L=_oEw!(?c3NavYaLlTTOwIUZ@-Cc zOfhBuVhJIm>@wZj5)m>4A!KIV2^JdmFqZ`v)O8n20A5`27J<}2*9=C4*wx^7E4~va zCUI;Svcyg%xlRbC5vW4L zPm(b@{21U6yUL2>-ZwH=_~B(TW7Wd)g+V`P@DCc4iA@9qp7`+%D`P>O>NI3K?Bq^& zyZapsaTUDbp@B-DNuINsG=@9q5L~C3R`jA}8tQ3nI$PRJDM~>$veAZ!kRjr99O02v zTtN;N?8wWm!7`L_We=EHmay)o0oX}w2~wa`q@*B}@~qKcNjX4VAR(byIc|Xlu%aU1 zII;!C(O{6c7bG+>v_TGRVFME+DgvP^UG}UpIXo>_aA+{z0UvpyCT7ksyRS*A`;;Wusj6=mg7;BRh`9TsglZpAL z$4^I#3mzD`2~B`RW1iwq#?F8XoOpp3@3`N6)|zz5XhjoQ=u0cz?jb}g$1cYt3(nL^J3qjaUh)75h)IHj z)c}hA1~(skD#~vWte6YtX9gOG;+h}$8v>-jmr8PiiuZ&jIT3U;UbwHEk%J^AdD6~z z#wSJZDGd%s;+Fa0Vi8^;Ck5~cM?dh>pGE(3D?neEGoAg>T^}1>4{UIQCb5#0NAPP9 zAu2s8T$B*mA1lB_!V^el zT>_|K4On=LR`q709yFJxS!=l7ElK|#dR1IAq;`ZL_KJq?_*~uLPP%gCWR6(%F@}Bj zT3mgoDh(Rsp$F>(GQPb4l(6#KCmQ5Wp&>{gnEfk@ zm3rYin{d{kF43VgPe~4^3mN3*smW0}7^s+%!Wdc0pE9gLglz;t5+p<_e3B)tgswp+ zC_&;v)oaCp2t>BDx*n?R0UUVdA_@Yc@$YPd8do!eo*2+XT%}kXO)J zT7X@-YpJT+3~>Bm0C&A7rc(dX37o9q9(%aSA;57Bg_Q|B(^NM|wiSg%v^%XvuC_|b zNaQaBO{-gdbcz8kA2*whUq4l@>F($qoY^e8{+zC4;f01ke+-^Nj%bOO6fHgUkidO? zg0p6w*lkGzA0B^^1ic`XS#e7V3Tm~CVRrne<<0^VrI49)s17c&w} z1YBnH`9RFrmYX>uD0P)lLlR-9P+;6-J-2qifiPUKOx+l$0v@K>XNvfN4=KomxodHc zAGiW1T1=r5I=62~dzZNVzB`I0{n8|uV2}G^OL*FnKTy!v(>9iC#1GL_J|wf^qFp8Q zfT)EaOH1RV)C0_a&BFgl5FvZHb~4Lj{&JBreX@*r&!!EB^LyB0Z>h*pYL#xP;~{~Y z5RyVNTHbJOjCn<}XpwTm{ef|}`zEYc3D^DL^#b65BAQnBsvLO_+f84K1{ik}NJbJ9 z^DGPgE{(vamb{wYT;13Fb8R_$-P?Jr7sebI z>XHFc0xEHt-prIHOy17TnZEelr6f+6tx2INN(TlTeTCHY>CrdEmI>O0)}h@a{1VqC z;?_|FhBSc`S_mR4V%Du&ZJ`s@cpnr14HpEV)cD|5#Y7GyA`0pm=v+)(%#2%50Q)&% z(Y)W`b&_WdoA7{Le_0`5i5HN891vlgl>DDaF+%6@Mk^5D(xG7j{+~O%N{R&u3KUq1 zRUsx>N71Q?Gf4^>k(08N5xt38elcJ#HPu|Kq9CHm@Xa6wxB?UK0WRo(31$Fgq$7s3 zV7!?heu4i3qTypCMaCIfW88rP_c@PPjYr-6pc*VvNXXAI)x_VNn<%DG2@C=|xYy~K zh9{&0EW#hCL0nC_nIBLbL8+JxIh6Q0lrB;p_NW0chM@pP!pb=Y0;XZi=~u8&QO#vx zl8}Ha&A^MLQ4*5iWW-|ul*(S{mQnISUyy?=D2+BT%rj=722Mb}*&$Mhk_BcHD=ydx z^}w=pS7%(o1B7K*x&RKzNjo8ACY<1893g8#g#egGOy$!B(3>s{45Xaf zoOPiTcG;gu%ArJMoGwb94f)tgVuvucZ}G7}_hSKdfls^wocUZXkv-DI2_ zO%VU(B`_Wvuz_yj2|Ba_`nZacIL|}P&m?)61lh?CJO$P1hfxBOS!G2;ib6ofg;+iS zqgY&z70PC$)t`JsfHdAw!H<}ULO!|0Az&mbr~%J-!$l4TUp`!4=FYa9pztWoa3#q| z@>O7|6IXHEO4e5Z;@U3Y80DA2H0a0W#Vah%Vz6U?yVs+iRtdIGU|0uWRJVr(aOzEH$@ zB!D_zX5wd%W+t?(MoPYhWZq6=x+jvx=aPz^A6Di~N?V*}K}GG*XNkp?@+Yje&n1*q zCyWr68mO1tNN zY~vDSm!|PWpyEd+b!4F~VRH1IEtpGv?WLnmSEJJ8Xs$uU(ZEhd4;k=Z8~*>0Cq7RP z5$UIf!h1gEsH!0g`Ju4=Nwoe^u>3*ZD3Z?cW^-xCSHw!VLPdALgcVdkseCD}=BhZb z!Z@bSg6=?OrcknYOe&aNxS^@Lj07IY#U7@pBxqv8r0R&QLA3fx7f>tsi6Ujl&$>l{ zD9~2uNC9(}4i+$BcRH#@s%wkDNawIYhAhp;VXBa-k~3P~J0)WR35JpOYm!>ozhW)7 z5>Hlv))dVlt3*hNYQ%G#?I<1s@yOIAU_e&{71!0m5th{=bZKcOlg{W3I!VISfUF#H zP|N7ks)WQ(pyO@MCJT}r2!_QXC;~fG=81VHeW4FBYf0YC)%JEti0*zshIUCSxLvEnSq^N4;!S@Y$fS1p=L! zMoP@8Kbz1uklQ882l>R`OXHVB+kY>`F)&QZCYy zMIc&5Zu3-&sg~vY&ad5#Z>{cWI~rPmR;A>akOe%#(Xs=&rmKxoW?zt|6c`N5#w=O z!FCyyUS|48OkT|}132vBKIn#VN}}{YW~7?)Nbb}$W6w-$herRSP8?T-Ve-6l>UKx-7B*QVN(aABDvTuiB3pqyo!to7t}xQLibY zsaln;>DFlK3hUE9`~2n5!;)a_)^_eu&H!G}ZVHfCps)U0CTN|BIV0uxlDg)C8fN z2o27b6&qcSC5rB9DHr-w@g{S!(#A-c=m_ij733Mu)u8{fk3q`^)<`eyYst-lA`}%L zi&K?_V@drXWwNYW@En`5+#oCvHJ>bHeR45>6f)aFZ7Om$5#57qS|b3{(WIWX@)koQ z4v^4qcS-W0M4CoOpXa9EomG;!S~2Q&GGch(h_ddFO0CnJr{u+27_f3K35$}Uv6HRW z7dCHwUfDqN6VYsNS|!5EzQNDZnMGSwZ4RsX5^su?YhqP{oo%= zWM`oKG7@IYHJ7yIo*ob3DH5L;I0sX^_6PcT*m73m|30BQW3hg11h!EjE}n*$Maf~# zYvu8xl+c&J+8WsSu?4%=@4eS5@0xL3L){E-2mAjQYVmMa+{7$ICupP7pM-)$-;Dq< zt~E*9gLv$bkp!*ooq!#e-P zA8keza4`*3s!q=HV$W_bW?>i%%QyCgM{MmMTN%AfSxt@}X1~e7)RcYd#AwZe$s$#Q zlo?=%)gX?{;jAK`>8VtLG;6<+2nd}w(d=fMj9T+IRE&j=$cGn?ClWmJzg3$%r)8i$ z8oX4*oBSTLDexO!N$>o%O!Lllz4HwylukTwJ?|oRf6c&lwF@ z-Ev)^33r

@qgav%N?9B6Cmu(oSDQ3cMa zQyFI%H5pW(#%@XA7~~)jdcGxwI4U$pa5gN`f{VWb4#kwRvfPD9~VNfy(~DA z2$R99_plMc7;_0JsDg|hbMS%!!m}%{Pkc47UMqmYpo5f+G<$M>^#EX!uhI&0h2DE_ z!rZWjq*Jw6xfXC$7tW2Bz4y z=3E=5JohMI6?;L?0Yvcz+QY&rlzs1abs*7*cjrWbE$GPwwI4VES0hQ|s)RvJ~9V zJqM@%Hv*x#TmghTb^-?$M7Qo>LR+sGmMhok5u7Gvrkset0po}mT8MCbf`dxPk0*e9 z4094?42>Q&wRCyOrKdo1($p;U36`o;p-lB8h0_oiGk=J-oau#zg$qlSmT>wMYE&8~ zL{vavm4=r=XeLk?Kmq>*RzcOs}nkvAVxrp=o=+5dEph$fwxeX`5%p;eL>I-1z-FHFfiG_Giiw^#PK>H|g? zlH{u>_80_0Bg!J{3{Lh_%d}ZIQ4p6h?2-aR9!?u=jYST@!mkwdyO5z*48Wt7HS#(n z5Jqx?EVaohiva%(@&aR}nR`6*AV=0lJZ!|-l2o!0T%2r#tR-!$?KT~%nn1u;fZNic z;Sw?q%rL(!2bDu0^dpJqmOwEM`l#FPBsaSY54@Q=icbqXGuu-$K4mJTCR%c8?>(T1 zx&qNel_&#}{seocF11)NF|Nj0Nm9m5OrESzEu)8BCYfaOFw%1jmzWgy;saJ$BgJf6g%o;jJ|HT zZjL*&BN+eBH{k^7q*WY#7-F6z7L-t*48_Ecsk%B~;~W-6)Q_{6H3JJpm$gr;A>nma z5=aGV=9N4iyitTcn4>0F)`rT1l1EAyv6FjV2E?gF+%d1ZVEr|KDW>_`gyWO@_~Ef| zX+^`tq?9NRz)7R-7F;C9UAJ6vWh1+6Jc4$G+`DF@lB^jQqqkms?atS3wqT=!iV$|l z?uZ~tx-QP{820WY6%7BE;))%&81kQdk+p}Fb=?^As5N)$2Ol!WoaPF2*u-UVV17f_ z8!Ii(XV(7EaFR&%A zoKXHF|KAjhD1^rl^tsJ-PAWC1TvQ5(D&f6CXtwhPC$=yl54a{7?~p~As^S4&p+hr2 z13{=FD2qhou6MF%1|^En1^rSvVyAVi&Ma;{5=TqMnkE0wwxa0}~i=TD!B%YCQrwts@h)Uoz zsK)^ij*3VFRanpjm{3G`Q6L@x&+;p!Wd)EB+8i2Y0S+>~y zRJJrm!k7I41+Uwna#~2IE#&EDyGx6IUg4E1Y()>55MkJ^X9Xl)APQXbMD zDDfk{c?=RXEYq*%J8s9A&M;k(sBhBtR+jYeRK+g z_!9@K@+VBBf-3-kn=$P&7n_zwnN$=D9yZAfYWx(7#-T=-TvpU*4upMxKtunEh>*=U zm@y-XQ(VEQgU0;%FP-XSXBsB*!Qa3PW(UcbBUM#S?~qJCmXXg=t}?y$#)hE-w21R5t7F;yQ#I{OW(7<>@NIc9e)v*7!0?yL3rk`Dd zs2KEvTjq>P5<-+h$&pazbZt3!44Ii|Xjl(l3bjlO42GQ}2RvGbobpXveT1tKI~;uph}_BOMn7#ZqFkcZwAs{r4i2~q!w!(OLj_d@g2pwr z@u=%C5sxiFv;$S3byG12?GBb?+>LWu?B$Ihtryo?Tyn10G2Z{BT9zqH4V;koOdiH0 z0c>HmuVg}14bKorCb0|fw?yd)PQe*wZo!3@d1&X)*5zHR7I&h?!pO2Ly0oa4kh&WU zq7p%;3h{RIyovar*X*IyYCPQFtjc1k{(Hp@3$>_kj0t`{i89m51TI>I>P=)!oIyrh zU*urf1bJ6XyNE~naBb`JffbKZ5w;6?lMEz3(MVtx4MxWf> zj~YlDVmNf?Cdzhe!pbB6iNnd)?+dH=^ z7=ac9n9qmF1+wcuOtMFmbl$Fc?jFu|6IPe+|dWR~(#s^$@5EoJqaH6dc znr~ad71|da9?)H4ozkWeBtk?&r1QW}=>jnHOiuxyjtL$l6CxsvRN@WXCx3eD-}ViD zTujqy&-UJ<)Y!~PcrVcy&U*4r1+Jm@NHA>r$q#~N=gLH}CO78fIQ;;a*3Wbj%M z8PgAJI`Pm*aa?S%Tf~tK2(swhuMxTnO5mmuF98@kZ-k)ZA@^Vm7oia-0|A*a0Ra&@ zlxk`84J78L-{@x~#%LS)aMW_LP?P{18^9;=D|D0%EU*R|&18Xw61t89n3PY~5<-_| zaD(y#besYOGtq!la4NA<1j26-nt<`#sfoZ)+>nGMPc7N3MBRe&-NqtZ_+W{4Di~{_ z7j>!}Gg1s`#=#)a+>GEPA4CvUvKjwbk_CQcxGJ!WRE+D+2#vV04=b)GZK{qUlLdlO zDE&+;n*_;nZ7Gp&65;V1p3*bN%U7HosCU zKXTl1lMx>;-7M4b9xpJ1lO@;0k7AOW4(GQdaO(^+VHi^S2!(ysLoNOJ9A(6a9A%v+jmuwf(BQ>|l0!;HY6X-fUXYu^tQ!c?XEDprF64*GW z1}vaIHSyQv6FxA&i2iTUh4D1ssM6Ao8LJ4y0RG@E37mqX@ zEMqVywFZRkqSOW}>G{Tuw8TveQ4QX#tSBxNL~<*fU{WKrv3?B4ti-7ejteIVZmD|m z1stKd7=S$^v*#*A&;V;dl)pC2!UFWq9+5N8llegNM>8W3tdlB<)KiIcyLfOl+sqG) zw7ZN2Hs8#6YT!Pd)FYp7O0P6m9P|N0hJwVjR{3*GA1(!iM}cD19350iHzDN2jVKp1 zKBcrbi_Ss3G6xxe2vGDn1d}lTvy8V?kDIt7#Wo^g$UsmBwZ`13Cz*f{67?c30!zZN zY+S&^p!8QKZXW4~bl{F1oANv{mB}G(YJYJn+hGj$T+ zqfWebu+*+NUk!MF*I%NRcY|lhuB&9GPdeCiMZETu4kbQ^PDu|6d8be46a{3C^}`OU z!?qVvw>Qpy!%`>Z(60f#1c_JQZkWVVmUp;kTinMYYxCL}e8X?$$FPKHeh=MoRCMTE&EBHE! zGi*gzc#)TQi}%O>QW#}EO?*MjK3Ld=?E_H>0b7^=ed}lj)>m^4lZ`wSbCWY^<`-z` zSGk6yC&eIUgF^KfRZ6PBM6hOeHdTJ6xJBo6fM?f$@0Ea?Y%^q`bz4AHE;t%BWd|xZ zheM#8N|)+(un!(MgH%?IUUZrdsV}MWGcbt^;6p`IFwqeSVpQ`Q}{F*rI{} znfGF<+!&Gn@oAYA*^iew`p8&^t2Ym9z>MLwavS-ZCoq_?Ie90U4Q!YRFu-G=7OJAT z4ighkfS{YIY8X6uh#8X)D*+6k#ZWQeFPT=A*%b~n1Wy-Pe%`=}XSslBS!)$o6nvwq zRG_SWnVAvBB33jj!d6u*nrFlKnX6Tge;Jy$;FvYKq|+#*MLKB-Ae*tSjFZ-y?j|rRj1il4M>2k)9S6E7Nk*Hr1KE3 z`*;ig>e{_xdat#)rW2W%cRH?hTAZJ`d1;!Yf$k9OR0b zIj{Ztr%`$@^rHoYdm0rRSa`azcRHgr+NY7=T!A@J;`oGTqgtP7%n;Vd@x~^!Mf>-%azN1M?wWz2Ra>IQwyWXmmCG3kREvtsdAAcfm~)UQ zb~dN6u6Jizkl|Uk`374JIksKfsfv4~eVCh%dz;Q$XJxv&pURs#e6dTn!>t>bgAPsq z)Z|_N*dxBXvg3IgKnZkFnj@hU!q!x%QfXN zpukdR?7*Vf}aPyo7(uI1xF8so2-C_a*VzvhRel3v--c?<0yK(G|VXL3!){JUkn( zYuYimbfwv{K^4=(${&(AB@V+zFZ0bvw5~xXas}j`Nr%Jzd<-y`<@V ztAn@J^L-3{9jEPl;>4Dm|9$P+{2D{}eR!6s5q_L9yV*DU*%y6ZWipQ4{e#uGUoBqC zvHb}Egu1eY$bAaRfJ)?ns;RVb1T^|j0fvVKUBDAu%UTH6&m4riR_25EzfaxXbAInU ziRS}eg@3-aR@hBoIOdM`UI9ykzLEu7~Q>IoLcXFVVpVZ;F6d9=^Z}mYtLbVLf7xzkBAxY|JdNixP{u@@4>t6 zAOGx2{^ZsE-2)x;)7nEV`t6H=!*zI#_qftjv>MG=aY~=_4L<7uJL{+W>@nW-|6c1Y z9{1HA?|)y|z$qm(%{sp+1C_H}AAK66A@YS`@`*}@p+JoXDIVE!qD96d5sR3N$YBe^6CG3H7=e&L$bvKf7iuBNu_Qqt0;!BV zS(1#JKx&>4F&bu$(4R%0%t3iH4AK};r%59=44BqjT)Wcx8VhV#v9Mq%N|Pms$(uJW zF0}zAZq1NvF_QdA!miq=BM$;es}X13zJIAsA$Ro=?BK^^?}fHOG$5 z`1FXUp4*#_j2eJ#>BpOErM*^!U&D#koMoA{$J%g?u_a&sW<}XXAWae~gdATAF4j{; zs?E3DbAz#U;X~^Um0TajsimNFRCI`hPgB{r--}3HbscuveFx-NXT^oZYBt>^5>4g znP1+(2y=2~h~kP$hPlITOTC7WUNANogPtYUIOu4HIq4*iSxuVVcH9LiIrt}+Q)MC?b*U*sgej0Pq`K- zDx8xkM^%QWfdS@Rgo3DL5G24l+n=$?hb3J}`F7v`MNID2ZMnlr+GC}bUb^mBhCnwS zl9BDTUWoQBscMXb#kbLwI6}GKOJ73f9D_!>S>=ZZS}SfKf!MVxlZCzk+MTP>x*5N^ zs^KMYqgJadtGt;S;7tf?EUTkzhFYpqDNFp)zS4b6m38U5) zro5eM`zXlSuDtJmP~2f-(ZWjm=EC_+$zsq)i%LX&6*s2eUxnVAY`*~K>ngzu)@W~l zXZ?q))4D~=vfPj^*(YCl8z!8}JQIPgkLt1uWO(Gw`txy7+e&oC45yc59B@HOXWU>E zrDM~LdyA^rb{e{$$E1M^EQSJJ8F$%f<0`QKsGPEWCxJ8T5iw9!-el&6y29>Ou!)v7 z9oeJ1_hg^6Q^+?VVk%{0;DYCjso?`Ao+RkvPH43CM-p%3&{fOGwbh)@?D+6X&uJjY zx2MXP(_DL<6qT2;b~fu*FW-9A+>YNNmxLy3BF9imWdIb}_~xZOzU6NYj<^sZ;sCW6 z*o;zxBV4;eAw2}Xhf#F#8wb@uBkujHJZkZu-Rd?h@ujFvTRMrpZpT8@g`_`^tIXLb zmaiKkN?4Lh;ro~+FPD^#X5Mm{3|r+l;SDi%aw39i#z&=n?F=dz{FWAufyH9n40;#L z)CQ|p#4qB_d-N)n)XFmkq&+E$N>p3_8HYn3&Q)e?s?(fbvarOJMQ9@{TvfsDXsrSA z33~_gBiQtKt*ue5Oh<&+l%PnJzD4qNwIbh9Cb-24cF|oZOyJ=_)hQy4u^}tbo|=%S zD&7UgV4_1Jf)c4a!Z2}i;oICS;W*33(M=1MS(55sC;1G&7QMT)?wlEX@@#VHq%* zrxPM9m8dxemC}AHe9!-gi65mU=zSq19T*Fex!5JJcrNl?1h=QPY#GsqSF{EuHOWQq zJf)b|%i%bA=N52@lARreC<#gb3PuQWFfSj1q1664&(ti^eg4sxLE^K~lj;d*G;L!u zd16UrHV12p+?b*AnM*n?<%vO{*hz#IKn3QMd=|YZMlJXWZZ4Fj=9y+&-Y_?mjuf3B zW!d7AmXng^GJE3NmbUygP_wpEp!rjnM}Nb{m%)^*`ZTEgKv%hbKIg1wH5WbcCl#w^ zlyFl7=i#Uqr42rIuoDXFN*(*k;=OTcA%SKsHCVz&WfqWRGnthrX+paC%AS?|t4uuW zR)1o4tdC7)VSMrk%+}4P-3r5K)krM=z^Ex~II>SEA$iaPqO)^)vz}1$@UL$gC4rg5W;XAO!I%~k zBx+43`=U5K?LIQ1@txiXnfp-L!ql`MX`!hKe8ZP6%XZ>q8b5tGK-2gYQ4|YV_BceU zlcG0fcGT2-OX?qNW112;*x8gM#=Ci{YpTPO&ocs5MpZImPIl1Zk?0sN&_J z{u|XEj@71iUFUr3x!&(}@QlELQcyu0&&9ZSWFL-PS#>-~{`P^XK$dDG6{xqR$au{( z&L(*N*5$lH(PwNcUvzad*#TGAVgRmQigO%5EhJdPpv&;Q4o2mBvWi$Fl`tQ@JDT(A z3V|-AZf8)V=OXL>)y`fH@mk>9zoGrvy}-3%RjP^%RM!{3Lz--0?!Alxo@anomPb=n1nhwSS4={2o>6HJZ(gaAm2j{R}8R*=R&{1{19OB(tdA&D0oC zb*f`L<5dr?)t$s~tqCb>(kSmo0W@+S|RPZ@>XzsI|zx3M1X$V*=9aK3514tj3? zVm`E?o%`Y`UFp6981X-6$-wus|%i#Qp4F!IM#x33n_SZ5i;t^Igc>Pds2OOz0>7vxVuF z*9s-+(zr{MOB2^MQi=U1NxyrrP}S^>vW7IP3-|akE`-pBrF#AKMJ*R`-8XuvCs!bc zTbbwoOA4l7B4U3AXl+4ddyZmo$3%BKG-cGrR-0isa&aehU}lRXM{j#rf-1*kL{x)}BZbtpcLgVjjKps%CN0bZihosl zGx&mafm1mMWG)wl-iQ*p=!jmp3cZMTsMlu;G>Ze$f!ei>xx;5)hjTL1igc)m0f%?n zczLRLaw&9;C87s>bzsJrZJI}XA*fl$I9K>5RqO{oPtiAFK#scDh*VuKwRiQU&ChBsZRQah{&F9XJqJA_w)m5xdXgd+8DaOH$_Gib9RBgO}m z7TJQ$I9(@+l3Nvz+=GXr*n`~^Ypw`dvgLk$<8;WVe)|=EHy0HjlZqP^WifdFZ?>m+ zjP_($h)#&udljjcS63F1PzZ17mM+LM%LixTp@S0miSkyK`p8c5g^lwTldMNTuVa-t z$y?&6PW?EE6g8FexR>q&FTC|^j9FN^)pHMoBWoFy`(K#u?pUSpwy}Vpb1)?+3BF$ z`JfP52-!KI4_cn!d7&BlE*OfMX`rEhp%qG++$p7|DW$1NmwOte#Fv^`g`t2NrHe|QU&=SD>7_?i7-edvoQbAsim5?r zr<=;Do$9G|3aUgJs-vo;rAnkt>Y!@+Bhv{MeTtn1Y7Dnq%lzOJQ$_zWo zs{|Si(P=@M`l*C7ti@`quX?O>O009rrp-DQ2MVoMHF?t76}Kvss1^?0pskgvtG?Q+ zz#6RK>a5F}tmSI1mtw5qs;<$>uI*Am?TRzgDzDZ$ud#}&_i7IJiZ0#iJl^`Nm+GVI zO0ENYuIWm!%nGLhtFZ1$tz4uo^ZKy$DzRUPui0v`7pt%Q+OOZbu>&fs3yZKIE3%q8 zvL$P>47;!x1+gj{u@g(L7Fn?vE3@2srXFjv9~-dada^sqvpwsxDC@8+yRt5OtwkHN z+nTRQOS2t&vmD#BKnt}v`?FJNNL|0=dm>$GHRwo=O- z*dP{a8?GLEwNY!eZwt2#yR~&|xAuy+d8@Zwc(hC#wq-lEe+#$(o3(VCwhKD9Xsftx z+qP@VwsBjyiQBR-d$*K3d0m@}UTd*n%ej;Sw(DZJk^8uko2h3@x~J>7s2jSI`#hHG zx|O@PMjN{@tGO84xwpHEjcB^8tGc4wyQ>SlmNL4#tF^@oyT>cL$*a7x8?*h2xq_R! zz)HNJyB*WZyNsK<)~g-byS>Z6y{&7!$h$?aySy3ew<%b=&WpR}%f5trz2ED;!7IPv zOTX`nrsb%Dt2Vp&yT9eD3+AhWnM`q!h*xYAUwo9tiCP`z$~1@GEBoqoWM)$!Tu}7KAgfRY{WpEbyW<# zSggcdoW$wN#5vr-P#mCS9K}<-yhW_VJz}X;oWOvDj<#fNOOhwQ}y9L8YG$B*2{2K>Z>jJ|cO!)mO_Ykb0P49DKP z#V_o{&#A|F48=+t$*6n|tK7ttY{sK($(W4)#+#hTpG(WPe71wj$d?Ssy)4P9%*w$G z$gvE|X>CI;cU=fOwJeG z&%&(A9bL=??b7u;&m}$59(~Uy4bwC|(G{K2{H)R|-O2_%(>YzsLv7O}jMFs@(n-D4 z2Yt;b4b{K=(-|$)BF)s#tkYL*)km%W3|r0BF%8z-ywgv;&p>U~Qq9C#?a@Td)(7p? zU;Wl`oz-(a)>Wm!WZk#ueAaut&T`$>e+}5XD%fgW)lMDQbv;4t%+sbk)iK=9gk9K& zjn$I9mR)VxVa?9Xe8xYm*N@%Vd=1$rZP=Ji+0H=Pr#;%44b4&Q*o$q>jQ!fN4aSrG z(xuJJv>n-g4cDn1%G&J8Jq_Ary~nT}+M(|t=;ME-2~mp?>*kFE!F7Vh?XqT?5*DU z-QDN?-Tm#;P6e=j!oVH z4dMQ6;q+bM6mHrH-oYP!;SAp3rwq^;p4w3j;&1KS<~`jqPUAK1;Wz%9T zFJ9&To#f5k*aDufL@wqG9p-92oIdAtPO)7Z82j*vo6bGj=`+%$oc9zuxCiZtM`9>y^IX$gb#TUAWEu?0$aJ*-q{5-R%Qy?aMyFv<~dtF6~c#>C?XI zEIz! z@+MF5=DzBx{^;8p5C9?h1O*fT{{Soi0000$0YCu&2>$@}2^_euAi;wM6DrJykYU4z z5F_$i=x?IMixeyN+lW!4$BrK}N)$=5q{EXa>(N`uGUZ5oEn&)xNz>)anmA|bw7IjV zPnAG(2E9qNC{L3{K_(5!6sc3COOFbb`g3YkqgJnK9m;iU)~H~;hJ8x5Y1y%8zfu*; zwyoQ@YU8GzyO!=*vv=**mCLs;+`oPU>)o4oaABy26BCvvxbb4gfQ7b$thn-H!ZQAz&l6=wgYkop_arB(BKUgE;2+qG2}DSYwZB?I@&eu(5_Q$m@fhE8UwC5%>f8CsRSf$8O#usylunXM^FlbBzsc_o%+hUg|v zY(}YKon+d0V4P^?$z^PKzUkSX;Du)>pA_2ZCuo5-x~QOp_8BRpDdOqqoQhh=o_>fX z`ev1vVyfq)ol=@9ouJ++>Zz)ZO6sd*J^%V&r8I`hs;svD3G1%929|59tGe3ju!~7* zDrCU^8Z58OHhZhGv@R>Fv(z&C?6t{e8?AQOPFwA_2IdKEx7t2eu79N(3+|EUwrMMp z;|7@Sx}+*v=eY20+pebS(mRj8=2?m_fVJ*x@Mita`!B%puzT>p1V1dS!W1uDuf7mR zEHSzCReY$&%5l7Lz|m=(v2!4&Z1Kv(iac_|FxzYL$t$YN^2avE?5)EtuSxU74Bsp? z&O;Y%^v+1@%=61kC+&35N*^tC)Kph3wAE8%t#!{_cP;JHS%;yIH)+p(wcWU~EqBv;^Z&i~!Eo2D_TPdNes|$`C*HQ`T`RJLGj```1w|ma&`F8I5v89VHJL|O1{&zUGn^QaPd+VON z@4uTK`|y>&F8uAg&wTvy#=jlAE~~H-%Jk7o4}J8QP_KRU)^ET4^{aSuy*9f*Z_F>{ zmoK_5!MC5h`|rcgyur1_UvmAx=Wn|G@6PZ1nBv=B9p*s50vf`AX-FU$45)_#IuH>J zMBo4i_`pg~5EB)IAR?w03GRVUdfD?}`9kqQ+?9}gC}iLIc*i~z^6!N()E^6H7{eOk z4|$}4q3T+fI`m*7fe}=o9sdyc1~mv#h&Zdb%@;Tu?y_o^Z*K zk^!9JY(pO^+0K~))t(!*CsH@62O_YvoZBqrQa1|F4Lb0j1g*sN5SmNvosgIiJqB2f zNl`!QuZK&^mM$_nQYcKasa+5%OmDi)mckXEDoyH8IU3Vys*|rbWkg>C@ltb&w2AEe z=@qBy*S+>Nt~vFjMxzN%ST>fjU^oFP3CL3i$~CDk{VN!Tiqp9sm4JxoCneLkimVc} zmIdu4Fdxc7LjPK{Rcpm8NR#N;wz@NjXY;(IeXbTOt-h>q-$~Gnb@MHH>wIeZG5%4&k0`DiyY)$ zS7-asjIrW?B>nCY^dML$1mV5AHR*BLJ6xYe*QeA)FkpWw-7kQY2zekZOY`vAFMRi{ zgncY_pKDj&9(KalWiEZ4Yf=_xQ=3B!@kgT=V%@H|yrn!URI{*CFM{{Sqm-;swOC*C z7CDZs4c|roD`Mgjcf==-DlBgd*aJ8Dqj9zJc)few6f<$g-2Lz*ZCc(37csHN6*FJy zdDoya*#E6q9;bX|?ADIfImIwGty4=%MV3}xNx59TLWzD$?l+|J+=X=E8XS@L?GBP)ik`DUFHw(G|-?{Z?20iU*#s-48r|) zlZ}dHYTsDZ9<`^Ry$#C-x_G5KMyj;6K?(O3wFRLj?hPROSYYqF)rDO+V*7yW=H~m( zfB!zYg7@0s<lS{vVJk4)of&UEsx@;?X=d3Ug{2GT;K_gXS(k#bAk)KxeVa9MvuS&5ag||{wS;e=W%nk0>t%H~w1)p>S&^23dAM?Zmw##o zfD%S?e&>C+h=;D{X2aWe0h9HHf7cgvR$#(`bv! zb$NVPjKvpWYA1qhM}fVjZjJbdYS4=$M*w)(Z>|QA%a(NW)@$*Rij0PioCS$@c8pQifI>$H{gC47H>N#UP{<;1h{jFNR68)X|vdo+NY63DSGuJk!Sdi zUU^d3Vnfgaf7|zli^qqf*nH~LSpx`85@?P32$vh#iBI`nRY-x>MT^&@T@V>*>Xwu` z*;w~CQa$-)9@ut3<$b5dg+vI3iaHygX2bx_K1_?$Z9v3gsIkdb`XBeq&vAa zd{=(U|4&?c6i;S zoL?Abftin#6`8!qlimn>%9oyXC}@^hl)%NHHi&vR$&vdgkRy4KFG`Th8Kg3WZaUSI zZxthEsZ1aWYv@IUBuZiHXPGtTr5$I9tm$(oT9v_eoHmd|4H=R0S&_>aPwxhxI0}1$ zhlkxLmZ$iWC5Cs51enCgbX|IJB{+|gnTZT0m1CePSO3~aS$e3a7pZ#+cDEUb?MSA_ z29W!wlBf3v0vMm5IdnuCp|}~FH937|;D6FsZgCcbyy>3!=}woKjauM#sX3<$L}p*K zonB~goT>&tSBNn>ojaG7hU$w(WJrxlK5PYX8ycP~<%#Slk5t!{;TCj`$Z1Zessw7S zoA`D22aW`qlaH9KwmF2vdVY^InWc&Zw%DTSWub0*q)N)J*XetC`JI6Id~~Uz=44w~ z8nDbdoPN4=?Yfyoq>E*!fu`uJOgWR>DpY=Fk%IVj3_t?g<&liAr1g}QTd;n+N3JPH z27nloC@Ple>7+tBU|pb{=$dJSr>by>drewvBmb#*A&aDM3a|n@QI2+^ZPlK+3X$}g z2cKqpCF)^!SdhO(o6uNgdg-P@IH!c@tvh&=-L+k_n6w_YoaRcUb;*63)|QHOl)>79 zZ)%0;Iin-jkNOCJ!x)u8yQ1-l1KOvoJ-a)PdT_#Nok$CoFxUrxxNG=Fmx?)H@DzXT zs!aLhoH>Q7!Wxc8>zF~ifMGUw7$#`4 z=lW;`*o8(akS`miI+}2mh`LVrYdm_U%>T)Hq{gNGs*z&Lsu${^jcP`;8(L0;pDq`v z4d`828M^t2x%Nr9fr)3zds1~}KcDVbt zn_U_Lfv2|a$b;H2}h`}f7nQeNwd<(cBw!2!YzOn15 z&Dw8ddygV2qn4?{5(!sSyOy7+!&ix#MX09dR*A|8wA44Mi#fx$3V2RRxYIbMMKHKH zaGz%Dz7N=+h)A#d=b{lzYfv0~ZU4%$Y8#7piD~&(!dc9Dt?0%atE9ik#bZgZerk)i z*}E;3dyqji!K!V`w#pHyigJ_HMJCuEDufjIL#ixsMrO2R3 zs!9rfUdy^p%K?*Ymd$FrC~S&g8@=lKxz!18J9%a>e4v@@Qm1Uk)~o^-yN59Os~H=6 zI+($l7_QFRZdTirz1+y;G)i@+qCey94BGpf!w6Hfw-)de4#ToNoZKmmF&x$&eaMui43o2DZ8)H?dor zqh1VkikgJ~sf#BdfB9CUm;acF;QOu(`+gsd%#tifK5d$w0ASy%xQpA(FnYL1oWMnU zlb9R6xrvxKymS9dSFMU|Q>t;^*VG*S!2-y^K#OAYi^?xryTlv^#3_Bjo1UGyltBD@ zqYSE-m(`!#zU11|0u9tcy^Dq@#h1yLh>X(~U3A0R(rj&s)mYf=CDS3`li`Wc*SULs z=BYEO)Ys>FLzHh7H+rqdp$e9?*q4dchqCS(mFnqoliJ6Ce6Z5>h-IzXZxwHLeWj`8 z+S9t45lX&E=dJnqilpqgPMEp>2E)zyZoeIhT&H2Z+@+=*ykos~4;y^I8ndN6%}iaY zqh^K6>(o>G!L*Ftm~SjG{>mRYMQ7JMo;?7JZ#XW-NGO9?`q8={kIyue9^t8I{U`ITBlJ+jS(if4jiC>?Txnm&N*k@m@Htz zY_jKhrcS$NZ{Qm1-A9d9-f7Cla@No6sFD?|xtNMrU~1e3E@u@jWVoEifN6x4N^bUu zcqneO`sR=ms++j&kxd%H)zph0tg;q%){HHj7fO@c%uPjpeR8PY%OtQt?FA%lw`c6c zAh)Idm}?a+<1IX*mFavI_`L*thLR1<(7eqJ4T;&h(c1{!8J=>E$iYjgoje-Ag?)nr zP6Z4aobO$|7ys(a6aIR&f&aaN`fa|G&-_Q81UCY!F@aQ+*prnqy z)kp1kJmmJL(*?@cYzw*po`<-N*y8@kpljt4I__z##*2Pp%KkzPv6;itXW&KrF5X<&X%!OuzO_#3Th*MNj< z$NTA1T*;VxT@@LivOZ$0IC;P!9N>{$KsWgO24UA59J*N_N?1d!%UMUZgI z*%hz%*#*)|uGrp8^fg<}1f2Mx-_Di(&9p3tw+^wS_tfpV>EynQ1E|CHZIws<*nqm% z0RJr6v<;bUAmSrlM-}(_oNt{#C(LpF)$d98*$!rTT6=!n{uphG!-&rAdj2nK>(mSo zQ2Jzn(&b1HGHT)|w9%!{LoNhV+-P`$#6=Zv5^m&ZgCZA#9Eg0;a*&KijSM{&Nf@N! z8%8m?OuT4tVxTu8$0%&mQ_q$bRPOBLIh4o=DmG2t=xDLvLL?naGL;FHCBrQnT{_eV zPubqpv}d*bs$_@`$(|v%Zk>8iXUUyTkzPF+H13YM1mzZtP#B|AqfbL< zxx>h+$e(+QOpFSZXkV2tfdV}Y#joTTp)E#8*!XW?zEWMT1X)w$WHo=(*ySKmHUAYz zZKA#&Q-r~Yy(v%cHXM{UN1QG9Iu;zVDR6|kTTnJ>l_bUB$UoZA(;Xt-mhHrwC7;%O zT3yVAuhyATuidOSeix)qSvC;VfCmx{tr{y_)DnxMKia5^EfBt(DiA({`T{Ko1~{|9 zsKxBaDn8fPvu{23c+(5Ar3PycHYz^CF2m}40!b?aLj><1Il{U->PhGDi!39AUZakK((Jg&GnW1&RLT$ilK;)J=eDen zDY){=(MpC$`;MomUaTuf6=%bd&Aa-W4M|dyG-egIyll|85<>-*CY@Z(wACifxJ@)x z7kpJvP zf0qqP!!su{7Tssl75FU_L@SZocQJ#QyFgEPXk5mA%&}1}XUOi}g*r=F;zzlivs@47 zLk>axew2486lYB=&0B@b&Sbz2PFOI68Rgg1wVKXG7hgWz5@qC)zW-H9CwBljkNdjZ zG1Nzo&WTe*gXDNpGX8+gIzS10acCV`_PJqZ34=`4KgxACzjcuCHJmHJG3N^MWHYaA@{TGh5xX*g=6-dAiYu zgbCiOyQsL#y)m8vJh<}k*|6JtLmgZ@ax-@^Gku3ramh(f{yHJAlbQ6|!bNs)2JxK! z?T0@T_-qr8cYSr$r8+R#fO%xE4b3}u|7rN4t{jUp-nzn8`lqPB;i8tjOhs*NEk+LV zf*ic?x+&x>cDB-8#`G7v-93;7Dhn3=B513aMeA7SDvn4H_y4u&!7D;Gde7Dx6~3KO z?lIT`UxcRMD-Z4NQy7)yfhEa>;D2EpDfQBWcu?bE@BOBX@#{IpqiEzB% z{%YWZJK|A~d+dQ8QE~1_=eqVQ}}m4-LwYF$eas2a$B-C3N>fN)ECN zn4Baf>41hC)=-j}ETkjn$VWliv5u>hBo;Lq(ftQHf5Jq8Pns6LT<1-f<6WwZarv#x_9YFb#>pOd>LVVZda@ zv;oItf+T9;Ol+n@r#VFjPoF6cUL@ll)nGyuUvN~U%0Q`BfWcBbP|zuo^pjJq-zrsk zRXJW&6I8xC@4X!YaDA?&3aZ=q7|%wOr;?~nO3-tE{<5GDqTBT*Sp?z4@>~3 z8chbWDTNd(rYYDiRl3raa?YJH#Xtrc>sZIuwEr4E^=V}-TMjGefUz@0W;{88i=aN@ ztuvt#3QbE(#);B&CyL|>1N6OEItxGJlI2b3L+Y;qW+^BI`@siS zISdE<`{4UJSin2#uL;~)#{$>J2xpY*XsXvus+zbkZuDJ+Q{YhK@|6d8%++X^N05c- zP<6z@Np-1P-KH8-yM!RJ9=QA13`mx;=Kp1Kd8_e?Mj*Mp>!q?So;(OEYmpNhb_{?u zoC24Kt2k%W@5W@d2h_IMFesj!m$_L(;GuyJKd4TJ-wY(+>7Wq^b_fhZaE3kCQa*+u zrx8HRUqBUll_ti6qID+XMKl5qp!AjsBJhJkUo^%{z<>!*o7)`A%C{Gb$uFxb3s5MN z2}CAD4CX@!PR|qrG5|y`npxiQ=33WX1VwtMoPY*AfXW`Q-xNSmMs5sZfoGD)i}4@= zJt2qy(-r}lMW=q zwE}KQ+k08~W`kJq_nU6#yQ(eV_x~AE3~)GPIuU>XL^x&K>DYZRbZW}WYVy2&b6`72N%1rYoi*#G2au|?rUZ!BteIPo$Z4>B@t zAHdoNs-C(hO5 zzJ!4sUFn-pKU-IK2A5B?!qvM*Db^Y!Cfh5=h6_S|2 zB$}}cZ$LVpTeCBmfTtKh0;o3CV;2a^Hmv)k-un%21G?L*y?+b74+ygZ;6?PYQ-SI;xC63;?87@2I6L8KH>&s_rtaE`oFMpfeo;~VFR|N(g8H6s4}>@_$#(_v492B zKv%s#<1V5fAZsi$5}8I2+r}DrKpiVXVyvamyRBlRG+Ro6GVsDN7y(Rd zxa871Ly;C`37y6{iX$w=ra?u%A~H*xh*n$z4?r?UfWr2ZLh_QoR;WDn!bOMdfcKJx zOyIIZvMAlD$p4Bgot?8t1!}rlNwGPk9gh?)HJqywB(8E4z;d*ot6Ctr!mG8Ss=LyN zkAz8dJf%3wh)00OEzAK+Lq4q_n5khKbUBTD)JJ4M#gB`OA@G4Fkg^PL15VI2S!6PA z;G$Y&$cLOcER=(rN`_}Mv^@Gcf72h3I!Bj8I+x5yuIWgWOfi=%$uyHIb1bvAGDo?z zIX~n*n>dAkUF0nZYdi*UdI7)p~%E%frCxXhTWTvXLECAF$ z9neay6adit%=Q8(`l|p1n1BN?fYeL@)KpE^ya4|Sq}4M33$p;*ET!Ar1l`a$0|5L6BmIga zl{_U~QZBkmg|x+iBql5D%qTS~D+MSl_0I#{QU&eO5{1rM2&!2aQ!-735-@-dtA*Yy zQUB@GurCeK*4)rLI#EcHQ`St;Hq}lEYSB_Mf-C(eK+RHR+P+B&C5v;L)RLX%@=+ga z1yA4s?~2q^e9BjpGVMyzU1$X-9R-}z)F$;jtZY(+q{2Hg&=2r53NSnTD@`hE)g^F+ zPM|+o0|3;FG_O<88rVs9YRv&y%^E<%*1Uo+4O0jmR##xnOu$fFsHkf=RuKr+D|lAa zl-3IXJr1Q!6}?v1yi;n;#yq7^1QP*O&7-hFCAA#OJqpi2{Zn?OQ6>;Dn!$@hmDidm z!rH<|My)PZNIx-PgDm<6(CP&tP^whiE=7XSB&CNU{XfZzzfgVHCT-FxJXMO_Q~y1J ztk4{%P5_4}tG@#1RiSf*$!Y@VOo4M!(+l9z130G(SXR|6gJJE$wRNq%fJ&-%g#hH5G}Y?F$O#xA3Pt~c&-Vm{A;ZjD>#l;$B0}KQZx{tu8!}69 z)OQRyOaui*2!u==T*Cbq2~*k^+p@F_PH>2V1O3lE$XYA-SXu?yRK>DONWBEjfsNIs z15nvD$l6~;!(qLG&P9W22-A>t1)JIgEIU@$6xwS*0|(8#_>uzyG*JJlh5xTjg%xnx zNAOwE9o^OJSyv$1X)WEMy;>tEx2#RQ@Bl|ZxW8oGOMuXb8nH5kr?R!ury^TU9I1P} zwlJWMk$VuGfZNC<1R)RuAsB@+y0vm>UkCs>BNc?a?FB$^tOIRDPL%{Q&|5E{ge+o= z0?vb}HG`6!fPt*gT_Ooeh}CYwTy~w(;a!2%U0ejkyyR;Hr`=gM8eN#xJca69N6OX} zjZFn5GhDR*&Wl>2^;|AJRW!X-iUr|QlGZzVBj~+g9{wnD-K6gv*B_8Gh4NEDC0ix# zN0Un%d;FHQt+2WY#iUT*QgqaT9Rfly(tUCV_bu2)aNiwt#mBNwfR%*-28aK5;F|er z3|q^aQ~(G5i`o_Nxn6pqi3NcET-SNBLJH{F+f~gije>yf1XBfBmUY>kEo4Kpq)Gx; zZcR{fGTIFd*-cPn-^JiZFj~{((&&R-b==u!-Gf40-l%0&{B~qbE zgqxFNo5JQSi{7a9%1vk<`ay+jNWFoi)hndnPR87jrMi-h)j=?1(luImUgR)M!$wnG zM$k|cxMb$t;ALH>lucS#J%m7}*3ykMot@n4edj&U;YGe)Q$FNX2IBuZc4b>^Z!6*650cQCXg4k7fgFU9EwMu8|m+ zdA#GeAOj>^X=`v)PJL;9V(c%TPYp25#$p1AOavu31Wi!3O%^T8cBi$g&?K{KndK7F zfg_F1EC(>>a!zNpe(InVuX!G1pT^y+hS>@|Fg@@xpzTtq9&7&!a8Tc7?bdef;ce?1 z9%Q1H>r~BXygn%BX70RB;=TUpO|}U@K1K7idbDDDxi2G6#n zuhnNscm>+-Z3&<7;9lwgC+_5iYW7a+6;ST?#%mCN?u~Blb_J-WYHD3vqv@dH?N}nK ziJU>HobG0+T^NF02=6ZbhE!l?Ep7*Fc;ESY1ooEi$OiB#oo^)fYPipEA^*_|3l zZ3{PY3nr$>=1<`cYq$>XX03%Tzg>6wW;qBiJP>5KzVQE#(yr9~BV6uev&07G(21XYIWTsyX7}!irKkVyR zDqq$86wR5!@}Vx&@fmSF2GArAR3eYLEN^WJ&uY-Zg@QI~=&kTG$6T~F?lnhe5Qy_m zhjmo`aEq>99)50HrKddS<14K`knStlwQli2)YcG~^3`SNF!c99^kH6RWnSiGZf3~x zt{pU0&8l)P3@-&=a!sFanENMKecU!6^K!pMQ7=#hNOFcW^&+SD)yDK6j`KG^bu^!H ze!uiwU+(3`s0a7DA#TYU$b*H}gbS@+gihA5!UO+_Z$VpKgce+b;`MT4-AT~9m>%># zT%uRtsGc4*MU!XWB*1w0{TR<87}pBA4pV)V@@FVS;h7Fma}9*4%iL#b}#n; zQ0;oZcPy{Za%K3l4aJuQoNM#Vr!@M%Jbb%}}}O>OfhfsY;}a+CNa( zdQUKWL>dH%m(YaoK#_NKEka!Dx3Q&sO-MAjp<@k^DN3+3VWbA6SDHj=N~NjKq;2EKvE%6z-A|xGd%7!X^ys^C zNw0NsHA|;XPLDKAWfW(JimW(pjM0LG2o$emo`m@d_5@k5W67?K(d;cXxNxyy6C|jR zAX3JxW$VlLZ?smPK=o33@9;N3Lxlf`Atc0cA~%FaeOvb`5N26WaiLkDqh_^U86=Q) z(ZRHlN1=19*_2Ak2bEL4U>R%M&YB+{Iuh(H;J0t!89k#eUYy}a(*r;lZ+`rwbLbj| zFE4p<2}?a?4j}+dMlw2lPz)SnW24s zqN0czUeAR-g@vRTiffE_hZ_hpCuUU4lSZ z2Z$sFwyG#67WR!dl_e%b5{v%A-K z_TnXb^10K$?8xE?BLx=pY`js7-Qm7z;po%&& zVt0X@YBwpLlX5XFvl9OaA-(LfOFOcTB7`8j7=;)fE%m^X0T(d)?zRnZn{5y5y@-Mr zB0=;s4N?D$#UGiJ;4O4g9!pRZ5o^1MB;5_W^_9>bN+q^qr@P$>>bZ^N+wP{_$baWr z_cR(xq>wk<;a$+LT=M}OaG&=XUO2!JlXpc796ZS6shjHYho?nY{^?dBH$DQ%B)4O# zRw%3dR;;D#Tytfi+9T^Xf)yr*&q53B!l5m^^ zM-mg52kw@-wcY=>+jsg)H~^J`xWPS4aY_rFfr3DWj%8|67mLsnv=Xw;jqDqD@lalV zF*BMq!yXhwO6Tk`Fv^spSBH>_?^4K%cVuFJTnd+Oc4U(j;K4{`YMz$7b1iP2AtXz9 zi3W`Iy%Hqr55B3*#9-*7r_F>4a?2hR@wNymY{Cvzyv$8FVK7PDgc5m}$)7S|29c=A zJD0!(Z_e}_7w(G&(fc0(1sA|{94?M@w8bMN*2CFI5OZo^;KwLcmdJr@1Q9sZ2TO*R zbF5BvAWY;VW%jCtxkh?F^>LN;)kjM@S1bHqa82F!aS0PVp@2^ zOWVMf2KBLVXR+!8LwF5V($f<7>=RfgK~S|e6q7Uo1zG>eo~q5HuT~l31d!l|xk~6n z<7C>;(7=>W@y|>ZWe(S*VA8@amJfSa#1S81(}>`(0LqEweSPVH`T=ulowZmoJ@&_h zffjbWk;0KMTM@XecCA8%<7<&Cl*2r84^>$07#t|b-C{0tRBcPA$}%dVy2BQA01N-j z2sW5xB!xzbK*Z#Lg`(#A6BS^LZWCl@fC%ik$3a8hA(|nIyuLyfgZQhm9+io^E^x1b zRpJKLu%;RF!FKEgANX8A5E(EhZ#~q3dUHcYFtZVeV?N1BpE!u$NeG!u2JIJY)Sdv} zHWlYt$TQLW!abAr143IEgzppR2oo^6khSm+ncU$^S*#BqQvpp8v?)TVfvCf6?mJ|0 z;yUEzo~YPHi}NE^<^B__V1+9eZ-QMB>=A=wO#*l%K^C5 zm|`LDq_<$3>)N4U32d_6l6_$GvTc6`m~p@Y=h~tg)r(#RXO8g-)prV*;Cj%lS=5=a zj#-+<`{a}%Dq#s2dxFI@hOnt?+$t1=zyd$sl>lMKujc-Ap?d@D4|DJ$|61`8zn^lT#}2|D4$`M_$_L00@~tD1+*1#Z63YDJ63oED=82!0{!rntTX~E!ChsB*xKA& zfLtID?pcLNigKA^gk_r`5Y5klNf=~n({pgHQ zp$`ssxTZfe85IkZ*p@uJsatGgF3$N6N9r1j-_gBe8zw#I!UHZ)5vl){>~xjv22UOO`eblFlVv=hiC*gEQCZ&ZIcUHD1&n zo@sqJW-;zW2S3ZNfH=qjHqk{VP=n@%Ut6sQV!Ij?BMj9O8mWvBhbW@%69& z#=TZ8S*IX7Gz1IF;d8|yY+F22UdQwL*$-R*pzfU)06be|IfZ!PR3j9?wN+aM#7mCF z9t+V)x4i>j;hye6MAfC+b@kqDpqrC`ovvI1**OKD^^!!`8}g(=zD1usVlm&>KpK_%i;<5i<#wEqZnZ=(uOnb~u zHL=N<-6099Yj8;S?IF;c)pHI5dJH z`Wn))knu>wnmHPstwk5S#A!@ShH2dgUP5Oi0VmXro;m;G2Gk+ZtQ-q=RUo8=W>BK^ zWuhjg0~VD>Zh+ImcuXOA3c@v&!qpw5u|+Cc(ucHS#1Td^%Ayk{6iOw6$93EeN|57l z9`isMpYg#vI0ZO*fF@J}7tsyfxYwdM+R7bSF6Jbxfs6f_-ZJ&17M>C?+D?$j!G?te z*G0htEad~3hB|hP*L4LV>P<7zVV`Lsw9JAgun`5!KtD>t8~kG^7{aHy;6TD)^&!_K z*}}~kP#J_Jrx4sQl_G%o5h@y?MGBr07M?!IVljGA=IG?!8ItTgSu8ORA50l6)#FUE zfn#>Znw5hc7!NO)0a`E!A|1jFdC&LM4^W!ke?b32lHoy>V1b>HlQC_~X)t9|{s2wA z8+=UVv&bIVCBdzEX72b8BjAspeI+5b!7kVn=n3Qt7FDRY0w`_XrvTSfks`(rLFGtJ zMV3S0VPv7KP-v(hM?w}^c@fwxOK)A~#|X_SsS6YZTe7r`25^#lIp$*mm1J^Z4l>$& zq6Ob<=4W;bJ_^SfH3KGKf`$wN3|&Bh%q3>9=29Yy{$BU7(~lZbN!FJV^`byrqXODafVBUo6S!x0u;3l6TtMYr4pvIv`M^*@QGVji zmRU(744E1jN+fbgU!mp#F-Zp?Xx=O+ZXt}5$kz`v#q7bUK=sTc*2HeQP*?U&C^QjZ z>cUr*Xj<}{Ostl<09+41&PX=rj8f^{(dBarjw)V~S7c;}Ugt*cs3r9%$Jrv&Xjn-S zm`R!!N`inDR2VcpU@f?3lRBvyMkyrLP#`f$2($qSW+_T?=5OeZyWlDl=|P)u3}&>! zNibyx9LS(>!K9!glRPLNd;}cW>4{v{bR^m2_0&_~)Bg!4in3jC`bj5tlP5+E1JGyq z?N)S#SP&9Y1aYLGL|kxYBrIy`rvCrs3?Y$>Z9+?=8ZTZ%i=>uDsK;=uk9@7?sG~@X?{!9Fe@=>n6B*6Q3_bI9@%C& zN;8>2Yf!_~l}l@UY)i2qTAC<12kxqTUA20&7enu_C zZtOlqYyik~iPkI-2iUeO27DTUIBB#Zg5FLh(45CeF35vOh2Qwq>*CfDEJwy_EIxjz zB+Qx^_38k~0a$8oyIqdp8scZQ+un?i2-xEfl*d8{z;SFLFiFp#_Jk#R>xsVI`b2BW zr5kKAWWoXMT!w*PrU5IK?pS(NR5anF#^U0^q`o8z#gIg7D1ok=QWANujx1T!+@@e1 zRs*V-?H*j|rh`D~E8KW$rxvD`OaTLs1Tcj_`&O^I>|d7LSI7pd_&QqQ5*S?eFm8 z=>8>ON=-SUU3qE`Z`zx%xJ#78i>UplAw+`lLYI)OK70RRv@p^2(bb$^RgJq!aY<&8eg8<{=l42tA3g1 zn4BXHW6Fp6@PG|#;AX)~#wxzF(Cu`lG-a10E3y4rGS6b}580M@w#6YSoNwVBuzIpv zAl^WkGOCOvs!;zz5)Txq6er7;!t3B!MJ3kaI z=O;{X@^Y959I)PdiIX=C^I>Mz0TeR$g6>T9FK#3aBVQRla%Ch>vgGdU{Ni&mBHCO2 zZ*zvuTq=mAfS*IwD<=2cje^Kl-P63nCi#6h7PkO@)fi#m!qzNKzUFO zBT$2i>4ySQ4wlC7milS|LLq?C(GNxSd7Y3Q1aPIUuAH4|oC%^pCE8RFaVmMl;`~7I zbh9^4E;xg$bt%<2rxOptDBoSxxq9vb8QP)jGXcjljA3e^+Eq=uG>`66CE;EQHbLHe zs>msG0t)}=Fk@Nqq>rcl7*B3##%4zibE)qPSxh*tk*-UY$Qn|9Z~21aAvf#ZaHA`P z8CX#gCoPmhp+QUFwS;{$6JK)nCK(jh){DMVj8?I_S}}0hD;OPN(jxGtim;*Vv-HGa z767XWjA@Y9osrL;-oWw;&q?kJMuL z-nN;3)|{}KGy?C79a&LB)~{e!&=B`F8TVdGSPkNeWriYQ&uZnQvlLTlV!Lj2dx{+E zWuP*b7^^5Tf;Wmh*Fp;*Ad89isawCU0D%m`ofU+kZslj2HfY!IFjsJWZ}cDs4Ii=d zwABCh@V&(8StTC~v=UGkR6NR|$=`z)WN}M4LD~m}g`HnLH-|gwqdGT=tutBNm@%fB ziIrq=8E^^Xvs$@1EAO<5v$&@XbP3^dz$U84^y?CAcb<|4B2I#aay5O6$$s)TtChB) z7dn-!urS%>F_EHY=}kw=q}X7#B9I*&*z6t3cGFb(S*kLjfvcJ8FH?ml(2{S4X9}bd z?3o{98ZbcjeMl^v0$1Hrspqqt7Hvd=VMt$VvE6*?X3!C@1LeC(0iJQE`}~b%+W4 zz&9us`?SKx!d!8c98V4%-fq$5;&jQ$qWp=VW5RYX-aei|b@3wkATDDWQMCp;GvSD9 zGrH|~K}-d{`mB(FSxGXIeBE;F@HEpi*&snH7=ukPBnpfE_%oWs9Bo1#CYAq3V)vSq z%Y^*Vkb{kS;W#~oyOj+xw^e;E=Mr{cB7EDvI>By?Re8Owf4wPu!Y8z;&Dj{}qj0$a zJ}3^&tI@zjf!BUZ3^lcJwI0gp7BL7ISYBXZ*3Xw0nL=ugUF@K)b0=^u{iN4R9yP4>koZ~+97Fj|n{7)XUkj1dA2 z9wew@A;J@yC|0zn=uVm^bvV*e1RqO(0uprP`ZfOmP@^Ug7A%V- z52RO6QMoF!6aa+*Ys=a4@DuD<2Lt?6dld7MtW~TCi*sdQD_*@C;xgT^U~lBd3M4e1 zOxdy#JdO5rfGqj410AE2UZ~-x?>2I)(Xnpr`gLq}?q<`rT^o07RGJah(*8zWw3 zKrwklAqj?k2F7rB_PCno&3B8c4hlPVj2a7lJQd12De<2o>5=ZTkNG_7Y5dWY2a8&( zgKS;jv}uZ;-Ekny2-}m-!9p>lqwvNXlsrWK@B-09eK11~O;TCUmU8atk0n&rlB5hr zUlIjHxd4;{)EXYSMzr9XMyqa%oU&C<4WwpE zF1=g_%rNDRmzQnsndK3unkWlS0hqh*5JY?g$a(J+<|0qbQ4^);Y zFaSBgfq5IBb){w$ROSHoteS2tc9u9W_5xHoR&ESy$_Cj)jvEij&XZThSoSAw!)m}= zv(qLLG6a^m#1(KJDUxn&6I0jOcHMn9-g)VT*EXv-!E^@Y;A1G^I`1Ut2_U9(4sn6v z0EzPOJi1)s?+6X%Jr;b@*h89 z=zB*|qJ&X+m8BUy7-HJvk1wgg20a*=;bIHbS^@uK)iFDb`Vzk0Q5H3>W?4qKtVgo4 zZ_Dm5>-4Gg^Z--Y)tm3W{hlB2)?OKWlSIrYD1^puaNGt>TbRNgm!~$g0dtiDPv#~@ zDEM$}Wz~~l^<>8ro{-Ffp}U&r1`(!R_<;sLXw|67VV$J7%pHO`R#S+EyEG9GhD(di z7HBB4tK9DzGb7vt2@f9Xgs1E zffJ-+mo4fQsahyO33z~nA!*ppPiD{rsbPy4z41XLKyYKInZ#}g*{B^62X><1WD0*r z15t8HY#}PAKi5Nv`#_>Xj7m_;22lnOQZO!yxs*(#HOcOwbZ3$%4<=97o@p6^6{jMf z6}@7vo?b==*(%Kx0z=PwLeq=Xw2l85l^VGG(acRJFd*ZS1G#guiH+z?r#jh5sCH6q z4|GWeQn{fe|6D0iHB8g~M08T1-B2*HnCLf}anVp-Czp4vlq~N0GKxktuH8FADU($O zlpG6mW&0j7i?mWAPI94>hyu{s_Bll^1%uR6?D-h$)1N*j3Qt@R8q$SUSk{M4*4hmt z?%LY57W?6XDN5v&4gw3V7J(60=w$fa;^anV$~Zx70J&}ZuGf3 z0g$8ahsxPBqoDPnEz%}|i0$I85|yYcZEM3<8W5{LN*gTWpbNd7z!Y4_Tnv^L;k9W| zi${k^W@6Q)m$3+ONXq=|WxD@}+cV7KQ8g1`85X6`v9h)$H+&F2uh1Lmxdf1;MMKp% z9AS_85H)*9nQy(R1v~~UI*v0`b95^(s%~UE!X0jLHRr%U00Oy>xX(vHhS(Iahpo_s zXt1mXG)G#uOJ!AZZ0RZvvO0OmQLb$Wi^CF9`pcgt39+MUFuwhQFD@WGgnc2Kj#^OG z1=sY=CUjr`Jh);JC}E;z0c=dqe#WG67NPVqTQlX|Z_So!A3)U9>>aQ!!1b7L3tke8Av zf$o~Xnw5GikP>re30)!+4yr*X1Wj6IG96t|m-^~ikM*m&-t{6DSsKUb^%O%+axw%$ zrlVtawEy0hEE!ci-`*r={m}6`QNGh|WI8?~nd5%v`>u__YiYiwpq|L!;9>pT3YP#} z;7CFvaG|rkh;~H?*p?15gxn;kzii|(fWuL`hw|HMq?mD%%R0eqzTl$&0<0?{ z_Vf)kS|#;3L-8DK0j)0U7;s8pj{++Y_5gto1_zD$r(c4Az3iaicrQ7^00f(Z2z(Ft z&@Lm+;1T~C?%`Z8sQhb!n$KxW2h!^0<91;cu5X%jQ2Lfj`vmXghK&ffk6G+b#LACu z#>Hin&|CK49oAq}7O!k{KwQvZ<$z8BGC};7bYTalZ#H%T6T;!|vd;)Hh6Yb@2EPXZIzbG@FI$>K30iQo z$nO!>0347HDB$270PqSGYh14B7UA#z4$T!p!wdy50IY!Rb}Ch1(E}uITXKcr3vB@1LR8;gYIh3 zAQR|d23RY>JcSNsVG~(FAAF4m0Y?`?F$Z_y6(*q+nXd-<3vZZc8}~r57U37g#r(#l z7LJd72oPVkaq_J21rwxg9*zqOa5BiR7U0latT7}f^3Cu-0fA#1UEvHyC zF@AI-60EPupiGq9s&r5>bs!RTEH5ZFY{P!TvP1ysLae6RP{B$E52%qD+l?j7O|<`z z(Fz!Fkw|YFr64cmLJEI!@@(gA76CCMi3Kl#4Q`STgR&75geW5??4V`sih63(lTA}TRpkMO8}Qqyz5Mg4%|=JJukMv5z;kNUu}Hl+zS$#OUMqkop~ zH$wn88x%Q>vtxR~G~iM$Er`%S1*%eqc_b8}vXerm(=A0*L_<#|LDVf#G)4d0koERb z#Ow|0%JU3d;`R>ln+PsW((yb-jjP~OJghP*Wxx^gfCCs{L9yWdz@;k4#X6tG=Qsg) zp0PJ@kOv1t4uRi}CEZMPaL#lQ=A$1m zZ)@@_gAT+ld%{KW(Z_H?^neC+2B|tr3PDm7giKCPQS{>Sa#J#uU%l*VGLqz~S%N_BxB zXtN+~^ED0>`v9*MZYl(Z@Ij09y=KBVu`*Mh(=zb?rP9FtJ zW>tIKR1>N*^zJfHsntwxV&5`_P0_Sbey&^_ichywKiHMs{1SK?Gz?%wJSp}4+>2l^ z6(S;nfZk}EK=puROo4iookrskn;qh4R*@FD)7;*B5G;fQVM03Vv)9Rw+w{ZH7-F< zWF~ZL&NbB*Olm1CObta+%hPin_INTPJ-YTeZ1fxh#utKDY{&Ls*9klv;}O#q35-;0 zBX(N~geZt*#44`tqAB`tHOY82OU28qKsH#nBMT;Wtq8B50QaBx$$Uek2*(%Q4r*@S z7k>YC3W!2{>#k}-=n7Exutuh^{J;$I7l04zH&)lM3RqbS*npoFffpEo(Wq)`QARx{ zw~AF@Xmn`=k=XKt9S?C1fI)bDuXxo7NFR|29Dxv)hHd|OfK|7pDhSnBIIRV6PZPUD z7k*HC;fEEtm&v+Ry)F#}RMkO?kW9nLcZXOxwtxhHc!+mEiQzVlmY4|Pc727I2n%U_ zv#fpN_kYuhfkkFLycmo-3NF8xjQ97j9@q^cxE#;%cQ<02+PF@P(kMTecthBZBf%$v z6h;sMg;_uhOtWt2Z4Imu3}E-W?GuQJBZH0j+-^XXq4>N|If*@aL}0mmuQ;vH(|mc?*gOr3 zW0_G@B9^dLg2!%ycPmo?mbX~UgNL_{?O24tFG&AO*b&4)5rjj)GGHq4Y>nxzz`i` z80h$!%}#-otVJYg3>QS6OS5`9VQCA?TVRgaM^Rm9PIRr%lwa0p{~GVLGN4HJ?kl1@a@2ZyKki zY`oazc_1n!!W5sI8lI8bvS+!n4>}|I`JbkmvUi!Hts1LaIkC4Iv(N2t#M+x*!mAko zep|q^(^{a{DymZ(qXqgu-C9PIz_vB`YmJ$f$8NaSQLpzprB%9+L+CRGo3QDTkcLoA z7MrmfJ3;3oaa$s^EgMxyTOSiT2=o~~z`K-Jo2j9C466FPg_@yH+YV&AvWHBq;~SVq zo2kKiwz*ih&zin-8=x6Xr0_srnE*dwqP@_us*0PqJ(XZix|y9hxy>$9^T-I6$+@4~ zIj0k2-37adCwr|Msknw|efmp}t-SvuThk7kpclHKKY0VA>IQ!Le-zqRK-;Kk@3X&v zKTvfh*tbNOxJVPX#Zwim_eWLJ8i^Acfp1%>x45jw*!{#akUINFpLmVwnj^e6xHrPV zj~l}Kn$#xzpPF;ASTn=PJSXCEN?k`G&l$vH3$-Y=$_xpFgxcc90193l#oLOaLm<>z z`@37<#VNT5THN+-oU?;^pAR~hT`bW7y^sK0t!(ePN>G>8WTLBh$y0oj7d6Ty*a8s7 zpVQMu6T+8~8P!j)%aI$*F~^M!M#ZH&W#jg+X+0)tRX4Ni&7%zDWC0Pvdswsh%Cvl$ z0gh>m-HJKzqfh&wO?(Fq2Yvqui>;phmWw0EYW&zTozls=+UM8F)fdxfTbEbW$tSed zH&s24nNwNJ1Xo$rS3Sa$d$|cc))701e;9YGnxr+`SUGU9JN(0~TM^X#z`@zsn>{@_ zJ=_s~;l~}^dpxckKG{`#peerK8J_Os7uif=*oEvoq`f^ozJQSuoD|%5*=PhfVuOi+ z(@f3)_>2FB100@%Mf;oaQ> ze>~|c9(^&M>ZQKhr~a1_KI=Kk$v?h=Jq_%?K8(wF=1PP;l>X9h>uo-jH#RjqUnh*jV(3zlkINcaNW+bH1VZ6BU( z2WytQAR>&O^fwN#nJ@nhx+F<+<_UrfNRY__^Cpg<5g0rnvoj`@BRxkDuo;2KlcF$dCN)`* zXilv$pC%>HR3}rGVvBSQ+cOMQH8+xeZ5iRGmK#rzT!lOGE?TE>jS^9MS8JP;eOd0k zi-m2A!d0nqkZNPAR5fuzKA9|cX+eQGqh`iTWm-gFq$_gNh${<@@ z!!9Wq@olkuU-#}hH0WF#vLQP@P6fHnyoaYYW_%p1`p&v4;yjO;^|)ml z+?zx78ufSj5rbpgG48fY>!utO7v-*Yz3YKOF z;&vjIdUZFP76(o?RDy1@NrZzBnie626ONf7nlso2*ohz}x0!;FeYsN>no+4DbzhdG z9!=N<8Rc2wrB?)sF}9c}Z+KOSqf9qiC1YVP+KAtIeK{$fA_xLU0+C$;VQGnE=C|s6 zQc`MWp%(&q(-@x#>Sb;h#xUBPb=pbcnu+mx6`xc=A!|vCM#+||i3Ymqp*-ycWox!h z3f%vgS$3CRdBt71B&(l}wP=5jZkr^gz}_S;eQ;{KgLBpjnX8}kewS{uWazPAk%LN! zu&~BS#IUi)O32!@vVDPthBrW)CyRio;2Vf}`37T-fYJ*id;t^rnOWJ5nBvNB5FJqQ@Sm$8o$^ z+dJQg9g-Jox_2Klndh`-?S368$9CL{x0`HVCX!?gLensvS|{5taoA<0L3Nh#W7NcO12zHg{4T~YiXk-8R&_}xX zX$V8E6H(U!q_i}NP^JpO^LNGEF&}wuDxcCn1VK@iNuQl4P`yS?~y#I~l?T z_cG;yCx}D*)LZcQw+Yt7lLp#hpB&hbp83#yPY_M9AjHCleWHeSyiKG^cgxwtW?;1> z)2l)TziaU*YT0XABtto>A_|2uO+=pBz&A+}MX-v)tPcn`)J(zTFqHed=J+OAC~>9o zddgyDoLu?FSk@^L{M%q4ZOFBvm=fKj(F>Jx=7_)zJT$PB_lejvrH9 z441;Ea%#|}8GFrgeh9U8ndh3ROQZXqX*OZr%suc!<(Ud9B(5+qKkIWQd)|{zfyVS- zK>eim*p|Uc%@3SA{GMd2fzqt06b$VP>I3WfABxR}moW+{S(-YLg(Vy_ zf(f#w4WsWwpj&C^Em}$}vvDoyy-az=B#Lrom))i(1(Q@PX2Ef=`qngO=P4WjGPbtu zQR$T0+CTc}B#BjRM)O)(uBu78AkCu;FQYkVnVn%t)TAl(tadF@ zA>pxas`Av)t5%fT@h*G+J#kpVw^#`~Z~SFk zhyK*dje3)Q2OBRdQ|93y;~*MDE^>o|;oy&~FS!X#ta)c@-Xd!6HS5(QKN})muCfP4 zH*T^$M+`W>eEGRLwwexMSKY#b%1k9PWMCC&(K5<;#`+~`5UylpQyy%y3SMI?v-?UU zCmACVrlXcm+*)$ZM95{$GmHDjN4bJ?h&N_0h&^^&zOw&xV=`_tV@Lem*~)I9Zx*FF zP5WHd2J&=YU~a2vlLxp>m&(*~-$(^Q*D!!}*T`_ASO*=kH3^usn(khi=~=%X^L1Uw z7E4xL+%(*-<(N}$EL|0MWq)>3DIaB~L@N#Lk_FVDGlX6iF^oyuTr{m}vICyjn`~__ zRv>Q52CsV^=p&ova%M>MsWpt_-sbb3hi)RI^$YLkqF38bUaZ}0_HJ_zkIaS&1aJ9; zP))(`+z*8BsB62pjcZi6kT|fdQXBH&9%S9QA+Nn(j&FSn8kQr)b}l!n(GDTFjEKJR zyVM=x6?1e+f;A)A{K_TN{bz63%-cJs)lHo^Ju?5`=3%yH9X$?a8>E>1(_`aZ1VzNU zf>Kxd$g4)_0dwR1gGp#Z-4Of=jW7Q-DQv>o;c^!$~bzQ_7Ws;bv?dwi#{bb`2P4mEaJ9 zS9=OVD6S@caCJwU=4^Q6 zfO1;=r-k+Rg@CAh2Izj?wR8EkRaen^)-)!h20y7cht|i2zUGGb28YKMfODp2FKCJ! zM@!B^g`VbGItMM9gou=&Tn zFn42?d?~hzWGIKev0RPkdSF+DDQPI!Sa$Dte)))H;Ras*Q-T?p2gGJ|0;z#GsCY+I zUKJU2NfUJYms1~zk1zR;Us?YVX<(I)HjImCR=H?Xa(G@unUhude$yt9?FWQ-a%o=} zWlYGCSRszx6in{}mH4MC(MM_d_m@wlU>V4XUa6R|fC-V{n1x^nkNJZSIGN2CmlJuG zKtqu7BUA`kKsPyxwfK~rf{;gnQTAs#dRUB-=pA+>THlz46Lf1*IF2w?gd;Q=CWJlO zz?=N&n{233r0@yCIh?`?nUX1!l&PG@nVgzfba8lQubGgTNpVc(fmb<9w1$d}0f6CG zQWV*XrCDt%7<6#*As~iw<#~%phdH_CZVRHAzv+?>_7Sk43if%Q_(`0_xu2%spZp1+ zjftGhNubR+nqpa*4)OnkWyzrK*pZ%jblwG_>S>N&Buu3kSt}@U!(v^abeWd%8D1rN zP-!OeX`+kSU=uPr`T3tM%A)-lpfSpyg)p2nDx;1mqsghG%UK7Ish|t0oIJXcuu=yT zs$NE#6%{#+W;vcrdZdrII(4Z{-s7A#hmsH2o86G5CyJtQ7o6$yqG2kgFiNInYNlr@ zqb`c3Icl6V`lAIJnagRQ&sn8?1*A`EmrLrFdWt>RlcA*;j0HA?UILxFvZY+wr7NU; zpM@;?>8Ou7rjaVClS-+TN}QIerfZs}$62HI>81cGcgtC$1lprMTB-xOqngO0rAnwA zW2&dRqo<0Xlu7??IH{bE_-BV2pNZ~s z;i9Vmt!nC{%(@D?@C&~%s?J&p(yFVwx`WT)Fybn6zS=^;`m4x_uIkFJ$GWcV3aQ2# zua7FNz*?^lv#9i$t;X<`q!$kVim2ea4CG3%*vF{vnyv^7uL`@c?8>m`3b7FzvGkJ`Vu)S)o2z#&(E3YYwvSPZjCVR3JtFJH{vl)A_=~OHn zi?bdpu-F%_pkri;3bZXNv@BbxLrb(0i?m6rv`bsDGwZZ83$<_5uQ}_n=3uo{yR$*7 z49u{#TKoU9OMA2|%Cuu^v|?MfWQ(>OSo3?w4 z8raacd<(cG>$ibRxPSY!gZ8$IDz}R}w^f_9J`1^#i?<`2w_jVhVmr8cYq*+wxt+_o z6MMK8o4BL9v5X6ijoY)5`?EX?v|Y=(BwM+ko4K>=5wvT&zM8q1Tf3fXuD2_@mW#M< zOS;2bx(#EurH8t8tFy2BxT?Fl$|&4z117B)BC&Do4uQ>sN5U6zec>K0k`2> zv&Wmf%iFW&>%8gPtI>Nb*sBrl>%H3>A@Xa#+Izpk>tN#>jd81eSZliKtGvxiz`W|d zunf z^!(1sEY7xE&FS3Eovg*FyvY1~&HwDr`3%no9n1Bs4hF5yh>Wvj9L%K*&FoCk>O9f? z?7`z4&k2pt-mK6A-OvRs%@4iLr%b&Bz0N6p(JKwoIGo8Ez0n*U)AlUSAN|cPEz%*) zwHBSzDxJ=|i^V>T(JW2WJB`vq4bz_7(K2n*0L#rr&Co3^(m*WH70uIGT)RkpyIO7k z)liMoQvK3VJ=06g)J+Z6I(^o^{J=+D)Poz;4&B!N{MKn*)nZN7cWul!P1nFl)o9(- zYz^3PZM$`S)g$f8c8%AGo!5Rn*L!W!f(_Y1?bs1b*k6sQcuDGuW|zT!4c;1*uf_if^cz2oW&;w`S@MV{m%&g9K~ z;qyG?K2F}Bed0Zi2og7j(*<6{N?ri=iNQ&b{^`Y9^O)};7UE;yvEw7&gp&*=_-Ez=^7pDuO8B; zF4JvZ;jBL6S$^2FPU%Vx?3Hcnxo+x=?&$|T>dF4=l}_uQF6XMP>-!zk3@z5>Dd7?(LXf>dL{>ybj>)Ughu3-~34L`tI*rPRj#N@Hrmu43F^e9@`8~@flz2XAIop zuJQlA@DqPN4gc{lUDh5i@}N%YCBO2gZRi#s@&MoQC@b*AVZ2A=}{uS zdMHmiG|6(M%9k+d#dJw?rp=o;Yv$aUlBdp=KZBM8I<)9Vqa>3mJu1|v%$`l1KAjp> zs@1AjscOYaw5!*pNWn4Y#U?cT!UOj)z%(wR}qCHww*9)9`lXCQwFM)#k9 z1Rhx6fDZC>UV;nG2VsO4Hi)5r%AHnWho@SI3iz1Q;V~Z)C z_~DIz2^QmyGlEB9jX>fkWR7vkSYnU&(fH$QEH23)k4Q#&q>@6SIAoAYLfMazTAFy} zmsUFIqLf^g*`=3YDww92Q=UntaRIKWRg+wWi6xuzwfQEMa;|9Soqz_)=a_m<8E2n} z3aaR$gNEp)nP@U9XZiEUrs}G`_G+wZ#r~S>vc&T0?6FRP1%Dj2(LLXyN3E}aKaGN3oygd2HS4FtwyZz!^uuevBwyDyRnrccf6RuAg4@j$R)ST z^2ZCS{4&P{pNFE$Cc`|B&ML+;GtKVK3G~m6@x1fVNg}N@oIW@0G|>k~4Q_%mll1h` zSZ96p)>?Djwbx&RE%n%Bmo2v0Ij5cFjAv^UcH3{mE%)4XqdjzGciW9;)p6JV-M8O= z2QIkYc@yrpI)$sf^591!&iLby11|aGI45qo;tf--`R0*dZg%CKPu+RwoR@C;>5iw) zwd$<5ezxnR$By~v=hO~+>fYdPd+xiN)BEkf%WnAVs|Rm<>%Sily703<9nSOG=)#Kh zyBMPj^s8h~z4Y2w@BJ^+hd({`(NmxM_22Ujz52Sd@BaJAD=+{2^w%H1{rEHA{Ojdc zuZST4oI?N+F~B+)uz&;vU;-Byi3AF;5(x}NC=@tB0%8J!7n~j@uGhWoNsxQ!6QB9K zvAq$l?|krcT=eeeyB4zWg)oev3uTBro8@m~`*U6e9ms_-1W*kIB*Ps4z(B+>7*U9N z;NcJjXooi>(T73=!xNWS#3%w$fK@aE1nY3YNF4Bh4E)~!BjJf1MzD-!L`3+=*a}u) zqJf_H1@tl)!tZG>d*nOe`udlRJg!iWGwcQ+`#4BJ29kz0yrKSRmo;yAk%@OW;uWn( z#Y)DZiBrtv6SFwQCwTIcl#F5wfKZ1`W}%XvoFo9Nn93}Ml7NgTASeq_iR?WRfw)v- z7rQ9IUDooAY^0+wJ4nn9%F%krG^8>K`Nw58vyg{OWcw63ylP@Xm05rS5uk9z7(mjM zmkehlx!Hy$LUEEkbRsCziB3_@l9Hb!Wg5;I2un6`oA$(H5*5M!N+~W;iSzs>JY5-3 zRJ!tqxx^(XcG*BJu5lH-yyG#EY0QZtZXf&OvOcGXbpy1HOkQ!R+DnO5UEU$Fw!`X zv#wNdsW`)FQY4}js5&j_UDZ(3r2>|xN(G``6B~$FdeW3P7_3o8&`N!#v#91A=qNL5 zP|LnDiJMhHXNihb!IqY=Ni`@|jatNgB4QF|Y^VZfE7o8C7BiX8o96PGri)wZv=2Q! zEMry4S#%b*uX!b|4}%-Z;U-qNPgUv@ic8vTs?)D0XsTrQic})j6tgn@DJj8A(&CPx zrrI^{9gaKGte)1oR9x>6Npb4t1VRO>9BGNI^E*(w1}tpIN=f!kTfcw?4fmN*mh; zZNl`q*WIT|`$}I3lXnY-Jt;wRnp`~~!JC#P?Ni+wUG++LyBX##JJ*>?5}%l{^(?|l z8S#f0PdL3&jp|Mpo6^k2lZlcbHcOfRua&LmdMTXc9iua)t&Or}*Iei4 z27w9+&aO_oJZKJU8J`tSv3FT)WJ3hXc%mR6hOM|}_1fXjW!7Yswa~-JDp<=8ezF=U zi)Kxav(I2|1F7TeW>j;5&?6->Vp(lv5}eY!=auY?^~&7AhFZ?JhNlfydgH(18NEgh z@==4W>>@K7$$fY$Z#^MtT~9j1Y`*85vt8m)bD6#5>~Wb<8)X(-dc4-Ywp|5%f(R^^ z!RU^Fpr31N=h~UC5%%)19W3adqITYu^>$!K&1t_n_{;eP?4MC=>~to)z4rz-w^5B_ z#eVju(ng#%sLkFm3)#@34t9EFOyNLay25k+HFm9`T=0$;{NU~NH?c>+ftQCI&J({i zUXlIma1z^yKhJ5sPh8xE+mz_1hPk>?-exf?7Tp4WcXJbdRHSFTc9eK;rc<78SV#QD zMvttWZGLYoOV{1semVz+-sR(d;n-l$D~KuG0(;B1#0~Z|xhYplRNUV zf0)lvcerQ;yZD`k*3|;%`i5s3TA)qS7?aTAAMkEh!e4gWUQhfM=6z|S6P)CyN4ud_ zzhH2G-N$V20>Fcgc!VGO;HUIMeb;&OL<#(9@UMGF$Tc3sL%bnX|e|kAL@AmfB zJ@2=_G^*>p{>8T-{wEc7lq6nlS8*qXcEDA7u*QAsMP|z9be1-LwzqHQW?FwWe3BPt zJePj1L}+$%dA}e=`ovuKH*S8`W3q>W5~yzU_GuenIqr+bulf%^7j?Y3_x zKvH!n+S%PrHE@ciE(Iui3f#(D1*$mf>a27A@F8&WrY}MV>|d*O|^)R z_kx%PjOiAJ%lB{HrhW zxPEzOb~2f7R~U<(SZEsm)@~7Lc!`*U8cBO^hlp#YUCKCojIffk7j)T(eCv3LA6JTH zHjywSJ(*>UzW9*$*l`g?UEL>o>jw#Vc$4>Ndc%lZsU?3ZiH7Rrh&M2f@hDVi_j75f zYBm^)#YO^lIgcW^Wet~%UdI4qXNk->bv)*eDaeY0xs>ggk}3$6s7NcJXolm*i6sYA ztC)r}sfcvNU4z$o9(I!r)`l(Vj%i1iHW_T88Iw=|n5bw5Bk7S(c!Wu4fQ&htK4*v9 zRF>+8ioJGfffjB^5R)#rZn#)!hWT*OIB3uacCT4ig$0=#m3;`7dq8ZP>2cZ=er`C3zDJ!q`F660mb(}TYsZF+iCUN`m;>1b$EAt<1ef#_ zW4Cu}Hp!ZS=%3KIX+-IjE3Vz-CVNR&KipYIo0 zT{(N3xuUzsO>QZ6jpvtlIHY-4 zpTF3THh_(`xsfNyrTUkd2g!8J>5^9Ze8`oHPidKrhjKs}kSc(rN*bYZwUC4siko<9g7)GCY6SPCK~RoD zcxr2>cLzsV{;7-R=ce77wWs8-IFN#|IF9RPjJvyhkBg@Lh@Ic4c8c4hiljc$V|U{J zw{tC(l?6M0gWHUR>WseWX8fnKzov5r%WotQ2S@3??AxWvn!SE!kKzS-zB-_{hp$Jg zTBNz4u4<^E%8)XPmn!$@W7o@;SfkB+5!3uyd zOhp2Wq9>PG2{dKmb&5u3kccf>as#2IR?06Kc%*L_7Ly~$g^ZF|DaSD4(SP4+v( zXQ#-Pd%pBGvH94e#wxa!EWvsSiPRaq1iNDK2({R?qv1)#Fk3<07HbD6ymgqfsq1mf z)_?BFf*iV>g;jv}=bb0~o4aYiz3a%gtjQ~wxE`#GsOrn*7{dZ=iibCX_|~>d+_Q+A z!x<}IpS;S*Du&2hk%(YbYm2SE?2lPnh@?EM&?eTS_A@0YA9K`Z6~+*S-c0EdR9w`llaAeyUnqB!=T5ax<`*u3c*DGFuE%z$1$+M z5L%Iy5JVg8#?Z=<35=0!hNn$<$m#gHwYYY1*`a`$fI{_|wMoxq*Nbb|(jJYy<0(#Z ziNn#ng#HVco>{L@=bW{>r@6ex;Q4=0#$09Vr)Jur^r@f)th`K+!PxC%Y6kOy-Q`)Q78qZZAk#=BEziKWYozIvV3 zH~hgf`__hCk72;LQL4B2WY>4SJ8J8xKbXA0jM|dw*jdQf3hl${NuPMx%?5pC$5yfd zT~ee+prO3m^Jb~p`<7zucAdG)5!`ToZMq_?d?wqx*_O@Ye9_?lTfzFApZ!abXj{ee zT8TRe*>VS*pk2~`oS4GR*6-}iD$90K-IS%gYh%mOMtjhdipZ3ehv1#L);$K2ox*pE z#S)vcAbZmoYR-jv)6DFvOv{S++S1Q>hvfa*{w#<^h?Gyd*GLT0rR!32d6G6ahrEbR zj_aco%-`kNdsdy)N%xYzXO)ZSoSA6@xar)#Si(WAm@&Y5D!G!Hyt;>($mm_LN^8@M z_nlgNQX3AlgX9T5UB=Zc$XamW$w%D+djkQEXwj|EmK=d>Ez$z~2kh*`Pc^|3{hcuu zzXOhVVawwOLw6}8{}dCIOQDvJd_;Q&0NVY{9)}| zv-d0D4N2i2$^f!CbC`?fO>E-dExiTU>B=pQo1LbdTHTrZ(0ojD!KLb4yj;aClE(ef z@C|4M8O#{m-^lHz4wk;BCO?7N=)`=DwB5X5?ygz=;5#da9WO}B5%PJw*u&|ut$P%D;O9>bjbwUtfA68^qhc}4WI zQ5}4dhq|DlccrWS=PEsu`)Z9XtjY}?cCl-w_KVs=b${-5&Ye!OGG>XBIRd<0t|yLZ z-wg3}hoIOU)jA8&ty_ay%gtVy(!2ah_;X00PzYZCfa0!eb1^NqT)w}pe5MV&W#?S- zW_-Bem$>+u^3^@cBOJb9i-2owa74`A!~mq3x#8=f&-Kp3X=obahPc-8^YF_vS|5_h)W)vuvlUip@n&$ZfyV z9Zcr(hV-Pny)@;<3Hs*-3Hfyk@PtP6-QA)X-r&(bwCLC42Mf#}toM=5uhuvD+1N#-+lB4Ao@R7~}*)Bo2eLc^LyG%!?Is8Z3Fx zBow0?d0Hf6G^4t9=cG=hYR>A_a9FYa@}wst7oaF@Y5^K^C&oKeICLzS^Gm}PErad= z3)ZK_2y69@h=7)?%bR@!za{AP=7=3X%Nm|5GHDBxB!(WnVe>G9i7Oc)6d9ukXU9tG zb_p3#?P0n^Q`#i(u%|;ZkT_4=vJvzD&Cz8chR7RP1&drGS&@J{*<<5QqoWiw>{%*u$_u z1LJy*xY|CH?==q>v`jAjf)g&I^ki(2x|CMLFDf9t6B4|wCQ-`93bUJ#j0+)Ts-mCR zGfT9yLb8u10Bszt!nX2o?}P@)3dA|wRJd&}+|E3ZAn8zRDi0M;NXxRyGDPS=>y%5c zO*b|2kGVKgW3Mef>zpqRFL`wT?lIVE>?t@7O$4d0L=#dHDH_kBQo~CD^~_H0h7`3( zMCS0b(mi!dE=RMTY_-x1?~+xh4&!Te3_U_PuERpN=&Q`Qc$)zQ1{$?>Fcb4auh$Y~ zQf{dwZv_m}g<@2((+Ua8$;-st%#1%xpEJ(K_8^RM41?707EdklODa-ly*#y#c<&tb zULsfV3d7m-v(nk+OsWp80g>I-PhZ;|Q54M3>{U~I%ca$0GT(iY& z3s}wgW2KmF9*gI*F=dbP5x72o&&kO8}czXKl9C1=X?tG+owT& z&|Q@ur_-O~dTZzfuYC$#g{Am?=wXU~Vd4VSi)%cFMo z5sx4#c~S$>Zgxlio0T!JKc;Kn|Mtf@*rm{YQ$yj~(3U{@T@60NbJh6DcfM@oP=|O! zgB*Ie1|TjWh(cTgAT)srStJpOtyrQGo9IL(+Cqv`qy!q25QQOL@rPyb0T!chh4)?Y z3R@Io7<&LlGp-PgW&~asEdaa!y^#*hn`0fzHXk{PFnRr(!5<_M#~uu;FK?Ma3Het7 z8lWJ6VPw=I0|`JyUOtPqMkj60Lag3X^;u+Vt$w$Jm405=n7d&ao(vd5JR|{Q) zEJ!6A-cX0LTpCxhkb@-1K$j5UWiNY~1U9_l8t<@&JB)cub|e#-%VcITjoFDqz#*4# zpk_7Mz=uZvVDk};$V4~2$<1$aq7&U*L=uKr1u39Yo$F+0JI|2LC2#^9#GGe6-vN#_ z=)jy_T!KE&xz8pLQK0>FK@k5r(1E_woOn>EK-YOrgYJQ$5e4EI{7_LjOcbLUQi+5G^ol1 zDl)mji)4T!73|!l6=2|0CqUJy3VkONChEt)`B1A|#4vNGL66K?9un$~m)ImLxDi7M2gg7&kX73w|8QH2a#HnSbbEL**qipr{teu;Z5 zxoTTI#+glxDoWUK+ed3!5;6dRaqt!L*kN_(8c3=8YHC zx0ezN7zu2Ozy%mON&|v;s}!!V0bmSW>E^fwk`&a3*Vb64c`Y^f0}ObLj6)`r*NBAw za6}>i;Rar+x4k?aZG5S03pRit$_c=4Yj-*evL2HYcO26KJB&yz%!j~um@u4hhs77J zISO&^O$`=d0<_6F1)low2O7b}v%Q%Q6J`?bu*t~6MZgH1u!1SewwEavw zB>d~%Dg*n#&~L07MsybZ2}NL2iD#Rwo$-PifnCe zgFY#RvuyxSf8b+w;FY%dfCmvw2h!xO(z$2wZXYo6!wiT*HVv5<0-#&c|33HsrX8)W zY`YuZk4`wK7tU^gA0gCO1q2&Nje%0W2X!XZ?$2EF!f^Wu*0ILiDN4tC9tO6*x{S=p<#LTuaD3TiZ?8Qkh)nK2^*Zd=FA*Jk*3^EYPU3KtIPov@JMjqa5m>lJRu!JELoNLR%+}Fakxs{74 z^q}kD<;mUv58^3B5d+;DvM34JmLBk=W82-24Ez_eeg?F?-p)G6{q5ENt-!LM`RolJ z`^OmM_iNUmglM<=-AxI0AGF>1Gl=^OM(p=^%H4piX@kQ%{6D|{-QZ*+#3L&HR0ywsb;Fd_38r~)9qG4syjyk$Bnp+kiX(7B!CG6%4;WIzQ; zpaD=Qz$pl<*E@jNd%YkcDcO6c01^zyu(}`6z^`Mws~a#7jJFoaJ0~cDWkan6s;t(A{USV1uR2l>pEwXK;`uNdxDh}RZ3l1zK zaJi5Y{5rk}F}35EzS98~bRe+{LB8pTD=<62K#5F1M9R1TG{C{USr6%3zDV-3KRY-n z$vz>{utKssAmSKFFv3zBDfe@|jMV4@D9_ClLrc+cQ3kfe^FC zNOA}YC_z3#fo+RJ%DAMrQk_ZCx)#c$-6|m~YR43SF`9|S9x5{?6U8P7BoX?Q$*~l8 z@t2O9!ta_tT5|-;`wtdKFI%JpT+BsXl)zt1z+kkmGq{HTZ9t++AhW>9Dm=;`k8GYq z0!cdZp$^0jz}m6=d9INRtUIbMm&`OxIiU+t36F|>%uXtEJQSe1ztVptKfCcci zpu4j#(yP5XI8MSzouo9Ngv*r7$-J`5HB3VFgGabL6ik}Hn$*jhe94&9NuTtpIr@zz zgEI6xN&*W5q6wFdQl~traD3&;1Uu+jB~t_)Hy{AA`;D%tXIs>(c8}q5Q6^t27ka( zEhRuM?NV{@QqUVv0+j->B+~*Yz5GPbAN|GuG30>=wb0sxQx-K*Ge9dlO$9tX(M#~5 zS?Ew2ebML?u{oX7IDJu-ECE8T(cC;#934*{-L2mI03$FxN+qvKrBw7I(zxKRFsPMV zX+Nd_nofPvsGJ0wno=q?P%+TTAW^;513e}xKvp2YS*2B75Yq~fR07pif?};pH9c35 zv|S>@0*FwFLP8780gXyl5tsmEwJ}_{)3U0DPN+?2T7+uw17Ax8Nh;Jp;?3Q>DBm>E z6iC!$9VJHPPA+oR5NuW+ZLmkx%XkIMxPqq__*Hz()R>74CxM*j>8?RR(k8V}Sp&_K z>ja%LqVu{}u<}fuY6mS%O)Levph}|uiltaEB|uzS0@|F%Q(8+8NKh`@yZ&l~1}(z@ zxXmzgg&Y9a9OwkIY6RO%*$9QE3#bJ&uuVFcR;}WM{jydOtA$H=woK^ON5s<)jnfp^ z13XQIB0T`!L|RTzTBseRL0BrPO@T~^*>9E5GH6;-G6Ht}fkgmYkL6LM6)+xMgrN;q zA;*#5saW$O{;PRb^l`h(J_`jbi%(MtFq^ z&I1jeL*4{m*bF(P@Yza2U3g;CWeZtJ{arNZ*a+>nGB^a8rP&vrsOaTcAZp($y3HRb zBOCjH*-hHp4Wy*~0I#*NB7R{bt}N*F+K);BcLiUA8e3kSt9KPKOX^`Q{(&MY-ye-q&zV9TSfnstW^NbA@I+cdk4^jMgIc?A&3KmP0(1J*f^l$ zJP?KXiUUaSufVefe4^3WRI%C;fiF_iHg%_3^Z>NFhA8-8-aR7ZebWO7))K%4IpAUK zeFQGrNE!5l?Cc__%}}zv+UlLqw!4JqrA?JB+e`4|7iL>mSOXLInn;a?yyQY38-47yCnRMJ+=%vNBsUf9ZrZCpa= z1cppiUb9kYn_g(^1^?TrL{Q}El_y?!1<{)TTb5KdtE~xc-&{RY7f9JJW~Z^vXd3P} zydrB)zQI8N;*{lGY89s$e4kK+8N24_j@1FXmeeL@-oZv}E~a6EK7d*r;@w^BmS${U zr0JQ~C(X8L%-(Fa%2%n%REx@}3kn(R0g-HJ%FrQ?Z-yyL7~ubkZDWG6DQ#UYa=hcPSe3C5?jR#N~V?x2010-znj`8u1@qj+;^DghhUhm9yZyt9m_wMle7VYRd zN<|XbI_YM9*&ZQllGQl}AiMxX4?7OQ%_wD-=}VcxWAGG$I=JmQ#7i=a>30Rm9RhE^yA zgB1cnn0G-CXL}z_(A-B|pX4!50vU!TS`T;yRnth*1PeEVe?~U@68MD|_><-JT@7>p ziDztz%BLtMbZF1`j6ZY)*!YPb>pE}bdmLo3jz)W=M)C8YllbKHlJi@>gNlG zw^}F5hmUhO2VX@ubUN>Pu`l$w=J>H^ZZfC#kPqgq#Zyt(hHc>T9)5Ka)r1F}z`$t< z!F$3wGGiZd@w&fUzz@ZVD%(`x6Tf1d2^l z4qyll@LdoDP8bCvO0Nv``m_guG4g3M5BPurUp7}sylfv{&w8m(P%xkPWX*m5-v4;r z_js;9d)Np2vbRK^o})w9;v<5hJ1|~qa6obhx?)R)=&ye3&uS=&BI^G#692Ml;068y zg!Gk=XsMAz^9dsJ^vVo_R}ItE+XYKlxleF+s0?*cw}f3Y`p)0#Zvcp0-@d(b2j~?+ zPJiUcIrK1d1{3jVOECT5g@)wDGgH!X zjC-7M#uPKe&}@1^aE4wNI1}yc&^z`>B$iQwgPOHN7g9=;-AFkKB94gp?KfHmw+`6_ zrLso8SM;#91*k@$<_8UD$CCpzq>ZWt7A2+j+5kh6$*C2?86XezzGuWE5&I-k;u98L zXh#4Xc9@~?HY|8g411MwR}w1vp9`RZwK+yTFw_*P&QY^Eh^)Fw;)r_5K0B)o{;X#V z##-j%vRWvu1&&Q?Mr7{SQXw^#?Fdpl5LeR{PCM~r`k%o2*13DrdxP)Kr0wMLAY)D%>cdg9O(9!A7qLhPHehWd@efAyRX*T*E|G zhHA*+Y_n(}x(2a{2o`5@nfL=Ad{6;CRDmLJpp?%rwiT6iYJNOpUF%Z#I@qzWg|6zA z6!L&OtF7w>Y7iEBaLBtIGS6vhKpu;Bu_Pie$$75PUbM(owt^&Y9aX~3NbZFghd5+Y z4lNfaR{azkKLOdvrl zyuxIGcwj0RVGCOcQWB7$gsmVVNVSa)Q)jqjejPMpyJXAg{0zCp&7=WlE)d&Lr8}#5FE|)0FUB06ioY)luM`fqj*$FV^ zI>RE8>47yg4skKzq8qtboFp^~js&yc3L4QdFG#{W{EE-`6lMWqO@JzJtj`p7g$2yr z4=4jTfcVT)gx%dJ3-zQYKKrDEH~vW_i%cVWUqTEOyZf6q0*G91(*yOMY;sJQs^`R4gQkF zBj|t%v|zGhL@^Z_f@(1^mJfm2Nt6z>sfi%`VJ}xa6cvN=w@2jbQU!zRZTx`L$2m?x zBKTFGddH_(jWw)Z@IqPOaE)uYLS7>OtQ7gi!X_2c)pZF?$3hY5ke~f5Xv8~#?uwvM z%#dKPJv{7=Kw6|iux2@3%gsuup_ZMlDy7Lli^`a1Iz*Z};`<8Bj z;#RjA5kq4ZTFM49vuIIEhDwlHPaxt~m8irO+GaP?)gl#^ksoX;_b4gg>9_IuRO56~ z2KHR6ngKYfgbVP^GSUQ|d{eIfb5mT_h1_Aezycsseiq?{iBGk4)$Vo`3Og1>lCLss zFp;CcPh?;A|%Nb zpU$|12EdTFIC(<(>bO;tt&Y#mP*XzRXvno19u1e3$|8LlSj4n!8a!$V+whmA2i6%( zBfC;7%c#s3C}$aJ$~F^!wIQAugc%MwLF(9AL%sE|eJZ?(Ub3zeCD4HtW>VAuV#?VY zlL9!C>IdM4jni*kA7XL^OGb&J>3udXU zT;P|K#__-LDsY%VVhuqxcoHeEHjf{jq2eur%q`Rul8yxm#DHKnssOy z;U@(vArsXAOdh_B2f>^TCi(J1h46sdTcdy_ZD~X~p>l?8WC_dIV_-KQD5wTFDkb1} z#5*#T?X`XY>HbO+v^fD7alfNiN@&qN*n{<_nuLmimLa^KYqK~UGFgAyvs)?lG0q*p z27L(lq;Xv7bkG_?aCoT0e{@)8RVJTLh^S^N9tO)+IWvW5GA10saal8-@RT(q7HY2N zCYT)MU%mtr*(|C51}>FoT)~(&JI*>3sniynm6)B4ZZ_3B0Rc!y7dWn1i!>o%_O7S} z+Om#@tPf(_KFE4p?ZtQC$y?-PeTgY^_!(R2sS~^TqEYwFZxhnBE~AfojzhHGcfeWK z=5?AYM~xmFNL8;L*l*T=Lhf-WlN{rj>vEH|JTmLy9+_=KhY1(>?hwlNOR4D{L*@3)$-Jm7k zWh@`ytXyZ|g2!Od=6u;_u@LrAnkT)`uJBmnw8XI#R%2ik0d$o3`H=aUpZX1x;XTRw zbxSfZoBS33p-N>5z2yL^>>v2hoE9~L2E`UO`G8n(8C*R@=p`U!C7=S%LWfn#h0&fB zEgd*ufYX5kF1Q{X0zw91U^2x-(qZ7kWZkl5(1093u#F$PwZRDf*q$uk8vt3mu}8I; z+#?(eb{v)F#c#7(+rLLq(#bgB^ET8 za){X-cta?ZWk90kjv*N*cp^R_7SlK+sr8QFwZUC7mM*lSMlvB^ib?&+kRj!yDdtG0 z(OfU8WMWd)H_am(yeE7%$hgHpS14F!<{c4`fEpYE7kZ{>Vvd~!fXOw(f#H=_Q3U+B z5o9QrV!-B72GtzcfrN@6I%WnaOy$OmjBVwLgXEd|+`-y?<#xWMtwaI}9pG6q=ZU0+ zcO;I?(PxmM*C_T-_(fci&EI=rBrC8&Bq$y)KnEhggseG1$9aLs=IiyAbBN~3@e_oi2B;tjpz#r1Yy*P_x;7|uRD1=TZ*~toE zG6#b-Sf=F5X~m6A=m0kj(p5}=h}wpl(c+0NCv%d+G(}P2K<9K;r{D>pdFkhL5V)pZ9+lvz2;K7z&obny37hhtfL=t(AsoM+PN8G z)DPQbPQO-;i$0o+)Tc(;6IHKCUM;c_?RUPkU(y0p@PU?*j>2@rjQj`(~y^3l3$grAFb#p zkGd;pedjQBq^9o4nRRF16cU-*2X`!lY4T=69Ng3bMX6e)8_vQ^7Az)(0nJ^2nt56X zXaW5Y#gh?Nsa9CU8ceB@A!@$UnHJdya)Fz=TU2&zSh>YTB-uCS6injj0pZ){nE~O+ zpjpBlTE44yf!93I)4L5{d8X;(`rY8kn!6lfyY3~?z9Jn74Oy7~4z5H<%J?eE>_>M% ztxUP(*+4B@DM;vHty@Tts*(pQkOO7XT<1FM7BuY1>}lJ>U=@rejLnls}hljT5KX4gpymDU*0WNq{^sA z0Z*t|u5CS>lr7VUQesb7BNg0+^PHieNJ#vj0FS8k1O#c!pK&AHf^Oyn*{ypWRPs#2Ew{r=Y%fHPAO}qJu7_>%=UIK z3!y0DhA+1kBtzI%xXKe~^f39ME8uZdMiGz1sX&Cr0sQj+Mvl&JSX!!bVrr%qA_-Nd ztsY8cnv=YU&*(f#6ck|0Bpyb>-}J}=_fc_S0;Bl+;zEqkqy0g_b!l-WiU@IHwUUYD6phpJ}B3@{W>WHE04HukD7R%}L1a0NkOy zKn%vT6F~!_9K>iH-!Y${O%uT3TD{hMoDYbx-B;Sx(AIFbiZ5E2Ue7#KErw$5KyKtZ zuKFUe0s4`W-whAzxP z(iGeklx3+z|m@PP)3i`nwhlC=;1a;qLc|@K8sDUiXPH#7}Lbb@nM)PCPwGn?} zM9$@LdxpHSs49jtk81ePUfB~LW5?Z~A+4x>fkFb}h46BA0rZWE319=e$(izIn~}1L z3vZVWfS%=CEsHm=QLb3In2Uu)HO4kQhMq*Y|M8Q(LYygi(Cjvnpfzs`q=G*oTZWQ@ z7Z!y}IONK;M{(?J1GoRA8fGvSnX{T0$}~9_M_zB%h>tHT#u{WnpB1-V59D2_{t12k z0gMw8p6&Eb_cUn!$xcu#dc*TCI(UWWGA@T*Qsa0`Wb(+awgHcY49O|MhSWrYQ;{&Kj8r6Nv@UWZAWP_&HA~#KLsf z7c!2x!K=DorbW+LHY=7P59ARQdW_0-?o^rXZt%i{^aU|KI~z&Y|9&ut;lw{h5+9Q` z#+aDjj*1^KP%0p?AGKT}^dsPO*pTNYSlkSNKaeM+J9{5=Id!wUtMytFBuzhOC$hU^ z;HQm}(nv$@z9$nQ32_WCmMT>JDr_gLgK^pGF%D;aC zELgqlW5OqlC3URDWs)brh^5`u6Yuj)Q%HuZ{**FZ#Gc9HeF4g_)%~)=RclMA`k=^d zWsa3eTjbo&CBQO&lb$HBvW1E2!m!iMMUtFlfXwXyo|Vc(RP6;k;vZZuoc zqpQ+@)PuD9J~GY%M2r(uv~UfuOy***T_Ty zsZ*&2uJW%*Mdgy$oVP+7WDIPDO`Ll53prR$v6fLu8VNfFpUHLe$84o$uh@yrMILoY) zk#0n6aZx$h|9#THR;*)S76ej0$0_yl_kF%;=K3BFK3 zX~i8^{##5a$^e5)|E|h#Y%Hk)mcr#4TrAWP$+j3Q@W1-FA??Z2R-3Xl+Ge}5N_Kbw zMZQM{gkuli&U+|0fs#{$%;U~{j*Rfa5NM7#m7C*W&on8i%+ktuvCNVeNO^f=k1&h7PR)V3qLa8a z1tzXvf%QBX|KUF)@sqpmyeo7D&m{iLJvk2Dgp>ACd;`5aHuS+gEY>@u3|e7iccQiz5HS2>RU;;FOAeLoNv2cJo(+j3D1bEyrk3cq)r|j?EqNy%&FZqHL%6IK z3}%tsky{pX1oGU*W|AV4TEM}>>B_vt@Wf}KboX6(=Y2b!xM@p}6F@lPpo^0FHFMxP z`;M+oIb#8|VZsa7$QMy2QNje1Pn?*^iD6>=ah)6=>Pl}&HX#R3Wl`J~nK>7Q*d=aW z01%x+^iPv=Dzyb*p||*gc9B88VvRvkIKs_!N7_VPtpD->)e@wggldxyv)WwYxo%bi z-nRWQ|J|Jqz;tc4>4g{W?&rqXIwUv} zQn`VI7uIEqSIFWF${PTu2c91m@Avp!}+ug=Tu=cqxeuopIM(jWY&iQ6@Cn`|q zL{%&ryb+FZ9GQRAII3egMRh2&;13Q2h|ekRjBEJgC?d#$P259etGi%`GPuPKfutN_ z|G9=jW)TSAy+IRr_|IjqW(8dtZ$`;$8f2RBr%HYD61yUp8(WFowr2nb(B@j2(~;&BdQ5ac+dL&70W911drFphD4fofC}ns5msnec?!>w0cQ_Q)u5ETWhS*=!eC@~6ndUYJC6U9AWhl;=1j%FS)Ape@y*6BHG7Ilf1 z;*ckj&Ll}EHuDT0S5gLp0x@e<5v*hK<*HB_2(da$t0F*=L5`6wlpw_CWJ_Av%9fQ_ zo6zA5iiy52+7uC7jL7=>MvM3T6sQ})K@;#mDk@PU6~8gUPQTef;6Qb+i^^tI3AhW) zzHSnF7)38i+D^2q(q13u3xGL=XSuy1wIA?(sl??v83QRs(!SxmHe(;#gl|KhPM;xuUc zGB6lX8%_C<=fAirE^g$QQz`DVLs30qsgV1Pqn@-<;$S3dM*LwDv*wR+R2;vKpr-VF z(7!fx9xRu4fJ3BW5Y>$=Qn~f1MOX|}HcrrdO$5CefAlRKof)Y3h*h=%g){X5pL?c? zV>B$`%31wBzPIj!!qZwy3E<-RFS8|gwuD*WYamZ$vNU7!)a6N;D>35K=P&I zlUwTr+sF%K*C78GoS$xX^v={HYxY@!pMkq>i7PFSE1 zo3O4Y*1;Fc@F6UC5ld$lQ7Y1nhDqS%D3X=j6FJu4sUl?%p8IEn|0@#+%TfR{Z>&IU zV^;y$!VWfmew;5~av~3A)?!z_snY~AN2V@56P@cUKW{q(5OUD+BQmjN`MAP#QRwrv zF{~9VNNfq2qUWLgk!XJ`kzV%BcPS-3X-n@*w~yk4bT^$CAq_&4`SnVuGvQDWq=;IO zxS_h~>ckWr*<7^lXplm6*xpRRW{6E!g)$Rt3w>GHC{MYvlihNc7aQg-cQ#A~oMtu) zjG1U^voUL{6>guR+ukk(r#68LYZeLVuCQTdAmitD!?9z7uGgb|D>r&85uM+-`cWrg zxrq%;3#+;+F8*+2ZCQ=G-Q^#s4XEi_#yah5)RnBG`_Uj_|12^;pj#%&Vz!c#T;(el z{NQ6=c*Dc|zkcTI2`u~sM;xrqHB?F84Ak%(t0koyOcuxc=0|e2a5u$G~|G=RW(7*-TDieB7Y@#4*s?VA( z0c|K?%dWwbkZjAk&#ejx@T9;6T21|m&NN^&hW0V27mBp3O;VW3JE_rjNVl3(Il`nbiooTurvq=1K02X zR00tHFn^S%6I@VSa$+!wgm^l^WM+_u#)iAt;2vt>Sx#_Qh=klA5e3JEy!rz$pl|v% zYAT!n-Ner?q#*l74%L+IR+fM=TtIBBN$HY8mc*qKw9E{pu>GpQ4W1~RcHpMWr?cvB zU=G3yc}@+&Pz*88Vj!S-0BA7Q@B`{Y4rTx(|8T5n#_0jA?iXled+ZR!FhLJR44wMW z4+D_^D@O*&&D>}O1pm(yhv%9c@pKTN+-hMFwoK%b03KOzCVFrN84>sz;T>w>Tapa= zbk9J7u+=1kmUx3yUXT`jOctTA z9eFKV_~_T#Y6TiWArZq0tgX4Z`Yv~j!b;S^IvX! zN6k{@KJGkzi7f+D+5*fNe{vV{`*AwXcSZ6XKIY^TWTeG*h!o&n`d(r4$?a z%*upxB}T@FEp3k!@*{}Rl!C05ghZa zMNy19Spk`b1CCB_JnQwj=95&->k;Q77i9o9f0r1NKq|R?7b1RM`qY z5>_4QLkES`f=V^yaE%(pZ4s^jN4rE<0W?WHwi(SbHZR~?31JfQ#R+z!SsB0!V`o~? z#BY8@UWG}fnCTX-RcEnvKpdfKVP`S(ktX6nB_pst;JqH%BX& z>~a_pjKOv`|F(lbe3n}gfe40ne2g|4osL5j7tr_?3!K*Kp!QIu_8T6PJNXcqBrj2J z5BHo-o_ZJex>k67mv|2)sdkrmK@PyIu=bY9P?)!31n_As2Es3=M~HEYxT1ln~lLXepIK zJhX5DHnu3O1r9g}ZsJ2^Z*~*affIF6xehNXK@_fSQZJZZXOwF-n1dzPUNM+!r^r~T zXlsEngG;#5{}4EV%_lC0*L7bQc(eC;%OkD6w{iufhG|%>Ji@>hixSwt4OW(#h-Pz# zX=V-L7@G-Z;TH_;_vbt}4D~kyWWWtF6i;hP2o4f3_kjCa3!tV=feTeJ*Y$xT&~_yl zUYmib%%Fqm^&uv=2#6+c>{V!tHQK)Q7|91xHTZIO;0{{Abd#8N@Yqs!vyJ0e4GtNR z6LB{ z;*ytL4>HU6mm%#@h#3WpnFOjiiB&+EMf#%`Zwuboq>0#kS6YP2=$mH%kvAx&WvZrI zfMZm^rDd85RJxiW*`?t*k+~Up)#ICqmpr4HW1WtkL2jv&b)R{l*DS4Oc}~qn*s7&t zq@SP|L$ z|Jit_8IPI%8VQX0uk}W-VY;w~daqv?uYb4AM7nslAfy)?saM)|xTu;xdZ~k^v7dSm z5Oj#&>@BV63!~MT0NRKP+N-}BtStgh^J=00(yZ}HQn9%W+l6-F`gXbQMHf_o?I4g@ zI*Jb4uU%G}@7S8bS+Vg(xv7A-MVh%y&z={Xrvp2v7x}ue&9aFXx}AD=w;8fG`)5pKUo;!TfSMFzUd4ih=X&`I<{$zPa}3!#h4pE3_BS$m}zFS zkDz1FjIZ59!Lhln-Ruevc)`rsxdy_w5xBy~n3rzJKn%g~qQo z1LJh9ts+*cfK)_m1D#keqjkHt<*Bdym=S>GJ1tyq7Bq^SxwgJM1TI|3*_eE~{NKhL z1uD1AO+2LmZKf>^QtiB$&v^?!=C+!fpGdi=bJ>I=JB8=@i#Qy=&KvofRXq#+h@n-% za}IqTuat}Y$XgpC{!DA{HE;1%csZR06roV7ywPxpz@H?=P`bjr+>Kk;-Q@hvT^++^ zy}~J3&vBa8$-u$m(hMrvl7SjC->r=dFrleuF<`1Mq?84^q&9x3l^;M`Z?EQ z_UYD#!g=@GW9r}){@8`S=YQVe8D8huEKxi}!#!q(qkif$H)~r=*WDf8JF9OX-Tp#e zL(-4< z%=8Inj|pOku^3n%7`EN@e~$Ep>6~fb`@Mem0Rl)7A~87MA&d7N|iwd+JKUAi@_it|6h{0DRJY8Fg6>Gfp|0G z#|UJK+91#}B+?c$50YGokz!DdH>ptl_~s3gp*b;jt@y%h5#Lb{rH{ z;epy2s2foX`Zf}L!*QUJZ(y7x5prvN7#jm^(W3F!kMs-+os5SJ{A)g%IJE2Wm+m5GVFD6N*2*7-otxE?5(VS5-xkOEIR%SBSPf z2Dh`nqKRIK0)0WQXioX&u7gr5ayi(3v@0Dj_#$|%P z{)B0UEig2*obN7N^R7su493eQN}UjSB&V2dpB}oZjSmn0GL0v)u_#;%(zYDw3muhQuuSaQZQH~DnC zi|;)ZZ6vK+Y;7ZQhIVz6#uPJ$F>q{o|I9>stuTUtKR zkz-iX0%(_6z0Nxvq}7sahA`uGuWsduAIRdDw$#;4g`k7qRJJz4geB)|_&Cq0P#dCy}9yLdI&_|NLAwxBe}sN8#Gk9SEnBBhhGg2kTMjrUpl0`D8O@ zgHzO|_q!q;a%XXghR+^|xchu!idEzym}(cm&xB=zgu7V=!}Y>%4YE*hdt(jJ$hNA* zs)Z0U-4I<_LxTKBV)D}i88zn;F&@iELA+BUNeHH+rACA;)85mbAdO@qZ%UD5%p_mt zrG^!4B3lbfszjs30v3=>x5Q&HiL*xPz-*NV#8KQ@N5LD`%7)zx#5hHBmi&#fn-$^Z zKprQYPu_?$?^KT5$|=lr8PiFg(!w+j`a~!u?mCfkis(96J1<@lmjYrMCNl1Zwh%=-oS*J&|3J1vV(=j%z=@MOh>&`t{HBm}3^rlW5W0~@RIe(2bW#D-f62mD(+D-0XNgdyB zPB$|EdKH{gMNbML#kpcmCzRg&7+90IKhoXPfZH0JlX|91NH$Y3&#WSV+C^8sSTH?! zgWA3{cfnXHD65C zEP-ZK-tT(vtCYMiFw2OnO0HI=;f&{3#JUrzaSf*zB+oVp+}h(t3P2Wat|l+MR)HEW zu+Vs~sN$=ipemT6%ta(qC;P!*+IGJ#swRR))Tzm2=RvwF>DQnMx(uh#jrmRMfr67= z1G5&1(cnmLDjyqrKpc%H5RoRvsz@fLwn1-xRs8zmr#x^&AyiznQE%IQk{)on+Q#7NSe!D zMllp0V}|4||2Ic){^j+|lh;q*j&}{^WR4xmL(1gDV`0p(E%j)uJYy%gO?!;6q&m7> z(isRm7BX}-99GAP);o5b@qk}!77!+B(v+^Wvo8&XIrI0)hVGKzZ0c(-lGVp;zVbn; ztKTszSvAUCwR`mD$Mv)_q%#acAKW><&9?hZ_(Y+!3nu2Le%5al+hi_{EzNWH?VANH z9htlpwSvNT^@ zw}zCVI0GF~nm(1?!*)p^u$qN|-{9WlI<;b<3G{3<+r%hF1T2aw(KGL4b}69UY;H0Qm*Us8ifU(B(+N@?DJl-*w;!(bjMi`O zeU8hbb(>`_o_mR(sk_}19*Vp(O~9FVF|oz&;hHrT-BCU!u5~^3Kd<_H;Dw2q@QZ4K z^P0^xEGL6=-_*Txm^+}=eB@cLW{H0Ozl8(1w_{-X(?2p4b!cnwu{gqU2RMDu$1lEY z4Clqu7`8A+KI5Mx{JyIQ@awNX+5)}$)7BjP5S0^1lTLHq_doV$b#`M=*(Waur+~Y>m;r(9M_l;_8-J5~jb$mDmuujLY)+;-#TQc8#y*dh zI$c0zdY60RV`X6UX8Sc&S+{38bq5Hja1Hni^wE9%2XW_>!GiOqg$o9Ij>cn6czt$;P0P1ywAE{hmxxeB zVOa$@okT#PrXtgKc!~5??jdxe2!}(5aGp>Hci4aa#fH@eckI_t8q$7=WPRor|9HT5 zd+Rrg?3Qtom4zIKWv_Trb!CD%cqui=X~Bq$W~LVBXAudMNR^jcs!@uMG>6wnDv;rd zBFJWAxQEt5ULeLP{%1;(7;DidIZ|kh?vr1W*odHpXp%F5z?eyY!i2u)haA^n{}P4g z6=PG_dIjcgIXFRO$Vuv9Gu2p)u@DJen24?=h2rOaI#@yBD2`O~a+~vz^~R2eNL9i{ z5gQnC9>jGhHB6xTR*Ir+kD5j()``()f}kq>$K%lY(J9DVc|@h=QutVs|BT z16YI@_KmJcegwl_;T2ULXNm9EkNSve@j-#~M}T}pj8f@&HlkYbM_I{6|C9gNB&2wg zLq`jrPzVj#eQ7y_P@|3HW>tMCmzfil={HX}(|j5Ehes4j0?9}LgC!HkJ&r<>@VHmo z#fA__YCox1DYcC0h>$S$XpWF(+CY|fwU^>TM=m(RcHjVTZaEMq|GTE4%Vq513Y|ayz zY4L7J@;L6LnbS#~&-PpTft{ZTnxn~`q~M+3*`3{anrmsAs5zc($&;$7o^$z&9*JPL zIc_JHl}pK#;>M3)NRd>7UWf;mGEs&mcO0K*Jaac511g=>S(aV)|7AvHCD|F8+6kfH zDWTy>q2h^_+?kdYYN4nJ8L8=^AIc;hDxyR9mZ`{|VK&Mp-L*AY)PKwd7kFUa3dOL zbvPaznx4Nnk1$GOX#$&Zd3>!FZZ^85i(-@AV3|5fX*vpD*V#-E8l*`|r*&$lce#+1{uM69${d%wd%CXNThXqTpAbYS0tDwu^ zttD2n7;CW*OR*~}t1jEJFuSrFOS9S9vG#GZIs2^ui8~vsQbxSev!y$_ywwv|L-WdmFZW>$gEOwniJcW{V$#>$K>~w`$9_iwn1J z`>}HixslsjcuTpJJGqw&UzlsRdYiFgTeyY`x^F6{p&Lb{d$^~IxNEDrjLW)>>$uh! zw~{NnC0n=Ms<~Rbx}2N1yGyi%>$$w^yTJ>*-1@o0tGIgWx~=PMubaF;JG&rTyIM=T zxU0L>YrIc;y~KOG*{i+rg}j_vwaV*d&Kth5%e5;|GoBGzYDCu4a~p>Tp#+|vFF>r6zsps z1i*AVu5dW81MI;N{JkK&wG!;WB`m^E^n)jx!ugBB7W}yDYrY*U!x~(-{QJQKe8DD6 z!Y3@gJnX|OtinL-!Y@q3IXu1S8@WF_vLq(M*wDnn3d1+dzB~NGRBXk|B*Z=HKrH-a zSsTSjti&TL#x|UNPi)4fcg9~_#8@oFL7c@2c*Sx2#w}dHU2Md4_q+lO#xu;vW1Pll zd~kn^$7)Q-cig^iEXP%>#dVy>hJ3+!49R^g$#ko}gDl8QT*#Ts$mNU3jqJ&6JjX}e z#+H1^k?hA|e6W5@%B2jun(WGJ%*BhW|9zv}$v9lgmb=L@%%*=#!%2+FsT{zo%*vHq z!(=?faNNqde9ND_xrZ#v%$&@6yvwX?%D;RLzuk;MtjZi5&$e98qY}mZ zoX+{o&<||84b9NPEYaG`&KVrg0=>kvEYAnM&=9TA_v;J~t#jMYDV)o0DpVNFyo&DBQT)fc_g zA|2OJjn;Kd)nNV69j(=S?bdDm)<}KVaxK_{t=4m$(}iu1Az1ex~!IG@Ydn~t;ebl{-*kjGvoekBeo!Xbp+C$yVd_CHvP1&*i zGp()KsIA+sz1w2#&$12Nl1T@b-OTOH$L!va z{odz5-SWNK)~(t14bJCH|KP|i-O&x$+AZJ>KHv||;DP<$5H7{fo#3S{-3xx;20q~w zF5w-{+Z-<99-iD5uHYE0;Vtdp$W7ANJ>o0A;nEDt7f#tojp7;2;O%YSQ3B)j?c<&u z;=wKCCvM{yj^8~F;sLJOEza0HP2E4PV$0Txh(8!O_r~& z>Ax=N&3^394(+TC+R8raTz=}q?&}o};HWO`qiE-{p6!`U>ERyg`3>wPuItXu?YRED z-!AIpZtCzZ(u6VT(Jt?ouI-Ob*-T#T=f3RaZs@XJ>AZgJZ*OU+MtQ z@v}bV0Nw558V~>>`2+fFh*r_Y}_ zg9;t$@uX3aN0Xvdx|AqPn=64zg-Z45(5hIcW{t|VtJkeforbkaR_xfbNSUHtYgTJn zw_xAGol7^aSGIWD>dmV*ZC}27>k1xh_p4lag%c|V9Qbf!#(w`op1hc_<;s{ZXLkIU zGv&{bJB!||nKS9qTj8lb4Z8Jc*RNxUOwq=hiJ-uH@dpe+M^Q+qQAUyi-%% zJzRM6=dg)4E`D6Pb>)hgR}Kw5x@Yg*tBe1qjyyT}@Xpi6=8pb5S@Y+w$Dc1%J$U=~ z*|&E;p8ss^{p}}Uegpa!plPSUcVK}r>E|GR3@%8YeAp={VT1%`h#`dtPUzu<8iokr zgCZJeB8MNQc%oY+j#yNSFs?`*fGC0}V~rb<=;Dqy+NdIqKbAL`jyV!(WRX4sDIJYL zDj6e>FFyHPlT-@HWRh0e1?84p=ICUPRbGkZW?hzfq?ulRDP~=3j=3V5qD3iXnrhP5 zW}R6!3Fn-9P8VjLce*JkoqV3D*PnR$6lkJVf*G8lgfc2=g&iU~DTjqNC}*aQ0xIF6 zo`QPmpq<9qXNi_l8tSU4j#_G@u#*3(Dyyw3Y2mG}ntCdypUz6FsI@klt8cvS>Z-55 zCL1iS!VVj2gvv&n8L-kW>+H3~#>(um)K<&wm(ON9Pl&d5yDhTT>KX33*zRQRyXSg~ zuDa~jJEgq%j!PW8`F?BfrvK(uF2C`@M{vKiiL3Cx2pgOrfOj4&p0NSj+pxsiU5RnQ z;4bX($MFn&@sb%cX)?$3f~>N>E7$1o#wVjZ@`No{eDBQFmK-z8$JXpKpCP|0bH6;N z>>t2Cv#hkuI5$1Cza1yN^wd;WZS~b`1&y`VTz9>-(_p8|sMKGVZT8uDqV2WXYlrPN z)KROAG-z$JZTH=EOjE=LQoGERKx`zXhBRcuo9kV;3O{SK}LAc6RZfuD!h?G_pJ|pFqEGQ!B@ZH$!~Ty ztRVsS$HN`=@Q2s~q7VOvSimB}t#GU0AQ-|hK?ka#iE~&Y6rqSgCz8R6Sd`)h#~{Tg zQZb4Z)L;jvxW!6@@q?A{AO}MTLPUgOd<+bq8Zq(4`9%VKbzGkcX_&+GDe#Ye3}iM0 zNk2duvKxp@qzx4*#6}X)h|W7=)x2SfEm)9?RJbA*^>9Tdf>D!psNy9<$;l^d@{?V} zq!*v40w7Eff@vVa1Z5e^DOz!src`AGJy{0`rcr{me5DH2NX88oqJwBWVJPOf%w*D0 zj^#TA`_2~&@cBZ4d^83j2f58dF4B>Y3}+;v7rG=q5tT>KWDG#b#3i1xl%C{d95xAs zHbjw@y3D5(v#|fleY*0Ws_X+SS9wo+zEhO^d;&f}*#~`|6P*jSr9`v1%QJHDm~@<@ z2%*``jILsj+O%f-t{Kv7b~BLG`z9fMNX|)xBNX!_vX74g_=uEGQtkI48$AAfCHT7K@A9XCp@Jp&rzmQl?vqoQF(e$re?CJLhWfPw+Ke1 zu5p(dEng(=xXd)Rb&nuTA2E=@&AQUH7}hvxA+6Waf$ajOQDvtV49ZijhVr3?MFL`7 zS=hw#Qm6e~s$`*h*sVrYsp%}j4hTxvdg>IXIelngx#-i*Hnyrredh_n09wR~bquJb zYG_A$)pP$2l%f{(#4nS1(P4VDnk5ZJNcRd!m#OZrll7`+kNR5Jing*}4QN6IYuKDN z)V0{9Zg#Vp-Rox821{J+b45wfo4Qh<-i2;#lUv%LhM}sP;WJ z-e8{6pU%Z4Sn~@^3f^(8A5HK_5iHVDtn_%3(gh~;tJ9}Wm$^2bX%}K@TK!TOvfAx0 zdoSFEpLVjJvTSHjAxqv7(=)6f*06GM3)UE)c&G36ZjGgj*&wu_4H(vL7r@J7(e8J; zk@cx~r&?S8%F?~i4$g1u$u!C)CpjKDKiH2>exmxSNqWRh}z%_Kau!B~! znbL=LsEDzO&szg{tobS(C@v`K5T9DI_$9TAslCrCzWU2XUFx^b?CVhX_|1cDb)eU5 z0}jLhy+KAQ1JXQ#Wb1du%}y?OvmIV$1NsQ=Ey7l>+URNbnyB7ZcEHa)=!mwK*1rE{ za-PAx(=Y46a`D?@yO)~Yhhy8ariEv@O}yxNBbDHp+_uG+d-LfoIoDB)0F?C|^m$iz z*pOZAq!SA6oon3DL=dl6MGkbVS~$8--msg+PWF2<{Lh&lZE7ja^;H^hc@A8UvUJW z9ktGP-g7FD;P73yeXl{hX)V;W>EwoZ?n|pwNPuzTNsYC^-(Xh$ZrkQK2fhDtL(X)~ zBLC5ihCb;DfAhh=dY{10nf^X)UwQL;#&32%VWZv2#AE#UK%jKW_k8TNPCtiJoflQ? z$4}KJc6oU|~40Y;1;g9`|d_R&c*(X8YA+J|=w{cV27vg65`oxA%q~$A@@_ zT{c#0LYQrbr)f-;EJWmk^rdV4=Xff3RBU*MWVeawcLzvlRt2|x3&@0~n25>-X+daW zZK!{X6>73bdx)2W3-xY^b^?)Tj)mrk8c2=%CXU9&YbS_=DrklCC~J$ijNoXDQPxUm zhhnbCgun=4`11?Q^?hVUh={0&-M59yXL;z@kRw)y+Gu{O2vH{|kKVR>%LsjnHhIwX zV<%9C9l3`W2?SNe00J0n(@1=Vc50>fe&UyjEvSqHg>P5bR9ye4c##%{h1iIh$Bx7p zkOE0PaO6Z$SZWXmhew!?S_fzxmwZ)l+9Rl8#!(6*Nc_}f;Tyq zBFTSt*o9fwd^D*~t*C|gXNXTIZp3zE!B=}|X?9)|j!TGS%bjGmZ8U5Zkcn$xQ9TvDS_X$Bj0STFKdsS(%e0wwg#*Q(FI61S(c^{Ft4Z*?a%Dj+@C; zz`1pL=aI%Y1?72w&efaiS$T}uhh9gD(g}sn)|x-1kW~~a;6#+fGn z>3~=%UU`R-J~oyRnqycMnO@kHW{s#)lw^q4Ol447iyxrHVFa zYr$8Q4V97`_ys@s1#UQ5HJ6DO$b@WJm3^3qey3sPF`T=DTyR#Jopqii>Wv>qanspY z8abcN2c_F+mvq^f_BNj_nouK{m=7px1!gk6}G_o!;InS?%CY;M?@LuZd! zC6iDZqkH#`YBqg%Ntq$Jl^vKIN{T~(wU9}sndbkMcV9N5a^`$4sG!yuat-QvCE1&F z76N;@i)9L)5!!B`nWlExlwf9~H1(qESB<8(qy1TTo!FcX=a7}jbq^VDT-l?fn0^d5 zJwth*!P7nZ$*Z<%i3HVD-4>`odZl1FfrI*tQfQhY8UoyBeMK;Mx2ltB$fG7IrFQ3z z3n-GB7>er^i?BABU&f6+2A$eDYqh7SplFz@q>+pYmv#83hj>%IDnP`gQh*hr@@TIj z8l}(*cp0gd>glA=7ODRUrcpRJb2CG}zfwowkZCRgCWvyZ9 zq5_A5BY>iFiI1CDbIKQ;fk&f6nw6u8i5QBijXICVS&^;krwZ49DVwzZr?QSWlNSqo zTPdJail;0Km&H?qi~6zb!)1V*|h+Ad?|@+(i(uT#)+-CuhY}7KP0!Nr?bv# ziEId;2isG;TB(+}eaLvLxapzJ1$&{Iic^}APrIzH%8cIXnGE}mTl$5#%YyO=nx>@! z8`xi)+nrXqs+!ujuQ{G$dz_~TV-Noq1F18dvH(-gd$-`4rL3Bt8)kNG`ZsmzRYZW(iz=P}+M1Ntz32xk`4h9Lka` zC!q0Let{TH+F(w7%AAzjr9N9<&KS8KDTE^|ory_VCfT&k$+VU^x1x)OcfhF^dWF(P zWG8&Mx0!AGJ8Zw3n`S_8tCg-}+r8?mxo6tAK1PLYtD93vU?1FGTo9?eF|)+En7N6K zUudsYgrAFwtr&@N4au^#i&+NOxDafm+`E|skhrvp0<46t9*TAnjAx-ax$sMUmZ`kC zI-=@pmB!h=HXNr|c$4?Kf?@xxNq~a7J0!QGSii36$HjY}Ap%N2NnbnKYG+maTir5g#7 zIvkgIc%p5&z_Cfmq6)hJ*tdDhktFJ{0B5YVm$DnXzR8H4y9il&B}XpOjX0g9D`|0 zYPs9gi21q}dWjFYxl#(vkSTmp3!W6rqZ7KH@ENj8xyN4ne1n>;Z5z3U$Go)EqjzVA z(HMXGs=9P-3?4kk?)6ox6zBN{cPFy49%-|&~e+ruvE9?+I|s* zqgBnODIK1LOk{+Motr$wruf5VnSh)+aD^!e-W*g~>zjNmhU~oB zy8N`ejoPiupez5^tl!GNLfdfEd&42>+8jJ#?df^FJKjg!)7xxa?TU)u&C4T+y`Ut$ zhCRKxc+&6e(7P$NG-=+Phs3HZv=586NWGl!Ow=zcz>>X^3a+*-J&CoetEYXWKpBs} zs-)>tv$9Fn5*fuF?VTul-7U(_myD&C&5S|Ym+dUuUVUA<_}?kcRJZD!DsI)t2;;l! zj2#(mcPz#_TAut!*jM||ll;*o9LE!Vv#&dm3!ZGH)ZwqKKr**+qb-R58l_*IwbXp2 zr#-7D`DnJSSnplUMGTzjmu;~S>m2mF#RDAKnL${D-Ln$ESe`3H37J*7+0DnUVSUx! z_3EoUsk|((Y#L7tZI65WcpGf!IlV=Hmy}GY$;2$<+AFx7&fCnZbW6*!`0I=dH{GC) z<2xR)L~ONKP?+&O($mJ=g0A28t!st+urLc`4eQUnPVK>s+=D!sIGORk&hGh~*7h4j zhMv&HyYd158^?Ev(Y?LA1UqP%iC>@_$|?xqH*B{G$-|_JhM11vthDT^8q?!NQe?dT3EyXQ=P2Vc+- z`|X*H6e$e};bWi!LF6KV@=J4Lj`zy>(rTHnPQg%I1)u`;- z_qAaEhqPC#$-iIQ%ii3#FYqFngIl{{f`6UZ+VhBPx&BSU1`O_t-k$V-=9qunu(!Yp z3J@o_3~Zr-N)a`E;vo+Db++)<=PlZIoeO&2IbGjTql${@l+gD~5O z#3_&=Llqnfsv%g?V9T;UcajimvY^!wJ1i=^W7ek)gD3Q^BtjqtN3k0TvW;O?E*X(( zTNrk0XA4Qect6fv8G^$^z?3W7xp~5`K*C`Mt|VK!XA6&GyRwv6`EQmtP*mGw6vu7< zh>l)an|6%aWyapR6LWnN-0N?a9PiXFZf0lfl2g@w9jes&^`=OTd5X&WNo3;E6))6V zm}Bafn@^5^eRTv@!f`r7=WKk$m^8itS z$EBF!t~uivJWZwHY`ZGDyUtQ@$iG@Mk;(IXBdf6!Gwg_f&5r!3#1&5z@GvZ3lg%;^ z3!IWnpt>p$G8-`R?jklt+{-~U-Rv+<$@2V4q2zXu@w_yp0`s@9REx1g#Nq<~&aoN) z3uz0PTAU0e2a#y!QXYHEPP?ZJnbAJ2T+2<$_P!*`(H0*Q^(EVGSQLj)os3Y?y%dx& zkTxP9($$gv+*J*%dMyGz+_JjN3|vVnQ^*v_)AdU-b#?Ms7pqNDSo2WAl@3Nz^0Qf9 zKl-rC+f=Oe%~%5@Ra|tz9Fsmw2r&JW?-Y-da3_CEkiE&>LJ++pJXxe7Pk#M^>-B55_r7 z)fIYc2GJw>0@0Ph9eiWjCnB#tHwVY8;lF{@cJ5_2pKer)B>979!leY2@h2YzcHPGM zuzoV!lZQT}kfT1A@_8-K#+YC11%toz?EEWY>R|%R?!ktGu))>Z8IYh%*@0+ z`5j7p?xNI8lJ%q@m8m%7qnA^PB`B;Il*mBC8zxebILx6U)`x~QM6wBzq~sFTum?$O z0u`Ga1t&fENl<#y7P629DA0g~C_qvPtMq{?Sy_fu(mWBr4!OD5ZB<91W;L8I4mZ6y2ye6{rXiRF7agQ16 zfii=5&S92Q4q*afEz|kS9<=h7B|yPDG1EJse2Lp?+;l94$?1~MoCh8#3O z1`CBiLmlc+DX@VPwYY~p+;LHivO}XAwP;2=>d}k3;t*$$0uPt~2R5KorH|kPCi##F zm^Ol?mx$?1Yg)-9lr*O#62^%`Gg8 zdt6A4R=H>$Epy2V)m7dTe@N699%W^r29X7?0?n%?%JA2|VgS5=AZ%e9dtUS|HofH7 z#x^R-2OWSHu=DLeeO=I3JlqtE4asHIVir7h1{k0GIw$1%JK+Ahm6`_AXBxc*M7byt zoUTiuKikR7Jd((qJWHi!&oIxb6_RdnWl+A*_>%2%myt<*s2F76Vj+ZgzJl$=JIt$I z9Oqc0)F5g(*zg1S(zm`7&BP*7QBk5kj|Kt%1mn$QS;8+mc~s(raEOpusLKH$YgkS;-=u2LZZo1PRZH8e%0qRkU zS^>1V0wc=7UQT2fjH(1=hO9z>3E+X&Ze+kxpNuO4zy+l*2ex61Eti^GMGiOjPhe8M z2WICc<^<4zyHoIytHj(2Mt7XenLeeV(;(eVZ~4^kjB-C%9SKRVflNQ_*-BqF0}1sW z=0-Ri64G_iHMC(V4K4&HSR&!i4ucy49o|Hsn$r^Bd&Q|C3OM8-;}4et5IydVW*n5A zFmH7fe7!hZ58VJNPiL}*zJxuPnKNG)dIU&H^Tos*-BKvy7*0NUo-32um`>{R3tHZ*4c9KEJ@v;_u(>2Mkeq-Y|f`z4N`qe`4#{*rMnuV% z4?N8~pIOk0y434}(?bJL^Ci_oIoOLmqFcb&_&;Mir{C)~sPl_<1Gg3tJp&}Zt4k$! zBe%x$I^+WbMEESR`@R-BvjQ@!|pTi#YO= zxMmN{PDvrTeg)@RQZUK-UmGt?NQw;yoi+5e>YO}3cnf~zaYdh#2W;@0s$mUfKjxs4=}*c z3WQODKjnzEy}*MHFo}GL3t>}4TL}SW^8qcgJlvQ$SW`U%7(E1}y<$5+URt*maKH!D zHltIF_kaYebB@rEs3^!i!|{RJ^8rbT0c5(vtk^jYbg)IRHYoZFgj>Fb!;yjk9!4aw zCpew|kwF_8I7PZX!J0cQnSmG+FY)?8PVB_S8baOL08%VSgCsn%dIer+g;IJo1OftB zJGvRr9E<@mTZyp$l{15j>9v=8jDh*Q2&xDzTeii~Gtn4HiFBrd__>s%v$5#4mE)!l zNW-m3MqOJlEu*|KQ!qQ5B8%v_NU}8`K!iao#3XP&kzB#;*}Q%$mA{BY6`MqelDljf zf>z?ke%wSG144kLxFQ5H2RKNvYybf;NFJ~SaNr~*O9&2XOC=Kt+JGV`%A$I5%NuFM z)3L~G@{VJ~uzbRrIa4FQl%Na?u)-`b;tH*|k2d=08l&v|4t(zG-gb5gvE()v`@R9jVX zXjNE^)p)4V0nO4Zy)W7-fcU&G{^V6AwbfrmfHED{VKq~$5Y#j!GS+IsS(qp^2*y;n zfK$8BX9d(2UC*K*lwcvdvXO(vXKTzJ+L zKu~K*=VCFZll~3_yN-$gfSh|(9HzLOgY)vU3jB1C`< z-$^@IDcl8yTwjKq1W)Ls?E?k)ec$+UFwPjh>=igA=I7F$-(!69JHi$XFs7=rS3g;5{`E)HX?WC1pS zTR6U`&#H!dsEHa&ggAJGv@BO229$P6-JHr;b-t>QMZe|6Vbtve_H%_EM&L7Wg~_{I z1XbW>%{9*oM%?YCouy!vqvyv3=&aj>lSRH%NCL{O+)GL-4?t!A3J$GiOypBuVWq0) z-Cbqm1?7*1+K{g0T$WczW4~>Q;%M@>_gU%d-4`Tfq*XxXArOT06$ES;f*9LrQ*}}y zXoM&&W9LCjaAr;KnkaXO0x{iKXhu#(fGKreX8@33U8U-1BVbNwgvTx9OR!<3#%K_5 zg_d1}1m)*BO=ZvGm^K~Ml>Oii4pSS>1Ogsb&F$c-9snblVS`416+qBN2;n)+XeC7D zjvi^l-so74RFQsOTsCPZm}PJ_l(1=Ou078e;g4b_)lm=xh;vo-jpjxO*i$vdCg|); zONPPXX7>BINd$-1s${Fi#=U}_F!_Vfjq3Xv00*$;Th49&i%r>m2D&8xQ;fz?%H?O6 zf@B(i1n(l7*d^}7<>UX2?XQ0B1m#;z#%PD0Ns8v{-=#m_4e#Fu07qc$F9X_(caJz(l^1bK*($vTm?bU${y=)J??1*2yg-K!CvSEPT*YUnW$IT<^`97 z2WA9OJjy@*TiPxlPIdtRaDkBZ?LUU@hfdHErsvp(DN`$J*E-~+lG(@;y{$&j0{C#N z9%vW`YZ~usyasN;-dOV1U-edO+#2llW^eaiVwa-gU%6L@hzs$29qYmG8XDU#Ml$~< zgoc84*pRM=1dG0*VHiduE1abf*ysJ3bsV8TxD z0|aJVTK;EBXzn~1U4M;QUYCpSBLdje|0n`$Yc~sb4I0W z8}JOTbzmiQHb%l4N@0RnSNch2~9M$+lG?I`rc25GKxAy8jJ$b(T}Eh8{s z0|sO*9r*pcYa4&Ql{`>1*KG{<&)shCk_veLj2`xhhjG`s;cIw@OdwfPuITKBc4~KV z4~K4p&-POUdGY4D30xD};^KnE$@d>fR0cHYGa-RqmcweNM z76Y9Y1f)-To#x^eqh=nLc>>+Jt|oYgPjel}aLjYU!c@g&v#WZPQDbg^_xtH2~oQ*RAJDE1r_; zi>Gq|H8BcnY4V)!tqo@P7o!)$%2G{`?FG9C^`%c-7wUfthW zz%Gf2tgx>yu&2$`x7D{V`?w!@xmWc6+wc9||9#)TeLkjOS^ojCUPx46hah&i#Z7QK zwY6kQkPceQ%t`4j^ZN_U{=8={1)tWO4_=>al38&_$S===i2;axa!V+MTQKHUu!KO{ zY0ck!hU9E4-$B%@*w6<7TLK74ngmElVDO-W04$PJD6rvSLlHBkV5vyP%NLA3d^ik1 zumebhB1d-Ak-;R&lps7D7|`b9RjA(qw@FD}?r}093(F3N)1t zMeq}-t4Eq31)92Xg_%^ZP;KQT$&4mmwjfcmC2K4rYGT!z?X^~|C{S_*N;aflE;p+%3DA~$0H98Mf#L+s9|sS;L-^$YCZVZZ;3F{1`a1oW&;vq zw8+^-FK#vhjG5Iq*=ITCXv{8xlvbKFskLSek-Dr>7jETz(+eQAjY7ziqX0)pl7!@u z2r7MT1CAd?A*a!$Wwal8g~QEON;iUVh~rcot*qd;-N$eS5fE1Ac|xGXAn=Er6o70 zswyFc6RNdghaYYy;(~x2M+as=q`0e#h0&; z-^FQ>fi47v-k}fFa~Wj(@vvxD1ao%n>*ZM04~|zE~M@#V&KdOR~O2G6m@Gvfz!dzrSN5? zWPvQg8P%APn7?$?(xszy&wDuHb}GL3#f>M*pWGHRE-~elV|;n0ItdKNHJNa7&8F%V zq{EqhV38J`pQd0zzU#IMH0(52NQq#W#c_MFxay8%stxzWFFmtXUtJ2b+VIG=C61_@+itnZMrF3ujQ1_@6j?x^8N`{8cmrZ0#C{(C zt^7mC9}KPkGDR4jTt?vz=cj@h?gLT?ox-Ak2jh&*V3dnOCm^v2flZEK55pV=-_w%J zE#MKj_*hdMr35r|sSK&}k3_DZARWQ1cP^Zn>|*!A7Geu%nt;Ranvey_sDOA#*bLwf zBN!X`@G-hVny;QPz3F{J5Ow%f9io+*(3oQv+LIFa!grxwSkWK$09!c#L4yex-~wel zBYhHeo&;E`E@}D$B=#jA`$@!%Gk8}D2xhqoO3Vng5DObR;)hw7Z*dcoSPATf3J$6< zOchg;9M%*`Kd^}t??S^P@^#5fXcB>ZkV%^;;2!*l5`X!tW9xnLQw-_s}{tD z9R5xJMJ^i5@|uROquEkeP%O+6rPstKF7GtO;E}nuXrb_F5mv(ahW8XC#z|VInvXai z-1r!sGW5(A_}Uq!+Ji!u94DNKW1N2$@B+h~L4p)4fC&fDu54DUBL5^bwAhDoo{DC{50Ql%|Zc2cCF~3}*<#R=%#I4jK;*mvgLM zc7Y2lx?LWcm!izzk~2yCWiYkXL?4x+9s1aZ7x|H`nDUf~SbWDJ$`BUUDTx=_+SC#j z2nYS`A#^!3fpJQ>mA&C)lxaLC1^VVs!ObF+i6bK!>a)HJG;Tc>&$VYKrU?Wif zc95SxrmG_?w} zSA^M#bAI!}mRv!T`Kumm9C_!ZbhU^9an$@9J9#J zd>(~D8Z6>ehCT0LMVr_MHNz9tT!Ulxw=&06#(5=0;t?r>Um$ii5^1PIX%t}yL{!4O zd>KpI#4_5heDAc$<(;YmOkkwKtg3M8yk!y1VY3XTq`Jtz+Ue)6~T+Yh4N0H0Hq}(ttctGlbD@zT?PbLME7XNXXYf)y{@qFcyp z=L^>m&5AxYSKN}m9Q0tU&IW5iQSrA6Pvi=8;P8k1CAo(sq^TtQu>%bMcPDw-ZEnm? zC#AQ|R){=Bj}y*A(ROS*>?$wQ@)YkHz)=NtTd<<{#H6Z~9PlPL*)GCiQx5+S#LON6 zcZxyQ@ib#2ha0OmrQOCHEE8Z~pQ@;6A+U)OeBd|7mazc*PjfVaMscBGF72bsmy+A5 zlw+wAImn=){NP;$UxQPIvJ;6#0sutgVg-GxLMA@;#~*O_gC_al2jvF!p6_)pp#+i5 zN4Y7v*7-^ix9cv~Ht#T4WhtLL6xxMsN*e5?l`Kqkde9o>}h~Jcw;o|37UiUCzEu)&REp4gBf=AAkH2c|(HP}ON2wDUGGXgvn-@Zh1DB0m# zkx|cf$kOHk&7aSG3=-v#g>KA+TSBIe+=nv(E_}mXt$uC~Kj>~J-eL1bfao6Z^-ywB zkO~iVSNZo?zAMD(rl!GZm$uGbN*MuXP2XF*_PL;TQF+n0!(HHtjf-`ljQ7T<6ykw@ zIIUlrOUTo|{<^jVD?pxk++bA9kZ15DTtNu5mDzaEQ9)aWvkIN#!6;zHvvjja7}+hi@fYn3zPMluMj}s;*NJfdT@l>DJ#ylAR+w){W}DuF!imA33l!%$)%;3E%uK}ujBz0tu?B;DWyok(EP0xbo~ zwV+Q95hbugLzW*CdX&w4*b?L=6A(vBZ6rr#`-9 z8*Hj04_HB@7^PRW+a*8(T*;O4tkVMorwdde#TDKr@J-J^8BKOuH!VR$$pJc0-VhZ+ zvaI4fPBu{V?+jl zY@4VFS`{dwa8*EOu1O=XL4b;8mZX*aDB49*reU1VH`e5#!BecT68s9<4{?dvZk!G}%yfq%lj9eW&0kn+&rv zxt4Mq#OB~di(aZY%-Nk;*t4+zm#2Pe-M!cPy`<9Y9Xz%qMO@55lin-7a@ZuzW~oNN zE(&R@3ap)m*N$iCyWS!2@?cC0iB#?us5J;Mufkt{q zp1mfiaaATL#cNG~(2X1Omc@b~(iOZcMhkOO|M9n?O2xUYONS1>5|H-?INrLBW!Z5BmrJqvZ?Le2*h)E z$;Si&4)$JF;R3o4$C*%&2lnS5T!TPFuLz@qA~+jWB&cTmE*&YF9~fC2iSPX22@oGz zH(k{Ej*_)qSE=0pV@7nEh-xDCO@QSt0$(|x5lCx24lQ&BaFQ`3UACfjdQM0cFapEc zUq)@bvFTxqVGY^HVGiT%9x3i>pDlbZ`fT61p^v#O3g4I} zl~m&?S|b)`sigI-rLBQE;h46>h2jkuV&b7YSn5l(#LB@030;AuwT};DvX)NHId7}H zq4F+kqWh%(An;kJ$_9#bv5qXy@{;vz$thL1P$n;D2J~2|WkIYj4XosWq-C^mS-i0^ zP8H0lDIJ!Qw{?m&r&xa44Fkd{8=zz>XmcoxG{^3zrm-X#klaYI6lBUhOW;|(qO0(8T{fW;;&QXC3ZNgG&48`{M9EhS~$Pt$}w z|Eg8>s5|#2%-T*9yN?IKOP>U__kF-nr>|< zlOClYGET1{8WQ|qm0R|;57iDzY*} zaR9U!U$$Q!^mvZc7<8f*C@daAFtk{U??zs2zk>ISp~yWp9a|a>!XzbGD#g?!$N21W zPE6h{7MZm|El{aY@^+2+w?~{waQ_S*=(R=%#iex}pC)(YQdf1k&l0d+n~;uGk98t% z3ZMNLc9(8rA9c+z?E=lFwnnTG?W#bxuB{dH(Jb><^F_f{f-oj6n#t%~WQhf;p^*yz z-v}iyYFgX94fybuPU>apblhYoTL&o6cV64gxDj|0cyfA>9=@QgTtkfX;liG>FDdIX z4j}gp4h|F#PV?sFB6HYanN}HGoBO1=b{BOQe=4?|$q7DIv&tVyxvq_)YGy>_jw8mB zYV|RfWSSlLeOIvKp=8aO+2VEFQaR`(x|I!D4Dr5=bSMyt4a`Krv=!T=6j&%JiGc8; zG^(c%MKm>=W7>6*hX`2R=fEbLwd6yq7UiuDD(*QA_4(W}8HH>J^YJLBL~=)(u|XHK zl#a2YU%4DkEp>htH(KCAo5kdvmQ!N-ZNrQl?NK{J;233!+2Whs-3f#LaBS!QNm3!e zyL0+Yvd+I6aHe@>{%l7rdi7~$KWdzHIu zQ@sbky48E6!u!mTtM2gQhObR2AJ5Gxg;ccsq;SQ+pNfaj_9DD*=@EG<8oR@{xZSOv zvrAlgmoZ|rs^B#j$J@-q0<$W>0?2Q?dzO933q+b3+ngOq?6C{IN5=-gd^t}9s3FD3 zY{*@2z+;nO&TpyO4egk-QKG@^_`*6O-6U*jpUWWI|9I0ZSeW1-h;M`c$ca*;1ewmr z>%ksUoj@pDTOr3j6MNPl`|M19V?p7H82Uh4MwX%RzK=ZF!x@}u)(AWcEs!*fl5{8t z7~2%4=Ei9Iz)}0wMQ;R#wOG4d*NR=Rogw)wdUo{tQrQl@_RA~NB>g54Mlp01VPyjK2f~!oaYVFYh z2i2xjgUVcb1&Lg>S#l(Gq$A=2npxDgjpLRPWy+J{R;pz85~jPD=w{Zurx4B=G*^(c zIB|kX97Jux6b))r#giCGnJOi+bVN`hTAqZtr0Hr_k4hr|DfCJIRajV|#A5msXpSFE z46(i{C&@zGDOkMJmGcKLq&#=XP~96yju#UK1AJIv_kzJ1doB@NL$Lq`gPhjhV?$UH z<#MhHCJ^BHbLhg65qPLO7d2`~5oLi(g%H5#LmML!79G2=$V+}w;38Shovqe~4k*yD z;5c&9osB1l?csUz=m?o?nl$P6 z`hAm=!}A=vM#W(iDXO0OX~4duf~qK}w89E2M)-QFj<3oxg~AH?%0n;KB>4ji6yo|2 z#Kc4lQLgDcT%&~NbO4Eh2?i@jhmPEW12auL(a3`aK-=;Ew$eVJqNgBx2#OI+G!jmd z>HfIxj0t+g4FDfpsii|dWMKtMT~N8FjOhySQaR}ca6vi;$}|(94SrYxjz)M06HVuG z1c@LQ!xKd#_}bf3KAHTSPn3?fdt{HCl<;9Gr$*>+Dm?@Y5D5S!{ZGIcG>nKrw9<$$ zlCdTv1=O>~@B*+>?J%QD8upqCubmFjt5x1=h;@)d$TUo?i4NE`Jc0)LPBK0?$zw|z zHL#)@8`vY6X1N0%g!)x4=98Gqxbk+xX(XplI2xD)<`0WLV40+ zRW%CqOXC})BC0PXjy2TrwF=goGkUjRSdw>5VR6 zXpk0FeZvkgq4dMIE4mF-5qH6{o4Gf|gtFekZp9byzz4wPohjEaoL<7eqcc5)^<=nV zhaX1!twO61%;P@hur^ULE_BMkIEs3Zz(=iJh4t25H>*=mPno^KP!Wl_rxaZ*VIYoX z)=;fdVfDdxB>vdVV8ujO!7V?j(WD|23*HO=lxn<2;IBYS_>2Wt?vbhM6!Bo^%0R|G zpN1h*oPjL_c-_l3bImP}6B;6jB2afjsewhnzg!MCj9bp&1csaHxW;h-BN%gbcd(`X zW^(nBVC5_~pO@eV5SieR3PNzIO8Mt>22$MzH&zvmy^buja9QnGNTEPctAYL7-4FN| zL*3Od1vyOL?ijcPUVIOH(4%3&OpuwOT`yZj+CyR%5Cz16PYxkDk}W6^gd<$>V;$mM z4tLT)Q1sz_cVNWz02r`Br6@h^O4%e_1VGrxO@7g7jsnp%rZ(Nme^H_%Y-nH+$x)7i z6|^8tQbCzbY_KaNETIWWcOVFrj102>ljI~@__AD3LJurxmistqLq6INhb}Te43|@a zHEaWj-e`u{h&ZcdMZ{~&OPnW&a~wH9YwL2^9CTcKu}!C zVwIyPnkLoL3Q!CJ1(I135K)nb3-GdhMG)Z@SQ-aNV5A%vtp>4bbFheIVtII!W4#WB zH)$ph1l-XDCT@g>#Nc6-M5WUIP6H^;Gc{8O30%>xkP1OSs*{})qzoXq>IRU7j!|R7 zK~hL&MWPHSbq4b1KWW#(p){chZkW>50_D!D?PLv`;9RxPhbkKE%171to-O{QN}LU} z2TfQbDh@RmXyz@7AQ>qXPl_Ze^0TEFa}e4xGgCo@YY}2xA{?EFwUFdbn$xTRO=C7u zb55dDwfKkwzDdV_ISaRblh@U=k6$_C|9Q?@dJhQ1RHzq_P!d zK*7*Pfl);4z>ysDU9F=32*WS&)<;hbz$J{Su8N@66a7<86f`ykKh)_kr@^M&2v?Zg ziiU7!02XfzcBhv@6BVqmFG(z$3>I@2knd>*FPN4eL^cJIVXp z%0fNFfCYHi#CB&4LA4nn4|F*nawBC1-i)#|R?<*md)eg~(^3)kVX;qz)8zCf^|Y)p zUygIPJ|@C(UQjASy6y&MPqgL=ZY1v;feL26&pn{b zUyrB8RSYM=Mi@`;q{oMG_ex$+1!rh(`-4W*#+9k??htco#GR?7Eh8Dv60SK{eNT+v z0T(#GlZd%wSuA~~gUXIU7rN0o+;l5dM5%^+T~=QlEE5Mf3+9b9kJCx7D+RI|^a9uz z8S{t0p%1fFIot`Z1Lo~ac4K)r1&>X;E~|+M0@K~hr;22$i%_GCxMZ#)Q37qoszr+z z;pcVpC`>dIjMTl`N~$kHVppi##7JTOLcXQ+bM$u5Pmhi4Jo$qJ!fJx@x7F2JH| zv^`6$+_|s++LC#H>G) z`+=Js_!Jxa@QGi1&9HFW$UgT4VLKc+q4MPy85nOOW;KMq6 zFo9gF#V`Gj{%FQ7=HYUCaD)94T?s1+I+zCH{A@1vnVx0Oe;f{ENW37ltj#!Bgxd~c zdDO>g3dZMT54YyX-L@kN#O07i3}1Xt_<~OZH}Lp6&O!yQ8DOkh%OLd25sN=Fsd4X3^P#pY%mUafDqRa`QV8nu0@d`1t>@;krc&>Dh>wq zF5}uj?AXBE{;^QbXbL-#ymo@24DzA~t0rJE9wJZjyrC3+VJ6h;N}SLWGmk_@WY4Ct z3Cz$L!NFFfQE;d+7nj3(7z`v;#5Hb74AG=Il!lCw;5r89HvBHCl;=zEz!kj4Z^#kw zaKSr3Kq&L75p=Q~g_1nx(3tSx1=NupJ+M&3uR{_9tQJN3Zh!<(kSnZm;Q$OD^U-DS z0s&N|8eI4Nn23up>y8|fewD#)W6QWaV8Au&NB^#m42!NFwlA~TYdI!S5&sslR` z3~{Ou9Jr|f*#MUqk68+E38K-MQnE8_FdM1SF(GpnBokn)18Z_C8VTqj#BM2(G6?++ z9VsTN)Ns(2jt@{%+y-qS%25WM(htWD>rm%Ex^j`YAn`bDQS31WsNgHX5-dU`JjPN- z=EW=r5`e@-RqU{n<_s=fF$w9CCUzkbbU_tK0v01nhDO9ML1`Po<7rAvDI0>B7^4aa zjhZ%2#5zgv)*ud@u?}xCIrxQYO;a4a~LVh$8ex~^!{QitJ~YdDFsICla$lhdL!i8=o!HH{zub;w2kgyk(&aj4|7 zI_VNSwG-HYjn-J{J2SFG3<|I`l1FdR097=349e=DgMt3CMbgARy+l9_B0VgV#Byq* z5-|4yu1LG?I2u$-d1CqAj&ve!H^~O>=1zntQ6G!*6ZkDeF(3onfCXk?L={Z6MC|{Z zbgtwtF1NErjZM9>lL^^H`T8;hMln2n)KE3Du@FW@EkH%P=tp-&hw6_?CG}A^(o%D@ zvNQ;>gmhAylv6j=Fw9X)9n?Wbu)=_|1zWH;L&j0Yz)a5+R0JYK#UK%=fB;C%qD*u) z$PgI=@3U4w@S4XuTcH&A)L5;v741@jP6AMO06Z@MBSkS;~*A~gl7?d#emQUMR^ zFohp>$6OleUk5YmKn-lV5dwKYdY03mJ7nhfnzKdxTu^j=3y1M%Q#UM=ik zpi~Vni|A=l>JhW>>0rt9Rbll|kn;r`p$}BxRzD%$d`iRz2i0W>)z+%zTXl#s(!;3StULna5;%3+2!(g3gpqph>Xjq#bumKg<2H#r-_Zv-0uV@*30xqd8m=q3 za$q-95hJdj#Nq|>5w09oWeau`D>gAbf_NsaV|d6d=Q2SX6n#cl^HWwvEu z7G|N<&4AEn5z5Wd!|-a2X%_(reD+)AK;DFQW}y~&R8(j^A~9;nognu+Uj+a^?oLdX z+|)u=0B~aHX9PWt>+-`_64V4>h(8#B3J8I#oWMaLK=Bx$#~#tt#()J?=QkITpWfgW z-qu4G2RYpUVq<^-CN>1@Ryh#2acxnDJl1bf5pefZS?6#40wqGS00km6abH#xd2qZs zmpH(0W(BPWsupyOONbImbDf5Mx7BhrRiPx1b4BFAOgDy5cV-z%Yx5W1IxcDLkmm4B zP#|~*k!A)WU}7!dcY{}5u9wut3S><1cuk0d&Q|*Wa^ZQQ*G!`bIhmjm2w^MvBX`NN zloC!Z%9h%kCgqS&OO6$A1-IE|tbEc}QBh?DaZQN@*5i0$1YbA2qS$HV>4^g{Jg`N| zE@ndnFv|EBdV)ux4tVOK7VD~&cA1zirdVjT?u@~f!V>i^!qtM|c#4aFcYQZ(J@|vU zvMW4CQw(HzrR{_Vxp-CBkPq3qI+P$@7-Hu@VnLYWu+0eYb~d`WuX=dp7!p}Y7T7{z zaEVxr){K390wZ;dkf*hM-B*>}R}x9t7MJf3Us;xGv6ZLILzK7&mpF^17>R|VW`3Dw zvaZ_xg3^E)(jbk|9HE29)(q^2Ut+*>Emz5cO6U?9fHmBCIMnLFP2|+cr#B6I?a^lah2^@pF7QEN!bjf zIYUdCE(E%iTYwDynFpWwoLQNcUm4eO8Ps_BxD2nCH#*`p+M}V15GrArG2miQ7;TwL zHyN=iLg$*XS(~%DVRd9Ish1E;M-1e^hOKcA<&(zV;GFqsonems&?%idx#ff{FINfF zZj5*vjzcuKsSElix)`bhI`g6ogzHhCI|h<5vy%7VJ6Ihm-D8cWDW%4R@QZuM5x`l{m~t5JKb=XzpUYOA}T zs>K>^TEMhxx%qAzwHs{*bQ^wWTdl2#th@TGzrdg+F|%y7tM3}P!OO0-&adz}i{C3J zO7%xZ8JINGq^milfn>26yRnD!mr7$zMRfK$z_QC&tXuhP9?!GiyCp!|9mHlT@xqQ@ z+ZGY&w8iuUR>ijo#8H5opxeN<@tYQhNx2QYzX|-meZsX3T&2v%x67uXCp?&ZdbOX8 zw=JCfc9#GW?h4?B!CxD?t?Xa_nR%@d5miqbtxo!+g~6r4TgEBr$e4)Tk`wA4kTi2S zaiKt+CvTJC8@`FS38%Zj_gkO!TdP~Uw-x-W)gZ$UH{u$63&7gRG5pFCd_y;!#EW~> zx;(R*d;EOcxplk5rMk^YUJrcSl160gM7$S%6oV1 z+;I(zrrMH`Yi0nP{8Ie@Z9tO0;UiXc$ePLBIu7Dml;M4NEm`Ql7UxJEH?>`^FJ71J zd1hK_t`Ya?nV!!F{&0a}U2k3L#QLNUyPu`c>jm4_hauzfys=M}wA)>~dpwoH>Yw#l zA#K#m41LThyx_&Y5jC$%|7mgW*FNw2zLnuTqCx%d{a&+wTGK^%RjVb5oxbsRxtW`u z-;3L?_+IEMpT|2sL!%j@OPHDoxw^={?1#PLygsw{KA%BkwpU++8~y9!;N*u5$kWdy z_S`=PIj_|~u2P9R1(^kwO2U)B42LHV`0e=vZ$CF@+5No46@m zulK+G0pf%WfjBrsP;ufQk{u)(9tlJx1eJ&d35HNO@gfR}1h-@i!eL;M7B@tKB#E$1 zmysV)vLq6t<(Gv}|7d!wiSuO_FE`c9>`BH<(4jtm3Z?Vq=$klslqUU&kffHNNcz~( z0`rMh99c(r*|)EYD6jLJQ<^UXJQy+K&HQHzUW^%Z)BOeBe*ogwU3dxFXAy1& z65$|e4jLB`|7Zd>N1cU(5oRE3d@aWnb6gxI8HvJilv;=hZB$@_F#SaublLs3TyOr# zbQM!J5kb`xU#Ya8S7Y_a*m>!#wcdMnEg4FD(!E5IP=RseTXi(47$A2hc{rGbCW015 zkYNsopJ!b%cVTyDDpyc$F-GTCjfYicTYj)<$!1g#jhO{^UW%9|ZJLpx*Hj#8bXA+c zDQBjFhQ-F$YC2uXn2u@^#V1W;Im(!KPpE~YdPy?LWPCGB6e*O>SxISftQPvBhP<_i zC0J((w`hc%=9wi*x&rs+WU>AkM`w@<=x1k_-WQv0a4vY_pT#XiYoi+WF>GQVy|FA1 zr;YXm|B!bO3f5_<0+(rdVXB+zi|m3bOsJ!hdRH1o!8+TGjumlJmN!24;i3x$LF>S~ z`RJw!caG?8mAP)|UuC~?*&%@@GB$Cx!|Iyc#EFrBFs{~?wr5v@x~MI-A9qWzNGdUi zAdqTe7bKu$nagrx?3M+uyz|m~3e7umyWy)H>t`RT?ZPPSswhK?@P!I-%mtBoD%R1; z&2~p@Ox$9~tH}-j2ceC>e#RV(GOCt#!t2^mbD$u0I-^Ni`gCBUB^sLObUNSn*wed8 z)3c~bF8Nl_S?%;xe^Sw>tgA99yqex|G9ZYCoCA+5s*t`-U8tslD2{e>Q|vMZ+_tKmn#@kF@^u^-jYN1>9~IPMy}%I z$O=~HmjJ4oEb)=!+=ccP7Y{afdO*}N#GUmmvbT(ieng9$D;HU9qYo;w)`AmI^-{ps z%C5L2jdCojldb3$yv6ZpZCP2K;hJ|iRS1ng*!!IR2uPwf*PQ1r;|%mZ4`d`BFkQEFG}a|xk*$Gxz*4>@BaB8pCkvWztfVF5c)|JAV8 zFkp#^P_PpX4Wl+ctI0`XKO%_GBqgjFP9<Rl=A_NkPwUL69Efs)R&< zf%1)94C9z$mdhA1Q$(3+o7{%!N?)#!d>w%nnCe+eF-mKD>$>G2+4dDTI_#a|GgvU| zr=5;CQ-#m#z zdZi8Rxh%_lQI!UZ&;pO+!!u^^Vh#f##*T>*GV&9l70pUc-BnbwI1!@MtRh{&h_t9G zQjhDj+8{%kGc1 z$A>9WmmkF!k1A?Hb_G?M(>O*`otji|?sTMD>?8xD=h!IDlCPWO;1SVDP_&wF4k%=3 z7ma5SLzu5CGQ*`$F?pG^;_jUP!>IvzXC;juL6Kh76AqeqTCZVLCT~lqcM3brh&mM( zr})Hc?F8Cz&bD9t8YN{DT1;5Uwyo;pr7R2i*p<#r3pmuQzdAHq{~L;~SK<1RP8mW< zZ$?Z~ZUygUWyqxvQlvakq%URr$|K+&mXgHnTdU4%BOC#&y%qH9JVh!>kp7pQ73=0> z!}L>QnihdTWGfibNIvgEk78|b@84RhCN#Fdzsr2CG0(c&9fh-nfyHXL_B&j_29>cT zT^dyWmEygAma%rl6BTflNJMgzVJsveh&9_!cfq%5E7Nd3XB-oHhH-$>qp@Ol%hfWH zcv)0k?SjoBTq=s$XAcA&>sZp^^rg0=9i{R>nvCVUatbnD-sVZ;X6K6@STA#`9RuMz z5GLz$s{!IH(*#zu!G;&C8GW#iQ#wb=&9`BevL1!S6x_p(|2b4wtn7}RnCMJS7**S` zFJA-OVn2R1)LByuRDb#yK@aqDUIwU(#_H#LX1RsVGPOw|$KyTg`Kr+=*_Z1JmBboY zznO+Hd_GLUVK-Wfn55527I`=44pFO?wh(pcbW~JE(M%QEwS#Sm>uUcqR^4hLLRq}g zb)Orjr4F`-@!4FyoQ_D7dQ7T@*V`vr0~ySA_FJ69TrtX9w)XCEw&mTfJX3F9V9r`qpDR~ z&-6M~O%xL=JHay_P+i$*d%-mwDnl?Q5ef1n0~_H%ht7iGvJWtwC_sS|WFGHM5* zY%jDsM8{3|rgkb5gS%x_5{MUo6F_b^eqly%I@cr>r)ESrY9t6%x#xnu)^f`yf;n@0 zCa7me7idwjGOANEM#gZBMkV|=WLJo1Bj_#Q(|$RjR(wTx?6-X-R)Ql%SWj3uh9!uD zh$Mx8ef4&0`L|&+H+vM=OOywK4)|#O=W6a(ho@9buVrtJl6Q|dOD=~#_vVQgWrY5A zbb>d5>sNB3SUq<|27st~QTPiwXNZ>=|7=F5hzeGBNeFe^Cr_$qVcCR-LzV@?r+?ii zY@N4)802!_BZQN2cGRX@HTY4vD0AY%a>u7ENQkj0TIc41SLSI2h=QeNXLgk| z`hrh-)lJqlT|jn;1V(q?{k_=YnVZ?*`7_!wL6h=2bkA8CS%bw`GhMlfMVf0CG8COK*Pm`>x= zSM|q^z{pCFbB`;6jpN3V9z|OU5s}^~kw9rl^VojBhLR+?kw(`_Sw?50n1~liWeoF! zDtSVsRb5r}k41upkT`&~=2f5w|7{Jpb`{rqAH{g;VtD+BHs5sV2!?ful>VrRHn@yA*os6ykbk+FaYmgIADXPmXUj8sXRe8+mFNlPP1ZR@yz z(P?&wd1=^Vao`kL_tAOw2Vn#lWOt^O5vPrpRuvJ4c`kNP+R&EB*_jZe3dQva{pp{P zPzZ%!TmcH8&Iz55RtMpD|6xyAk}t_|(m7Vx1bD>QFPJ5Q)t7L$$C{Q%n?jhHv$vkt zX@*VleosU^!Wn0ql`eAEOZVxbE=pz-rJtriqcv(=HwvKt`JXw;qXBB5(eykDI-PeJ zomsh>PF9;ZhMoLaI&%1+4ysETd14)kdm)LLTjv#fA|HR0SfTWyUkam}CZ<+IT>D81 zW{Rd~s-|hWraJ1TJL;oxTA&0vrvloX3F@QVk$%u7q9b^q9$2JdsGVUrq5ij*M`&^y zx}cEBg5a4fK{``d;W?%Tq?_3d_6eqM$y5e6TxII1pUS48+NPsQs--Hbq-v*bnyNS| zr~SFAciO7v;enO<|5p#!rx#kN4ThTVDVIJrnS;5fhT4TDdXMIDshFCn_=z6nCsAa| zU!Z!b(JHOeYO2-BpVn%lr~0aIs;UDjr*w*^;0maEil;!TeB=6~fBInu>Z^eIqaP|j zFEyzMdY#CctjfBqW7?_D3a$N0t^dle04uQn%B=)ju-s~=+j_7E3$ApEunk+IcIv4M z>#C99s+Oc%;VP%A&?M>Fs`iSnValw{y01VpqoL}r0jshD%d#zttuO1cG261vTC(ji zi{^K;$Ql>3c&y5KuOYh(&ycUtu&E@wvNVgbM=P^RyR=Qav@*)H^fFu=OR_j?wGy>y zJ`1v1Yp>?u|Fu60v_s3OQA@T?i>76pv}T*OXbZJ%i#Sv}FID?MWOlW58;Dzrx8b0- zd+QBh>$gR#uQi*pgUhysOSp!6wuqa!hcmTKHMes6xOJPek{gtH+qHZPwt%atBP+L{ z>b8fgxQiRQqf5GsYr0hwxs;o_syn%ro1B*WwV0c^nj5%5Tf3i&yQ0gq*igE<+q=K( zxRATLtV_JFJCR(Ays&$@v&#%(%eg}Pyn>s%s=&Ma>K@fQy-JI{*$cdGE4<$;D#fcF ztxLYid%UxYtg-vIM9aR<+rH5oyzxuD^GmGy#EWp=S#T*Y`g(%zz00So0`B3 zjKX8f!4#~)Eu6a;?7}b%!7J>*K9igsT)ZKyz&cFAd#kwxoWeWI!W%rpLu|xHe7{J% zvr0_F;fuo}{KW85!g$NWRlK|>9K;Lkx(&?4G~C5CoFrcS#kU)aPRxN&e77Ks##2ni zX>7bbjKx_j#5W9gbMAF$BT@|?rX>T z+Q)nB$5HIYBP+*h424td!GnChihRhQ{B}Uh$edis^76-(2LXEjLaw9 z!zq0ZKMm9??ax2Vyg5D6zWmZOz0hZz)HZ$78a>k%9nn1f$trEaA`Q?j&CNya|ItnT z)L&iC2F=l6P1aZ~)htcbKCRYQ9o1Q_3}vm<-5kcYOV>F)*LJ3b z&AirC9n@H?$V)BPd!5%at=Nt2)NPH|nkvYCEzW?Q)`VTy34GXZ&Dou;)rjrcU47S& z{T83?*o=MJk*&a%P1%9X+E*;tm_5;ujhv~C+G0)G=Dft9&D*ve(~|w#t$o>3E!!sD z+olc7w4K<#&D_m>+@g)#wJhAj9o*oug(qLb*3@0y z#QnA5-QC_D*W~@&-`(B{4c+u@-e#TN=&j$fUEa#Q-Tn>W@eSYX9pLV*|I7J3-TPhK zw`|}7KHke+;PV~fzun+MJ=w5*;RycF6W-qy4&e^|;Laf8ASBBuwH0 zecbm+;3eMT9{%1aZsRG=;2Hk23l7a7PUAlQ;{)E|AO7R#o#5)sVWFE zE|FIb>U4h4{|)A_4(z)=>1V#vw2teTuIjv=+hHB-zb@>Bj^kK9>ngtI&|c%s?(Eq9 z>7u^qz&-7z4(`?-=-U45+-~lYF7DqR?bJTi=RWS$JL=s|?js)W>7MDt&hG2}+Vj5b ztj_P{Zs638?y?@v+wSWBF7Nf8?8>d{(tg$Z?(j(N*V*3h_73nAKk$rR>?(fmL*4C| zitx(r?iN4gi%jqlKkTg?>s%f3O#SW)ujy@m@+hD1c5c~8-14}c^Fki;BTw@z|MBYH zxVM4$Csj?@>g z^kjeSNPqBQKl4_u_ES&wXz%m+>F6ci>TbXEL67ec-}ZKY_V#V|86Wr9zV<&q(sj@G zci;D3zVk32_<5i2cpeY{A^8La6afDKEC2ui06+mi0RRa90QCtRNU&hOg9Z^QT!=5B z!+j9>MSLi+qQ8q6Eo$7@FC#~fA47_qND^d8iVf+hOo@`A%a<@&%ADDfrp=o;SLWQw zvuDkpKu4YwO4Ov$qDY4xjj7Y8)2B_LN{u?TD%Gn>gEHO9HLF*xNyB~>YqcI$uw{Xs zO>36zTC`)~%8g5RsolDG^LE|qR_)uqZ37D){Fm?C!-&c1RlL}+W5AFLOP(x`vgOB# zF=uAHxijI&pf8IaUAgq+#-25&zRMZ4>d&Dcw<8^Uc2nE%PP<;cTQzImuY=Q8je9ul z;>d631|GAwbKk#(Gv~TId2QomtzXste0q29t-)XaA5WgU=NAbte)x1WKr71*4B;Wc>RgYZSD9D?ULIADMnGI*ha5LQRw zhZ1rqVul)aXkvCChM3@p8@9+IVsDw(o?j`R2qA$bTKJ-kHEt)Pil!CjqK-Kh$zG2$ zruZY0x(!JriBLMZBacij38a5dW~t?t`Bmv7h*n;SrIb^WDQ1#qhKVMXYnoZ6n>4nW zrekW#Ip>>r4r%9}D9TsonQR7`C!8<}>gAV*@|h@)gmT&9qV4@Tp`?KB*XX5~-ifG( zYjO%Hd4p!^Xq<}rDH^4pmKy4)d8T@ttMPgN8tAIDstPHpuo{YMt)kjG>!!K_JL#^u zGRf<)z6LAovOyACY=y|$*sG?U%KEIa(cU;MU&{9A?6uuyEAFr1mTN7zxsIzYUAAhs zG?w+eJY5C5(ue^NPyP&{Rn(ObY{r;=)!l1EAaJ}5(3UI;MZ_Kg9@`CI!x*4}T9m*=l16#}?!%T0s|NYf~#XO&(RtVc;JgO-Z7T@5r-${PMBOPTcOh=py|u(^o(JD%NA4JuB2>uYEYxdo#WG<7)$c z?&znlKJUr5?>_Uu!yo+n%+oLb{qf`P{Nl!R&k81n2%rE5D8M=xkbsC7pdunM2Lc|j z5(zYfCm=Y7P*4zn7Q{p-xQB`8eUEw`^d1O9SibX_aDBZyq5W3K!WLF1g)WR?^Fl|w zsCgq4X$YVq(h!C$jG=*TP=gTvhX@8T@NkHFNForI$V4J45r{=JU=)RTha=|jf&e_A z78{5N3)Ufm0h}Nh!)QS78_TSXT)C_V6@(R$!Z;|HUs!aDZPj(E(Y`MSYJ_W7}o zGHjjx21&!ziH(6&)WZ~^Xa^AglbnHo zda5``jx5Q-zf9XG7>e86z>mD(K0Z|ho z)0xnGrv0=wO{4M7npHICH%IzO8+qaMF~gD^;aH?MY9e#B8{j%qI{=}l zES2X^+nLl+)^)E$mE;N5dP=6^)vZHKAZ7)c$U;bxmXD05EM0k34CeBe4n2hk!+KHQ z5wa%=Lu*lynn_0gw3V`!r6*>I3RQhZHK&D*WL#-_SGzWq3%5P0VG$e3rlNGQ+Z60i zof_G4Mpp%#lxlZBo zSGtb8!3`Se*O~^wrr;%RUEfK>bz;}CN~Nq|H|$TE4pe|&{RJAy$5FbmqM*iQFcgL> z;Ms0BxhU?!OO3nX7pqjoxg{rh=bGX%>@czV?XNnC99c#XR;Wfkae-?*-U>UJwqy9K zQ-jFcC)>6ExlpDjc2RuTlfo1Xjpb~MWenp4mlB14T#X34f$4GgYy7Z!No zJ%d`+NxdnHpGxXWv%12e#xtOon!uCg@2p}ZvX-}t+*6Z!(Nw;ysh5oEJUsW+)c$n? zZSZPFOL(x%9l?>+FyiI<)Xe(@>YXQy?(U9Y)vv^)yZy}PAc9!8TF5uPm5a?#dUxNw z9=3}0S!bC(xX)`9bFh!PLp@N_aN3}R4grh>4tILe9gg#a=dJJoTUgoA-LJk0{%mc# zdE_?#b!l89z+5KFwcBxZ>ab@l(hskA;6XqvZd>f{fL9ya{YC=2uNmed8W`gsk1)^E zsogaT`g3u-r>_m%1)bM?#C*;#x{*3#PQNspxBm7CB9L)>b70k6fOpN?aOM-oGzeEN zHO*a4gm*isZ7}EkikJfwCot-5-UlHU-m+;X!K7C&`n^ZH<1)TA+g18; zT+>#xb+@{`-~IP-&mZEv^*G`L)O7lVvvY3Am5ZIDwusCDW8Hi#m2 zfs0psYiLgQ_jQJ+WI33EKZk}Mh=h=rR4|h~|D##AS9DHjdxZ#q(ME+Ww`@wde|q;# zjfi?+=3?i@W}tTi3;=3t7kfqKdbUVzKjd>-Xk~5IQ|UHYUKfdZxNiMteoA(HSqFp5 zS7bqmc*F>B1qWp*7GaNeT_$ylm>6%7v@rA&J>mFe*Oqj82V9O9j&!Ar+h}rqm~SW7 zfkY^PWI%oxXn`v>f!{TQ!RxO%jw1_vo%dbo3vXns1GifHzV^B0l~r-T8Bi{`Y1ng(e8 zn2VA3Sx115u7x|^_k;@RY!z6KFz0OW#EA)Jevp8PpontLD1@;{x$b57ErYSUS0Xd4PoZ$(AWP+V335THVzO79 z7+GxrI9bxygKhbTH)@wtn2aQv1-B`PaCw-S*@lD}a$9MUJHVNrxSV{on2)D;Ey|JB z`3JE?gHsrN)>xcY34AoAj9&MPc=(l@M=D4eUxs5+IQ8!C8Is%f-Jg~OU&9_gU`XNOT5RXRwHq$jGQ znx?W~s+XymCF-f#`Ijupsz3;7Ti~VINUf4Key0kfL+W|EDVOqjqfpqLo7qzbnWzdX zW#j3X!#Ho?7-ZU7t-(5?G1r03cA4F%U^nNmljlR-sz)k{35dF6wFavwe5s9`{+%X+YL__M+4 zaNJ6&>*<{&OP;m&u%k(#D%+p2I;hr$p%(gqLV1jFDWv6iTw%$aI+cY9n6r;cfAETG zd|S76=(8G$d)-;1%4wqFxsM=uyPEd~ciEGdii*GLa`id2i|azN)u}Nmjof&LkeP;& z3Vy&SqOJLsjOK+S_mLurdHB|>*;ZGm*Q_G{nq0fOchV)Lv_*QqTe5z7g#eg<@Y$z0 zpq7=1lsU()BXzBYyLAW~22@+TRf|LM8oQ|KnMHMOTl<{%>5BiFf^v(!722{$>Anq^ zgShL51$%9 zEu;7G+_-;1LLMIt~qdBr0>Bt>=e-w$xQs%+4NO+n{r3_qt zhfB%mTEFrdsgVoDohZq=Jjp8}C(E~w3a5Tpvp}c5Z>ztAD$QvOcvE?S4V;_F z$i#7smH>>!t6RTSh`_a+SO;da#F&p29KNvUWIOxFL1=D;3x9&DWKKHCAi1aAt4$Tm zfQgFFMj4G?umb~^f?*4Pc*nB)Dv)6Mhk!hN(QJKH%eZE!XF8a+N=lmuT!KS6#j5P4 zfJ@K~e7VEwuVg^Plv{Os%4`h(x})-VxDh}AqvwCasFYp1zE&JhzN(dXx31NErUqvBeU%N{bMP3*pZv8&rwkB)VFpA09jLI;Ld_t+iPwj~7YNaBUp-XL{?s#^F^0wFPYpVun$Q|m8BSy@tn6l+qlr)&D z;)kG*>21e+ha4E!fjhl$cH5SkzIMC6Kj_SA_q8PZk_PW&2p!G+5Oe5(wdn;H>nrlz{&`j0J z8y#`9M4amF;@Ii8rhU*?tgi8yyS8iO^liqL`Lwp>Wj;LxL95$4s;A_q%NVSQ+@;Lo zxvnFM)*u?+C@acUZn)tNSehpPHJk}elzYz5Fd1&#G|lGN;qeU2ENs?^POtY|)(TtZCS125-nl@n z#(g@)^^C}E=Yggt=W}jH1g^FnHm%IOQekP$EWJ&BKF*QL012wcMG5M|O}9v^&XKNj zB;dL;YP2pL-WKU)TvwU3NXXKeW!ufo6#d7+{>`u+-v;Q}S_^Z!p5Z^3UaZczhF0CI zitbSSdg_bDJy_u`%EV7C%$bXXDyPq{9hh#~>y$2tAr7X6TE6H@zVU7b*$%(PDbhRL z#3o$R@J*tA{@7&h;Y1*_0Ugp3ieKeUs&&3^){d-&-L4VoqBJ(PLtxRN}a8h)$q&7_C8-f8IHoQk(kuH+-0ey3}Qkota`6;B(lrgg5n zYFNar7od-*pPTfcB_EsiJqD1ex%qe3?FZV=jPCn`T9DQ~g_8lBaL>{ZQj4!P7{=hI0VwS4UJ zRG-0vUz3QZ_6<&4hL1;2f7QJ1!nu-qKqulCj64yl zX=!I=L;_i7PON!QWZ987u)18D)g_rc2uZ4yoAnQ)KU_^}Z5UVU8Xq_$t%{YX?a4HOt~fh z-^Ym=Zob{yEsh8}_a3hMwP%DNfe8z-~$E@!ET0uLv$>hdi%$$SbBHUss5=)u|~ za|pE3DpYN?3w2Q>B#sE%F1yHb>kX`eW+N#KMh1(CpogUM@2$o>+DJg2l*7Tix{&iP zuOPk<3@G;0I#9%&no24TlicHJD4CcNQakdl zA4Qa+F=6cM&Dd;kYSCSHGjz?8zU9?d0gI(GI=xD@jnp#hm6c2yg9Wg_8>yRgS6bhK z?!;clzv!L@U zdy7)=${F(Dw_x37b=GBlUCmi6qP;{m!HK^Uxo9Y@8qsNQxiagqF$4Yoa0Z}1#FZqk zWKFoe=kSyfVyum>b7kJj^p@O-Bity&nAN!Ox$?Dug!so{;f#1jiw|ZJFvLBwBYz~gia@S2ZsKXVXg8u2F8a?!xC4U3Hrp*X@ zUK>cIR04^I4GdDS%9zkV6~CbEZaNFXftQN)y^*=9QRZ^fO6bP_yyiL28qfn&8Tf|< z6W;HBP;*?2T==%(sfD z@IyM7u}~-oOCsa6hcngvq*qHB%jg1VF1tu%@9YuJ@4XvP$5TIv0?MY8IyrLZMu!lR?@lSvTRGF{ndbgCBls)T0WOs7uA93DD@uJK?5-%(4oc zme!c>V5pt%bmu!kc~7v~lL@2P=Q_-ajR>gIicN4rv5TF>BRc8QzYbRavXeD}B`a&$Y*I6`QP6B>R~b&v zhIW>L{VQQj`$^QM7PSv$f=ew172*g{SnIoL>cq#+t#;M1Va-5q?>W|M^wX`wB`!gQ zI}U%!)djq5z&(Sy$VJKyEvy4$b;YK__)QR0ubnM+X)xRG&Y*%8 zxWzSYg~ef3aMjRM4j`XS9rIk`*@@4OLW*jD4_(8XQ^1JC$aq-}iw= zw3mPaIRhWOE8`M0;tZFDB9~LG#5PcR%YxK_-~_UQh#CQR0~v@EI)P;msW=31ko6`n zhMb5hb`Co$q4#S4WznEF!(E)PV4LjYT^ zV82ErfCnhPKrL6{4 zZD(8A)(6HzA|wF_KroaE)h@FQ3$?WIBJF%rui8{T_-(ws0uuD3Ga+Wx9yee>&yx6t zO5IIig9g+a-0+7i#9eQDC*Z@(mczdJ?Qeh!+|fonL7_`Yk%iY>3b%m|$7zsIf$_Zz zPF5YK41Zt|fV}1-FUsRxeey2|1mk6RdFoh>heo)-MV8liSuP#{V^cg%IS=-0V-EBk z^4$;7#=`-VKIYykQ@evB+|4Ne_%a&KvB2--L?XD(2}{&L*#!{;f_uyEw4WXA{Y)e` zWT6AN%U%2T#y%Q4ferBAy`R`1MIGue@W+!9@VAc$IU_m%&=b7@QM}g656yc5n-jLx+X*&E!2d%boeR2xn?0iIh&n?$ClIp? zNIXTMoax{lb%7r|i3uw3w&5$j<4ZnRV}a%yyRwSDcXPM3^M-H(0qom85A%Qx__VqK zKSd)yfJ21$W4s;-ye~5{@%oPR<1`nWzxrFC$74bT)ITaTB1mI{OM3&&v$X)knkZy| zgPS>7nLNt-LMxEKS`mo<3dFe(xPmu~3!xhX*lP$!Kq!WyCsGQYf-sJ6S(`RMph&rs zJixl)10fcK0KIZSSc^figFfk-z6rxY&C0$(le<4cgH^Z%Aw(%66u2}v4AL}d<_rif zsM6ybAkYCmK(<~=B4d-rha;sqgca}0z?~ojJtQ+K=)*OGAv2N3Qk1$(LMC`Yfn`)R2lBh}`jDc!jcH^hEc31`g0xUmNvH^|{qx04 zQZlzl0Zek5YuqL}n6jkE!v=$Z{1PuBS)-mX#2x|{br~GwVaMEBL>_PhM-YR(Fab!E zfOR9ge5}FZTDw({r+^el?(4os7(a7pg}l291WL<42rpIfgIltU=s-)k46?%Us~w^g zDVYMhtSU*{OJm|6i{uy`l9m&aFVX@`zv?US@k?i8OcUZsL&(EvdjT}^v<4BMqv@Tq zValeI1AFqu>_LK-vPXL)gkI1`N<>Jm{70(x!>ASVyLG%4$AS&x)WOyifcsp4{DiyZ0x4TMgUeb0 zr#b*+nt&8&r2V`Wz#>HPzk++SE$eng@jaaOAU2VKGh}$b<+j~)IWVwCL&Y?{n0axCZWtp z8%@sK%z?}5sXa)jdFs*1G@lYg%Ht{0&_ux%F;8|}(k6A%)l{X=Myfve zWIOuoN=WrV9iTTb#Zs86DWDpu4?uu4HPc@eRAJ?SMW}^YIM!r9)>%-+6mSJdONIhu5&>W(&|ySI0v&)d<*XuW&|*baIyKg6ScC(p0cSndGDroD zwE&S#*^wpC8VK1yWm9l1foZLQV9l&>9WA1TSOcvAm~`3{=uN1dTIswg*t}Y-?a?Mc zqww+6C7Y6|i^pd1nRZFi;cGrOXaqvS1w!ft1)D26lLJ8Dgk4xwfi2jAty}9;SiX%1 zwqu1^jaXIVP0qqZ9_R#Dn1y>_f-j{2lof%)x&~(z+5_-DV_kxi^;Md+BwQ%dOE6Xq z-6))`1y#IRIgDAHWmW=&*;@owB}m*_$b>)5SWY0_Up)jtN(GNCQ$K*&K`_++m(2uf zGJ&RD0>pBKOHkQUdW6Orgqd8WjN&Q0%38YOPC(dNPV-cl&{XgOPjqV`&de=ISTHfD z-5KnKd=&z6L)CIa)lu+9L8!BGv)7dv1x9lOP@n`Oq=ZS>-}mK^we&@vJGKU5gm3J%CQQ29U*tVa!>d z%>)x(Q=g69iuzTHIs_C>QzO8Ii%MYLWn0@#Q`kMW1U3Q@u-pd5Q91o9#7)+nQdcXW zUg`}@5~K|&z8+h-qVNSbLQvl!AS9KNFd@iSwMA7y0ED`|Ud?5J_C+B7_|#Cc(gYOQ z22`K|JhcV3pj-i+vI5RfG@w7n?LL|U(>37G65G;y?+BHxms)f{D z^(wQyD90>Pw&i0lc_o4?Zl_iF;vs-ilv;-`#@F)wVi+9F(oBW269qcJ z$JjaGIKawnhATyw05^u7FRz*mPa4Pre3rt_BD`*+(#86;9-9$fRokR}s!r zdc4rZtI%+wT#~h@8g^Zba^VR8y+K-oL`DE$qti}+R5T6bVs*9ug>G8nWo3yq*H_+X zip5cmzU3yyWs%0^Nlh$3IL8pNG1-cjy@*%FC}yr2Ax}VNLJ)+36$G4a=H7}#thLAT zP1Quur)qf55etM-P~12@O25`7VbIxD~u4=6QWI+PG31Cx{wL>KU z*_riKMNV8%K7dB{fjxd?Mqq%X_T-N};UvIdJRn`f(pf^D;T|UIB(Up;9%}-vsEm!Y zJ}&HX)@Y0l+>iEXF=gWFUDuJGRHS1Ws+)`Ji7$|7A#7U+m(#e?gLj&5$XZHrdm;`RW*ZsqLmV5YY1jQ)TtWo+@jRjY;UkRE9mNT@{U z(3o&Ww)xtYAxF*D0&eMSuCnQ}ODX+EVSL3miPcXg=!MC~T~Jai+Ro4eKe}5iRX|SO z0@iI8FlQHlsR!oIEv;(kw%m8%|m5@^);ILhl~Ur$hh{uZ^OWj&C1tqouU(5yA$1)bEr+U#>gVa?9O%3{!vv zZ^fqaDyLP8-2fJb1d@S^2%~BTU{5YRZw8NW7w>TYS4b%0)?DO0WwhpDJq0yAcIy&j z@#s!*J3sSGQVUDrm1oMM2PUjJ+Ah8QKcMFme$Zg)Q!4 z{r(0{=-2rD25b<6R^TUu+s_>w)2}4;?EA+>XH@M**)ZR}0qEEW?^rHvaa+H0bLMWx z6=9PysXQ&Z-_6ij=yUI;ZWJ$eTW96!-rev1ZjScoiWPL$6fEDCsmHd0Nxe(WVf6Rb zK)J9LAPNaem+3DqH$4GAoE8Kz=+`gq1$E!hN5FDZm+(@Db%Ct#3NZ8R62mE1XZ-Y1 zu!PNdzxCqA^~~jUTuAs%I8&xwWrHX7ll55tJ}2v12Z2v^_7}I|9B}xh< zU?_&J8lmNh%Ji-IoKI#UU;!r;dov&NpuYw{0Qjfx?T#&~$Vf9hrP!zc{1(6PiAVOr zCic~DeJu6*bH?|HJ@c*)`v|639xgHe6Q)?ChkF!zdoB-&mDmtmI{v#baQ;D-dD z3f>aS`{J_^_y(MmU>z09?3MRa`ixa$umnNSw*q@#%a43yCS!pmSb`;Oz0%N8YU27vmhAI;lGP6jF53I~V;9Rw0AXfS|-0vdFHXz0+xiiZskP^`!>BF2jnH*$1{ zpn}2yB6W;B;E_TFjT#he{D=|(MTj4CG{UopP5@Xyr@xs$Xi)k<^Xey1NO^c|qXw#( zIBMfWp(;o2DO9IYW!a(%)K;>hS#dS$B-JUWON%z8qLhbBjI=Sxs6hf`2a&c~h~VHb zLTv;UNbJacLf0=~q=LO6-Gxn%phA{z|8>jexUsd!k}J8X4Ct~LLPCV}>a}avw{PI8 z0TTUAmd-?EIaTY#A|g&dJ`gStP}{|V95!Xk1i9#%CMaKq6OY3S92(pJSjsr)oObPy zy=P7b`O>9y>)Ew)-|k)H0E{IsZdXaY_XL(Fm()mqfu~PYk|d!*rBms+sQJ#Ds2jp`l(R zjM$fl8n*Evt15&@;-lYSb66U6D8?9!v$jagD3wUk%Q^&>b0Z5dxi>)xO_cV{D0lp! zno|dM2||2>kU3BVgHU-%3XfFfCmw$!s^uE#W%ncx=%R@&neCPdo<$%f^qsr)MyF<- z`L49@pYke%r$d?axyChkh+u9)c;I47A6r0D3vWKOXF(u8VufFZ02`HL~ z{KzGoP^Bp!d&t4U410OH|JN8Qn2I5Y8P3(}g=*;)qNN3vbC@81t(dDZyAC6YufT@3 zv=}Ui@$_X8t}MqWkBA|ckf`v&M-2&y*FXp9N@B9f?}mvTopak69^DF@aZN3dPrGKx9sj!1-Yd{>!5<6i zhjD)yb36$nq?fKpAgQZfdZrET0mm9mg!<`QYmLEb$8+hS#;PKwHP>Ak_S|zjf(#M7 zVv?EDj$trb;v;CF0VxC_rdis{*(UTL_j}#va9*4TZLd@JW>Ck|=G)VNrO5Hxt?rZI zKRk#WgoovtocMz2|6nKeFSqX4-lsQy!RLEFC{*SSXoT}*4q-`9(-|^hwGUk9I!8)D z4KUY012AG1B?4WeM7J>mkx*DA{M;*E7dtw*B`;c7)mv_-yWEM$cN_`>U4lo1;aRG9 zI=fY42vZ^(jvDTLh;Sa>Rr@uhb!qn*ukc0)DrP(UcM z)rv}SqN0Ij|3JgD2Z34{qm!+SWK{YkYrxS+I_b|h3X)C9UZKD-ZgB}xh)KV|bOt;u z5O47S3lvU|F42t{=pYoKXrd32sF*tG@CrMuGZ5{3r^FUG z00r%>UQJg)m^!rBz;+)R(OPuo!nmxemG}5Y zVgzvxTo%ZqB|YhYfKmrWr@BP1p_&moT|q+;D6vv}5d;8b zFxgJF34%=(N>103PoMBM)Gw7mVr9*emPR#LtQF@=jab+o_O@E|QIG;~?O!cs_;9J{QhtVh8H1#1t%mg7x)467N8&TbNkk z-r!9pZ4%oZp_2rgEMbgtuog6d3NZDA=b48qG`!f%VizM9$*40Wx!f3@MbgVVPIPc4 zwQE=J9(2emlntb7IM^S8)n=;FP&q~exvA74E|KsRtShzBWT0kuh;RhR?8}RwXt@_i z?VbVhqUK;5fFf8x<0NwOEiXJvEH;@iJ^e?_b6sL#N<~RA?FdJ~6~GGV0;S3V|20j9 zJn#%!bfz;CD23jb8{54p%c8T?hcG9%4HIqjaN=eiI3R~#{x)%Sy;#~FW9);L{P74k z1r$(5LBbenuXzIzvx>Ih38fS@M_m>owoG=VP_EUhA4?Sg{a56YwQqmFT-^8kSrdbR z>U(m~aI=X*HqG|KYLsvRuewbW!mP1~C;bTa=$ajin80|kjc483W&n>kD#04@bT=c% z6$5HyT1VrO4zHQOY+FguYtU7xOCDL>1NEF5MM7Zuc)A%v?!4bZW z@A?4^eosyn3M|&N!`K?7E>z5Y2^1Mbt2#PIm|F6i@iJTBn zpT?p z-8yO}?g6FJ0)ISIIkvaQRFA@Im+u=?3-%)pFL0$1XJ@(z!#-j8`)8LOx;we&zz}; z)m256b=EcHR^e2e*5#h6jg8d-0xl$=72E?>xj+U2A6o4m8bpo(Je{EdfU&LESP2E% zN#DFtA9FE6x$HsMjmG3Ki4O)86Ac9>ZOq+?AEzDRT=mR0bV9g^{~3DW#mt=AsKpQt zVU*g@U;VinD}+M+k(}jyA+3oFQ_PqFx&cr$5!JLt*W?*NjD;w*3^Wl_1X=|h(jht= zRT+U(09Bg=afJsOl@%y})d>h2AOOL*#zzz%$(X?EZJkk-z$GZ3^Uag{=>ZMZTPaoJ4A%6e`=+#7sQ541D1p&BF6sDaO!rxqAAxy~v$SsB^ zV8SFwo)>~)<&lQO!A19!mNH>pviycEXrCUL6BB8kOKiX+c?D3kLLCMf9tPaM2|)7o zlhnOg@qGXwI^vKu(t!xf8t}nUjRqr%;ArTN7dat|&D(J3|AeQM4Hm3jS>Yffu3B*A zQQT1rc|npKfkhEQMJbvhDpHxq&`;Ync-;Nr#jg<{|z zt_dS9;vzn&lNYGjS1IE@;*;D|+86~I;=n<4g@nR52vdy3Sh$oZAfhDEWSvCNO0Z(+ z^+{J?z-!Gy)?wWTBFE$4RP$M#S}D#CJRd48UY4}g1Hixnk<}YuVhUx@9)v}^Ns(@` zfmzNSvdC61f)`+T0y}ipLh4RXr?Is2@GcCqv?PYJb-RKfR`+nvz!_w7))AKLuW4CHc5fj0A;|r<$|e>FQ#H$ zG9(g87L|ZfI?Z5R99FB`fM86$LqiY1Wbj0EE_if798fqU7|D|XKV%H-j6 z0u#axOVFCsv4MW7Wy%x<6rcp!q=J%>N>f}x9NtZuFdb7>+dYZQpdrA+^y7-{=5FdE zr1eD`*hitHRU<9I>ny?>*@1jS6?(`PRWO?+%0ej2Wf4YacPUh-d0LT)3SjDxd9`PI zkrEYX;$e>GVw$JYP^U2Rnmu7pYQ~?;z-ML>|C$1Bl}*Y)A%a~h>?f7hgnxEXu&r5= zwM`T4i-IyKQ;o^(>4G7!LOS5jyokgU914rYMP^Z`Zr0!%)L`2wmxf{>mF!6}0niG# zC5cv-3AiPKz2vRoRWqE7i#Ft3DqJl<2w$y1OHLe%kRr(+W7 zLh%^& zga=mAsGP#-3xs7HEbA7?X_i2U2$Us#A|hyk!Vi9cgK-ObOh6xn4C;6S5Ugm6CaR*k zTxP9R;N<{~LL6WrnGFplT)>bjiDyWX|K}EVVa9;rHKJx}+M1_iUV?QDYc6S%W+oiC z0wB~xL%8az@+Ylvfh3hoj%s9t!VAFVO_)AMjY*@wpcn-RiG(6S*yyGaK&shkq8hkd zw;0S4G!o|^O~)?5$owgQn&_cU%27Zkx&GMo%}3=ts`gmqMy}Ew;LHnc++dEHap)TE zy#g56D=7S^W74aB$w#go!rRyrkz9hB-B!)TrW0MzhEbPv(MA<8A^J(F6Rhp4K8!9- z%+3v{We)5IoZrFhK^4*7-M~Rjw<+@&?B&Bic|SO z2OXWi;9P3>5lVzbY}k~vsp@G}QOY#z9LYip5Zf1p!25A0Bb=52Ud;Chj52+bnBqb@ zjG=OZNA(#iz1d^5jlpjgCUcFYH4R_+m+mW*o8x{4%>-RQ)g z(x^n-hmfwt_^6ci;xA5Zde+*6#T>{`f?3QiK9;Qu;8tox$qGClPPmyQ{LOPQrQj6R zCsC>2Ap&8rh4jE&4u5RgGvFp+YFYMGz zh*D6BZ5;KcQwa)6pz#!_tcAsu2nw*A7OGKHps~fR9S>We&42}{D@6d7zdU2TBw8L4 zvAEQ4^x;H2End+sGGUJ8ln|Ld-BJ9wTU(wa3LP;LkB&9%S-MuD(Qc{|B?CCFbS&}0!4whv{SVX8Z+PS*aGqZZi9YpAuH$tFw>zhms6Ue zQ5dbkPyAGN4Dz8>G&p_0d~kvL(ad!!mmBmpDAz3=!b>SSSV zCV&zYtl9GV#bH(C6z`0V(hgs+^DiN$#rdKk*otfgqkVYsPpD^7@0t=XlRj_4S!BvE z{c|x_)iR$!$)Q3hIF>G4Kr9EX~}o*-bOzqO4F% z0Pv!zy_|`hr8FtBv~)HkRqU9nN@SN%B$6sh6}N@0CN|OgwE6_d^yFwxU zrcz)=@nbi|coH=tu(D;dPG@9x-ElTMzghXLjFxsvK;Lq<@L*6avqcceFA`WT-_4hL z$$TSc4bO*iXq82i5roQyaDcW)CRc+a7x~zpFsZimK3Fo@t-!Fc0Z85uDkOC$oU=Kb zxJ7}e{1r~Or*~H2r22&zw)jfz;x2Gur%Fn)S)u|jbyl3lehFBFys~|OFC<`*8mUJh zD}ivE|Ifid$Jay$BxMj9VBdW+^MAkQAj@~L@pdS#FR|VPSi%saJ-A%^lP6YNTlv8{ zJ+-8ymX&XKUw=3!!L&?cUOE%ja@8W3Q$dUS?5@d|P?IO3Vp)s&^u=)Yo8|dIYYjUW zxVwZca$%;-K8v(%0DdKNZ=<%xmPD0XAXUp|AK&U*-+7BQx(T?#Q>s;55k*byxKC8k z%CUKedvi-$uB=ygGQfF}MPW{tTB_Kn{1GNwq@14Hw4x|R$eCPtAA8lB7VFf9j5wBH z6rZ9GP#xFi_DGX{pEP2y3U!zI-(yNpj+uA z|3?w0qng7U2N~3#$NH>S_d|}%yF1j}e)oBOa!$9}=RSzf`fM7EvJ1_ZYa4o=NBnA- zUkWj`W4ZRlKs)8+=?n67OyuV}mk_ zZvw(AyeJFSC#VA0uYxQfbjZjE$N>BQC42%Ohd401L7$Aqr#3Uf7P8nu6+KnZt4H#s z+E=rqrKm5jhSOIbiP++%xGA6%2pKq}0%?1VME_4WR<#^GxtXKy5>w5S-_{A=|Geh^ z?S^4W!N5BXRtw)F^1N*hGaB*IXL5kBzB^!9qmCDe>#@v$L3x`R71FuxC)U=>d?1{_ zC@ad>hyBHy0x4jEv&7y}2lF4qx9L;7CisdBmwj;s@fi^KF*YgT(JRRsN|QVIR|aS%k=gaCt?D&2LN3QA5RUERzuQv+L?a!GTvY@2$cUds zkWz8FgeKLjS7=I;zCHAz(FYctcr>(1f#kGE37{0%9IF6neY=ew#eg;K%|<= z47Au{s|U4wfQzma+#+$i;7o`^p}huBO*_Jji$W$*7DFwEPRx2`|FYmR%M1X(MCc5O zB0tN}y-q69K{F%ZyC#trPn&YKS;{bAj9FOmWEEXx>FogEh!c}a364wd%+!DzNJkgU zd^5Q?<%E&1HTn?Gyz=y1Pd%4fvSmX#@K^&5`IM;3r>d+H$_@tk3o1ZHe*%!FNtGf( zlmtQ1RFOKsa1a&@CTGk9!(>uXQAb8lDaQ&5V8h^yJqmT~0}9q~$R;Db{pvl^F7Tt4J^0cV!ycrB zjk(QeW1tMO7zrg)Fnd+NP3OjhbILjO)wh#t1m@#kGMh7&|H(S(z;n-r`RvnA_D*sr z5vK?>bOtp-v+yQJ8x8PL3pMVP2>;>`*--$uT)9h7RtW_WP)`YU=20I+1ZONpwCjqn zSj-4YvF1%sj)h;TOK7}QV|7Jd-dX1(YKm^euwWaNNW~@~v4fI{q+MVQT%y(_+X)^i z#}-t+@b+6Q6dHt)C@jEelNS;cZ^p1Q2VLXrZq_KnB7-p!7JoDoar=GIr09 zYUAbJeV0&F;728V^>dBq?yZabDC1BPQ5=!^FqcLq>}>GxIi-T06EKvV2bYM zxe979gMl2|Q#^=*{N!#wCw!eEFJ-%);Gl(E|AC?KVpub(*bs<36wLHWv@=fnz(qGS z&3xizzHb!qEt>NHMA*@csK96sAITIHoi;`EL1Z_sgaQ{z$f!W<>REJ%Ox-SEg4>{i zjdM!lX0UJxPYg{HC?F0T79^TMyeKh@2u~z7q%uIv)n_VWKkFR`Va?h8b^V6 zAlLz7+*pw!r4*~!a}NCMCnYJAsxzbl9OeOjEM_#;+ej-q z#DeGvQxTp(g(IK`375%KXJ{~;E&F0ov4G-z^c+K@T;;@6^z08t&|%SxaHVB7ZZUEL z;A5k?zp0qYnJ)ruPmoDRaZc+;(~@A>`UuP`=7uyx{E{HeXT5bwWFdLmL0?MtA4}b* zKm*cPr7{aYoxD)3Y^5Zuv<6p?T=8@FtAU+B<1xMB^$-wkLKuHViOFJbsJZ;rjuH#p z_7={*!gWM3*#{`uoi|bjHKHh$|909qYKt_gb)#ZKIwcqVCX_4iA%ShXFWlmGAP6ym z4$gp3?0j)msLBnhz_3jxLUVAPJ?4Df!{WS4>kE-2;^H{%yu*^ zZGubnwU>iOS+*ZQ5P}sRPHrz;g}*&i37Mz`*|t-xVhOWr<#23@0aehRDQFM^CM8Ws zTQsRIo+66MH6tM>}5UR6@P)LSETg=dYtgIc< zqNatkt+8AX%cs<}IMB)K|8x2pokLYbTgsqKCZE@#PMHPO4&3b1{_09^(A1{Z{DDT~ zsDJw1gPBj!89)QL`EJeXP0BRY#>6ipHrmD?ihUu zq?}LEnJxu;SK;YVx3>%8y^ujw3p$$kU_J>%DS}fs2q}-~YEcl% zwY}|{6BZU=eQ-!6|GL&hCOOG1dOciQ)gRuFg+KNlHB{n|esvK$Sv52oTf9RK29|sU z3_1bQenp#{SVDnUz}-CTq2hvK5xRZ8I65Q!i@`#-2Z@e!xhDbkRCoI9V-I!P-yZk5 zmpasEqH3FPpqGiVyB5ZZlz|i6TRt8&+D8?Oo73iR27wQ~U;l@NGL$Zb9j#0MzW2Tt z9{k?_IQXxK2-7^cnAoTL=cNJ)B@t%;@wT=Up?h z#7(zUW7#qxAsT|9iY{|hN%kVJ_GmBnc)$vD?*cPW12d2fQYU&khC@O|QVi}Z$WHjW zC=AYy`Izs-{~Snc1m;O_VE_rNhiWh_3;+iK12P&SJ>tPj{4V^!@B2zl@cN+1&PDy? zA}*o_ob)UTaDmz2Lu?ZvzLB5Iayh;7KlkZ{d_;QVNdK zVCSs5iV@Au;Z~5;$e>y}VcRs36Ss}G5{QQY!v@`A!^nlI_O45SP-0e*6}N;Ri16?b zZxp|*2EUCt3W7$;FfYci-=L5<*lq=YaSa>c9W<&(aB2xA#|xwIFeHZ{s!&S8Mi|8n z3ZnoN{}U*IFiAm^-vnEMGyIRGo}_^}hWjS!2{533+`5YdpL%RkIY9#IhNDlK=$jto@L5=W&b8US1LsKEU3 zAD4y`T1BP^Ont~tA%pN?xZ)vYDHhMqcq-B&BWydYZaUH}TilK(hmq|91|$<|5s;-5 z|I|=8{<0G^a;G?>FblH^nJObM2Q#R!FB?G&+a!;u4l)Vm8#{6j_z~-P@*Is41DSvc zJn(k1saqi>%T9I4Oue01uF-AL-=HFsVD1X$9tnw;JMRW2|HZed9u(O7oje3yhcvgTQ zPLVyl;Ver|Pk>W6b7Pihk$!LxIWcq=H#9kc=#!{3K}AYEzfBfPbcf{VL^TOS|7(B~ z?b1XQ4N^|W7?DzFN5}ZHKOF-=sl^TI zBLp%gE1hjy0&zMDszp?TFb{Ga%pnwZ0WE6;-kRq_8&yypRW7wd_w=;h zc=Q6V^R5)dKsbN_x=wyDZUo5+Wsp=!`{`f+)Dh+&0yQSM_=P!mWi1>uzM zH6z40P&HLomByx&Iktsc+|7SLi7?#DEXQwG#gty=L9T=iSZUEHh&5O{G+Gs+2IB?i zG>ltmARUWIUI!r+meq)sWIC?kTVGToedc(s6jAYlJyYo~lt+1twXQVJljby8dv-)D z#zl(`Qw?cV*$-YTB1$i%UQLKKjqe0V=tv8WU;8!b@=Mog%isnmL$qAXM!~mF_y-J4qWTxek5&fpOtxTQ73CDKvA8r>8PZIFtykC9l{~>zP9~n{ z&MuO0{g^j$(`ylm(|Hcz3Jw=8vUdvLCRWPrSXJd!cqWWkwsCj15B@URvM-GAtWj$= zPH7-IJE^WFw`U#go#qrh({Xy4b36ODd0OCeMRydEc2kcpW;88?Jf^t{SSsx)7allv zT~l^x=3i?!5n={+qx3q;#PW=-2e|jxkOp{hgZzf~Bh>bs|8}DiJZ*Usc)1vAKCe$~ zXPAbUGRfq#x^6gAWf)X-CN;CKu!7ik(`ATv=JI^lFIwfWk{E9cP0zNVmFI*yBHDyga~{Af+P6s_DMScbTqZLC@cUAHaJcA;C>zG zhhumLVr7JzL3mBrL07?eRd`eSBT5e$kqHE2Stn`-1d$WD2r@VXO!G}0*@oYii(5C7 z-v^S9Z|g2uY!xqub-0L`xI@gq^HLdaRylvF*wI*7bXAQFO1F6I6^z3wgbvBM8c9g2 zGL0uVy9@(hwN@xtU_ebT3Hnf0ZDo{aO%wX~k4+eb|Eo+Cn&FgjV8$$2o~$o!e=h_a zf`II-RpA8QDPHJK^J`HN9Et$taJf!SYaKr6A92UN{=3-GplWSOI>nZ4LX z9u|;KnwoD;$+#JJaSRDEWRgD@;FR}s12_@)m<%{orazf>GPqmjHJd}%LoRJdecGNG zq?_eg){I&S-#MO>Sf!l$O`&?zo*Ic6Szz(%nys4d96=IDcb2i`g#(Uty{dKXlk0W| zjVs!sTPP65wlM;fj<>R3;solhO}Q42tV%je|4$mFuZR_rVH;Z5b!u9lqZFoVI*(~u zvBw#)bGoq;kz*oTF1`SqcUrRB0Dv*Osi^>WJ5`i>nzLu^3v`+QcDSfN_nWC&O&go4 zm#d|-cdK{$dO<^>Me}eDg=xpwtm3w5d%I~hX00i@twTi(PGCp^RJjeXuI-SM=T(oN z*`)tyc(3fDwk}-^TcyZ(ohh3H7<;s9%CzNqKJj_9p}>Pao4i|Ksn0uwC|jU6n*f+g zskOkIU;8FhTedaMwfkAIWm~Fg+n_2{GzTNeCOe;q6&`=Pn}vb6*;;}rv3Ar21D4yu zTQ2BokS?Tqx>@nB!;5G>FuPZkz7e~e|2g{vSj4_{c`03oi-+0~yH@OG48=v9{?4*`?aU(sMH=dwquQwjN7wI}o8!D~cAXIz z>DSj$(S?1nXUy1_E1m?Mq4K%}|1&OSM0KV0v%H>ntZ$&wul|n_WF5G2jn+wGH_SbtWvz$xS+9rs z#~Yfxv;DcFR(BSg;dwgYowwkf9pc}okfi zKC=xOx}#py!;b1*nxG$^ypz6%y{yqWp0!7H_`KfkT6nz!ox#U=)T_M8d->&o!PJYJ zD~hzN^tiH{Qs?g;y1KxU|GU_J7HcX9hc#L(@CRR)VtYV98|+7(>RUIDrK>5oz;#!* z@gLtvqyC_)p6>rk>9L;kGZh7G?Ao7Kc}ozLKL(Z)4)ssphxHgL!C3J}oj=g})3?^{ z+y3ok-ZkYN(KFwh`B~?ip5+~ztbO8kbbo5unN@FD!IA%TrQEQ0Uc1x2l3`z_fiI>5 z#Jj^;@0EYNxBIl^hv%Ii`x)4`GvC^Q|E#sX<1zomMJ82;V!>11xdhIuY~S5|Y`EiI zj4j_}00PB`8%4H+Ai$yEi4Z$*sA$;Y;lhYP6izfqB8EhZ6(vrL5P`^%i$q2+02$Jx z%9063dhD2RApI{LZ;0oxt!SH0iCgzG0}3_1VfzMJ8G{{LTi5Di$KNa`hrpj6k5(Oy!n z!S+>w5gr8u|AJpVRn}RPk)hOtHbfVOU0R@I0$_8C>c@#9FQM5rw0s~`A zhSqm;wL}wAWDqwaWpT(iq<^Eu0GMfT5l3W6Pt0Lc7?v$b41L1fho64=VPeN$U4S>` zNMaU9#TQ(q6rho6jwc#}XilZ3c)7XR9cn9{hf+@+`i9zbBO!!Yg9vd)`1d0(;E;ff-SziM2C|S;0*C(8@A#34- zY8B_;|4E60R8k;zfCgNNyM3yWue}|`>uK#dhbfYqG6&dO#m0ADzw!yz>Z`Hlr!ROv za%CG(%hAhYy+avFC`nL}bls5{vUY5HneqwUy*Xyc0cK<{GEt!9miQjVegT`Yg3=xu zv2Y&4jG?9pZtI{~r|Kl&lSN%jV3}K446eTxLwh8uH~<{Xl>$4aWhkF)94A=QR;yvq zmQmc)MiaAmbk9L@t8cuA5{*^5wAn#2yo)ya9ZxSm$Z2sVXK3?AE#rnH+;C!zug(q| zT{FV&riw1%{9dcxchUsxD%6s1`NqMeX;#sfL-TsjLfciHS8x5H%{UZsr+nb$8rzF4 z|B1<-d~LmPi~KIisN@=(U^`ZQc zj+-*Irw`=|PVQ<1Eb&<^Z*}9E1O4^A|DCK3dud*hIAFb}VXtxw6G#D>cdK4KEM8*L z5!yCaA+Q;0S=!rjZC;2nxE7)kmLz2 ze|lRP4WC85)a`75vw~hzqERKMT@QN%bDj$^mn)?yve4ML`j08&`@-c}QKoiT(rYF4R6Nva$+Cx{J8))_|^vsz^=h7*Eec23|SM%D%l zg?yW*NZ2j?4Qq!FOwk+5IJXHJNtc!j&1-Yid%Pyj&%L+7vodg3~!(1fe&5|JK6wea@eD z1f>H*q|j0!O)Fz_UF%Hpmp$^aWDI0qvf7oxnb}hfFB*{hyfe&w4)la*WYP%&Dl}*o zj0jL_Dl}17hgDtll9|lremd&Cf_~I%&RAp?XD(QOZ zHt+E9p>LJv0oxhFs@*H6_v4u-mv=g=rR97V=_3`0XF-Zobz?eAXq>dlRr_d^i;*Dg zTIqFHv*Jp!UNj0?m4((wrWK{9BwT8Cs#dT%vUG>Us`8x3KV#|gf%|0bj_8-Pt9~ne zhh=H&3~J9#t!J7dYol}c$x0C? zcX=>1B#W!OPC<@VzlJ1JnB42&la{hU;*6|CG3w(V2N}QwZtNEs9NW=N3tBMhTN*#O zI8^q{%_`A@kSO%1cLuMjaTD&dPj2gu&yngNH%Y( z(#PiUN{kFfhayjXd+w5sJ_~GBHnq}gv4*!*&FXNs+A&tNpI-B}Ypdy5th62@h8<02 zuaOtewsS0!{fWZTt}n{wrj{apH{iTF+Eo0GcX1giq&%3=lh;nO_sq-Ss4jae+4wdx z;=F1gm%C2;&GW-EH`XB^)y|6r`IGnUOjia-hw~LA37Q$umdjAXp}sI`vvqFe9xb7l z6Q!C(?w@HX;txggBelWImvOLy}BO{?V z^7Lv(J(Gsu=8XgV&2+Duq_YJasUlSDh%?#PHAUykN!6t{;e548rPQH|uykefI>e2R zV%FAvbAzLsIwJ+Wt*g$>X)4^3;Fh|$Gm44ULRhDUh5Xm|9CxeRaPX>?JVmcQWhHkl zyR{|0wEyft`)b)n+-CctnyK~Tr~92R=Q-JF&z35Bw7bgB_d}o#XWwHy_}c3x^f|sY zicMaOOb-7F^{rNS5msh)dml77a+7=`rgPK8|5JAcYtct}mEtbswK8iKPvBQ_K37VE zB~$sLeU}D*@)v))*n%$AYdB^nN@jka)h1zQ zVZO9s4WVNW1zLjk5a!Z;tQS6URV#u8Dv@VnP-u5e0T>~;co!FELKc2YrfFejaxR!! z_BU;>RD&eeF{c+w8{&T|ux4_Xe_yC%MrU&gIEQR_V?#KF&lZFL_kri0;htmT`SQZ~j@fOH8ikxUS zy|`-Z2#m7keESxG$QWGu7mpXFJ3vQcZb)*ZD0?`hdd5c{;Mu=7hGkgQpgV-gruLkdYb*Zn0op*k*q>r;lD{T3feY z%NI#7Nsvf)bbV!sJ}8Yld3J#q|CPF?L_vsH07!`D6orZd6;rfuF&JER_gp{8O?q&Y zby**U(0>m_U*QFI#l&X9=yv^haxtluBT1IVWR-{bOAy#-3`k006*9*djIdQ>L1I^A zRc)|`kXuM~r$}s1pbfkzns#}Yrtk@;c?zdl2&=i8T#{9+$(oE-ihNm%Yj}`z#g~K! zVuUG+{MdQ@5&`bVnZpT_C^;<+b9ekVl2KWPh$c5d5o3y?ty zs*sx9iJFlhp0L@P;W?h<8Jj#QQ(}jk?RkwwXqdifPyyF~uK0#=2$-5#G{q>7C3%O& z=`ZJKk$fSP-3J8MiJ;iI{|uX@olygx4+^2-DWT>$o)mhXujwUqu$t7Oq3GFJ8Jbm( zXl5%opR!qiWfqCjSBNFnbPD)_FIu90!IC?7Faj#0h{Jmcs-TJ|XVkTw-ua_J3Zz3C zp+zd8L^`2Ks-zZ5o}W;jot2&)>Y=dt7lweBo`s(=8jV2JdXvdmjK*mus+?j9osBjt z7D=qdO{sJ{mP#B&0}Mq;y)Rc8aHYYMOg`r+Uhy7OI+s5U3Y=f*2a5QW~Q3 zd8H~!hs|M#SUQv1(`*BGY>8?&kcvj3*``S8rVSdWa{8%$Dyl>(r+qrAr)sLGx}-_k zq<`w9PWq%33ajWT|D}T(n;TjO-5IOr`Kq%@tCl)ZIh6+G39N-$rFF2WoyscAz^u*M zqoE3|(Mqb*O0A_jr>csrshX|Y%B|hXr`swu*9xi|G_6ph3+GCLk#U-ek**||tjfBq z@#=eUInqa?Iu&&~+#-Om8%CMpt4s80a@XDhM zO0VMTRRN2!{+h8GyRZN1u^+1?o_etNVXz3>t|fa-8`-cc%d+P1un{Y<6DzV8E3gB* zu{R5{IIFWci?1{bD>DnUCX2E|Yp^QYvPbK(FB`Lqs0_})H1bNZKFhN^JGE77wO3oO zQX8}+OSD4k|FvCfv|)?15UaFhyR=T5vQ4|7Y8$m$>$X2Tq}Xt;SWCA#E4MxSws7lJ z)HSkT>$eMgvQh`QgFCilYqn>*wrTscZCkN<8!K{qw_Fstj$5}hJGqssxAfY#U(2~f zOSobSy6{)Hhl{wGl(=uYxs017m%F-*!Mb~!xvq=1uN%9yOS!i@Zk@Xdy8E{Y`+Ml> zyP*rSg^NYSYrMxhvB?Xy%e%PG+PK*SxwZ?t(kr{uOS{wyz1aJ_jGMdN+q>5Exx&k| z!%Mu#%cJD0yy=U&scXI5>!Z=Dy|C-P?Ap5YYrplowThcT!TY_w7{2@~z5!gmq-(zD ztG=AN|GrV{yj(lK*W19^`@r`b!QRWjJvzRqd%+njz#IIo1YE$Sd%*r{!G5d16a2a+ zjKUI}!W2ARBJ9HdTeJqO!QeK!9t^@EjKJ3OV+p*%D%`^?48-i~i=F$zo(sc^$g&&^ zzHMr_q?@#L39(Nr!#CW*JIupDT*B_#zgWD#Gn~aD$dQ6O#Y&u64tuyS`^0E$!()uV zY#g>%T*O-p$8+4pbsWQC3^^aH#{eA1UQEUo*TiO<#%i3#du)Y)48?})rZ8K@e@w@6 z49Sn|$E||HZ!E@++{PS?$b>w|g`B*btjLFaw2Qp5eN4q)%)^=7$aYN0lgzxPtizZ* z|G?x6%ALH%fDFosoXeT)#;$D1s2t0!tjeqm%f4*LzbwpuJj*(|%YkgmpWMu(+{wL<`yvhL`&jmft0Bz6VT+8~LwB@|d;2g@*EYSoF%ZVY-@NCfZ zOvFS?&-F~p3XRXUEYeDQ&J^9z?2OVFozW`I(k*?^B@NLZEzTj$(B|yVCcVv!ywW-C z(mlP=KK;rrebb`b!64ntH9gV~J=8dDB_|Ek+g!mf{nX5Bzf&#MCtb}(P18xu|JAn) z(-#faRxQ@o%(i1~*4Z4?9<9z=&DKb*)L(s+XYJH-P1k9C)n(1ocMZl%P1G~p)(Z{T zHXYV+UDtYD*oTeRcAeORJ=c8Q*JkY3f9=*uZP;jA*^ABCOs&|bi`aS1*_YkeYMsPP zyv8*h*?}$CLao`Mz0;u0+Mey&a4p)n9MJ+?+LFE1&ivMx{o1Ro*1bK$8FuxUD-PQ+?ze!+?~?@e7@wp-H)x^ z=^e%5{oU;i+w2|Q)g9kh?bz3i-Rhm&{S4maJ>1L<-@+~51YO+qZQk~+|Jz+{4yx_l z{{7zn?cUT(-~m415q{n?jNi$f;Jh8-`~Bd~>fzMA*Bu_SS0wUNke=q9PU@QOxv%h!%o}lZQyji)_-m5!G7k&4(&Vs?8y%5kxuPdJ>kOM z?9uM*){f}jzU$ep?6W=Y(%#~WuI-lI?cCnx?e6a4uGioG>*Kxd>>kkWp6lQq?#4dn zGTjY6obJR<@4#-=_3u5m;UoRZ{w!T1JD5h03rDV1rz}P04x9i z002M%Kmh;<{{Zy~9GH(_L4ybr9$cu9p~HO*A@ZBp&!5GM6){rWsIj9*jvpn897(by zNRue>seBlbWy_T?W73-`)1}RtG;!+O$+Ks@pE-fPygAfo(W4<#W;~g+snVxRi3*iE z6{*pnRI_5e%C&3NuTZUu)f%?!(x_;ij!kQptXr>dx5|w>7jE6Qc-QL9%T{gQy?|NX zwL6$@VZw*yB?cR~vE#>o{qlv3xU%KNmoaOmJUMRXvn?fae!RJKX49wTwu8sIHRjKw zV@DP(+i+^sM{(a??Yiz^*uZNGAHG|+@#DpP^L{;>xbxtiq3cDSyn4{(i?K_8-kke( z@78To{~r(cJME$8&8G)1UOV~r@8h$dPhUO!@b~58uOE7S_y7Fm_mzA4?Pp+s2!iHc zf&v!En1c;62OxtBDrg^t4?=iggyl_$;e{Y_xS@y|QWv6!7N%GsiXUb;B8ersm=%jO zs<>i}C%$;wSTRl~A&x)>nInTe!dPTwLMExCjW@=2WRCD4vHh3TU0DLHcN!kxBqVK)k=I+@6VQnyI6WUTP_&b9U+}pQegh=%}Bb zx=?SaPX7w4l&aQB>ZY?!gtudV`XY-Gj8m@Bcq9vhvr11_s)w8Kuz zY_%prJI}MMx+*QM!h&npmr zYcR+ow@h-p=rx@3%%8d}^3680tTD|zFAOKnK;KMl&M(8jz(+w>P?*6VCCxC$Bv6#53Q# z^Rh4h{O{DCUj6mCW3RpR+yn2u@LfBu3oEOHQi=KHhw@AMn6#h1C+L&^KKkx^GJg8I z+^_%o_dgFk^SUtJ{|4wk0Z#9L-P@k>u9rXsHt>NCwBGl?hc0gzBYude!yGhd2sLnU zgJ~Fp20a+U4BBCXC^X?55TS<=roj*_Twx_js6rUVuziuRU;N-#zDVSdhw{Tj`Tyj1 zydmN*f8ih>@szm4@o58q1cV|2yBEc781RY^%woZ`7(oRJZD?{@#Rowch7^*Jj4t$` z9@ZF#6Pl5YB^(1B(I`SP#4&_V0KyCBh{rUlF@tyPLk-6mL-l1)5%Bw<`o`x&A+n-= z=EET(S)mOg?(dT2dm{gesL4)3(UaK-B`B>(%2HMli_77nv{acrdWE8ncj!YMVflm= zx{;P@e8U`Vc}pVi?1n!FXpoO=*pK((|79R3$6<7XQ4CdE=G2 zJZ2HyjW=rAd2Qj|)QjEI%-Fj!@8OM1lEKVL}z5Ry}Af?HB`PJ~ay`ZRuBmInt0qHJfEr0Yuwe4VsIovm%mW^nQqhJTiLT}Qqem^ZLWfOR+bByn^s%1nV zHap9#nsp1KC98Q48e8a!6ufgiE@_*)*YooBqt2x7Lq(fe=)xAR^Cf69J9~(%647m9gn=+2~W5X*gWj{L_A_U%s8S6y(){ryW%z%nE%Udi0X@tdxMGA^~Vs7 za}bWZ-Z<~~j8uMej19~P8wXU$?=9~cw+dD)>sifDhO!99>sb_&@R~tzbVHd;1rH9G z(QvM`q0ww%EBAJdU<_RnF)P+RyE(@R1udo1tV{Af8q|C??vCFqWdr~D1cPR%R&yKy zU}AaJ8lA10>55ur))&5+jlpbr8(LXE+N0IxbAOMlX|fJ9suI1kO1tV%WHZ>*-Y&Io zIX5d&#BdC+g|@HZJ!D!-_}0SwHlP!2+6U8n%DP5xzW@CNF6_Ww5dh0jf2a=X#ie57S)Ta9OS z%MKJaG_$~KE%7?1d*tI{D zg(DC)p#6+&s)M`oun#ri!%BIBgIcY3X1lfT=a+USD&uqW17=ry~TkmCk z$^+TrCs6iPi*0<_id^aijySwwU3Thkz41g~Jpb!=`&CVAe(gn9!Re=pZyr|@)^G=hSBh6m zw8m^UAbsUmV;S{y*9S*l2XC%6aGrJp1BZUWmU;#UY?Bvtm#216;C-4E2Hoa-9@j=8 zCxD%%2wh-rZwG_CmSF#eR?1gy{|AIzHe9gQZx$GB2v>n6*LyZ6K4s>9_C-u4D0rr) zb0=kIc9&^r+l35Mw-MiU;U&r2lw(5EgHbH<0rNgMz4HiFT8qxQ`*YlKBXdANgNn z5H}0>3m~|QFQ*5=2zC}pc1}i$u||koh>kXRgJnlo@MmC-Hk60BbwlZpff#fz7>ivn z0+R=MvG;Q$sf`?Gl!uprYzc@RXL+=@<#+gLyfDTP0}`h zop**QwqiT?gUNSaTet&c010ofRt%+rW*35H7@0}Z;QmXL4gmO#go>9=|>D3X0?m&VzTA9a63H)G;eO*o@P-xY>NM}lVA zi6w=5)i-mEx0_P+kjWTuu>YuAHaT+H=w>=NljHfBi|B+X5CX56eXhxP?}&&%d5oIq znSYjV`52H(hF?s^gXI}{s8wi5cWoirg~SMcl}38{1zHNYU076&QAwK1Xn6hiojoav z*Y}2EN1yTObj0+Hc^Q`NIgfvMjRqQYs3@S^$eoB;qDV;w9_ecOh>CDXo1h4m{wYj? zWS}i-jX0Wkp&6vldTT7GUK0U?dfNxuQ=?)Kt;9e@8lsdx@YF7kb9{ zfamC>IU0U#DU&LQi-Ht~p%|v_c#2&Lj6@2U@Ftp1*m}{Il7R_{Y9ySqxR+wOmNnQUH1rPvRNua@L zd-At!^_Z*K_lD7Bs#*H1oa(KH*>SbGh_=~#1{j_l_XNqRtXDLtf7z`98L&_|qrVciAqmGz*df0PY(2VX{cc7=9H$bHL zsi!Jep`Qk-uK${*rYVM<8cnJ6l*-yvJUW1wd5lR{bSZ|eovE?c2AsXbjm_wwj@OS*$E~xvuC)4tqR9dH$cB8$ zv>1qBJu9gMJDq!&f@e6dp$etshNG*>ud(`q!>V%ttFQZMw-&m%aK~vZc%iU)mb127 zHfu-anO+3SRtR~nhAXc4s%R{Rn0@%61bL{t)V4XBxot&8!`Bzm<)>T~?I zrF12am;XqnNtCG>V?2b$j^whY#YG=sJ&nre$z6>s#8h1JHl-xs4*FLt|z>Z_LgVv4fU3x|iOibqxx5%kmt^Wt_UnS@`I+bY$YEQ^N;{fW3mvN6EWw!1t)828Po4WRiixOSW5!$z1%-O!LJUv$`!5y!!a z7|7@7!o=ySWoq2L8*7Y5*-{#<&pXp^JiK`Mqi8n8rP$i$8geE!j(DB0UH@6U<|o(q zJgA5Y*;Fvr+L?a~3yyKQ+WOnQWM{(Eh`YwTY-fA1V2fONu)K6CN9YB!PAk*21kb(6 zMzl#rA(Aade6lf&=TFt)~mhwjgLsV%hnsd6^cY*|g@ ztJATZ#ew_Zxq0K9DyFT)u^aA_K8@oB?buyRR4V z1+gx@otzIp? zW)Re5`j>4BjJ|ot0i2Ocj`1ueNYw0@kMyhqeeyOwhIO0YL+i&Oi>+=+#5Rod9n8A9 zF2&@I)*~y75@ge2(l0#CobddbO7#TZ=UQ%}L5O!wD$^twdY4UKbY?XST~ z(r5bPoo(WDzm~qL)HVz2o5tLRo}6O6;0%iAh+X4A4WI!kx~m<1nC;-6Z;xEQ!ys3; zeD1pNUGu1Ky5Vm3;+@WAdB4-6lsew)a8Z03?}-KYEbQ#zmo zZteHj@rl{@&M-s`Fygl`_ik^AC< zIs4RzJQuXv@aJ&r&C|B(($~DDQyrT%jH_wK@@fqb5r`DYawJ9&65gV8tR2?+jGPu!W6@Z`6n+NMOsClMxe+Ng0AnADCGlf~;t91V@%30T#Hm3d9+~BvL~B{9eGx1 z%QkW4s-t_(?p?fj!|mnU*ApVMWUxx5*|zFZvxK!4Bmvg2SFAuVN_99wYM!-`^(^Kf zd1r(;G%vb6LG@-9t4L3H1=3~pSdSzqhA16VqW@UUT(Rn&%(%1LsgHl7{kTIemBu2W zTCTl2_?VA`U-wa4R;XCx)KMnB4coKu!$iC8XuR_JXUA;23TA0XPhGw7&CRD@fA3#9 zX;mIk7*Z)tjwD-%BkCL*&oC`IV#%i1iUuAp_fvNlDPHYlW$A=?wf;1Y(G$yPg1LG_~-^3X(?w58G3_2dTtu z(B~!!#0>OSOmH?1pv($P_V8S_#TXrP)jciS;j&BnewpOUD9byOGoFBhEYs*z9jU?O z!YEO^2=yb*1_Ca6a=Fu#8m-mU9|XK4=~-(J@8avha-Yg z!N9T&Eo|*2Dj^hD9I;+TQI@bo3rVyvIj_jw7dx(`gO#k=_+$6xMy=%6SCEB+OWq#E zJPAc6*Q7AiRed#e&mDAzaB2{b>;H*B(nto7&WZzb7D9<}?TfEqHnV!!FyOUwyN#-4 z)?xF^HSs7tXxF9eG`!&1Mb^8h(>ZTb z|8@5twUBvZ7vZ&IUSa`G5NY2xmHZ3ZM7J1i-5sPbKO?7*v*YS#fXmz>omyMnMH?R& z685`p_WRzng|t5HWN13_2~J;vF*EgCoWBiT{jGPK-<+$S70C&78+=V^ObBo zicuT0Mz*HkW^3Z-1Mm=toaC90|9ED25zk`8}ae^w6_*%fW z^j(EPS}C091a~t>U2Nd{#i#?!$_`Z0>E7nnu-uv9lYVeiCyHO31+>W^vG0q#~gT-2@&}z|V*FV4(LTC`5g*hDB6V zjsU&rMCaK>jvB$A9rfr(?P*VoniQj8#N$cj$x<{-^p0DcX%_EDP@A68p$;sLE(_B^ z9AYLV3=`sy#1=)Q`Vtun+<-SB@KmTq^{Gs7W;Cz4j;m%>tLexlFB19)D}>VtVjU|V z*uaNIq%{+2T`OBJ@zzaTqMQPyYd61X*Sh8vuc>%PU+>}9zT!flcg3q-33~>79u~2R zJuF>S8qdT!R{ydyrEFy_yN1fbl%-)DCm6@+%^XC|eLG{7vAl+{IVDxKtNrC6%pg_R zMwP0yu!mK#dRwmIR=2yg?O(0HgAN2&xUtQE4(16@bZ&7g?D69RovX^{M%T1>d~S1J znah1;7Yi2D?s~l2T}%%0liXcozpM-0o%)D-=v{An5z7SlA{L87yKAMuNWX1a8dtne? z7SJ((kpJUd4fO>tGL4HyTi|bhlM2?UGJz0)AjBx#Acm(7c(x2yaDvVJ*J`iDrVcOvX!IAgsF;w%Y;aq5E79D0VmtQy*+c8Ys-xcqk<37esj0m zyoB3f0~`FP?LBl+2qbv?}Mho zo2Q7n6NXEb)LeGjxcO6+O$bY%?2q6ZsFbleABQpvty4spEYG?*%!@c0D02dfFdN81<`>S>!1#}a>>5D!-J3hF} zw^2#K^sor%OClr)JUsY->VrPHt3ht5JBTB^9c;J@u(eWqCA=#;mJLAm9rgGyj0Erftr>=E18ngAptG+`DpG`2H@fYYmkH3YhEi$mBmvoiAr zRv@Sk0Y)kkEs8NvBN(G(rRut<8CbDLamo$Jo<>R~_UaB;!LcrsuCR0^ z_o6QKGRyXAsjUpgEZnsubHL79o8{Ui-hqhsXvY3}MhMU{W|I_{j7dkBNtzr?Yofi{ z!$}FyNyS`D#%!v3YKC=~k5TxZcVWusYRdKEBc!}3@7Wa0q{y^frL#ne(9BG+BulX@ z&9R&>v{Xx-`b^kd%F0?w*`Nfl!aPoUnXM@v>{64E+&sL@%l~JCsx~A9F?dO8TowLgFa zQn^D|!paWg#UUG|NGlop%uoNj$9=I!3&_t_60Z*9qX7l4?g~qKaw&K`2>nAg{_8ay zcsl`%ns7&9_0W7Faw5)D+-INLHN&0m;yi0ut7+` zRWnj7&C&}%#2~deEiKY7#nM{*(q8k<9tBS{El>juss9z-r*G1!%4^Fk^MR@)8Z{Y> z)-aaOi#e1TJ(Nr!5B<;q6R;5_(MR}(1w+)_T2w}DR5`1|n-r-Qom3F8s~DZp4bagX z)zlmvQcvYkPz6&_?NT&gC{sm+R9FNk#iChAPcY@qGX+vPK!H)M)mZh>fD_Xn)zw@D zP*^q6G~GKjjVPQ1E@M5cWqMHUj8J5GvJ1sf{rOYYDm^wMRBIj7;xd5}{RUh4hF$np zZ3NeFtvy|kh3Bljb5&7*@&FG6ElxEia1xP)prgiUDGFg1XDy#OH1sQ*#5kXOyvf0bB(`c;bM*nq=MksaAR z+Rg)IQ^GpdluagOtRa`izMQG20eT0PdqFSiVtsSKU=%`DGf={JHPVj?|71Bevqfo_Fhpd55 zb=ZdTSS8Se!KG7u&4C|qC%LlPkYcAi3MnwEFe6i0(LL5(vpgI^8R3Xa)EyHHje#$b zS(~ldR^Zu25KKg610ev>N05c(?1iNLUH@+d*H#!_a~%a#-33t~URFq6axGp-U4j&d zsm??%by@)#bpclZhjO@wZF92)NZafb*tq=xA?@2tFjzLAVCSodEIGgeype zGHBnu#e=OqgljO{YRH5j72j*XgYr#*xJ6%2eT4npTTd<9g96-8#ot=k+Byo=^F`mv zwOoU3;Ef_E3u7n+?Sz93-O>F((mf_7*s~BSNtxZ#t#KQ4blun0g4k6pN`Sy^-Ci{N zvb`LIq3bfD6@oTofg#X?{F{R!Mug;@5Kp*6J@5on(1XHws0onTw^|$aNQQT?qN#OA zbkfnO1z+?F+kSoEPN*es$V~#C`vxx#`C%nPF79kOuHeL&FdCs zUBduWmKd1fD+A&oU``BU0wAWUp%Vn49b5rZTNV(ya^QqEK!pa214+={3EAGD7`nu{ z)kX^%4#`LzH-$vxw=?F0^f-}s%|g>APOD_?Q$TSyk* zLcXYJR@@qxTYg>Jeno@79b-Oj$SZKCx5ZVl&4KJ>(Rp5hk!4e2<>%++LrflBmObb! zWG6t7O(@$NWP!^}p&wI5D*s@Bz#$j{aQ)%j-DqC>r`#RX8}5bj++Jn7FwEcua7YBb zwS|q{=ZOnLwqa*4?j`_GfSRrVG{$LsjbOf7g7TFETyUmNXlFDAVTL8&dMsf1i_$Ox zjkm(p2TowS9oq!1g?k3oMv!WER@(v=EE6DS1L)vdnC35*XHGrqHl^gWR_lFg)0I7F zxDF@DO$1b0VNw$fl*l)l=ubxge4Lkdu9Y%WzAs9$V`UkYi>dzD1=Szfm(z!P+zs}?Y{+UZRKrq-fFY%XMv7x>5lHmDs8)p zYZKms6DG=QfmYAMANM)SzXohU_y+9-tk+BFYPD(h2F%M&1vjvTUSOxlPJ~w&Scko3 zNFeBbp4#&=FCTD$c7+2PMQbl^?REy?R7mEWF6vaBYPQ?oxXRlnmC~?Vy8*^m1fSNbV=e-ma-Uc5p0F>#xNiE zaWLEHFUv@MVpj~PJd+|RC9hwrEpQUXUsU*SNN89-yj}q>Xc9JU=U!ViR$zDD1D@vQ z1YTHsEKfbCzyCf)a1ZbB>?HBE1@Qs!=Mi6XMmFmdUuG7^b1XOL-|F*SlL2xZe^DXW-<38>iR9^s&<_BklO-Q)=7V)j#@?2+gT~~EC|8+hNcFVoswSMG0KlT?d zVL#9ADu7TH+8qlWNw?w6)LL{NZ}cB`TI3yqry2r4*ad7ry`j@)8NJj@?R0EC_YMqz z_Ws@-{iFJpizzNatR(mvn>ug!Ix_H{b{;s&SGI{)e| z4sl^WasN!8?qc6m$f|BWH&%INX3p&-yEYR>?G?o-d#2j)EQ5LHsLzEfS)=4sG+O{eW{j8D3YKbxo@5YPp+32< zx>uPsG|;yH`crAM1S<{^Uw$Osj2^iU}hBZko@L{voJkn}2*Ia-@el}aQG z5pT`Jk`=4VlAuq5plNICHk&oL(^^?-B7_A7A$s@r`u1&?-)i8*?CiyfP7@_)*8i+o zvest#?>_y#ckJTh>Qv2qxu1YMCi-LagbF(V!G#uGjd37XPbp|1YF<4>#aIzWXv{8x zq*X{Ma%mVBhpQyQiAsiz!iiph^wP>Gt@QN`A${4?izAGDXdW(|=`aX?x8W9B4S4)y zf)RM&@sLKEY++i1l3e45K>#@LLIO@|X4{m|v6cjuL=tJ`mdtIi!j=e$ng3jL$qm%o z0BN4-Tx(?h0LMd*QJ`IMYbcQ&o{CAr$QykY#f2^0P&3}3f}%$#e&w-uXnf?**9a;+ ziD%JhoW;@Af_(SNUH`rifVIgVVvSEkOpeWKe;K(N*iFXZw z7ZrFB78oLdR3zzO1RXNVHOXYK3i#`YX2cr7{)r6G#pl9 zVqo^*iWdeGOy;w8Zm2j)9z|>7A~-hY(3p)pURFj>ucWYsNkgUp>9yOM#)vD~-Rrct zU|Tl!xnti8oSN<)v_#ruyInTi^2)71y2AN5+L9mmi<>Le_{TMA2j`F-9mqxuF`|hd zI*A>qY3nCqe?TG%c-v6wMiet>zEputaggo)iM-X`>}ec*JWFZN*iVgE4HbaYF&?IeV_0V>L2 z27Czu19hrBzyW+SunG%w7+L5!r zpvW(`DhD8zcYzrifQI=KTWnwx1vyME1wJSPBa#55Gyx4y%acSSW>_3_8LwMz=k=0**|qQUC^! z1yo1^1?+>H+KdCmInEIhep}Kz02j?#R76Q0$;5TShc8?xgkNVXNSTP^v!mS+Kjl;$ zAJef%b+WUaisM805|gk=ddd!cXk6(Egin$MC@EeEl_ML4mJr^~g#kV2>O|C+L*Vj0 zwj$Ow^mQXFZH8@|Daax&!K6i~L<$U<7c1STODFMR77QIkz{ZrB8753h0|)^J;uuR> z{!*ALnSuI_sXT!k;7-(YTo(jM2PAP0s6+K70v1uDHSi0KVQP~&J#x-T*)g4Zv??_+ zafmZ8VMxKt;OU@p2T|F;bP)tl1SD9MryLTYk^fwa3pM#d_{_Cs@W~HLk^w_pu4Gum zV$vnLgG;R{)ud~i6JwYpQdy?a1w7E=lsa*bO+0}T@nHi(YD0s?$YD*w!owR?`n3WC zqH1p=A8Q&?!=EZrYx)c-Y+YL!g5bhdRAkfM-gmVCPPM97T_;w(>XB51v+!WDZhQLy!NRnvi zC1i<*(r~73x~$^(y0_VkB4IPAR0R{4?Ms)BWp zw{NytrUnS5Inf#DL5Fy|g89jbA^*{bY;7vp<^GDclAW?q9a+-pnZs8wGzk)$7gvK~ zoF4McX&P;}BP{`m4P{8f8v1O^p5TlmJi=3R7YqE< zR{nsu+CJcixjm8n?GHr)k@m=Q^BRy}5*k)#C;$3Hl$MqPUAD6RY@9E}JC0i;8)>cE3*>>EAV#`}xlJD7 zijufbhk9}`7*^$q?CQBM>eeL%?>eN#6PQSed|=Np^RQ%3G>Cu_f%N$J^Z{s(xYZy2 z$Yz8INzND!HXk0TKuf+PqVK|ZX7FHJSTj{H8&BFLk|vN;yhDh!Jw(4UzrV@GUt0Ho z6LKURi}6Vy$XVz$8yN`FwpfH>WI}Cq9K5L1(D_UVxZcs#QPRoY#A#J=O%HNK9SP9W zis|0fg^X6*fkSy0cyyhmA;P?&*Li6dzir@2JRB4qjM}xwA1Kk7K_32O*am3AWF1BD zOovE4p0^>&r8Psv<^P$v3_`|S9#h#0luTd7^h_iCUz$YTJwy-ZiN-xljTRCcVsS;Kz%7=sA>pMhVT4Sa zNV$#r4N2d$U&a&w5>6HZoPqr)1_jg>XQ^K;K;8sA-WQFbaVXpow3Y?o9BU!qv+=_H z?8N>JNzk3y^~jG1++o-;ih3v?0dbEL02UK$jlfaVg(}YUI{iDRA5jF=Hc++ z7}3OtBvN7?X8$89_y zTr{84HYsz4rO2jW$4_q(Oi_(&nuA& zZHUo^O+X%4Vklfjm)~;;|j)Gj#<|v z*uq2EBULh=D0~*%?W0%%U4~W#Xqgjy z$zEmvX1Tx~Ae=ZdK>={ph5ekC^ps9`4Py;Ox}D6~&Bg;f0J5RRp9GaN_Q=ZBHlOkKw<$(*BqpSP{y9p5fdl)-C$~;Q|4ZrC88oe<=wGiS0+O|wxA+3OncNL zSB_Z0h+7--quu#lRTPm<5o8^zTomdZ2&hCx=>*5cWuT}*5zPV?JlSz71vY)uXB1>l zecwvyRP?xHa}-+-s3hnqf~a(W6-?%20;0%_)wmgwlF8z<1?Jn#Q7W`#Xa)ja;T$7) z113;aYKnul0au8r)2-0p?LFoo%2;i#3=zheAm!$6wn8D$<8KBhC7RuBcIU_>SM;4F zTBa!7zyUTj$7jM58#nU@IoXeSc=L)HN1h2 z;{T@^$ftZp*;XiHwj@I+LJOQUMoDPdYk0sLctlGD=xBn%z#Jd#cv^xA8iPvAOKjkU z1tQfknpru>Rissv5@FV9=#BAaP1qZVmYvyIlW`6rR+uRKkPXK?Hfp1C z6gjj(5jlm5`Ch<8@3@zoZbW|RkBbK zq;3x$YG+PVyHFP8W%hsaEDkpT?kw z9t8v?QgU5uCl;!TYU&4sUww2x441pny( zgpYEb*(qvgyP9CLb%2vv!9UigQ!;BPUKl(bktb-F&XZmz|PKlvfdpy+R9u{!J!YG2uY^eD{Fw~MCuZUmFys`z#2S&z@pe0 z9%JauSrfeK{V>%U;OI!`4Q2r59)RZB;GrI3?5|=Kdco)l2#B^0(pO;JN~R+rNlBHl6Tl$)@ok1QctqEnh`>^At})1_FyD(j>9?Yi-8Q>-kC7(#El?8^qHm~tnS5lnaqPQzB2E)Cuw z5D&8Gjbj>3edtma3dSRFu1UdYx%TEZfC0hLfYdCjQto6b2-}cllfq#tFxkfIvH~@T zD_W`=309D0<}QLH8cV)vf+-dFKn?5GRA^#jRVA7(fHKT+#rbgHvNzT<-A>CbO zuGtvz+PYYV)1>4*9u5eHusaDCXL;+{{I@co-Z%V@3s9`dWtbW zbFk)0Xy5k9slG8B&v9TLLf&xMI?qzX9xoqrV>f0jdprsoJggw|%dG1?0! zp(A<-CG9CBmmgJvSImmq|8P`3j@douKtD4l5x314Zz<-Em@Mc5y`hLQr}Fy5%^zr# z&y3NR9RNiE2}9GXDURxeHnoJ2UcSBI+1?GK9#1h_wc1ozL!Z-mEHg6`W%8A9gnEwD zt>aU=>mfSKO1~0-hsULK>1Zc;sVmGR%&xM5NpYN~0bRjoY8O14{Dqn%wdoSILc7 z1gFKbEs%t#&TC9ln-)P{k8J}3duuK4BEKryYnA3NZ~x8IslgmW3k1sa_XvlzA+sM3 zrR*);!-=CsgCM4mKvqyQ^%|n7AdpGi$6LcQ*mZLaQjTA}a9)q5O8E2`{RxGw(ogS{ zP{2}QlZ)Wg040oOPi7VLF)e63wtBXW739Y%U_y9p+1V(yFn#uYDv~d=rvV<|;50(! zTERo$#E*DU)A_(o7;(YvF&}xgRXt>q4Hq<{EC|kNI%W#W(6&@GR8Jv?}O1^^xQ_gppY=@(Z#!-KH6tN*pn9CjX2Fqslfap=wI>e$$`*xOQUT?Z|4h zy4j{#KY_m`9OZ0-A>?T&Ojjj1q)!}fOjP1Km~@3#(@E?whzQ08)A-<=Fm`J|2AB3H z`->fSxQZ8KwSXsPly{3;_Hr{35cBwY$E0g%f)F_%jx3c#Xxa}j?tpT6^CbDHEIEIF z^?Eq(l*brSAS)sYIJ!~VHIu<_7fhsNopn)?3o0|+cid*+Rh36kGR8R0lc?(p?E}kox)d z0Xp1J4f+@=vXl315YZiW&nqw!k?N*)@-?VZhv9$v?=eWxEB}(vSG(@{+jK5W3^0JyF+Du%bdQCCdAgT_Zr zm@uWz`+2W$y{oWO1H>hP12suvr07Wm4nPWhQuQ#@DpM0NeF|h}kS1Q$Bx!UvC&`clz>(#6_3jp9r694QLuy#->we>Cv0)M)}1d7;njaxc*Xic2fGazFx#3N6$ErH!_crIsEB3rxdJ3X4##0q)wvfH9O}g+&JWFu;Ms z^kWRL4IHac#=~rEZ2!ja;&Rk}b9-4Rlhsa&Vh6N_K1!#h{KJ zfdh^vCKL&~;*Lx14(F~*jydR*I}?X7qk6MDPcBMBJcr0D&mr_qT1k#1j@9gb~LNf$cHIr*s4WTEV2? z!xda`Spvdnqy13?9cq~10MAl`4a#(@q;fBI6=c*iE`MOoPco1*Q%@VzbV-dlacF4{ zfcI^YVD>^WME|_;A~Fw8iwwyL;)s2kg%K}my@#%9-n6BfSthR7qZRzqG6eu~K#jjN zE=h&swuCY=2Rv4KrJBcBND5SH_IS_=x-#LU$JFwGl|os+(8w00O1_2@B@ieVCz*;Y z5rkPdO7Rk8BVvV}7mMVfT55-sRs*=a?z$7-t~o>GuWf>M)34}QDz7KwW|D4p+jSRa zTqNOgxHjit$vNqs%a^(?=;fEW7-C)X2|MPH2=d4eMi`dL?<58DJQ4ZBX+_EP!LvgM zMLCs3hPnvULjBOBq)Gqq53!P&x*OC=)e83_m?8ZDbyU2pNe;>ENyc}cFq1`;HjcQE z`5v4f*ML7dF5~~NJ-ztV>jI!kMW-1GT9Cao5NnovR=w>S{Q;urq!!5@Q`;%HjQkvp zS~OI*?se_Hn_b^H0fZB!35gyY7pCwuO>v219CoS`w16>3atot{1S!ZcJcZA5xAV{E z1QaRL)CYt?`CtY$I5H^>uNloqL>X9CGAu1+cUV$H@#gXl8}?!p;}RC~UNMIqH17*j z08`0cH8f4kAU}tK#Q3mhG9OUQU@SVq1_NUlsAcb2&NAO(AQL1k&PaYY`U|-vfiiUA zPk;O4AG?UBvy1E?JA#vA0%29a9}292EuoMdCuq4oUanzy@`ao>LzB;G@Q~!nPY6?x zgC7LQ0Ez!(;vU>_kG*(9g)?a3A*56mNG*XkIXn#rE@4CeIj?B4NZ|s-5=1kcL`&Bw z!%O@Ch5T$Hm0{GPvuIE?Myw)PG@wzUlz=w>7_&3)8(aF$w@dhaOJw2d4dRMmIFmfG75|E!AR)&yn5mRNw;ePK`e>F zT@(sjN_a#H+4G+@q*9cPc|||3G!j*|qM?g39EOzCv0K&5V@XWO8Pw;k@-3|+;ClrU z71Ib+R)iX;_)02TfeBC8EI-WL(HF1yq4zBaRx*Sygm4EDc^7M7EBlrBqakGnWuzH-T19k_b?r?XzTW{Hc@J z^?cWQHe^hVxEdx=p{wTLsjZ=qOOEV(I^29n8;J37E8>GzL<$^9iW$3N){?^ zK%+@jM3oeP>>^l!*+s-urX6Tzw_>Er#K=sVX2jo5S!+txYHX-sF%z3kU>rCqm4M1) z<(M8wRjXdrf|)Y~$dZB=YSw~2dS(T8dC`XtDugeQ;nvjSeD!!Md z1;C{rpxEQ7pr}=bWY%-XxLE8tGgV=RiYtJ!0+SA6Q<_4<6=n$<22mm=FH2pUVFD9J z-C2owUNCzG(dS3xgbE}OlL|C^rhX|;I@yUsd7YYD=#03;>2w$#@PNx9hf=Kcc`=Nc z=c<$GRBmzmv5=7m(NKz`8KX!Usdb}DPkc~GaLz(mIW3dqx)sV%)=wGHP%QriaW`2Q z1FZ)ry%|btmeOku-IKZUOaXt3WSCOQn#J{%uu6quqcMxegkuChkS;TQ?)JFJosqJz z<|k;L!4#AO)eMi5Op9()bsViG)Gay|zm0U}`s4H*k~ch}!c8O4j&!yCXF;s7=>ggc(Ts@I8aM!$GQ|0Q;`AluOLSj5dxh zPeMLgF^iGi0S^it+7{4FEdrpNxpOiL{FyxfJve7jx)heqbf*JJXGlN#a^W~{s~frv z1q}3TnL2C4DNo;$24tV^3>-iInLwHHb`Jr&+kBwUSYvaKc#iSjQ@!sU$CDu8-viIQlCSz~T>R*e2qVHqK2H*jGVlQE*_kds z@8e%S{LRhG>-sSQ75GeB>0Qu&|F8b_0MH6rj{pmB01eRJ3~nL(*VdJ8f~_G zPwaqCxYlE!RIFq0qI9aF^hPM~K&V2vukuvu?x?T&u#fL{VIKd0in_LMh?YuLAO>{la z2nZ0`M_c|-YQUi?xQP2ADUuX|0Sxg{cpWhMP@5!eSamN5Wt(g2yD3Wf^5iUr=XadDh#>x^=o#vmz`Qn);$DF+M#qVn(< zk#l0rLkuYoYGHV=511e^ja1Md4(9q$&D8+KgO)}h%Tkh_%uBSYIPGE)K8O~av(hr5=Sna*PRpP~@;!@jeaMUr zpiwh7lQX-72Cgm}^M$#zV@yOd4pUP#F(*@6(=}OOIoWZSiq4*_&7@{WH}Mguz;f>- zB84ziID_gv4yh~w?>IvUKg=?NU=Tz(Xhi?vN%ZEYdT6f8T;<6$%LCdoT#Uf6M$)|m zWEe$e=Xw-V3anG~6DRkRGg*K@%?s<2w9p_Z569606I4N)Kn$LTL0O=LK4b`kbc6&% z6z_-bcC$BqGss#^)Jp9SK6C=$ZqVX`AWIZZM^xt=1wZDLk?vH}&d=J8Bstr$PZi1h zwr@S(F;KfyPuu2D6E$`~06$&tJB^ex5yh2)@+h4OD8)%h=VT0?;7Oqr_|if_-GFlv z@i?y3nFfkI9V@#op+72ZNy5}ZFH|=xAvj%a58c#x=EF^KRZ;UwiOLNAD$FXalpvBO zgzgNk(&AVVWevJi+0?I26OO$ONl5?el!dCMP|r_OdR6DRwKx|O_LfsrAJqvWmHB!y z(FSc_?C{{4W;qP>KsBW`d7!$Aa)_|BSx^<|EU%0xL0Y}78Y)gqWp!5dE-!Kn6#67S z*fc)c)IRXEKc5pc*z-QFAP0NNtzeCNRwqzb=O>1hQ+CE<>nR1)k$do?GNpBcN|sd* zRUo`g-3;|PStvr`QN0RvW_i?+464}bV}OLpNU`*e2-Gx5lS1B&C1z!TD6mt>LrOD1 zRP~@J#e_k>Mh7ZQb?$7r=p$0FQUzCm1$~o4W8-QhETuXN5#C^78CE_VmizkiP8qSy zO7wp)7G;x0NosaurxI>UfF}R1pd&Ex+a^|kL}(zgl_z>OuV?^K>cXBrqfs_N|K_Q2 zD~xi5BEj5`K6mz2-~*`$0cayt1VVrTGOlq56g~XYn~b)%Zh>iE_i}`^DVe|$q?Q40 zw>YYHm(;Wn$x2T{MPS2r;>K1t<{=_BbkyV|J@f?&)~#xneV;AFiwX&l#c2KU>h*M){}aZj|*YWDn|ul@EFd)eviZuSb`iXein=hCd3 z2!H^>fDlB14cwqyHHCC-E5rO{Nw)*TpsICq!F3}TUdhAvnzc1Wl@Jobe?vlddC7TW zFy}%Ta0B*hg_n50wjck`)Dkg4JPvIvf=C_ulhjY%)?w5ys)f&x;%|Juvj5s5nHQ*fOWpAxfJT@M!cur?Y3nCyr9ASV97)hb` zsKS7N5W)?OGL0Gdn-I+xAQ*Pz_>o$Zb`=(PTOd>90(jbEgcZS;Urj zCo3$o&#wIV!rZNq$rzF&nT;j+w#fK{-vg5~S(7*Ul0jG#nHYyd*@o$0hl$vfwYJS7 z28fwe4o;bf4WYX3Q;dti-HO(M(6xaHEeuxo(coBu^YEPH^@6KwmldLeZHh7d3~B5I zX#SXcNjOYv^@RVw;dm1nO`8D%+f9;xrcGtgUZ!?!Im|e`aexBtfEzdt+%^a##W5W4t|OEV|VIi5q{3(AD2$EpYbC#PwdI>gzwgqoYx6>hbesWr}>n=h(eX{z{@N;=(w4tQ1=fa2KYJT6r?t zyTSViJo~VXIO?afD&P8N9*kU^^UX!d>&(v=mO^3YwLBLp5B5xuGDkc)TIO z417|l5h8j~-~?=X_SpHuy_;x%I>tSD1}Zxw>RgxOJE8B0R=n|kHJg@MJG6)Vuz|c- zxOh@|9IFM}(fJ#@pLOX(my2x~rhhW37Mhnylgg{y$_+HWofj7)9M!kTnVK^|7G?OP ztJNm4q{sHhI0ppxoDpd4l>qOrF$9qFTe}Yq&MT(&3g@PE9ZaB`#?71RocA;HAdUa` zd3n+szP&jt>`2g4`v%I{hAa7pA+WkFU0#H?lkfb%35TJLwsAt;mq*>oqulGlB3Iw3 zPXn?OEj+fpmK!k1)Wlq`OO2V|c7|b0oPz*x0_(&I$HUHx%_Yjty)W6VeFTS6$rAns z4j!cY<(^Nh;)iy?@m$yU`%h9Fs~=g1|M=I{FUdEtw$zxz4BF*!Ih-137;2u~-5p-p z)#^A|MB$v0w;7s$%uIKR=+6`qo_s(jd)lG;xT|iRS)Q_=9%0ExmA=#D8SEh*i1fMnSP7i=yX;3kgyZ=q(><%1B&A+6mb0?YrY9iNXI zdIcNsK{LViePakVe?Z~D6>ZV*NaqD_>~)@9A7AWaSmI;F>wzfa-&x5eAFIz^heh6> zl~(knoAgKjPkh)v>v?J^HRcPPz^j88?Eda~yK;Oe3-(*ezp`l&z8(^+v6fBLl_D4{&-$+_j>*(IPL zm&3pMU%%mV|LZwl*u@F-0cN?%ABMgCa2U|_lYU*Zk^KRJ*f3ziav3~$D1a*X|bcr4U9!jaHPV3feIWXN5(*6G9?uvELob=nDXRFH9-G5#`IV-h?a{S zBKR~(pr=q0IDdxZSv2U-8%jln{JHXI8=^a$ayX(TM9?o0qK4QBHKW#{WNh%Xs^aKK zkszOXWILki*N#S)E*0w&tV|n8ug3kmac)wWcHfp!D!P)qU#1Sv&5Q;N- zW*V8&2Kp+9kfCUZ22~-V=*4W2EH=&9oT0ZQWR5rY&ZgQEaZ%iW#~8+1H*MJAwcjdD zNr3L(;J7nx73?=UN4R{o66Sj(M$_w$DSKtW8Zuhpg^SC6Gu0?c@`EcYKVJQIYt3^O zqueRkapba&|4P)anSRlD23mokjkZcdr)@DGN9dujkyr7>=Fb0{K~9#q36$DnWWX}6RjjKO#UbtZC{TX1EakjH9M*>)9XM1<&; zYCT>k;Y=R=xF2348L1r+i~)FGW-rB5hkyhccpzyF!E#C;n@xcga7Zu+VS}w%#NATX)xX>t2H3`)nhV~Z&25?+v_wT`sJh{v1?_n1#n6Vz zEhjrD%I;N&G#O#coMgIp2JNb_NzaR6)sH27>%t9(7RqF-vF6&0z}^KJwh`kg=(HjK z_OXUP`v%3tyCu0ck{v_R9mF!zEHY4a+dV7VjRNtvxFk>Kw%?Zmm#5nBqRiycjhSb) z$5tDn#ei8e9A@Wa{>SEYu!*H~+_DiD`h_!|faU+(bzj6qdyjVrI<+gqXS0ijE**xa ztXqt;lb_a=DcR~_ob2ml0bZ=~Efi~?iiDGuaZb6j^dr7eP8sX5D$zRiPMvAZ`PL8B z8(s(&W>h+jjU)YY9Ggz zLxPn&$xQU40+sydp?gJbUsU_vUi9NDGf6OHunOFnC_}#TIZRAlV+@4gXD??p$8i-4 zmE*4GLFajIde$337W%h9x3Ow)oVuNb^d&bR7EfF6+Y^qeCODMM;b`|e(!Bf!z9M!E zZIY_o2QQ>0?_B0FmT8;9QfNMepeiPjiy{B%!ndZ>(Z(V>_#)JJD8)ssN?Ht*j1r-+ zu1RG9V>DA>&RPbu9{z29{mIG9oOK8Bu%wWDJekaRI41`tQh=*tUheK#!_)Ck1$tP} zl+eeL@oi3p=IfQ4B4j%oX0VN0aG}d?_%-cSX^+_fpbUK}leWn5`?L$nsOpRH_nJtn^$cR3Se# z;Zh^NqSzc^7fTl5Gl$az*#`%AJY*%XV22dsR)k3|O17swm}Hw-TEsI3CJtyf0p}yT zX~EMWQ;-?_I?pe> z^k;FP#FlD;0d5`*<_~X1xwI`UnGIC|zWP;ErD~^bKlv$CSrk>;O$ebLwHzc#iLslG z^r}FUU}h}WQk14trCf`M_39|P{%wpx?7JuBwo0&T-c@o-MQY2gqfD<_DQe;gs$A!~ zQE3uxkSt4Iv--%?shYK-`U`~IVCm7Yt_Eq8g(3<2ptlQZL60B|-&yO1FY0^41d~I=QTrB&ftqY+U6O7(F8Pv5VzwC($`ev3_*0?6p$8 z*2&iMb}hRDuIpWUiV-@#HICm^ByeGf%N1*Ki=pf*>zu{GN>0p%Ch+)Yjf4kQel_F_PmPO1oPz$^R%M?&MAGVY++8G zjjB%GmQL%LW9Ck`shDK*Qll$e#92?Iu>^CU;Ri!s(b9tQ^KzgYQw+_ak)4b#jVT2x z%7)_Bp-QqUoSkg2dJ_LNnt*=ReCJnr zNphJ?Ih!_;5oo@2$p$Ok@B*m3#J;jZFjrrUzPNf}A+r|onQXm&n$BasZ~A0ZlNqB@ zzyn=imv_sdHq#lH79FWljVk7vJ@~p8Zm|WW?8a=LdAxb$ZtWy!jbfX((F$s*+A2BY z&009a>)9ulH*9bpT}!dTA#?V|Z@qszWIzdMdt=oT|a8Vg*_5%(g-zfDmKS$yJg zo~s=~O*9fS7|#E^cAaep{@9;Lc;UF#*t{3q^;7+U zv(uDFcldn+mL}5nGJdCS`sJ`$cZzR%Ab9zjl75)nX=rJ`S{cDYRDd_Gt>%Vfx2_!Qy=sgJB*RW^i*=a`sWUwSBuc zdsD|FOt(mhXJ`>fc+s?MD)?I95+J$dOC{KIy@hEdcO|cNaPqco%NGOTXLD;MeoNST zE%bZ+2ZL}F9$%At%2#%DmRACJfTojhvi4{2$7#qiJDTTlj5Il4SZ1^(ea@DEi$i%` zxObuzW_DzN0;YON$b@;AQiV`)cNd0lxP|g%hBa0_Mi+(9S7ggpaYt2uPBMeg2Z#sr zh&rczv*msa_egh;c7kYh&7(b|SA{HQU0~>UgQW%)h=-|oMtta7I#`G)2!){7M&k5J z1vLLqvSNkvw1Dy@K-Y$8Np@3rH--0iYu#jK0k>Fdhn1`&Gi~q!Z7!iX$=Zf$3Qr_5V^hbkSxLh}2F=kg&sKadOxQKyhgYlMYn7hFdg@hUk#hb7{ADe>tsod5>n+M;|D328dd;#XTXDi1wIK*R*SOl6}OdPS}``a2IzYmWAG^N$dDo zXo!aC_ZrUTSHmZRFUX0&g)Yq~l7TpgmuQ4^g&#o@kwREFN~Mf1DUR%xF`L$W8rc6^ zp^|dgvmZ8@jW|hcp%4i!H+|)}i=lXKX=rUBSC2y{H9+=t=*X6cI4sTgjU<4PE7%gr z2a;RKkq%aj^CyKO7<4Szlk{|NGKnRDfQo2onQ6I@M(Acqd5B%qb#_G?WTRY-IF2MJ zYaAGw(&l@DR*|j9j~RG;jJTH-0#yX)f}%ELzk@DYx0v@8J!=tANjs#hdl683= zHcQ$tobPFqTSE%-`3b5ppOH|X`1uKuz=wt4pUH`V+4i3k5sD{Nphs4RB=-M{f4Q0m z^F&Wr9iFm}GVy+;iHlEJpxe2b8VHL5x_CpVW-OO!IPiJW)H@$jlkiELl__FmVxKR1 z3Z_7x`AMTPI-~lDpM|iW1KK5Z;Gf9Gp9I>R1}dZrYNT4&R0gD^vbLQ*xs&L)P*6HQ zd*yPCpnm z`lBrcpdpy2L|Ub1GMmhac+6Rsoz;=~6N?pEIRMyuprfc>zJ zscEXIZn~+R+NPh%siBIaa4M?)DW}KDqdlssdHSCjYM|b^oZ@DbvC99K%Q>r#SA~h$ zrxno+k~*oBs%VzlqM7QV#pbm^s?ln#bqcVhs;Z=#qxhL1zaXOnx}(fWul0IH9b2YmimCr9 zvH(l6C2O)L`>80)ubR5DDjTv7QL!MaD+QvBml+N>>#H2Avma}(GCQ(u^|L|CvP0Xl zE*r2$d#pkmv@z?mD)kFb3$-EkB^zrFRg1Gbi?x-CuS(l6M2r8lUJJHiJF;TSwPoA1 zokM<7o3?8kwQTFQHk-3?d$m@ZwRPLGTWhvttG9W(w|wij`kF@PrM83nwrzWsH7mD? zOSf~ovFpU5%7C|j+qaTSxs_|Vf_t_qWVnUPxtn{qaI3iUy10yMw^~c49viuVtGcRy z4XkUQtqZ#%ySlM^xwBil!%4@^ySnSUz5BV@3%uJKy1pvBcWb=FTfE>atjDXm(Tl#*tG?@-zS7IS)C<4Z z`@8eYz4Y6?-#fhGOTOi6zFX72Guyun@xSN$z5$H8p-zXvJ0`TMv# zdmzVaz@l-%1B|&F%)uD!!2n#n9L&HAJi_(cz?BKX4;;bv+PL~V!Q*Re{0q5wyS22d z!6F>L3^Btv9Kty~wPK36B+S5Z`@)lozpB*2`zyiYC&M+I!B6bN73{=4T(hCU#3o$A zC#=PlYQ)03!b{A;jtj;Xc*P?e#Z-L8Fr39@tb{iS#BRLBNzBD6Ou9#mjbE%ecs$17 zJDhCn$4&glYpljM7sP_By=;rchpfnAOvj_^$VjZm^ICI}46JfYuY>HugpA3W9LV8o z$Yk8caJu2rjsneiCfSA%+LD_(WtA;_x#WlO>DjV%L1*@R{PE!ZO{je(DA&-6OFEj`patm!Wla1JPz1WQ{+IVfpKb_W^9m{J?*{jXjtxe9DP12$L z(xXk=@NC$ojoYXl*r~nQpY7Yf?F_CB+=&g^ie1}wJ-1?Q+qx~+xy{z6z1z&~+|pgx z!kx?0P243d+lg$=M~&Ry4c^~P-ND`5(GA_h4cpdz&a7?T>Alb)-QC9>-j4m;?Ty{% zjo$7}*!X?l^?lwjebJ1)-5D+4NNxY${H@>lec%Wl*+%WkUTxq4KHn3L(L&AO37*P+ zz2F$W;jw+;-fi5a9p44s;UAve*WKUho#JNg+$Ij-#U0{wP1+w#-tFz+8m`?OuHq%$ z-#4D)FAm>69^i~k-^&fzIW6Qn?%^(eS;z^F=P+sLV&g0`<-6xLV zQ$FK)?d3tPqFR3CPoCvl&gDza=4qbR6#jPYOyM4lj_E=U=E^ZIP|oZjhz?&%`_;^kfHrVjt>h92vquGf;D>P61#CO+%_{pyIm>yr-EQU2$*j_Z+5 z)x4eAc>e2VPV0)!;_oftBQEW5p6iv4*s87Ut}g4$KEs9n<?A-3{ z)(-069`5Zf?B>4gz`p3`KJU*C?e9L|`7YHD9^UJY)4d+=-(K^L+?(P$B@f08M+pgyWkM3`d>i&N3>Hgvk5ApEM@f!c|FrM=A4)QEt z^6ak1m2L7ff9#3A@y?F%7r*i%-}3z~^GYu9Jdg4bZ|_9k@j~D8)Q;^mAN1>e^1S}? zP4DnKfAmIQ@$ihtm!4knOfT?KfAd_gjU*rSE)VkHz4cT7^+ONzRZsS1efCZc;l};; zV4wDDulB}%_rH$s)2{E0j_`9&_ZyB7c0cxGKll)T_1jGCbi4F{@A!{j__ECQkx%s> zKkVxq5C9?h1O*fT{{Soi0000$0YCu&2>$@}2^>f;pTU9%6CT93P~pCZ4kPmG=dYs0 zixe|zoQScbM~EChf*cvLq{)*K>#1b8vY|_sDPPKjNsp$@nK*CO)R~i~&!0eJ_7pml zD9MdTQ6^Q&wCT~OPK^fLS#+w^s#mjWZMjt{RIgaUMqN5q;1 zYYr_Mp4@h#8Gl~Q+Oz7>fJw`qjTyD++qiSDb&L%?!!Avmc=5NklPkB}Tex%I zz@Z31Jl0}5zhf8$XGAXwKCSfEk))py{82qvgqh1ESM;f4`vNTGii1{flJ9+qfg zV<3JgB7-AZ$Q*|-#@L~Xpk3JFjV_)jqk}x|=v#^_;<%uYDcUxnk3H)6V~|8rH6)WH zBB|t*H0Ic(lUEk`qLf@xxg?ZYBKTyORz}&SmuUW$C7C0Rx#nDF#+fExY`*DcRd?12 zW`c9h*e8p2?kT9AQ2r@rjnoB-T%L(8$|#|VR%vLGtwm}hq>m0d+@zfPS*E3pT9qlL zno4RarMuZh>Z%G7H!7$&9{=g-sh_g?>aC<HdR&qDt$R=H)7^o4CRX7vat@YAOJF2tL$05D-*kqS&c06cwKGDIOM9=zWVLB%MQEkl%Lc3?!f0ReC)gz zZ@f3Po1?t%%QN4+^U$x2{PDwAZ~foY{~rDJ*lT}%@S3Ww%`Uo>|BLzPr@xB&?3=$n z`R#iHKl;;$Q~&+=bDzKd+4JB3|10w!00lTe0)o$Vk~^RI5MhW0I`DyosDlJ2SO*4L z@PZFy-~=(aiVI@Gfgk*!Cs+}RP{_}O_G3l)O1QrF&5wR5grWUrD8KSSkA~vgpAMhL zKOFMVfI0jj5dWVx#Lofnd&SYk2OpRQFwCKWZFpi6r>F)eR?!Z6$f6Xls6`D{(TQH1 z!xN}T#XC?i4T@Oe63@uN4K}b6CCp$XJb?*1&QTT}WMK$PI71ey@Q*QUp$l#3LmG|` z8Hdb9A{8k_Lpm~%jVz)h12{>^p-vYQjG`E!Fa}LpF^ggNBpJAfMKkuHl&Bn|9X@GB zFpTk)W&ENIVjxOY!ZMbtw4y7|ctr|ku@Yf)r7F|dK`D9>gBye+11AwjJMJ+QZ(M~R zr#VPL3UZN%JO(4XflWw0lAGU*WC1IQ#CXX9igDN^EOTkfPG-TA@J#10N14ht9D$xi zc;_fIA|Uk;+enLR6vx73wbCSy6NnHJE68 z;ws;$!9pxE5`{2f9`#7q^RcjxCNh7P6exG6yTr+0UI~m7ZMnDO8`z*ia(Y zt7LtwJZt(-j)GOKuM9#$k7~vf(9)=BWoTk;U|6m0l%`VD>Io9tPIfthPmj9@&aPxYm)Rd`xLxUwXisf#SA9Z9_y2N(IrDwv(5AtUl`*2&ZnAwUcG7 zX0Ln7>3&iLswJ&-Ws6zLUbVdBjptLt`dZuCQ?-kot#wIBRHc4(wiKmmeaC9ij3!fr zf2^wl2MJEp;E*)o@Pq_?+TAVWv$}8CE>F8qR0^lJvFerYcq!b=dsf)Nh21AXBTCCV zocF>}RD^dk4BNvtc*5LuY*MFKTP*U|#xsU)gJ-#18LKn7A#SQX9coL(%JQ@A+;3s+ z>tp`*cZzYXqcNa)&Cn!x!K}4zWjQR__$FDbTWzR!$6Qn!fO*EPjPaLEOwaVbxc|pR zeQ}XLyj+RSG`i(gD4Zh;V=>Q?&)fB|mYbVh$L`p)OZKo(S4&y&Hd?Dr2Ca*=i&Xd$ zxxZOFmo_NDz%d^|vw27-iq$;dM)w)R>UF3JID29uOPI@9fUq7Mt!N#;7RiCGaUWjI z=Rpfq&NIGgho73}=#u!bWZg5ek4)%74>Y+-rR}oaJ6ah-o4@w8;N;FB)Aq7j)V6-_ zs0;mTV%u}l96olin;c(2!}`oaUbJcZz|>EtlLQffH?6sFYw_YV&WR>ySUr1ZJM6pG z2n{Vv!^3HXRB^I8+EQYRe_3YDy)0rUH=Y4XHD*B zc%QNbcLp=bYs_yl1cBf}rYwcQyxP`^7rZogIEeq9(}Pyl#5d2ZSm9gVrRy}{I7c&d zg+B0|3!3FfF!>F1?d`6co8nvrcD!30X3QRNd383!(5IW`iU)h)SO@gM5&GvY6uAX; z5AqlouN&E_!B5y}N~nv>GVi zaL+2b?w7u|m-8;--1i;|%vUUuDbEO9ep==FB|h6jz2j-STkk;yc>g}5?&;8jyTay& zG0>xZ_h)zQhO1}&hnFAsLpOV|(XaW;1-|<;7ykTG7X!vOdu^wAvX*sUGy+8zenqfy zftGI_hIbAaUJ;jUp$2-0CUUPw0;F{ZkT7cl^?zibZVNSdedm3!mP$t9*QMp%QCJ1>1_ysN3dFq#R8z*))H-LgRfjl(=T1zy~cf#HHKO@gTi-e{q`{e zR6c1(2VnjUxo45sfD2D=7g@{;fcsK@v7=)QvVZa!AgeZvZ6@nSaf-i*ls`vhL*Tm%cy)_MF3>@bGOHZy_R`y z2Ys|Bjr!Mmf{1IpH(|FYUH+DaO1EXWxO_|le`}9{a|eV5Cw&Y6gy-0RNlAkP$&l;yY|uD%u*hnT=Zxw2 zkS8~PwTOZG=6W#*Rt&R?K0eqKvZg_fV_->kYa~AntKR1He7nTPXnFyJD5IJ=cxt2v~WIsrfXJ(ar zh?ixCE<>a|Bou*+=x_kZjcNx~e))z(=Yl4=hK~7XBlUgZwhyRF0v~~iKnQm_Aie9H({6~kcx0+~4 ze;`9#vWZ;uh>IY}d+nHYIAD12Sc*Goa-ivXTTqA1$ZKRNi8!@uA<&jtd2vGc2Lkwx z%7|FP2bfs-pz>F5b+=Z-sF_f}puxwScZPTEm{^3@imQoyV;Pe%=!El0pM$lEKQ?XP zC5mZ@fAMyRuUCMN`B`9DbtxL3X6cD(`JF}zfC{>#r74a$$7m)hiD)Q`gvXpg7l|kr zjD9(p@04*}$)ay)l-q}j*Jp%S>X34`lrrgA|0AR0BSCzbn$;<0+^3+133Z4VbR&9} z!-u0!;DjBhdu3Xnb~vcgiJJRIs56#tOaE$S1{q;SScu>_TI>8wEsy{AeuKO5)&l+|*=Az6ff#^A3q6MyC zHL-cRnWE-+-$h2&IJic|3T6;;FI+S)m^LVT}r)#QBrX z+GhMZaPwMGb-9pHdyM6}svHYNBFUmAR+iBkj27FW>)B5iTCli^rdoP|t+!7D%BNDgsWt_uBualuDSCpLPO9pr zt*WdmYqTvInwu(GD>=EHcmr9;q1za*7xuH3J5?#mczH{b_Q$CiI#gFVw)HxAK1GGU zCt}EXvj7`)muhb~u6G`j8(At4RB{MaZO5DNHkqwpswF$p?D>im(YN zzqbmcB^p#KijL!Syl7Q_R|UFgs)@*&zLhtfO*N^jq=&zHr^2X&`Nn$M`kRH-b+QG9 z-}-W;2dRE4Q4DahQb@U6_po0oibRWubz7c+*So)~yocLyxw~H08Ck}wv2S{^LFlE= zn{d7Qazguswad3qS&G|ufq>V4YKyzq*QZg+b_5_>@C#m`Ij1COsXTf9{bY-C|bpctFD4u;5k3%eBRtCmZ}cPGA5myW)6pkVmF zIILh^iJ(-vvbgGO(+hXx`>{=YnVBhnJW8-+{Fgy{WFMTgVr*#cD4`8{x|__hg6pI{ z2(Up`%RQKKZW+U|ENY{=mh$VBw7I;ydt_&*!75O-$|0j+#F(uos5B_fEOx!;#>(S* zlHjR|K+JDo%6B0k$cNU*TG++0+{nP&vmq*k&KF((d3;y5!j+qw#CL}CsJt^*#)WB( z7Hhbd{9d5h%~tzEzkpw!=e+$Jg34vk6q&rTyvqklt<+nNi;RWH4ACt>%hSA|@O!`r z+qw>1zmF`r!T-jAYeJiU~90o)x7hDgHX*pz_hfq3W}~_1 zn0&rX+OTblfz>*hNE@v|{hr7f)4m9ep2Y#c3d^N=mE9)^7C6R<$iG;*PsrQ}HN1mpYyT$#gsLWg8kC@;r=ZXLL z%ENr2{|(?#?S+bZv8O!Q0*v5V@W5zo;cOj;B2K)etD9(*l!p`Ebk~y-75{qF8*!M=!H1#-JRsH-FMYjd0i*n=1Yv{;Qu>? z>0O{gUC6zOswED7?COhCe5mCsgR%?3synIFYx8X&@ijhCA-#u(9m>h2Wk7522zvDW zCeFKUnH=eibq%2&9LSa4zO0?739P{1d$`n?^_@F>dS{4)9JltK<)y{tY7O66PWR;w zssoF2Nt%#lr}U3B+`>xNb1v5ejIzr-&rhf7Kr5@l_q3QB$PLxj1bEuU9PWXs@6|`5 zlnn(goNENlfm=`2=w761oY`nCmH;*HhM%vy9rve>_0rq)sOzDue#J{|p=rLVhcvgP z$N+1*$KX1;Tu6>mOTQe+_BO_|5?t$=OKYP)$=b~N&j0k4x3+ap&yru%L;vjHz@3L1 z{g{=1_wmlGK%cMk4d`_r@P(#@sf~`+Erh#2>M7Ls0-E+?JB1I|{YefGdfq^pLqvrN zE_TLb0dg=%3lcF#Fo?k6gqAmfG%h^Sks%j}2+3q25CmjMktZ)&Y@tR?Lz5#>cI05k zMhu!Nw{Xlt$7PWaLHg|c0(7B5nSGRQq4?102!m}#PUU%%;LeCep%8=^N1&F77p2}X zxx>yMMt7i`m}n!+$3QQOn(<0kZAv6k?+kQWm8{D+MYEC#?D3?*mT5D}%uAJT$HIH_ zsB3r5@@35CG;a>ane!?|U#a4Ja?&r%j|-tXEYdSI|%J${p(6c#|{1SQX7D#eet6p!O zJIk9D0VA#cJS}h}h8s(_`W^4^F3P9N~x`VRJ8uYBQ2etVH5#tK;jxhCF z`pc~`+DeTzMM@L#yyeW}5J3PH($6>G9>dB*z!*s|FsPs-aW|HDVqhzaENp4Jr-JMX zDeSU~PQZ{<|#bW z72qgqaL5*!i_QN4s!9r4Ta26G52;LYrhv=MDZsp_XID?Td|V0 zs=9bhvrj22GuBOH;e=3^Q_jlH$Lrpjv^YMejq1STW@XJzBr9#mrl=Ci^iNhDlC+{D zG5ljU-fHR)T~)RHa;^>){SXVD97Aw0CnbBX)T{~w7vM`nkSs?vWvp~GFG7{IOvkp} z_F28SU3kAe(e#(Fq1L20CS^ljRzgffgBBw{5k(Qci(9QP!0Mx3dlR>>){= zSI>2E;FV?|3ESHXTA9|^C(G}wGU}#bUDMKZf81D7^dPA&)xWmnx8utn4x6YUDbI3t z+Ph4-NX@Zi*ixD0?q+!6>o%TQVZn4cY4IQp6L7?8T(U2?m%^6efO~xU%kI1ak$19t zhVoOv*uF^XIEsdyYs`~xpP)KhzD>%3&^1(Rc`MhQL~iNLAJEgx4rpytSI4v>tMr8h zdY8Myx`bD}G@>E>@W>8EGLlN5qy{Uw!3<*Z0#~>~ z2{P$P9~iL*KSYZpR9DJDtm_JqQ{e?{)-`;&k_M!_gdBc2%O6^R1{a(qCoiZ=T!OL( zllLNpZu#lsrTaM3wpRHGZUC`aA6hkJxnq$6zvA_w`#lfn_DD`ja)Roc>yW;CWV zb*V==TG5W;RHr-5DNlRqQ%DB&2Q`SL|MaGp>y;;|DLN3{&Z))-Ho*w;q$*XVzy?_W z6dmbU2UoZH)vktBtp98VFYb8>9=xCfYGrF%=eW|KGP99$Wuz}9Syx6b@~#}zL|^^- z*GHOEs)Ma0VObD`!WvewiCrvWv$@U3YBT?`&cq;I?Mhd?VivP<)ASr!N0ME=N{~-Rk9`+!MnZ2CW8Cg-$Hl^oj`{z zWI>IP;`2V8iZ4r0c!v)6jFVdgONg(Tz(x76U@S4fiJ=(cHC$JS=wg5pVr&+sz*q_U zp>YICFo+#@F1{^}W)~C^;saDPn@Im&1R;588v^_wuw`ot2RbST_O{ovEC6P0W{5KM z+UtPXm7dm8yI%;Rf)(*(f*3|Y2tw!q&SD^N5@@SiJm)#L|Kw*g@cD{YJUGIGRzfD| zK@V)SBg69uk(eg&u8L5TlzD)06wV_fmH~Nb9Zs?>r9|Tu@Bwp;5}!p3ipxknss<(9 zbVVY~YDH0m0~A;&QDw-0By<7~w(eSo6G}}VAx@!()Xu4fjRxjCY7fT-1Rk7_WejAp zyM>GH0-xiR>KyCLXGTRL;3;4@*TK~+a0Cx95ZgM|cENgP_qx;Jjjo!Kz=MvkxjRS- zY+%D4A@#>6>X3#8&H+jny=DI%{y>Bs?r7KFikz!gXat5JE#a$Pr%6)1!>h`8#yhGt zDZle-DR?X46X>|gR4{RceqcDVX27q9`jdpW;Mh4sIUk5kqlDvO=Dprw4ldD%i$2nn zfOxilZH=U+WidV^29`L@soG4q;oFa}gdwU-1TjcGws&@S*WK-BZpZ=>8z4Kqe;ag0 zd_ooZUNkHU@dQJJ23{5g?^0sE@YXaO zzC)7pK?j6tSI;|s#h5z*=)8V0$2Wfz9a`Szb#-+L{IGM0SbpY;Xg0LD+{2rRs5g(H zmNyx-qo?aP6HO>^xLyBUBdwpC30d_zpN-ECZzR7Ol#sx@e=A!D=s*X9NJhTd&TnkI z-9RiyWZgMz`5$m7atY^P3dRX|G>Cw2+!yuXJC6c3DE{#+hBcTHAAZnHMD#1T^%kt~ zd5);LK>|IR748u;Cz-wHDvfPJGYM$~ zI)eZ}c(c?)7jio{*jl#YJ3c;3zFp9SJSe;8BQ)uI1?$td+RDBJN^D&o;HFQxx&U3h(BaSdU zK#D+vVJk9#bAW0*21LMt++t%x6wc&~xEB~e60 zrwB!b+>rkZn8+ZU4UIGm8)Hc3GQDJ!!y1q@Sc0yS#4Dx?#VJ`xC+h>Yhy*uuy+`53 zrt73O?+^Kdd7{5VzG30y#)W3~&QfXh)@t2Y8Icd8|aJj7s6+0Xawodw@Yl zD2N713r094!7C-S2+L8!f>R1hRVpCr@S$lMQrK!4-6beOyG$G`?nFN?T9_ zr+i9!q(K+3H?U)cx+*4a!Y0siE{$s+gA+?UVHzJVc`;gagaWM$9u9 z>`V}l$9Y=-0SL{}M7uU}EE9OoWb!0evH%09&-#>4`g~685;j2F1poZcO_%};c(_+k zor%JwE-QiiR8U}IC5w=?{A5rcv(E@!(B=B23VkK;LMCPEQ29hnm$W1!ki?WKq$_GR zsnasp+D{)Fi%e6SZDbrk(JvZJqenPI35zYtOoDR*gzpSbAoa&)@J#bW&k7jL@{B$u z)qo{6PZvm19-V*#kkTowQY)=e3eD0Bpwj!ig#H9j|LlPXK!ZU*0Z!NiNH8S{#n1l; zWzdStP%VwqWWv(?ysr6dQx4rEHq9mMQcXKWO%a8upbI=1a4Q*L0RaUVMU}6<*ccH} zI*ABJpUhFCtW*pT0#5h_O_hgDEi2{oR8S37&Kyr7Wl|47DS4y->GLg_vMCUt00a;L zF1=D(mD4Sy&{UWOT{W&}efX983cSXEXVPDT z)iX#1O`}x^{5klf(oi~p^t;bq{kh|+h93adTEYcMkOSj7Rw*^TU!B!WYQ;OXP$P)e z1+7^w-6bHC*<^jV2`EWfB`U#kCTp@a;MA*d{Z^>evMtLCg{iO7d4eR7Q8Ic0cRi<8 z@Pv7F0}8{1UeHudm{+O{Fr?%Jf9=$^1=xQb*ie;)fHl~7NCrm8ff1EehtDl#RcXb8i3(aP{eVp% zRsoLH0q}uNXx%6<09kF_`wiAIuwW4|g3+Da-5cFyr7(k}(~q5n)vW0{y*>o*hv( z3?faysroHZJobQ9t0iHTRR>n!4l3P0{>X)!QrB%VGA+|-3ZAk*euP^nT=rT=s8>M{1c1eclOBS4&Czd7&kVrgcPQ3(C{Q-$1V3P76TTy7#!pYqRs1Cb zPDZSs4b|0gly%S_56HUn%9_U>)n)-Qa@m)`-q)h?eLja7v5DXpDA(djtfY6Cfv& z9_R(3Fc_088ELwCGk+c2+WO_P?F4((Q6p7BH#ji=EQekw!LyQtYuJWaI0SscY3aM< z{tYri38B3$02hD*2Y_wlrRcQ=>y1u7Om=IWz3TsJC@!6A-ZW(kNdfD~J#M&GXq#PV zOkV4uBH88!<>>Zgz2#fI9#M+M?uy1^6z@dd>OjWFGN|KS(*S$(RtUsRIw8~}MPmNlW~El{1)oyt8(gD~ zRU^oRYfxNv9$n9U*&Ra*{WNamPT*Q)@acx_2DB(GcJcHO==|l z%)$lD%wE!I{T~$IUH(n*C8h06f7J$7bvqw%ST!zKH!egL*8cwLR+n?-Zu90A;08D6 z2%qXZ4`nC!>pd^_?4D@u_HJcQtH67*tzE(eT5J&VkkLByMn@;}bu)mijC?z3K|tGj z^-)5QszkU1*-mvM)o!U=_t_p`ryNG#DauHkaZ`_Sa4lS1clF4nU<%tN$7Z+$d?sFtosbzbpU^FN{JTINV zai+_zw;=T2JBS+XNnI9$U1n*SXK8tTK6|Zms9n!pmBa>s>8SklEWbh3rE*9h>MLI> zpJ(@^XLzR1SWb8Pr+;_8e)5W6@KtYcg}-{OcXy~SOHNhG_snc>Ug(pLr#o z%mpZD!$oR$H{d&Jf~Eg?35QZ_?wdLmfhBza9k_g@Z~CqW_FF&wtxtXA27A@#daH+c zhcB)L28FUZgon*lP%x#wYMB47911*yU*MPDZleU^k6+^Vt3w!U+((61__-3q>0~2; zSL6}+f->E4B8AQVUl0Vf%Z2NugmE{-OO1IS6<@e@*{nO$3wm;%e{M5~F;7oVi;w(L z|41sQi`QP#(I0&U2Z$9Q1QG->-~d8|9&{9JIPf9Fhzuhls8I3XzyTgGTGUAKBL#sI zI10qW$B!Q`X4d!+v7(10Td!ow45W$NHd(UL;iM*N9J-+FgbpoAt=Unewzd%kSL>21 zrsc}b+Vlv}D^85!5K-ZkNEI(F$H zuXES#z5Dm-;J1U?Cg+iBjVI`yW1JQ<794(i>38xb;jb)PH|Px%19|1+sGw zfeCgJl~h$-W#1oU;Gou6W*rg4g=J+}mt1E_NXCa>5rG$8efd?0Ag~}-*kRI4GmT<8 z6^LLjLJT397=)Ao8Dy7zBZ(f4#3a=tCGpbBHF&i4nrp=QF-tXa#Fok;*!kcamR*=Z zMnTa<_ndNJLg)V_aZH+NWg<`ePVOk=hYARe| zg$UMJ|4H?YU_+!)2VgF?xT}kz2#KUUn|>n%3oMLu>`0JJ<;We3+@$HAesa=D6w?{E zkS3|jB4sAFXmZ;nTxpplBNLI?B)aK3XYH8nt|?tY>&fXZz4gvZ=R)C4)M*q@)j-~A zxoKjSwq>{^-vu-@x?iLXI~vRI_OJsMbKEg55*)l$1Qj5P z3+t*W&$0j3#I0_cpgXksmj8321}FKcs)t-#&o$(!3X z*EI5PpBl(ow;BoM!7F)1S%>00$~rEhPTC2%I5PCL)o2Exl};z><6U z%h&(j{tcyF+ncU}u#4E~U+qC21r*1y#yO6GjuVi9A_zfAam;f^n-I=Yr=cJzDrR$u z9SBv`x=rn32(imqIP{`I-~}yMGzwZ}T2T#D9IsHys{#Ltvn~gOX@@hZMT9n?zKkHt z1PSp&{N(l;0AQqv9%0iD{Qw<4Wa@A9JKJ~kvxbqht87?wpKih!lQBAoa8;p_D{k^6 zhMWNo18GFuPU4b~$SZF)%$5Tm*T9GIWF38+;2s0jxh8~VbSfju$U5jM4|*t7Wbxn# z5!9D7{G}9viJ>>h5y=Q{aulAVN*R!-EhvR16@80aca}&d*~IS(r6f<@p6J9Y{s8}O zu|cKTqL-QkROAL&OfGeD4wzc6FvtzC`<@deOQ?mx_(JTQCP+5aTJ=GGAl|hhlOQ837AO-NPvTviB zN)n;-JbKc`d>=rWdcs3L1pEXK6X2o!ayqvN8~{rwqbE>V#kHXl6$|+TK~zi`t#FQw zq+K%N1sYM#bD9&K?7Zq#wSrL#Dq%|}W6Pqva)Q*^U}9orzzF1WR+ibJleqs%Q7b|* zImzM879tWH7iRqoVO`vcD@(9qu8T{?E1V^Y^j$Z&m8ujd8+um) zWknNBNWmSp`_Icw;jzwI+97sRhW}YliR>fO^6VRcG47HA@9Q2B^~EH@xlsbObs{pW zGM6l{w!xaexpKra3(lcNIz_Ywxind$tOT z)Ts?nrAq%~MPB;XV6!CP-M$>#?$`z+3l1uSOAt-gJahsKAt~!*G5TvNg@)<(uO=?M~#Z&ldHZfhfT~a_L8!*tuP`a^W zodZD*d9#&? zK~#ab8A>sWpU{Rrp!hBLG$dO!4Hw|_ts4H0F{C3Q*Xmk2Co++ojK;m6bW^U#6JWt1 z2p6FnV${_%w748XQzbjQ)|B+s*CI%Rhf8M?9qu^^ICpq9T-?K=?>Ga?7ZFh%%#0;9 zmwAF}z;XmVVgQZclzYU}HGP()S&Lk=(d(l~l}z}+I*oSXF7Z%@m0NS-u+RhN41<$pY zah_Xr5T%I%;+qXjLk)d!^|hlbXyjvEl)RMgDAKM~S_}WpoD$cYM>aUtJ@Xa7T*7>q zIc->82B-WWIB2+sEAEDeS>Olh)*cs|^N>hcCu5{h*nU$UlC! z8kQYoSGVLU7qiSRWi7k)RcB(oFOU-QHUxF)-l$9GUE#3jvC^*zi1Q+GK(1Kw;3GQ6 z_Kl5gx7N*15%^_4^v$6pxBB~C5wA(d&Vd6sR2F#^01r3?Cxjl}MBaJ*oB{4h%&}fX zT|nQ6hY0x81~I}`u$JzHptvl`O0-Ah_05*GjfMR{LfqXRCEq#qnWGpK=ZqNGIZ|81 zo0{d*Bbi;yupRd?Ox)c{_(@v9kiFjc%5J>e1lbCeMv>f2(vEBk2AUI@z0{mRu#92fMPyrsz1+t!Z%$_8m8H0Ub zn~|W=9i0}$1V$a8D+z}H*3CH0-#FPBQe+(rUfl!9Ab63Nv31oBl$FVx6)}h#y|zT@sF)X4D=E#G=(e;U3B&_h`lzsUVnuRH|{&m^slJ7~ps1 z*bz#YM+6g@gq{JM!OP)7r_9_Owp=he1uiVX&yms(RNy6;zy!))fO%CYvK<(R;FVm> z?onG={6TtjLcftlGA#!i(Hfkk0AJAr7%Kl^Bu*kFLI_H1mGo>u3#?TSs#U#-Vse?x zs*qwi3?cZXV%@o5D=tvYFw069o&_*jaoOT$NXYz=p_d%Qmn8=r84&`+20P#wfh__k z%*07*-3=8Hd0d48QcENJ4W8geu%*$Hj1u^Go^UJ(1-L@A;93%KgdUz5OD&QTxW)tU zK#?t7sT5Xv3{I#J!~m{B1jY5Sde@*SG;Ku zgRPF};LL)>0m0-IFE|=l2%JOO-6}>TUIv$TZ9;pU8dw?TSw$KB>EiHBBd$5gLK03* zPA1`8UiMtTpU9$S8p;YF<5fb34hS?j8E6!KuiiP*ESLftg1x=G-iSPFjJV%n%&00erGSO|pVzf|O?%7xpn1 zRoIz+VOg2=+5o~Mu6agQY}qj?7@8qMqA5XZw&tyz4%!WxREj|R&4v!3#3QT$7TH}( z)z@$8L^XLsD8OL3nPs2-S!#tMc^)JVR$tleMyn{KDauY?LZm`=UwL_8`cw^hew7L+ zCVDcawZKvE!9k}uL43kzO{)J9_9O&yT_1%sqGx`lN>!n1Siv>Wo*^KGJF*RK@u&ys z0Y)`wQ{Gb`Djn{9%L?=xm0)GI7@kx03pMyb33%uaGy`D$6E$sx=8R>Dny9RxWO;SY zKLTXvJgId0AP9-$bYkZz?t&`*Wq0yIj^T&yS->W`MGl(Grh4Xi25EArXTPO~kzPW4 zBB^}x0h1a~U#ZjA?6vkD?4K8JH>8XjpnY zf}XVL3}j0mOxcaeBQfQGPhiSnMGP1ir}CMl)=3H1{o20P%a8<$m_|2)Z!2>MmLe7AEOhK!vXtOrL zsCfi^g_eQQs`KEKLBtD|&57=z=94ugYyJQlDgHmhf>(yK{p3iv=$ z;3;tGn6@70@{xm}zS9{ffv7>}Shbj4wv1W8r3QUX-aQ(PR>DKVD`3hi$28%z4JMsF zmL{0joOlQSh~{biSyJo}!2r0wj!Iyegbx_y$I1&OL0Cd)b0K;)Z8@h9p#DNKi=# z`ipb~qrV{{%!M3tCMh0_=pP_vh9M-Zs70pmYy!>RHaKhJR*^&ck9OIX6 zt|E-!_sZJ`2JX>03vD8ovHm zq^%^=c4y6?f|*X-(p-un6iO6qh5uB=o@#<2zE6b6K_E15?=FZ?wkIeE@9^H!hysaw zTtFFM0bNi9$TqtXAv-WY{yA?3_sKQK~4~aseQY()ufgD z;+3MOLLmR2pYj>E@}sdeCsF0wiLvc-Wvb`?Y|t(cDd}a>Hsk=8--aDrO!Sr8mM{Y0 zkn_?X!rW5VhFtc{#+0tcG%9fJDFS`Ui1sAUsF50fVuW$ME%FGdFd5hMG&2vI%*RaBnR!S?c5Rp`}`O-Og$c;1XJzX`iO%Akdo}q}~TH`gD$UVYe0vwhWBXzOJlN=|_T84A;OS&;;LYhz*$#OQyOFs$vo36&|uUX1=|FuyVU>E z#})M9l`!`JbE6Ed7j$L}4;_Cg?&nyd0e<(`)pLM>p8`2IA0?Y)RV&RVv8_ zv=Q7Cv1!;+5^rl>!*sWL>k@b!Caz8u2ezTL7;?E;PgiQD%xI(u_0n=DWP_UOQZ_YA zMPs3{JY4}~_2-z}DIS)GY9EoOWd$!6^pq)cRD0noE>bTzTSyXX{RIP|l-uJBnq;@wx zDs3N3nvaF?WKSgFyqZ(nE`#%{m@HERCsIr`U9a_znVE!r3$j#W6jgKC2blkH9H+Lg zc}7y&Z#b%!5c!u}Pw(H@E`H#FWb3X4pLT>R@t;;MR?!TVSWrev_f5CVb2U=SjJS*^ z11QLyq~Xdde<3}u^4_s4&_s!+c>=*Hg#&`>u?!uQy$hC`FLWfqB-n%$%{F~=c1Rle z(Ws4*L-Uv%00T^bYCXA=lT2sTW|b%SMD+xxS*Y=evnF%0UPBH8y1|IS+uG4ZPBW6& zozTjhSU*>-w=8O75A!g}tJ4Z=p8b*6j!;$7&2xKRW**?yc~D8*(=T%?WmhjFI5P^kZxa6NIW^PmTv z9Xb};TGRnLjw@fwc*C_tTWq&n4IG;b=D}k(ciqIN@j0eME+n@PmdkZQs8*fA#t8zE za(r!pNWzx@sG~C`M^y6jXzeai`m}$M0rZQ)nt|+Q7tGs%XVk8anht;^(D?X`EAT-E z3<85+_KjJhVTG$5c=@)%yC?gjyG=JD5u~O|u_*q#h}#DkpcT%1f-1nmJ%gyj4`!&? z2NPlr<)E&mXf?6ZbM+dBz;JDZ4FU>i7$Q0VK*OGFOMnruqxG2$G*R^jy9Wcg*;!}-Q4Nh(i8HOtU)t--BXDNGHFiK2G(9i#n;3fDJ}LYuYQ z)|@G`5trU6;YJ!B4Tf@pAVJBuB#a;E2l5@g1OWs)f#}wy8|bduJ%yYatwDnZN{a|1 zfP^742O||C8o>}z0ptY57#lf0Q86(L$&q0$X=>@xWuQ4AffOo4OI6NOrgVnFgoO?y zNvLL_G8%}61r$q}CUyFB>C*`V3MjBT0Yw?C6jm5n^FjYa4Y4LPRPgZ#)T?)ouI>>v zsaiD&3lh||d-tk>9zIG&q!K5PCSJ+ZJ~}{`?u0$mWTlb=z<~hCkpomdAcb&>8b)== z^2RD*1*&;TQ-&=1bZOBWs#ai)z{Mijnc%qKnq-OX)gHQveOTjyk~NG7a~o&;cyi>* znG-yS5c+fHMqLu7E#-=mA|XhOn23_3$tNX0YQbR1-i!2O5@FNpNPyy!eVS>%FI!#qqTG%ld>N60R#CY7XOoWw)W-k|_W-RAN!+$e41 zLCShxs{)*Bu9-!ODZ$mv)&sID^UO32e$)RQd)!>J)0Xg9!_FLfD56q9!)tWAK=srk zkCb8{RM3nRU1j7|_Ip%P{78Wm(*;>9V-S~MjH{Ml4D+=xQNEkX< z%D5y}49n1!*DK@e^ydXQ$yc0E*`1;m!YDbS#jd@Ean7U$+d~9yGyS$+cpJ&=NKIlS zS&~26)eYW%-L)5QT=eGGO4H`OkT)@Fc(u5L4PO{>gzu?klVjzW7zd6$W^d5Rxo$&a z%ArM%43cpQ#TU>;&oAXkeWFB>u@2ZOEE8Cy^EU^d?O{v@@tle@B|1?Ep=MfvUBg6B zuf~rE(QpKjxT9vX_dF)pj4)Xs2@n5@ExNFN2Bk_cV0#(_^Mm^wGJPCMypi5s)FhLn z1MMlxj~D(F<$sr|HL~Qx-_!1f@c9x(ExcfdQ(WTKR1k!uYG-kvLIlPm9vpmRB_NB| z^)zyx%?avr8PwqDN|zH>DDP^5TOI30Xr%6yAYddElksN3JL8~2JH7dU7dXKft7)JI zkEj|5Kahljyux{h7)N8M!H9}r?+Zy(B7m%hkt-N!4bt;j)RaZPZ5_ZvlThCp+LyK- zva4=^``7&Fccm{q>0TImo{Y@YMgbOZfCQWlt<~Go&U>b*cYQNG#{$8dSRG z1v#N@jrw|rwceqSW?XVFN^{%#z=8#|WU+sK=%+mxa|cxVPL)9b)b)ZAtZNDDXmOGx zB&O9XT&AWIvBKg&LC3|Yq2UqX@PvMQ}6>C zG!(nKL8%kuD%)HM8l*LHEO{ZZ@y}ME zwlS&%g)n=Os<&v7h{CdDUoViph?-SnWxX9D(U>C3E`o7JdK>Ckfmka3EPN$2!3fJ% z(`1TPv{<@CZ}|W4m0Q(dl0gYf5hbUHb3NFwLW%Hu66Kx>FUmd2%^pflFws3~!B02n z&PE1ATh5uMJHNeZI)O5)SDSLp3oKerlP0wS6*B^ z8U7%iozhyv0Q)%Myrff$jcd@B;E+l7dBQUG!yb+*t&@1?!#%Y{W#9My#_s#RUx>ym*6uFDisGls-Z$@vT= zNK9$1TZ-|yEzKc4_)QNx zgyTN;#e_L1wJi`Ak6jo9VM7(jj}^AsA14twCd4CQ3yWG@MB$@VZc9;6cr$b?jZ7F}NLnKo;65A1XOEU{G#<>0mI_>a6{U>Wl&xtk?Vk;TNM@1_#VaoH zX*&O$)ti8|W;A~Azfh|0b$} z+act07knP{MsK?4mtOiRoF4V5SH05jq-PXAnyOLdT!$%tcDr5|-z0vFu zUupX7zQ*&ic2D7Jox3rT6Y(WC!yj;j=oebIl^L{qf?$g<lj0#Vh zDn1hJFT7@tcx$R04eK_->vj(VVIgi@jO_G`bXnNP${@))a)l!8cr8d&F=po z&IQ>4?^^K3$m@`-ulUYtb{Nb2q~sbJ0vxJ9Z9dKwT;Z0K<>O{0St3MlCLprl)@TB^P%I?GiIuu;-*hjlHbo9Q!4MBo*+eRms3MVQEGvBAkaPg<7|!lg ziq&2a21B70>JH+jullTyDN0ZjN07=YLkhs56LP>6fDoCua1TB%T-E?YWN{IEL>132 zU#^e|@8MXk;S8Jv6|*E2gGAZR?&jb?OLTEH3akds&jE}D4_@)*h-LJWfLZ@)gXqxk zVJ?9R9|jHI5F85-n-nlmlh;1YBshRi`Vc(;F%SjuAzlCiFyKV)(H_<4vGB_T z4q@|PqSH78aV(J&G0_FRp%hr5Ah}@%&7%fIa1=*Th45=_7zY)n5JccBLm=Z4_DK$U z5WnysUtB?7lF$Uo?jjR|5Ar1wtPrI1g>0TpB!5wivN3Cf??Ob9afT2#sF5Tii*XpM z4fTc_*$@to(oW!M!}yS!>d5aj!aLZJawhNsGZ0d;stIzV0nBU>?+ae)M>eVlAgk{~ zFi{(LZy{Om1;Zf{Ht`@C<1-XU*Z7eHNs+z8Bmu{<`p#e{Z_-DG&?5iY;_aSpz^Kd+ zFJWy=k`JztL}p5tP*DrfEV8C?HL_+laMJU@4Kr5&VkQ$ajing(ARI^2^xS~ZuwW@` z?u0jVklG4~$=2F8U!qW@kYZdkJRtT*; zg9i^XNrt$M2da@YlMVryE_*hPdL+$)qH5|q>^$Hx9&^tg>k+gx2{#wiE5&6+AW=Gj zrX~4F;&cHm3rH>35)<4~Ev;)&8p%kw!vLgGd!(~28<9GLgGK*R)Hz#}I(se>MXA3~ zlo8d>Mj?ssW^_c>(~)u%M$r>U#jKg^6Djla^s-}{RI|1yXRz{+~tqP7#l19K-OE@nvLywa~m%}qq2SiJdL?zNiV=Y1ctVOx4B_K4p z28}}wuXcJgp(N%vW7JPqCs1cpMsw6U9i~SY)khh1%6_yfSTEDgz(|>Z2Rw~D*yFbB zaO)Zc4`iR!%vZL zI=Ky{@YFfSwNbD0T+J0yGfz8|!Zbw_0iD1FvV%XTlBo`EN%5^>blK~(fEM~}4(d?p}~25PYNTK)7{ca8?O>JoIY(-7=VcW&G05M5;! zNZ;%$mO?5W!Kj=70eK)P*wJI$%_(26)~L2gw>F$`FA(n$0>t1Dc=bE76gTZ9Pg=BB z%auY8@A51eRdyplp>Q1J1Ty>T;f3MmQwaZ7Wu8arK2fv=Tlw zukH{B`&PZ;3N?{bwovv!)XNX3?OAL`5&#Z+_abHtilY`p4pe1*Wv6E)S6j_D&*aoc z+qZDd*UnxhOpW$BL+xj6$14_A&qSAi!LWc)b9Ivk5R9~4ffa(#qg$lPP&CzJiQo=% z0fYNBWmIfb>+yHH={tIW3B*%G?rCQcq;LPLhf`*idDmfX-AH2tB(GAqS9idMZ@7kY zI23VsheZ?}cbJAzlZJ!X;AV%2hjs0`7kQU!h7(V$kd%A}_&4ZatE4!=2EmG_c+aTV zh@PNz7dUF^b;QQ$#Af$9oRqe1H$6f}ciUKly$%t|q8@pG2^^tdLAXZnrU~Y7vCM9I zGDdUyfD$LRgq;dDwavZxXO6UNL}PtF|eds`d=pwvy@{GnklnH;}}l0g@SF62Ua- zg_$5j3e@utY~Xk==9=j?#d zunSwe3A9C!c`Fgr@@zA)!wma&SditYveWyJ!;2E2yR&Co@zh|pgTPPVyJP12obB7Q zZ9A*O+P>vkw=?FxeLJ8DT%irTtZlou6&$sX8@8c%&?+@6!TM@@Ih{LttGS7sguxd$ zT%@=AjU{3(4^+HM-17f+!$gB)2A&DM4ac(U7A9ux@7UR(g<8Igdp{xV9Mw6#*Sf!V z9EAa6s`+~YGhD=$8_0(k!C#xfRrotrw|vb2xtknws5;i1*uj~4&-_5b7cimuu&<}v z_QXSt@5mwsJG()AkRsvGRvhHy#l%m%N(_~4VQe~PP`zK=y2NVdwjh0rZvsCJx`Chp zcl(iJx<7xs*4XEA7|n(O)S-|2$d8Y+Z?*n+PYOZdlkr%jhmdS z8q4Pz)#HlG?F2V!tPhW9>aJT)G+gVT49#;LC)8Yn*u1t7^)7K#7gY$@=ah;ib_Oxl z#dW$>mD*Rkw~POg=zAj>V;qgM%V5N!9eW;U4?#WCQyI}KeUXn>$~hIXug}pnebEKH zpCw9DM19=rY1^F?-e-8TZ}^GtaJ8MdYeGAU$r~!l{EqsXHN{B+tIEUEeAm^SKfoXl zCA~#Tw3Gv0;`Mf+DHO;IChw#NBrY5RKmMJ0T!-UboxeP?P5$I5J>?BPph;fjUw*Tx z*X7^+2Iab%=}n>|I_KqjMd3}JziG$Yae^gSJrG{k7oOK6r6b(E+^d_Bdw8GswxG{i zdGY**ZaPg?$&>Lt%GeInQ+cLoE$mr-rq6xFEAZZPe4f_c>|Gv*VczYJxB^>Af=qot z8XWIaUGM+>M9Qt!-FDd=c{x*WU5&$F7>+*adwtz6J?$HR>MQu?7s!!`n+*O6bvVXm zybHTFU&y2ftIxRW`FfK*9)fH@Yah6t8GobAU7|rc<$oBeLK)=SKJLR_>?x}5or~V` zeqV*$_kaKHRp0B=xEuw)>Y{t2iQ(`SKJgpA;8maTnHuVooVxWW)|p;FFD9zIy2GMA zoy!)gN8hNW5~|04{8c>|kjd$J>AXyMqtUf1VAR``O=afr2uge?xQZ@a)%5n(66y>aRG zt@>7EL%3IgB*ZJ&>RqmjITL&X*=%3NB%oN-oH&ME4v{7E)YvDE)0wni+bkP{kRF^h zSL*Z`l&J9GN>{;BYG*dskp<0k)Qcjn>*pT-;@0|kvUMOr8nU@p8e(?=cwPTf z`I$EzbIbWP({8qr!COe+c_$c8JLMDxaKiY)B5_F-x6~)f0cKiORZ&NyjUI)i)`qH8 zMi_eBsn^tctQ}|)W|>__--Y`f36X(7%12?76oTeidXL>v9(`O1xEEhRme@uN50Zx8 zk0B*jU~Mr;x6+wwhV>waGfGI}o41j8(^qIIR>w>#uIS>6Foq(dg*jrR<3-IGR#=UW zip0fJ6cVvziQ73j;25N3Ss|wo{wQ5V-2sW9ec?Sx-&LRuNNQY49;u*ZVIsMzU3x~F8;#6mqj`V6pRpm;ln?=NC>q}GC_Th=WMLXC{(fkx#p@$BIs7W>|M^*oPzAbd$ zR6cr|EUcVWDk56%EjTN#ijH}nxL^+0Ynf8bXR4RwfuSj$`l$=0mbQs1C6FmGsqVif zCVTL)t2T?_c5Iq;o3~#^45&;8589%(+jeW2R8keYDOrc*rDKs5_n4oPd5$_lyj>)y zvcVHMN$(M^lCZEw=*BA~Zug4T>3Iz|s}qE*UKw+{Q3||Tk7-7zafUjFx@*)MdmZq1 z;ei~o$Vf#3M%o%vgrm1t(<-spTC=5ZsWk8W@X7-VEiI_H*-V~#L9c!59Z17XFc+r^ z&F{}b_G)lMN$vEe4t97ZBdmFg5S(l3#`hIN{tk@G57xLVxGX#LCQ^|h~cH|F<`Zc`tsPYo|XEFs#E ztQM2z(F|P&v|HF(0zO!=N=ueopzh|Av^j;&Xvo{l{(fgW(}gW%T|r-;ct?nx2!e1b zfS=j88e;+bJu7X(6pD3 zVM$O_4G^Z-I3nRsHX~vfW^fpqb@>HxnG)f{!Z)I$=^=$Y;gbJ{WHJKOVT(|7WSwzh zw2^*ksD|g-W9%4pCp&ShQo9o30590U@hvV{KJ=jCl9-r9Mo&scVh{iyXS+!9=YkU( zVBmnqK{PEZjK)LcU$93+)>ttRO$-bGRYe6hw(EqWJCPjc7(bEG4}tlsl_mWI74tFj zf53DAcR+~0889Pd!^<)7Qhl@Dx19Doe_4lMduw3DmZY!bAdx#mQWCPPvOX}SM{Z+@ zW*o92NeDI5GnI1~7EQFha{cK~e455Bvz1Hv6{iqnsb6C}DNi)E(oy*7&yHZ0z1NXZ zhID)2@7^>*WTJ+00z>EAsOXkn4f2=LD;^@8rNvl{!Jk>;(hu|h2P$6*@?3R`iRZFc z%oN$vg}Ou=FSjzc8W}U7k3y&!*{8XOa#V+n8)^{gmbEDX=#iKcod)r?%|Q6HRRM(P z0u#xspHe3~ajPaM&19zaZIW8;Bx^X`WfL;Yv>3R|r=e^bOpU z!_=KMA&NUywuY2x^Xj|G#6za(4PY7(>M1o@$;W=~gJMMLA1!vZn_5zu>MScytEH^T z)v|BETx(ljcpQGxby7GL7Vf~(MB&-1n4tt{P-7QJFrOQ=X<<~n2 z7~CMK5th8_t&bjpC4z2hW}z&fO6S$bSDsLVIAJSI^+_E68Ocz&J!4QBcQm^`hEP(C zDq2F%ds+8gRdn=JVs-6n$ldnul87bKB@t7|4u({e!`%TGX-mGL%66so^&xz5)7{f* zD}FXb@5WEdji zU8}R#{o+@gdQsKi=ovY}=rOL(?XYn33Pd8DHM6UQZTlMBCli_Wo<`0V$&?J{+?7|D z&J8h?ISj4@!&H$?N;G!6XrBK-wp@j#pixU4AynD_ft!;hAv0++)YVeuhB=;VheU#s zm*$YRflTABtO4s-AEU@e22GBH1V)53TE{bN^*?cJVcc#sDjOaaSsc7%dgau7A8heo zZ@1gzl0=CAEgyb4fJ2rMR+SxVH5c39+0W{?m0_6ka%7DRTI16=2DK9d9dnPLiCJn?F-Ao?1GmJ|Hd8jHWKQEvZs%&v1G|{DYNOSym69 z8pf_ZT{h6I?vR-xw>WyY#4|qL5+b*0(!TAD6|Em8ay*KpJ81=#Bh|01jLJJ}cLUd} zVgETX;@ECB-c)@vb|vQKI)sFW;+b9qf8-ATP$%@uUGnighrC4|>$M-3joTjk+<`eQ zFxSi*bi+RS&>NSs79nYGd>6Z1F;V!ihm7hrr@hiDuQP~7y7XF^@b3?2h^?JF<>iQd zo(;}=y1lz~q0}7FakZCi>l$a)`g>FF(eEoUe|ByI+s$H)=(PbUBKN{Z<1`k1Nf%x1 zz-EE6`G#Veg#7D{Ul;d5y)#bc333;@&=5H+E{A&>carN~@1IX3ukH4C-s4o!uxH%A zj~iLzBKY2O@sKi|MN7~X9`yQ_y5DPf;TqW*=>x|ucRd&U)Xf_5^#7>xo7~&jL+c21 zCcm3U>122ZGn7Jm$7f0cs5rKDDk28|bC7j^1n6mwrB%J>Ms;>#*#=ZJl!1!(Wvm8T za&kWR5r6bIe~(v!!2lFUwr1v6e(aNgvS)1Mv}Ww(U|08HqnCSSCTN=VJzX?=HYi;C z_i{jBckJam36@NAXB`W3cl0(+9%zFxI8_(;S{xNnT6cn3NP;OCD1l~j00neGmxDOi zSMr5;FVlhfb_4J>c5YWAT7+zu28H7`ZN7JH6Y)WwhiVM>e_57>5f^Tq(^R3=ciw}A zhp2@pSP5MRgkXnwKuBfoMo2lsb4Fw-`~zK&=7`3!U<_q@=U0WM@`Vioir9C9QW%Ao zCyD_zMQ*cc=_iPR7l;8PgmV}FVUCB0Ca885{b}?f{QZW-d2N>P0-Vm=nk3YIi0%Eir;1vx~Y2 zSGBh&#z%};28<4cilO0b<|uD|cp1mocWQV=-FS^HIDoJyfZV8qT$OnZ_&Le*9Rrtq z8%SOONg_X?fg$%Wu(fEM_C6A2ck*GqC(kS*!sG5@jFxjV33I*0_q~qLML* zV|+)HAPJLAS%{igkJ!Zjf_QUQAw`qbXOK;}cJEezVO3wNW-Uqym)JrIfx?JUrZreO zgfkd-BzBk2NFbpXihbF9kR*^-H;-Xviz%6y{~~iNWQqodX{2Z-#}$KdIEM`4eg6o2 zxX77t2^_~^3ZD>~rx2QvPza|P2{MA3t4Wu2nS8F9ms;6~fBA^cNQS+XNq|L$S+#GC z^j>HgiO_YGF=ca_`86Rzky)8x0(F0xsS~hw={--Gbqkt|Yn|djgPN;_mNL}cOahw*8H0O~Cc9WCn8x`1l z$3vY3YM{{2XQ-wB3Z$u>3+kW^x}D$opb(mx6$+l<8J=~Zq2kG*+GP1Dr48M1pwr2bZgoc^!#Wcxq!fCgL~5i* zilhx{njUJQOd6gUx|-eTp&JUKuG58G=bAeMJy}a?m7_bF zlnD9^%aEW78l>1Mr$Y*%bZVz}il=#+qM@w+c4~)!j#>vgnx<;1scrhE*10Wl3aWf6s-sG(r7EGPimH2>s;lat8M>-~ z`llEQo*YX5sIXe0AL^Y_s-(8sq#fFY-T|qtnFbchh=aPCnYyVw>ZWYksiEqub1J0I z8m*>Et<_qm)~c!zimlpOs?ZvmpsG)&YMsBJ3&vosDX5*~daml4tjfx)d{#%EDy{QM zuiZMW+gh*Iny>rHuMhgE`GkT2E3g1Nus;?NEvGj^s6ZWGoLa^*wuq9it2#c}{dk*HXu7f$vm9HqH;c1& zM6n_JLL*BYCJVGdtBWbCvPIjnFT18NJFz*-v<}*|Ps_9)+q1beu)`s=?MSj%OSD^S zv`CBpv@t8MVJo&!OSWYzvttXgKC7;3d$m~$v>rwuh^=Zfme{o2e{&v~$Z0&TzMrOShEkw`n`INWr&y%ek9-6rS6; zq5HRjJF$kVI*5z9iCeC#tGbKJxLgal5&N+2`m>iSy5kC7c{{qfJGi&2xqX|rU~9Lj zOT4bzx{CX{$s4=Nd$p#EySAITy&JrkTPUC_KhTT4*IT^XtGFs+yvLh}LyNMk%bAZm zf(|RYNjtoqI=RoAy*|pj?n}M3%eM1dozBa>`TM=&o3zrYybX)Km7BiIOSSjAz6Xr| zz}ic{?~A_+?7%{6wfZZ6;Oo5?CEYq=#X!4rJJ zE4;rf?7wmg!{^Jvax}xt8^G*aM|}puDBQsLTQ(9FzA49w61H5%m{J~Sa!#W%sQH;e@Y{l%@!WHbo0Q|xnY{rg@#7&HDPF%)C?8Z-= z#c|xlT1>}M{Ka;xv}3Htdwj$&Y`#eh$ZWjEg1nIod&gW1$B0Zvhm1>!jK@TL#g8n; z-z&*{47+{o$A2uqfy~LBOvsjO!#jGqdCbVFTgP);$|nr7Zw$Vl+{T!k#$@dO%4*EY zwd}=VjKH_t$cbCUsI1GXJjssSuCWZunY^-SJj;VT%(m>oP`tRO?8~Vf$-i95(7em3 zyv&4L%*f2m$L!6T?8$S>%)-3KxlGOHY|S~W&gopu*qqJVoV?xq&E8DQ$-K$-46%j` z%H_Ps!5q!(+|P16y8%tg?9ibs(Q^d33S7~oT)!8M(gQ8a5`D=Yea12U(eq5x;yljg{LdKe$STd#0=) zrajqZUD{da)q?HXo6Xru9od(i+Ne$2s$JWwJlYwp*{t2#y$##4t=qXh+qT`-qHWxy zz1o-^*bMF4za8Aqjn9?c+)nMv$i3XfP29ts-OgOyBnZo|E#A_d!Q8#u2EEt={dd-{+0qFTLLGZP?wM-}h|)-_)Jn{B7C$z2FU= z+=&d|&dtC1Ea0(i;Vg^b25#OC?%(~*;ScW59-iRcJ>KIj*X%vw8lK`S4$LC%;u3!0 z!Hv``{^2y9;inDa0e;&B?$;<@;f+h*o&C`rZsR%5<44ZmM*ia8E#ERe;{iV18}8#u z&eu3z<5XVdL~i0u9@{l7&k+9KSI*+Xapg#E=4TG&KfdMg{Mlaa=4rm5Y!+e7@bU*?>WA*?c@F4{zUB>`>Z*R= zuwLt+PU^iL=~}4hSsmfI9^SM*>vA6Jzz*l9ZtJMt>;V4kpN#B?KI+Nt>|uV;(_ZY{ z&g#Lw=+%yeWPRf7$^+OF<;{@s=S>*8MSkiP8Gj_v3^>}Z|u8g0@F&g}Cp z=4(<#;od&qE}!!EuH!S0^I%T!1fTOX@ALAG@jox}`+mvf ze(^mo^feFhHgD{#9`iAc^hw{>7O(UGAN523-cP>+umJ%8A^8La6afDKEC2ui06+mi z0RRa90QCtRNU&hOg9Z~K#J5nP!+j7RM)b$8B1MZBCuZz-v7<+h8$W6sDUoEzlJyv- zY{*h&OO!5S%A`kg=F6KlbK=y=v!_p(K!5HuN%W-Aqezn$4JwkU(5FyqN*zjdYSgP( zS7u%6v}#wbN}GlS`?c&=t7y}nRm*m*Tdii}dX-CdtX;Zz=jwgCx2@m5fCJaf+n2E6 zyMq%K7R({VP%bqQk zHtN*4S#J&;ySH!Kmw^MVE!?o~+{k~k?QJ@^^XA2)8&`h3x=-rZ&pwxa8#?#y@T@2Q zzb+o@_V4JytM{6{{CM|%y{DIde*OEI?cwXUAAUYnkoD2$AAkGpRvmQ&_BY^y#0khC zegzsRVT7DHXkmaDUbrBI_jUN7h8c#K;E4H=$RS@Mrl{hICboFuiw+$J;*2Z;Sfh*L zF_&729?BCRjX>TQWRFC~$QzFnx;W&LHZo}&j8I0opO92KDVmf;B8jDTRa)7ll3;2% zp_oaQd7_tJqAAyw5t3=9j|#H+VVYIS`J$V47CEPyWaSB=jzi&@rG0!Bx}l$fCi>=} zVHQT`qm8zyD57y9x96pd9?9dRl)7dqp@tgTsivIjS1PEOrn=~zf1SE&rEtRkN~x-} zzN%ZPuG;Ent-MZ`XReU}+v|wKJ|(PY$I2?FsKrK_Bd*Ct`(v}z`Wh>q&`PUqwah-L zYk$~gyKR)+ayyTiM15>Zi)E5y6?aQ&#SCd z+I9nQ!0;6eZ?*o0>l?%5I?V66nDuM$!DCL0aJ|9qYVpMw%e%40Z&G~ns*W~UGRZH; zOcKiRtW0ywHs74{sJg=JtdTq8EcDMh2W_;$sG7{N%uEm6^wUrm9re^sOKtViS67Yo z(Zrgoj@DF%4Yt^06TS7>S)-ly+Fd7YZP{eQEqB~=*Uk39D&MU)(rwfK?YG)?2mZI< zUjtsa;d$?kIM;70e)!{rM=tr|lYS_$L{*=zc0~o@Pz7~?B#wo(i2v^Bqs-$%6PgmjKwoXK4oeD%YN$8pC25d-liA_W>yml zja23g83{;31_7ZBE#wiPsYxL!w1x-OW+bWk%@`_khc?h8Gy8DSKI#FQ#xy7k(^*k; zrc;M6ykSURI!r@mbBIK2ArzGu&w1L@iUADjJ%`87Vic8)0KH`|W%s>s#8ag)^CtAEbgCx>!c|e)(3)0tn;cbyBKrWzhaxko7?tJ-II7J-Qu3i-fa+PrX;rGC?*!^h zYh7(>(!LfFoEcRo9@kmS`#BMbrv&By>dD1m%rX|V9Ag}TEAx1vw@xK z46Axb-`XIhxHT_vMf+Xl<`#(YeQRrBt5D^3w6_1H>Ixa^Q9}L}r7p$heJy&>(MnRC zJ$$ZDG4UB);IzIU)oyu@JJJKM*Q=b>FKdh7)xG-G4%Q^YiC3&*^cL~N7|tke^IOA~ zau>WCCNT~1+TisDSir{xLrBr9+y1IRxhW>FafdwL+}d}T3=Zdj5v)S=k>_l!z^rON zoL>`XI1ex`au)okg1&}W$3#x+T2bpw8}iuytiNr;c2CTK5?{2;Vt#X)zYAOUF4?ck zr7ua*JZNRwIGY|0>=W>ekW417Y-#-GDJOZ#E8SxtBGy3y`q{331@sNl+~^$h zRtsI&t{O<(*+9#9ykjQv5e!;pgx1-&jF2-73BYJq-`DU?(x+~As%=f)Eho3G#x-=WJDk}>)0DKn)SM{PXNE0fdkcrw@LTPC=^FEy&~f&# zwYAOcPlFcSj&`e7voHb@WLnIn4lbNssB9gNn6(5~a}dr=Z7%=1-6DQ;i1%#fbrW34 zH5K;1iCx_D@)mO-XWh8z@M#W1Td$G-o;JR>N?;|g`I!$@HkY-_04P8?r2ek)lN*Wg zhI^OZIi`4qWo}w^=ljdFj<&;V++B(z)ZkQBIjk-I!j$iJQg=7=8-}fo3&p1n77f zc!0B2f8TUt;goln*JF!@WwR4@U^aFps9r_~SlI`B|K@G;w{$JQXu)@Iu}6c$_i_n1 zY-a|9eeX~)_kX}QfOlYgGo@XMdoF#n0pv#h0Iriq9$=)sDp5)Q9!VFTjh607hH)5e=?MJ3y4zu_i|Bq z0<>0hGU$txXo<4Mc9S@bRETgy2zV|wRYYUif#6bO=p0PMuC+WX^<$6mnVc9$c;gUgO0d+`4#~}7y*e#21n?7({_7f zSbcdmkj>bQL0E6_2wL)3h|efnvnPJmW{I2ygHMo(=h#XFn2Rtqg*>NfKX+~)ICFjU zc!$?o#kPzc1qci+SrH`nTQ=Y zel_T5L05$a2a5p*kYr|e2Y7fucr%lQJc>t#2`7PWs9D`ej>5NhP&j`D7lZ96XGBSh zA9Z#LX^{8FfFbCNQh1DDws(JcY_tbsaOZhCR)%1iT6FMqv=@wIWt0hdjKvs~H+GZu z=#N`CQAcQoj|gv;bb?=}OQ&;PVD*{N7nOXbkXA>DJQjeD`I^h70!5jWdr6M?r*(Ol znz4yzVnvEe$$eO9n_7U0XB7v2Xo3lug}3N@lxT_t)nUK6ZNK+^NC}+$M{$OTfKQle zbLow4_`D@F#B) znTYnrkWki+aY%moh?o`0i-Y(C-^8AH*pN84peL!306Lrp>6MpvWN5jSQW$lHiF<1G zWm=e-j`@sN=#76!X7~4+*yf}ON^hA{r5$RT_Gg3or=Z8AlHHhj^tp)7mt46SegBA| zo2W?dm3X-p0!PMwt`(DWMv*3prji$Ths10Ac8V5=r)tV~eg~yLmusjNY&z+e^qF0D z3YAR%X_fsqd=Te}R_Yp8DtfbMhzj{|6&jz`H=Yp5oNooE6IXbk`J}=bQ#opuH-=*q z`lst?g!>4Hhl!+T=Y0;T52k zrGfXE=0>Q`MyIg(ryA*5^+2mbf?9Muh(y)I!wUXovaz8wWX}Nq)X`OsJ1t&Nl2|KYLESDk(#)t5Ba73 z2%{X9jC$9Ucj~LcR|G6LKVIscVwHyrC#%r5hb;vKdK!?JsRrUzt{S^~h{~SwN|4t7 z1d#!&j{FLY8w-;8MzCvSo(Ic%;pm0ws&|h^oNqXtMk$3yhm?Soot9^EW~!Rg2vSH( zw1~KgQoEEuI-9Lojn7(_&nbU4P@6EQUjLe$NoTW6+Lax-wn}=UkUFJ;rv@B)j;lag zb6c-sdXM@TvH%CAUdyqA%aVrbjvyL`RXcbSH?m@jZgu5|3=5!eJD~$85H)hrX6fEC`OsC4GO#tJ)`o_6m;q2Xj%?0=C<^mYaG6I-F$x=#;zY za_EYMaOr-E3X>ekTJ$oOr3p;@)p+4cq6F!la@w9>Fl@&wZ5W7LP3ePa%9$A&cLOY< zW!jZvcaemda~TSNx@x~Z$gs{NwG^wek^8s67M>Ajt@o&i9%#OX>wWwyn(gbp%k!ar z*|-oWqZ{n1x7nknSetYUx=t%#oVR9lc>@cWzmO@PPkF9I(61CKmzB9?Jy)mL#I!Nn zt>tE!&8w;6WqjQGuFAQh3!JsKn1daxSz&5T5iq-*MwZcdaA7FEftSAc`@#c9w%ICT za0+Jc8l&8&ucHP|ym+i!TaEkKoJuIiHMGCv%Ynh^kU3kpO6!`GW~!L~ijfT_o+&(6 zgdCs9xN2;whie72mAXH_o3B7QjMquSF3M)2jK%=Dr?@A$C8eMHmYXDSx(j%A%(w;2 zcbrlu!|uw%891&V9I(LJd^wz>aH@<=Ov~L%yx@1pVp_APTdEa$s>uAErmK{ge5FI% z#=lyepz5>`h+EA3mlH|JcL&1>yR$r5gsWS8P+QIv>`c)quzE_MLih&0+rn&ok){QJ z1AK$v$+*M3n^v2&K!}gFhLGker8K-*sj7Sf3anP_h*ny#LCdW0TgVyti>gXa0!(Ox zYshk1&qvUEp`636d~!somZf{0khpxZ9JNv$&XR1#4cmANrLemHiDER`$46?Ymy4qN ztdz)RT0g3FVf)cbySz#}2h)6|TBe}>hs{^_p9Q?NSCzXE%!Q3ibc43K9P6h_dc1cC zu<{75R3~v;`<94{(o4vkhqh-iE5cytqyJ2{$Y`J}e7$7)z;kGMNlm+2#?$-djQu8{ z63N2q2fP!Rx<-v~kxZK)&7l4qnqUZN1)S4{{f5XotQ>2{i^{o4sjA8gml4^}_4%+j zhXXkGmM*)VR9Vq?3vkz0sDZTC|AeI&dusb;rnO1s?kc(l&E;RJ2T$=s)%yq_H{xATpVY9`f;+qZFDk3@c{&u!U8+~goG zn1#KiD_V#1JA0v5nirI16YG@gTHNOA**Cre7)*?VN6vyRuxOraM;zJC*_4r4bt_wi zT0P+xjL0+j+$d}289j+^3I)`Cx_vH}+$D@`M$`2Fnt73Di>!EvSMH9Y$cb5Qu$a~5 zV@gb{jOc0l%nA#;LB6wZItJbweOH``nl4&zDz!yyoMa{FWK~|@d)y9AsQNwJIKHq{ zPMetRw|u_o)HccZE!is2zvE3_jEJ|Beny@ajc^WxP%OQ%3*{;L=bk)+*%-yCE@W)1 ze4|YS=hfVzPS?_%-^1nVJUq;aT5;oBh}~+O0Ez-Y#f|UZlN`aBR&*`=Yn^n{;`+tDeh^y5z2^@h9tE+4;F? z4(b+enMkg2=Ki2KUdIxw(6;%nNTA>^-H@P`Qbr2nch>&U`Z(cXM)SWo760J2Y>n=tOvhp*~^JM-$unr@kW_x{ZhDAF0d zb7r6Ssb=*KN%J5WwdY)F&r9IK*4@1SUVpJ)VBqVUM2oW~wfd~DI>?~OR7LO6n(ZMj z`uDcB#5ezJOTY!XyNy`#aZcqD4iG9%Jh735hzc?V5f+KS;KUm`cm5Eo`~_G@lkiBV%b$s0RN8z6@Tzg&EGVHWJ{ zV-C6b;3JBmy-?cPQE5& zGpjql3r@c`87#EVti}lc@5MW(b1_2h96AcY9Gm+CF#<``@kg^tG?UFXhpccpPeC0t zyUngj&a)_6tw~bOO62nfNOABJz^J@qEYc8WL^Cx< zNo4f6B?CdB2oD|IbkZ6{rS-wiLd}grL3K+u4oVQi>?dlaBeh)1w4%<`9&1x}&0IxX zma9)c`m;!={AJQr_FiodRz!#_uS>dS12R=!^X0O@FL#I|x+~{g?k+vGjTm5wp{+Mf zZw-xCF_CL}bc=b3)YGV0N0XJ^c~zwBj*B4y7PQe!Jz3bB)4dcqi*&+|-J2HWxwn9W zlat2*Pc^YtdSxa5P3k5oK8oOOy7u~Duzv|gku-F?G3Jxgq?)USap-ryso@TAXsd7T z6|~d%%9aR~pbG6*&BQg^p@u4DQf;Vc`xQi}1$EeL$ZM=*ynF zpW)gm=*GhLDQIpUUfz9r`uQ-y{5>@kuU#Jdg}lG`-Jm4R2;92Ikjm}s-PKpOADQpI zd~v_0+F9FomtXwqO*c8Cq$TP&p9+-HPM4t3@u*<2W1#G22dhbd!3StqLj|jF!6Zl_ zgBf%K95!M93u=5L8QchA2~BuHacqNxD?~>NTQ~>`Zg2?}te_lXXu}&)&<~fwVMJo6 z1Rze(2tg!b@6b>K8r-e~N;F~xd$7byXhIW%fP^HfXvHfq5r`MyViqU(#3YuW2Yy(@ z*}%xWBQ~)NH0UB2*Z9LP7LkZzOhNB_;Kmxt(TlgU;U4+;M#M%^pvaR>Np6RRX9x5w6mox99%04 zS%mT)wpbL|3TKrou<;{=o2(%7&;sJJCQOt4iE$ai+!9O#BMnNf{=C542uuJEW_xaN z_h8O6d3RONZA27airq4lQF0?-?jF?ZEb`_NT2)~{6nJSvR#FrO6cuh;$HOpHiOF@> zL5jcP*;(1KL8>qiEg>9i2+<}%wpQ(@YFXRb)xLrcAfRoB!@2}-a046IK&UJXQHL}r z!X>}mt{A_t;)X9uDDp zuu2-kDacS>Z$l0+OCAs?#yY_AE*{GSLX3hCd}ECpSb#|iW7WbJX7oSf;SO7XBDP@lunU%8gh)g} z6(s)WJ*-g(6c_kHWot1IkUCic%$SXG2J#-(K;t|7gaGwr^0;P=09OwgS!1U45tnkJ z$?|~>Tl~(fX>Ellx0=e>Gxi6%EIDH*;n|llHkg-S7h@x#%-8N%wP&eKlLDg6FnBW- z6#G8mu{L-;QV49Y#GrwU1jkMo5Q^4qqx*fz5I3V0m#(z`Y0tn*Qf!*j zS#8HB%7%zQfT7u?=7Mx5F5IzX+m*oq+{AV89Ah6~<6enzs*Ox?V+*^EkZHEcc}MLF ztwGH+c94vP?Sevid*!442 z@LRBg=tF;WqW66Bs2>a=9>Ih*XMnKb`hG;q22W36)+!(8UezEP1+E`MnYv)}lMp_{sEDL^SRx(ZSZLvTHD)2}G2 zGo&gNQwx{EAgKtU7TrTIuJby2D}<60h>Mb~t4cn#OFnPV1@p4Bg8Q%;nuJvlguBBA zLD0K2c%1kvmJ_IfS6emdn2riEEW}YiwGlGKlZ_?#zaLzl(!;zV47|8no~ls5my;LX zh%P1c9w118&HI7z@{tudI^}7UR$4&!Ng~uk8AkxVrsFUF8Yqjo8WFGf7xlp$GSE5% z`5xRs;2j}7ds4&)>Ut1}^RyQy-7;|rxj#IQtEL<#6dN8A8# zyr>k~g>z_yz55j&!ay^^z;yJrBpNBC$f%t9ss2(Dv`M0#iky6`4x`D%dE6<03`l{z zDV{1wg9NIBL`V`yx(Dn-CNMV!=sEZ66;bIIx&ko&>}W>LYQ9YRx=)$L36m-}@T6=k zNk8-+bqhcq{)`LOM;WHL}0`JKa)KvB(!5$q(+jDu#<3N$}jxyp+4Z3{1j2PxMSr^mI+rlrhSD1;-j7~9V1re%5wi7soYK9fXEfXKfgASP5)o~xUl0Qb zD6M%Df@uWF44SGBn5rZogl{;HZ`cJC6`@CEQApK=aimm&Nxy#O!eK|dH$ z0#yPkQUWj?sw?2q6gZ+Iy{`q0E*MHv!BITBwG!;!IwZ z(gQ#Pku}*=$kmcPt80+e!mI&WJ=qk1Rhq5RRkh5RHG)+&*^~8I!ra+RNQRRYq#t$J zlMT%@O@Wntsx(^Ij!n|f8do;eT8Erg#aSIwywi1sn}jhicabvCzAcJit$`@|9|$3bcPN7b2+|6`gH$!o#2tVGr~xCu zOkVQR0d?7!eO1U!+%=F{kt^EtYE>#VfJbnJD1uUxwOKq!B_hoPD#`@hB+Ny4ge!nr z^!xx9CTvzZ|Vii*VUR9-G?Nl|D&#MF?;icNDT;4_K-AI2riTnf@fq`EI1P~7O<+77`!j^&sH$9dybo zxayO8yDMY)UP%%IK^+1S!bW`k(|L>6Hv>sSoz8l#-*@L5-lMh1VM1rT@VC87=j@{G!i6H3=rQXwnnT<1o#DqD*ELLD~CoN&>)oq zTtEd>$q8HN;!Ul90pLz*PUG*S0GW%SH_l@&O z-Z0nw=9YclTje5Roux8(-bD6*Oqij4hUYGPXAqub!Sqy2rsRUQ=HKLGX7Xe;0|fv4 zPjy4(4V@bQg~3RdVdbzH1wl|WWI`yB&DW3=&n94lfRa*s4dqWNhhp~3FO~$y4d@8G ziV5OQ?_5dmOz7S1(kL+9Lm1%41=2(#!(zr=!$x#H$6C@ggQCwl{`hx1d=n1&ggGIyN*64K&UDy zUgIVRWkb7GVD4!$f+0E(;5A*-PN1Y*(UCx)S{s#6@VrrLcIs12+IMJ#tKI`w00&&~ zQlfpTefCc?NSB}^(x?_~(uQoRZfdJWZLH2}gLdppc5QIoWZ0f2PyS?w?kjbbQ?Ojd zNVJ1qLtFJ8e=O`GE~Ul+coXcF74B3;BiNp!twua*dgn$xm5xV6+HEwKhu(A>MDniIZ?KMSv8HW_S~5ek!ri{B0bEW~Mvc6PaQB#7yWNEmlR;V*RCyC$2fL{M zTP-^l9P-%aOE-r;I9Gtjl~BU^lsqUu#(r}ar?iJ^L_eQ#ApdD`Cgep<1*R5qM1N@z zmUA|5ZGT>2Co*Un_SQ5XqDnuk=Vkd5OSSJKQRrB_p?-oB)&s5U?T;KI* zUiKdR^`~NSxO)H)KX$m&?mxG7YyWO(FZpV(YVPFrX&(8Lr}mW((g$wZWu{DUe+A#% zhRPHLDatb`YJ)zY1fW+jDH?j9NB5jBdZZ_MP~N;+g-UHAKmU-M=^?TY_!DbRCPQ)hYVxdpKx>?KSu_MSnBPYc|2z z0BRM4Md}vlTODjGz?F-jt^^HbT#;cvF=%pn=DkGC^1r`NLIO%)$W1ol!>NkRF?qx zVUci0moX3pE*wbk$#I2L9L|Z3Lq(YCt+#RKj?JPVAax>@3TZwXt5ASSx@*r{<QlfKeGLrJ-MI zE#TpYVih*xeDfJN7f^Lati)E2b_8}U$%)-bNkW>SSRga}$ zK_EMAcN&x3rM9FkiYNyLmC&hi#~=D_CJBGyRP)Lnd#us_#0)cBsRlw1Nk>K-qOF;b zL(4Hog`B*hWG6@AL6ng+AE8&wG<~L*Q%$?n^vxqLi2(>FjS`23eVd43h8Jf30m*<$ zocMuTO6`?`5Bik=DjiEy1&1V9Wwk09n$&WIiXHxy7p(vqAsCN2$h9d_WGMvz9)moB zhKMh=Sgf%mS{37AJ1Vx=3U1+YO>a>wd)OJ03=-rf#F|i~0eQ?qQIk%VyBd_J#I~-w zqO|KSex`*vCKf1=@Eo0THdLEIcAPoqbwEHgFm^=PdGI~$U^B!B*zw6xH}ITe48?$2 zLdY(LwsUANga~4&JqJHB>I5WPARnmL%35o$nQ=z{>a8)7+yl)t(2VH>HKH{utzZc$ z+s+ldqLZl&@a5tr3x>$y(JUMBm}F}}m4nn#LtQo07J>lBTb7xcn2ITjEdscIjC<~r z=&J1+J8iq|9^7xE3nm<6=2341L5%Unlx;+~x2XCyr$WI5Q-QFZ-w09%;W3Q&oi^ck zL+<309~$zVhmNxOHUxe0hd@@niX1D5m0HHS104kYgkGNkm)C;w&AUh$D(Cv>^;byb4tubf-G0@Wxs+0?V!f^cw}qf+~Os+hD+ly9N#CF^d=-%LFhg zN)@XGA@ISdl(ne=v|(t)dyDaEs4F^DDiCHEO=#i(BqZ{W4oo!C0G#r)=HX?0R*V)F zuayEn04IIx^WyvDhK}`hVi8QJ0v?|BnZo4@B6_%k8VuLQhRrc=ZqyD0EwQ*__#=Xt zi{PCym$_c}Zh4xKiY*Fhi}n;}5xUES2q8I$RFu&KqVtM_B%#9aL91&k`++O&fr>0> z!as^h%M9sIEJ}HcMr~nH9}*PA0)Zvp7oCi( ziW_((hBh!X9C>H}Oyg0J7`n@m{;i}Ym5>+yK$QcW;Wt&dmW#5eMFSXt4qbF7Grgp~ zXGXJqv&rXh>Xw5tWz%uM^rkk)89;D0OoEn3!+1h5lJqR{H9HxFBBrXyt7>bR^(;dL z;i(&UZHoq6tOS~7bGIot@otv?DNaf?g)2_RFrlOy8}I~h1>JFGXF}OS&X|Qe4TVXj z7*$rdIMGdCo)o2gJLxMLkwaW1^olaQ02+jq(`Gsonmyf)oyvHI`h8}9)FFatm0DD& z3eIqX)D0oT6A5;5ES|iegu6f?6;bT2JMU^2BIVG7%x0?C4As&1*cxW`?hXV916q0v^05ES=RP{@eqSQ92MQO|qpo8yOSBIRe zk4@PJCCYJDjM20OFLcugFFt^5dqbVRN|*qEy->D zWeD5c)3-wt4!i(LpOcYTb|o@v|7`X%%}Wg@v!=zMAHaKEj9%gz@8kp7fEctLtECZA z7yujUHefW+E?bXy1b0!z2R=|GlSpn*u)g+#CI6t46c`SBHEcx3Ch@><>Vhl}RLr_s zHj9|u>KFOj-*tR;nvl56cOQdI)%LHOcWg*&2Xd4CL6|7HF$8KUyb8`)6`z|!vpnDR zW+GPCGL*cVe6FS$OJD;kG1LHX!3$0ov;yGa9ypS2 zt2wb7BV@u(j;S>`Ty26}=XzKNi$WAo(&?~e2EqSoESBJy)g_+EgZ9PlL1UMKe0G*e zYVPc3+0hJ5WU07Pux4rP|I7iSs@AC15ttm`D3Ct;Sq~GoL?~|Ji3@vCo>v9v6c}J) zP|4aquUiB>Rr*4*c@T7EV3kTe9kxA$7C)6EM<`FoFT7ubBI}>O^t3pfQmXdiPdIR_Bf=L98t&Qd(DgSmeq7O$}2T z&qm;w?t)QJ-c#4Qw^dDtEXXB2WCagcyuLWEc~F45Ps>zn$hb$Tb&|}wI+R4dM+sUD zhXc&z0IVWuZPqGt|0e!T4^^}Hn3F{#ugFIc=1TL>b(-y>hw(kbswq+H7|wCWJ?47L z<=_~yKdTa@>sZeSE9A{}dwU(H+n6R$zeVIkPC?W6^kCojLgbg3jIIc+JG&;5_oM6D zSs;?UBVB>dnWRRH# zDBKE|Pq-AH7+G8Dt=`%Imya-68X*@%&|bsXQN94o;NThU;n~-n&F>AG*3Fx35g!Q_ zACLr42_Oihjmq>4iCneaSP;g(Sy|>B_WRLnjmsnR!i)Ez1Fz9=li= z-Yp=uWgEBg6F*f`u-K2`hzFWzAQBi39pNEk`ArAz1fWG#@MYETmEh^+1}b!$wvbrt z*^8wxT74Z00?6RPb;d?f)W{TLqjMwN9^rXiRWUV&0Pzym-^F5O|BC5O#@1t$(1i0lj;{KzGg0gRLk zX*8FV$kwQ_VH^GzrEyV_jn*|8n7EymF9i$$F-O7dOBnp&HeekgNL3`@L?ISm34)j- zycxGS|02OSoeky9WaU+h7!x1lnvYZkMQwsB9^@YIRyDlA>(qc^;bXv`hELqZXz?AU%z_z2;{>4LD2;|LU_xtP zqkb`9=7`o0sEW8*AZksC91W1}^@|-;-Pv4Spn+gL#-ltEpQ0T^^Vtg@62!gGM!?4Ujhup| zZJ|htBo=a^=1s>jPFeEofeQ%>Ukp#G929B9MoQU+E_ebo=_Kp1*=O~ZfGHhS-X6M1 z|A(9u)dXgTb-b54N?0MZV=KJlBtWG)-kY1*97Gmn!=Q{m=H}8EnJj96tdXTzvcZ6T z+#I~l1(1`1$(@Nz9CP-aQ{+b|kb@=GphxP`XR-+$1ST*Z7tGNH{%8RseBRrA&+@$j z)BKDVK<4pHLLa0QC~TPfU}k2{=Qe7K8J$4{N?;Kv#0EwkQ}v;O9YktA0r2fY2);sl zbX`{Z~nMJNT%9anaxhy*0HbQ%bZWssGnSvnFr6z9`Sk7(jj2;4y!#$|=d zW%PZDm_SzujmWq#7)}t8b_U}xKGu7UK_XewS^8K*fdY+CNQKPQG_sQl zNWlYGLA08O{{W{vq7aphB3I!%m{jhKSger=zVK50+BXdZ%Qo*5;74yA(;%pFEV?=2;t z4r`bqXrUP^nU=%Q#6j#St4=ru3zl7htbzD(m_uC|OpZqK1V$sg9XMP!a!IIsVu!RmZV=Dp{CJu?VkR=)f|H5poBR|Pb8Pw); z@sm}Yn9{0lVmNK8Ngo`>SQ#?Qyl#D>G9rXt!F6{e`Cp z?17Kef|%%-fy9Lty(ZB`E%Lgd@^T{cdPbHB4Akl2>}8-Gl7S&QSl`w!$daiG?~bBD zg2_6f!U0>Osc1z~r}cqI6WnENxMK*ile%W8Vkofffx*hinc((s{RfiXCw->XiylMEJ&4Zy#j8@f+X|-?(ne+X_pJwn0?y9 zbSCN!IcoC6EyZ%GN$GByya6fz)+0o)k7WQPTW%2>|0fqK>?tR22GyxIlCwUN@feE- zEDsK0X2&Bv5`RcoRQ;4fhAeGRkS0oHRUuNwjc+mM*N-Jj-}z{K<>t;Wx6G+!8;3RhhZ&}3iiteb%%5AbWoBsU`C)=RTWdN-?3fyY5wK&o)!Q*q!eOrzvSxplBwQEc^3NDg+%LwBzXV$MP~E+QJog-A28JBEe`F7t}@86}8I2 zv`E4Z|8Zg9*Ctq4S@YeyqSDhi@=I^y7Mw4%xbFbZ&O?4dF%xYUAP0NVbgO(`D)x|7|%hRYSaWefkbUTS8^JCT*tT#YGFS@_}PX+b|FNHo+MxU(a<$kIrFb2DQ^TwdH1tzV(B>MBN{K3--w-k<`W{M zch*^$4pw!019LLG1GnYYZ1DF3xx;_||8_%axcI0mBQS<+Qv&%kxJlcNgEy^|?@R{( zI1a5hIzRVG;SdEh!Mw1+Vsl9v6}4{20V7nj|3U%xp`PBYkAHFSW_S$~z*Tv-a>YLP zIOVcb_xLs@;t1$l!{oU33OS8anxX@Gd&BTL32}gx!_GDHtKgUJwgtl_=v(}FZsfMA zNjWWD^I3>F3{CljOJfI{jjAVu*<7!J#6>j12n-%+|B>0-Q@Szhq zg7QLcH^@#L?p5#=4It=1EdbTT|I<=b0o>qNC}%Ws=lc~D)PU(T59dY)vBj!=3N;M| zeipnHk%&c}ZzE|pWcxscL_B0eig8b(EBpo(IoX*@^J=Jh<{b$hd>ya{yRd(j>(NgX z9Ea28>rt67o$1~s#QdU;{}VZCV##hBca~dOwdmNDYwPYUT)A=AwbjJPkDN72s=(PhCF7hh zVPX^>eCN(!AY(AL(Q;U^V9A1?fT=7Br_Rit45g`4=u;?Ir$(D1bvm>nLv;EC(%72y zl|io{O&NHNb~V}@XMsG#OA;BGymTP=utXE5Og?LExsdzE4q>d-0;|@8%PHdGeho(olD1sg?;VhJ#?s=r7pj^Pp zs{R;M(5VF<{Aog0-l^rg75H0|yV+^v^s8B&iGH#4)j#V~Ol1WTD!-SU7Oq&ulC;z}gqAW}B0S!nl zxst?d?xL*^O747)};{58cDgHSdIW5|#zxXt(%c;wezLpaichH6yuqU;Cq8mmCB@j!S;1X+TPpM5Ty` zK7B}~{fcVPYmvm3v1^Ug zF+YsB-5Oj~!u@(QP!!=RZi`osDwkaaer*@jvbAqcIYI9|EU z(0paP+u<%QyaU95(4-h=MIZ)`d4lkS7Y80y?O7w>0Az?T2Il?nc~uHmAkx4B7L<%( z{BoSJU+KEyhXvZpZ)r61_pj25-FJg=bXe1JV4}h8B9tCHpszTeF}uS{~DnQ zPl&?0Trw`GSYZog_$~BOW%3NYug_5Lp>SAx7>DVPT6nNJx%rps7n( zupvcqgvJ?M?0SR1P~`?kH%4$OHhGYQBPQV@TqZ&k&1A(h>$I@?VF6;TXa>um7%9he z=YNpX)eq8lKvh+t8Z$we9vc`sIxr?L_Ig7XIJN{jYS2~*aY%>uRyMtu$rA&CASD=K zxS@HXBa{r5T4tdfK%nV()8e5HV@R!MJt1pVL!&C4#x+-d6r_E?s2eVE4QrARBoaC1 zJkz*1j8O4h0^Ln9cX7Y_5y%$3IGjfaYEYfZjeOUnL^fq%8I8)2VpU{U|Dnv;RC97< z4eMMc1F3<*oDIML*3+uY?uk!*eibSI`4=C)6iITbi#3XXPN!T$(3J{mfR2+Zd8Yb5 zy8?6&hd5TeIM7#8P7O)F^J|WPcNmdAW(2oTUMrn9#5AnIIlVDUSstY}z^F2*nY9Qo zw{i)|)>2=gt?5k>y3Lzvlc=;T8pJ#%i=1RZipZlUe3tq)oWV*dZDIiD^mw3yB_(@& znczt7HXYEZ>MCFb>v1Tzh&e5yq2+|v6^sQPx>9Z%?BR+M@PHEU;?iGB5RrJ3Fy1x^ zEw9-#!wx8#N~9XbMy4F)v?K-_Q!W;yjdi7J9@>O|=4HI`0#S9$|KJ78y3`_hifJuA zlA>=sSa(Ug6NE8YVYaZswc5*C6eXoh%JE_6i(lT zPM;c?&wYvqJq4~5b7J$a0UzhAUkWfD{yARAI;T9xI>VBsz+@+jD6b8z?p`leJX9vD zhQ0)BMSCO|{4!R*fb7wv9zoY=!COkK0Mu9-(z`HnoQ_ zWyKcIU&xP43(`6S8pSxX8H54y$X`TEq9DH&!2?l2)h)`j#FDP@$Zp(_2U)Dofbp?e z1BI)zhPS*;PPM98-D+2_n#q9Z1e-%J0vGOxS!wiEV-RFYjx$6i9qCui>A)is|jB4zBu{dRzUcY&0Juog@IXIrk0V0 z*X2&SSIZvF@32u5i;tAY6?(mfvTIQ+;)=Hhq5fqQ(>8&fg~`I&j_}OsEb|C=``c*? zw;Bbt2_J`QPn`hI%!op|vdsx@@HWn#F3<>jzHAW%WTJz>qXVZiC6@4OQy^-&pDj>p7-qF3e&B@RgjG{JTbSl~W<@+=UcQ{SD^gIKml8D15xzy@Q(NG;eg7%Ekj&zjoF_y;AZX6tg9gKvuDET zE^zg%{UZIhCw$>fju9v3j^zzYO>y(RwJ68`m@fl9@M4B+BcKoR_D3gXLeXkgN$)Oo z8K(@7fVs7Gv5L!QKEieZ6TIOqR14>-$Xpsv0c%YG7q9dbfD^!BM(!wQqJUPEVAc+( z6Hu>F9^?Q{FDfEUD0Bc((4gezB&=u!Ll!}1|CT`6ROiM_;`r*Ny9mVtV37UDsutj& zoMMR&gsBsZYdmx?Lk2S7JSwnk zMocC0&wMt5`+f{l#wP%O;r{@TMRdUu2yiWospX6#0UNLZy~o^+4JA;IsTKhoaKR76 zB={sS0+WEL3Xf+d;Pi}d*$}PpE`fD~4?>6!7iz_Q814ko11tDI_M#&CHc=`V*z4q~`oj!VIt(S0Ru(FT>`A4jJzb{~hqhN@p+5tX|r01DAt~IuRw5j^VyX>5#)Z zGJyvv1O%ck$)4^d6cG^F(IKQjDr_)4{BQ@s?sX>U9{Uk}sL%P{rxE-@6_XJA+<*t= zC8A;oYXD31;Onof?C;Vq3t7Mye-ZzTs4LKI1R)Ep&}9!~VnyabN%$dYoN?x$@!Dv| z8^VaQ>hK!r1sm%S+){3ka8f64^4s3zCaI^mrYGRO(4Rz%2BoV8L?jbfZ+k)naJZ`|;T2i$j=nrBtV;0Z={}|8${u1&W&@c&8-a-N~;e{FzGcIc@8-cQIwo0o$ z54!wqC!2D*GBYyKw;kVeu{49rKA?1n}TP4bq* zj`C!Gb_~gQ0S$8}9!~NuRWf(-a+YRipaRIUtkJIovoWW#-W*53(&f5fYGaZNjs|l} zCZz<*#j>7>Y;sa63B`JFBqJff5dt9w7+^MQ^QkiBOE$q3qKT)jAo3U?8MsY2+3-YIQYDj< z0SQwuAI~|LQ>jXY`}7J+|60przC=BfqYNZZUbx~?j78B#&qm8*IdfD~zzw>}O*8rN zJDDi2EK|)m6TsY)JlIm*NX^XLMOm0>5Ac%-kj^xHZ3<~|=lISmRjDG)vUx-#BblHQ z^gsm|lmXi^bYe+~x@3Un$+AQs5Hhqbhm$z@!8l#A|3U!~+JFSAj5$ZNL>)6@B#+D- zu&knO@np1QW^_i`!%$xoH%;a{*<&C`#m6>b>f|#-jua=MY*SOtyfy(3L@g=x;x_MZ zJx7zSkdoAz;Fn-%w-O;n@Uwdu@X8h=@0>*oB(e#c?$jg)kh0jPPuJvK=uS`v)eBaxQn>{Fl*-@OQ{Fg1HjeaN zxNk(fb4!keRQt?O837#ACjW#~@q83WP1FxORbHGZV9V-LtCL9!7DmUD)OZX97?c3i zfDkAF4s10AnsZm{U^T-sOM&MVDI*sYOJs+&BF!fw-GHEofLWWh2@vQ2rWN!|wQXOjtci!66_mr{%@ zcWXlrCYB9smGnY)B6$gFDC*YIKyRr982+{_o99an)E9YxnfkQ7nh(*Ef;$4%QQ(xl zK$i+2p<00x!>;vCnWRGpDsg%U1q4el=`Ie!)%$w&Px-b5=C?U#^UwD86@SSL@Yi)m zG*V1=1}utzyXkd<0CHPbm-h34iFV?Y_I4{+X&;SZ6;dmeAbF<{W5@4fPst%`=y{wdqv91vlOiPd8JnRWh-NQ0Pfs(08=`08u0&S(w*{^v zlpE{UI}%0vorvDkCr*A`_rG42Htx;Ry=m7h!mjZ)VB9Bz32md>4w6+u0@crP4|aF!EQE zLAi-Vik3xLiS0Rx{WPCvi-NTk+nP?Xpl#}*~Rls%; zT7q{epwE^(Dwb~XZX!L{h`F+3^_KCZFZ80^PNqL z9Moh*yOfldIE(9fje~llvv^Gm+Mt(}0EXJ4kD8&$_?L^em1p^>bNLI9`G9+Ry*4_S zR~d3}$g6eis>6Cxh>S{UgdritG+#)0f3;(8F_2L@rB(WH3*&BJ+L2i0^X!IZ|2waC z+&QPW$*quOT;JfB;x~0W#jl{41bWs_uQUdPp*$eGdyD2rO(YL`*px1Z``W&B`xnb0ox^M!wjY;t!ty3hW1%jiV67abNlLP|J!K{ z{I>C6gP9t#bG45HG{P&2xKBDtj9VA$TC=pIG9?q-8dE*bmb!iFS8$ro|FA?~-}@^U zCxF6Rvx>H{_0FaG`Mt{f1i)8X9@wEF*J45d#}&G?1I@;JR%qv0iobh5j~t46tdtxW zX~7$GpF9e?__ot0V?g(}K{^(nP`3Zr%OktOiyOnYrVfBr%gNOad;DS&cy#M{#3|Rr z0~-`9;={ohM|Nle6CA9TD;YP+x(Y{nAJzxWBZ_5?ylxNJcu&`i!FpLOI-Gb`t?CM(7Kq-(WF?-$i>!7%)FJ+j#x+i{nF4i7i8o6f0sH7Dj)M(z1>48 zicVYfk-n7^fA7yUvf;k-d0XhW-R~1WjIUbrO*xi{e&lJW77Gj4MP7B|E7OrT{oOj} z%lzb9Kjp`Lv9Qh7>;%97rH!3zZ~Asz`|gWgH?E|6!h(+45yblWzo$uxXR#7>^BW zl9>1;XOV+PBC`C5geC)_eL7ksBamquhFRxGJsJ_p9iK7Cj3uj61y-76!#ttlc4XX; zUgeI6JBQdXV0?Y~%>|g2;K76o2daZ*%9bq>XFyrH<4EJklDjw!5mjVX8*mjyEu!$^ z*U22_Dt%~Fv_aIK(-uwG8cM;8sY#b6I#nu8iL{@pP6#xz&faZ(?=(ub;^C37Vdh@l zrs`0xYauS|d^?6{u}Ia9wG9)mn(i_2cHHdHG#R~nh4sA!zp#A5(wl-_Tp9C9ms-LS z-LX}6pUKvkMM1H|oM{K5meNTLCS+L_#c{Bo|7&zHXPi!B43!5+oYlmIg-=blnq}2;Q31|C3kZ^p-;4o)=oVkZNh&6Vklf9;fogLT|mK zz^T{`9O-uhcR!vMsk7HjXQYTenwo2FL4Hf*zRKchm818G`|ONys+s4W%mwOj#}I0p zVT;|tI4o>uJuKkI7xow=s}iT@*0oP82eP`{wi{Bs?#*lOl6_RiZ@)sLdmXG77cJ_* z5h_jVt~lq~>Vz#zb{Bt#hCFqx8xedgZeiYa(8^$kj54UbGFt6VHhFbnzrm`Cw8)88 z8gOz84_s-uKdws`A#jdhh!}{`sOeuk`+O^xaJx+5qe!$7s&JNK=^^BO(s(YV8rdwX zaU6QhD6gS|4jRN^m>iqf72o*=|G;_!Ri&Xt+1Me_%3fVu&7+H0r=Po(2fU6^zUH@h z_$CFf&W1A?Unr{lEbxGp`)XhGq%N(mXO;#3 z+_bX(X$4`35P!X^w;Q?2tZuHaxj&V`%4EkIo7EQeHkX}X*ehAm zsu!yYQb|KYdKvr7hB?i3DN;PrnuNj=!ZVRgZvSCe2TRwg2PueJ5j0`Vf>x_A{O)$y z+EBPM)V&OzXC`h@6t{K+yaqb(fjv{j;3k+fs2Rz2;2YVOjqGHC&kA{|VuTAsyOqbw1=C7QJYruCov3X7O)CDDn#}KyhWsqT9!+xcHE&U<_Jz`X3stRTnUD35*Jy z78$`vNP`85X1Uu|*se$(Nu7}hv^*I^QfZ#a1hR-13WGHciK)UZ4{_65oaQ9gOep`l17!r~KlP$f2nwXf_MjcUVdj32KFD#~#Ui|Fel%m`+mbj~VndZQx# z(igQseJY6IJl%6n{~F1vRm*m0OyL2A_o{=6RHHG&r8K2DHjn<&p8#DXBOh49RX($d z46Tl;9B0v3rcp+*yj)W2C`+f}OqJ$TsYc&CYPp)-Y?oM3rcZ%NN9jdwN)7wh9Vsfrg6Z*X60Khtvf|4k7ji%Ta&6uO_GUdI66xuQ#RLfc_veKz3Y48vz+e@jZ%A! zBxU!Q*uxUFqlDGW+%DtJ|j*xcj92)YZ>D@pu%+iKp> zb&EAiTyiVK|2k$)n43YSd088bfWDSqh)}Kcgk!Yf!t#;xyr_L+r9F&lPQR&Qs_!;K zL#Sp|voBrMOF|pp6B-wrR8$;kQ>HQx)^E0r!_i|x3R$BZ`?J6%(7bpIPw(JFUa zyWC0cf)xThvMF1v^zeAE6;~xYYr^>zt6Po<=GLMZ7Qiw?i#NznQoZbs#Ov zO02I5c4ZzP98>XT*~Pch-##yM-3*`dX%yO_5nXJ(QjxAmf%t=06fHC&e)vDXB67iY zT-R)b|M|pXer=g47w5LlcPWxi^C8*&Y2vtz^rQFTSBVBf`SZm8={9h(3IA{ zBlWTZ%~4i?jhOp&?xCS*J3oFmD;;hqoxi#@xN0k(nAG&9JDuKa`vnte&~dC?ZD86? zEJ6Za+^Ay;Y&2ci#)$o!`$7^+;+9L|OtieUz(qU<_spM>O zy5G0%k1|%;N?oe7_1Hr^Gc@4k*0Owmi_g7XncX$l;XNyEacu)8;}CZ+R-Y|&@M8C7 z{~_OXXzO$G{Y+D;JwEryCwm`xri)oUN%_c!e%$fY7^mY3GS=hFZ_Dj@r`PL`)aP<- zC2ES(?7{dX#cjc0W74LZbf}xX3fZS;U4l9vDn<<(*|8(Lzve=tv$IxoqM3H_;RY}_ zC5~*+yNq+~0_X53uJbxn>Md&y6}-4budugfv-sb~GgUEgMl`dHLF&zz}ee|~o& z$aLv9e<#K6?-~>Tj8)ShrFFaVuOoSjO4+=-4NE^k;92igKI$PeF2{c^_A}gv|8&UU zeS8rKhbMjkM|laOed9Mz!}eIXZsC`OB6d}CXKR^NVKW0w&_!*|$1b4v9lhst zR_I>&7CczTiP}?3`!t6}IEoOMfQ2`Mf`^E!wp1Huf+mMlI;U8mR!zr-|9cW;XabPAsUlaUaUGFb=&Rg;C_dIC8(1sOy+$&=uNkg}DEV2FJO zsg2n9cV3qufslktNHLEGTpdYCY0!#tCPdA}PxlpSG#7{o6eTP6|2`}!meVLuE=dYA ziIy;F3TPP#H|ds!pp)Gemoo!?2Dz22Xq0AniaytiBl&XWSCvF*m+i)vmS=!@i4Z+F zY0XG_Ff*2qS%DVk{snxvVXX@Ha6BbVVRo^`2Yp;&*#_=H0FjoHYP1&NnK35Kni zjK-sq!%3XPxeUkoQ;ozt%=w)E37`QgpaV*s1R9{%iJ+!A|C$SWnr&GJJ?Wr2DPj^j zp5w`y;`x(oRF`uJia(j5leM0jRG;>FpT()4{fVOfX`n00qAlv8FAALn8lz{~pfhTt z30k8#s-2*zoHt6Nra7UhiJGeb7@^s14%(R|YNGjRoIQ1-DXOBB>6}nHrBqs_Fp8yF zs--gel3lu`Jwv2RGMSummhqtsbZ87|%6;LWrb@b`P1=3PX{AwGnREK3bvhq+s;785 zpmyq~`zfYlYB;~3bcO0uq}Zk-ny8A(rf&MCaVn>ODyez8r@b^W*V~SstW6Bu<>fL z4*Rex%d+j-ve~+;gF3M?Td@=yt`>{4{(7t(o2($4bel4=BI~jzi?AsBvJoq^NL#c= z%S0nvvrn6|HygDkrn6Pcv&`zVJsTKK%QMuP|FTKTwJaO9U`w`3JGNX4wP{d zxVzHGw!QnU%e%eJySmOxy5P&bJ&UN~ySaKtam8A_r+T~Xo4wboyYK6!?=igId%X8M zzTu0%=3Bq^`=-NszSHZzpsTF$3%{Fo|G?Ipz0a$?TARPFTfY0-zYeTk%!{*%G_ z$83DX8SKJe%%pXE$jHjbcZUi`?&>&b~6%6#jw zdmPD2Y{`?H%9#AbgUrdJe8-3^|HMKo%6SaSw+za*9LuiE%a6Irf6U6P{J#mT$&I|o z7(C0hyvtrIvi+;b%KXXDyv)6P%D;@t)*Q^T+*1dP%iXNM$L!7FEY8zR&64cL*xbbF zjLF7KySqEiq`b`UEYI@{&T(AM)r`*SoX!&=XzJ0&US3{mlgp$M&qy3H{LvZNm+X#uAOuuME%zozWDn(a3Dl zDc#0|%+NDU(wF?wH;vK=?a?s3GdZo(J+0CBEV~@s&>~IKne5Xm{nIVY)GWQ!%IwC> zAk|F`(g3W%My<-ejMYe;|I|;d)F*Ao#VpjO9MV^P%}T1qL|xPTEY@A^)H}`7Z%x%? z?bUOQzGHpXczxE{Y!0{V)^;7&fUVRlDc5zKJX~$oeq7b|$=B`+)Krbwe=XU9EybsN z*m{lGV!YQQ&DerG)prfjDm~eOjnkq{*@hj~mz~+Gebj<%#-k0}v3=U5E!$C@+Kb)V zYrWX0?b5Pc*n>^lvt8S{Ev~`r*}E;=rp?>J{mjg5$G*MXbgkN3ZEcrq+^>z?&JD{+ z-Q2$o*4n+@tWDk4?b?Cd%Au{?(e2h?JJ-$3(a|i=;%(by-P-EF*VDbg-VNP?{n+6B z-P)bq^L<{#t=l%e|KI5i+zI~K3;y2-4%Y#W%Jfa(_pROzp3DtS+xpGm8jj%=j@uKy z*_}<{{yp0re&8nl;PMS-`7GhqOyVAH;R=r8FfQFSuEUW%-c~E)91h8rz2iTA%qp(g z0*>Dhp4B=&<20`1ZGGZK9^y{k;;WtHOCIAue$e?X+(HiJ#LVQqJmpkg)=UeX3_4(*{uID_?VnSdwf^a}zUDaY3v4%-p^>luyeN3G^;?(21q?7iOR NvmW6$PSXJa06T^ThVlRa literal 0 HcmV?d00001 diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/images/sparkfun_edge.gif b/tensorflow/lite/experimental/micro/examples/hello_world/images/sparkfun_edge.gif new file mode 100644 index 0000000000000000000000000000000000000000..057a52d4d18dd56dd7024f6b1fdac2597b6fa349 GIT binary patch literal 639801 zcmV(uKe zHfu;%YjJsOA0TWbC2WPDZ8tw}92RkkrFj<>c^({kA|-k#Eq-chfH*{fc7B18s)9vU zf?8~XfrW!(e}is=gLaRDd1`}}vV)<#gdZS;R9%FFmxUuLg+M=rOId}VxrP-OhAT6M zGcSiiPKZHDh@rWQGB}GyP>V@Jj5Iuqj**O$pN&*cjbKBMp|X`YLzSYimMb%sN?Dd^ zYnLA)ms)0-Vtbl^W}1PDnuDC2qN$vxzMwclpp~qlk&L2cNurUQqd`ccNJyh|kfc3F zq@=8*rog2(J*9Y(rM1VVYlWvJDyLd!r*eI#cZR5TW2s77sbY+&YI~|PGOCMXt28>S zlxwbBNv@x&uB)f6tGceT%da^^uqQ0AgqyIz*0F(!v679lJw~#FXtXIWv_VL;U3;{Z zYPCg5wUnl|nr^p4R=Ah4xuSTxNm#p|Z@f=jyjo|xl#9HfbiAm%y{DnQywSbA!oE&H zzNdY@!rs5j-@qX>z^jD9h-<>8c*AjL!>^6Q)#t=uuf?i;#kG~kt%J(4hs$v+&9{)v zzShmubABwf|IlGVMI)xMe5w~p7ro!POA*~G2c#iH8F ztlO%E-O8rk!O-5#t=`w|-r(=w$>rbO@8IqH;nA_-*w5nW?c}|9<=xNb*SF@^!0Fw^ z>)gNV-NWtS%kJRD@7=ra;mGjd!SVO~^5)L-#|II^U?lPFD+T**?U%aj`7$(#w3rp=l-bKY#pv!}?PAb|$e>2s)2 zp)ZpvJz8|*&ZR`5KAlRnD$}b-n`YgLRVvr6UmaTYsShH}Uwsjko zZd|)~m9CA-?djaS7X#mYoA={jzJL=eUd)*7-^P#`KNcMMaNWw77w&aDIkRWU?g)P# zO}g}6%alc%UY+_fYuB)IqK?e@wQY9oZextCo44=Ty@R(_4ZQPj;lqy?SI!)Ha^lYQ z=58LHdiB%KuV;sDd^%Fx;H~ff)=u6e_VV1*FZUk4J^J_J&%@uoUj2CY`03A&&;Pjm z{hfzqCTJy%QubKomrib3BbEw|N#;6d9@!s%ht* zzR4LUm~;lprZqhb zYAL3grmCu`o`QPntFV^;dMK)YvbyS~vAWu7uA%noE3LL#3GA@JHfpS{ydtY?nZGjI ztgOUFt7ME_dPA+Wx3ZaSvdcc}?YGc|i*2;mT5E2(wT`>3ir4PSt+?-kE3dfNplk2F zdg80^x%}?iubln{tgfovbt~|o=#-;y!VE85YB%{_3vs;^Us&8OLPde?HW?)oCH(*e8ew9jt)Ba6g=`|iA3a(nN&2M_%1Vi#|G z*=`fw_VRX56XcgG>zB?lfb{N$?$~F~dmUg3v#O{rKOn|Nc?{(f|J}fC35xm;wS0(0>OkpaBWEKovCbfin=G1Sd$r3fAs- z7|h`C0%1K4p2CA5Bt;0TH$vWt&}4wi$f2m$LG1|-dtG~)_Y?=d^*yc@=)+<9a=}9# zs;?04<6Hp$DF6W?QXqjxEI|yE$iyZd(1=hhK@ycXMG{2Oic-X)7OQwg6m;>5Q7obb zCLl&Kju8WAynq6^mk2hxk$bJHLmcOLK@D!Og(U>xHr8dNT(K~FVlYD&1WCI=KEaS# zU}F#$r->Tsu!oTR;W!%kL-tv+5KL@95|_xyA$ZaVGU%izL+Qj)lG2o@Oro zF^pfF-yLLGzgpVzmTi2a91FPy4)*d4c?4!2_n0z+vFwjbIHWQY$;>Pua+ynG+$E;T zhxCmPl8>x~?AAG$<8K@vYqR6WjwQJMtUZZ58%Y7 zKI6Cl&wkSKpLgITAcg76f(lcR93&t zyIt+t^{|1ZE_lTY-U0+bxfB4vNmby4V30uu%gtd85b@FGPIkVU$ZjqQ!CY?glzg0( z9YV7ji?JD4v}{P~fCFJ#Dm^N-^IEMhxjIYPTKK{kro$m;`3DZq(tbIdhluk*;u729 z4O)nT3@*vya2~O^@YO>O0lZbpiuK+-~c@c=N+F#*iE;~e7{$c~2eqkkOaN*DRk zL!f|S&k5rO7(vUD&F?JQTMOtC)xMhz=Y7G6SwRSx5yL1jFJ>)^F`s!7Xddk{)-0qp zyLn!v5T7RG9PVyMdZj1-7k9Z;9@zCR zG2JmZE4w4R-)I7B}yocJ&0IwkydCSk-%8Xz@5}eJdffX$Dyyrg1whyeR7^xiI zg*`Jn;$J7keyx}TGafnQ zn{<2Jc}!^{mwwzU|8D3_P|q3SeNy3;zSz<7F6GcAIEw7`7r(`Q*% zc_h|#I3!%u1Wr>x0V%auMAS{=Q*z?hXx)c)+_!xrIC?!6b0g*nt zGlx%wbjgQ+b?AzObO?*b3n2py#L$kw@CkV+h_Z)I^i_z@$7H}rfz!ui>sO2>76Xuo z2$YzN^=5w5D0=00i3+)p3OS7{wE}K*jWQUDe!y?D$c-1N4cQQDsi=cq(jmm+g8^7G z&+v+%5OqkHk?y#XaTHC17*=^u4o--OidbkpHG$Lr1p)pTfy(t<%9wIoh>(?-atwKC z=1_4?fQ+A{PZA}H5;>7D1PpUCl`46W94T5Pw2H-KD&!b0z>qtGR0l!>ixjhxR4Hq6 z#9^YNmQyB^&-W2eNC-f1NzTS<^Kf%H>5t98Xv;W$(D;`{$#U9eS7}E95BP*i`CFBP z3Qb^!PdQ+o1xQ(AmQ)Eccmr^fxr0xFd-KwYQwNq+r!g-T7G-S(%pqc{jTUXIaUHTiKcO(v=TGbzH?ccEB^+ z$da@cQ1|niO7mf_Ic-(20yglU0Q#R(HkXUoWk>0W?PZJ*@M)nG3^HJtndYDkU}X?M zp*(d&!Ei+5uz@i!p;l0#SKw*7Afig-3>qp$_hy(OYM~i=1tw|&_ZD-YK~8Kj!3 z3Z1$LwN|9#NtOBcqrXrMn)xPB)udCGfH1Q$R|7<)1_g>^eC)WAN7Dp{kc-W?PyU%k zU+Se_I)RKtO0GX$6P@#orPmnc93bdo8@Tukrq}OAwt$svU{1f|RPO%A`-q zp0Qdq{x}Ow&^}jFI7b7Umo$Mj>1ADau_=~ubt#XEbDcF~4hqm%*QN%S^bL!a4_%6~ z2cV||;HLp_odp21d5{kW00cDuYqMjdvQJ0=Zu+w3KmdeV0NQE*II93Un+81l20jY_ zy%+$}NdQqmvlfsK77zd@mIoOZvIbB96N*0ba0yKbgF-5#0z0t#+6tcRkN~nPvI@`u27m~X3bvSPwrBgcL;69S%Dih3T8IN%GFVy# zE4RFYfPRRu+WU^(@O(o5(+RsI1&Khil66BIh`t_0c_U@Gv$)HGrTdn7rn~=aKpB!ovuY+!6ai37 z%A`!e7?8TD90eKNrLOBhyW6@7zz4}CyLn)`e1OX#n+B`g%D9`xu?)My477;G2QRPy zXe7$Y?8G+UnwI30fgEg5$;W<7&3|mo*xXc0DkXy~Dcp<5Gt7>(^9_63$jl@NSr^G# za6TH0DWit9dzN3+14X?;}HWnZwNNejvbt2(SQ>I{;v7wMZ+i0mrFeA3@Q#U2pQ z4v+yNAORu&Fw;ap12~=2%k;J|G|k!k(?EUAKP|{M*v+tFirU+~iF}S7=7M3H0#txj z7He-ZhnHkES<||#jqE>RuzLmUD=k6 z%|u<)bIX|~a>(F})b$bz6Smkps!@X*Sxr`t|8><~_XicA%u(!af`~O7`;txrd$t<8 zg&WvxEwpcq4-)VPgnb8d9m4+UtQerPuKk2}Jqr=w*EQ4E5$!k_z}6LT3EzML!c77L zI-qy|{McoSi*P(NkwDp(UClsk3DSU(*lX0^jIf|B+SL5rP_RrjkU9AVQg$g)>YSSJ z)5x?(+hHx%w{1&;#*4h#(wOkksPGN3$-|5(4jty;#C>_+a0%;-;W9G?juX|TRx@{? zO>|6X;lP?Dj@?Xl;3`hN;LX!?fXSKb!D_|g=Z)U!o!LfBEJ*mx;hYM(lT0{mh6;oN zVU@mQg`EdD8)d@_zteFRqXTmjZ4cZoy)F}y%^G(`9-al>s zkXc85)uC2u^!VgV?&QB}I?a}rJIqVcoW~ke-Y~RFn$GFxJb?cDxCvVQ7&44PyO17VBj*gV#f9hS{>HRWEm>Rs#Y zZR@!X=+Hpu(DdI`o?DCjmLe>|c|czZxa5p<^wPAF0l#kVfE@Y+rD(&9tTh3 zqof}3uA}Xlo;wl$N3!1WsQyCc{C+fk?htc%n1JC>oj>~|N6-rk#Q^W|o?5#9j>z^r z@AV=IRYlc+i=DbyKEvMcy%=aWoU9z%@5oAGs%_?a2%Dt?zwACc)*funo;F*M1U?$| z)f4r3i0u-u^^ZX9nx0Ii?(sk>w$hB@C+44Hisi7*1h#lR~g0Fk!fwQ292=-GyT zUM)7?_c-6{Q|5#OkG?k~t-YwXm1j9@d7%Fum(%K}7M=-N-fWaUy+E{l{adlgT7cRZM>-a>*nRmPWcS~-|cXeuVZ^X zoG(If00*yt`Cd=@>Cg4nzPvzc@C8Q6}!gGIxmlki%PXt%{QybpH5&gVl1}; zN}4oCoj$$V=#HSbhvJ5nTldhfQgtE~l(RPuEWmCCErKOj6{%L64#RQtDevRJbRaPxk?7FQm%iTNAOghyj3%vV3{B|K7DJ0~ zENE#X5#qq7w(g964HD4Ek}joB=Sf|i&FVR|Z|4r}86)RK|FEqU?9E#AQ^{^kzT&*n zC_qjC`4c5Ao+VV`+R8N0)(H&WQce%n+M2>Gxa^`!3cVU+5DEw*1gtVQIy)>eFe;4X zm%ljE?6bd~qmV-nI|GppsDh~_HPUFw2%I)NI!Fpy-q1z2<~s8sL?3!2LdM<_@-Gb_ zs#xKI1CnH5I7Bc~qbMMw>}Zi~tg-US?5xxBIxf5PvO6%tq{flYdTU~d;VMdR&82Rl zsw?-Z`U8Uq>V%;M+!)la53taR0=GYENz=Y39h}RpS(2ImrN7noLS;3u_}ehDgEm7W zu?s8nFvJrh+ib#CKxNF+NER|DMkyQ71u89o1hl^;nls~-K!Qxj#~*DSjW0A8C}7DY z37r6g3y3W?1!IxT^A0;Id1gy!qs>xG@2Uk8D3If;3t-kQR_C(mS;T?CmCm#|rqIu{?kJT>f*fi>L}Gm#1t@l%Es){Laod9Y54H*ORwRa)4hIP=J5~D3E#BnQK0h1PgKoA{(Bi zweFm0zkJCjYm2^?%ibit76?fdVY=mz)ZPkVU@D4&EiYS2yVK`2P_$IrzL47?j z8=Ph&GZhhAZiIqT%7jvaR?kKpIN^mcWq1#hdGI@g51IhysUK=1VuZu<8(bG{506(M zC`7Imj6|40MXx7%#A6lLI=&SlMmnLBfB*t8paGYYv>9iaOEBwo*Hwsu8Z5Q+S!nHU zn$~DBX$dM@tFdAeAL5DU3Ll-eI@`%Js}5dkv?`Enff*($E`_o0O|+tjY|)H+jj^c% z(=Hw)%sIsn7O_IXc%vILz2hg?aoZ*!PKFBTOg(@55y5{qz=0rAO^_T6%9afLFen@u zNnjmhgexC)q6@#GV9QVk0h+BYb{mY{vwCL#yW0f~ce^thF~}6U{Af=i>TwD^ig!GJ zDX&S)!;|y2MTMK(1QYP8pX|U!q+NB*Vu@OhElN|4vE1YiiJ{TvIF|*^O<@^3O5gkL z<~~4_0aWqpL^tYiIokwKNf*Rm2FXY#$5nw*vAPwl`t=_*C_^^P>kk;|2f@hs>U0&P zSsBf!!3`3_gCQiO2z5s-qn%JWkekg{-lHBDzR-9{p-1wPI76$j(S~=D0&v=P6$zvO zdNF7Qepuv@$hF`SZ?Il7G?%rB-~$Gs7|beI84H7;Vj@|{)M7?9h=j~<8C$$iBWhGG z2?Wqr7nH%wDM@`)eTaF@uX^1r^+RI+Ouf)v`_N{ zLL>`#ODj!@NkEGSP;#(BAPj-ZZ-SA@8h&nRGqpkV@N|b2{IEaTlG3({W07JVL;+|i zjuB{c7!iEpq9bjM0?0WkS{h>ir6F0xH^lLcPP`PRiUp+}ms!S}K2@3^kV9I!aX4$H zc7D1vS3daCnlyCfj!8XS1)Hi)(dKqIwDU%9e`ZyAW)+bh6yz{-b*vL??nz-y4_Rws z)_|I%3YBO>Gv2DAsm0ZXP>^Xsr?68Bm6G|T#AW%-Q(Zzs?r_*KbPA=Qc zg(5pYkTRhTF@%KtHUWnPfPxVJ@$4S$cT*KOGiQONEf{{9-#8V7H+(>w;gSkCCIIfX z7KEy9ohj9+1~*Lg)M^QrVF$s2wod?hWJJ+n3uM5f9`-;h&{$E|h+NmLR-p}ftH#|$ zt_nOe_(qUr+tgY+W>~MP6ohUn?0OcWbjVJA{4ZCid*7m6=Mws zLGH(7+Rr2g10CjIg)vZKj7n6(2Agif46*xGO#I2mL;-SQLdRSkh7Lx2$(AhsN?XTTW7OC!?TdWJ#@RZdO{$gNea(QZbUt44sBF|4U=#| z6Pxclyyo^3G0+7U+G0V*o=&{?Q1qUE|U&9DS zFGP&!!xW6@WTmQr0JK4%(6m?*o z6&kNka$OlS+UO@SE`rjkp&&HiEGRtwoXF0ZV3p+@$uDbckjrM=EMI0`3sAZVI3Dsx zvXiuF4nQuHfzCuwLNfA^h6vzrj^f~{Yzj9`hWs@+7HDeH%?)U$0~dsdE;ORqpkHQC z`xDCe%2$a0-PU`X@ZSI@zPYqnx>6_mlxQu+j>3Ni#_UCx1MOa5?iz~a0^#Sp*UI|vNIbG+%#Ybw~DX`vjer} zNhnm}I4yXKb=e$5K?To|t4E0ev`_}>6TQFksFRC?rGX9R_&7xnyl(hC%Zez(JGtfn z9Je3<c-bH+(uoin?vEKq1f~GZ3c@l$x#cB#UCWW+1!j;tx+?pcz2H8Cf8!C>IuZre>w+N!7gLn9{!)eTqa{yVMIn&6719Fu<2m<>NTK?hhCGY^ z==nbg!Wt;isP8&Pz&Q&(XeeWW6$3DxrTa4qoJI(oCw8kwYn+>wQkT2Z13~;lR?-@s zgr4xR5sC;q?Q)fY?5Q*vf+JZ7t@9oA^F;?kp=O##iJ^rR@IY)}0(~5nek7U2;6zd+ zLSkEme;h@PERS-~gev0?{iA_eSP0O=l||XBzhO$%ib0*)7G1QaaL@$;d9uAcfHG{E zloTy@l0d>Fp=hiYm{hGpP&SIum`>}%1p~q18$KBTpMem;x|5Vh)EAvQyU3$FQ*nw< zP=ZaAOU_$6B9N?Zn;RsInDz;#t7MFUB*KGKjD!TXYActFILmL!C^@(h*;+;a*g6x9 zY|8WFwnVT&stKb*)XT?|sn(;lzyzdpD@>M*I%-KwYbqZ^j2i#>tLxdL`JlJR)EJ7u zAbWhP7`lYNQ>8#8l4sjOxAFu=2m#zQO<(jfzyl_LjLm-BN~}z_uk0_N`p57~f`zQM zC$YgQQ-c^VMqq4%wA@Ze(Lrl7&PTM7%ejQ4xF4Ytio{AV`AD7U{FR!S&Y2M;>s(Rn z)QLp9JztCi^K*uCDY~7b2nS_E9hDCwm_An08K0DjL7bDJ;D8#zicRakA0QG;Oq(jB zGyJ^J0`)a24a?U=2#|BlCm6`c0jb&yh1*0>jqIvFs6%<|v9ItVIfEPj^6@f^7&jnD zJ#dmwj;KQ)*}t$uI&`8xKWnqlTElm`P8J<5sXGJQoY6c4D|%rY4)jJHJI?_)Fo{@| z{zyx?k~v>QoRs6Wn3PgaP0&y5&({3UO;kuo@S-jK1}gBxCm=t|T7+X$$l|oqq?*e$ z-2)`Z)h}QSkdrKB6F_m87(SROVO_a9#nU}&KsG~Brh7wZk-b4RG}=Q<7|m1;B|v&R z$&YCzadgax00KP~JA2u_ak#j)LkL|;*RV*}VQQEteOLSpJOFh~0L7t4+)`l>0mcB0 z6oJ+C<4SnNPXyhvg)9s#0M+}%xp4RsOpq;(#DH3uhdkQIdQ>t0b)*1ioEc{FFcpnf zXPu5|{RyeECqre*C(Vz^G#s(xpWA5J$b8wa3qh6@jYtp&RIs_^EVx>bATgZ-$?DWu z1(N@h8zejoY=ljtEsuMJDTKSMp0k_$QoC6dQ-b9iLpWH3?O7=0JEhGix)ZXyOfO~I zOTHA2$Kz9Gg~pHd*tHGWjxAKIWk_1J$S~>)mStJ6`5}lfx{<^KTdt0_`TQ+pVktJSf z#k7)@Tj#wnWNd)@VJ~iV2(NSv1ezGL;n|LbxnQ=-d}+Ut+6VL+GNea02@k%_pVZxb?MN1z-a1#d8fsI1Af(!JwHk!xHV& zK26bRgjSz1UJE|V57N%(om;y701wyz2iQOBVc8Lu;DCThA^W8g&@StVZ{uK-oFIij6Ko+4pV`ZG|brxR1A*eYN=M}2H>={W-*y9o4gHT%75->==$zG@^*H&}Qg3>=Ob}$tfHgzlnKB$2n#)GqL3QATK zVUU9NrC-m51j?~rW5ZBEV1j|96<6IDr;XfIIj!d{*e%A8@h#v!AS#U7P&iqoiCx7I zhCk0jTQm%2j)mjmCFT|tq$E8-mz^?kx?>JTfkbJh>=l$QLIF#D92vmm9eG?W&^bmn zO)zVci)&6{rB!$>WGy)(FlZHoo9DZV8W;;}Qbb zDpaSt?O{uyyV7P;UP0a~rIU4)J2+o_v(KLmS|?E2B5q}wGifTTyp;Ya^zCD7rfI*W>5Uly7HEhS*k(Xp0|`11 zrDke87%3m{SzLYIqezAm_FqA%3iYa#@0G@2;rGoNnVpT?JViUGL-9(d}o&XzU_$RUMb=*ADC09%E};ZWbRyUnw2>i%xQj&-gTH<$Z9sy>e@* zf-1joXag2z4qFY!qfJ6VO|=;RrsU600E^PXKv>mc$sS-ao!f5NZYeNuO!n|nz%UQ* zTVdz~NDy5v8->l?WPt33KX(H}e}z&N-T8flF%t%tqJ*W{)mKh}DQrd42;vi;^SG33RqW6H{^}sy#9i>&Zl@)05BDuD z_a-mkgy!$C?uLADPB6}bd&r2EW^$bhY*IHvC`USAe#Yaa2~`iVd4e$&6(s)v224<^ zK0x?Nh=OJ@61wv8FL#;$`x5{L$bewkb*F)(N!$1rW0Vw8fn&!IM`F_Mp)?5ygKhsW zO_y6#P-pveUQj&(avyiB{DNqh`Eut3n$P)f*Lii{_La|C7dbm|XpIH>k1V{i)H10; zFarGGo75P;lpY#b5kTNcTY z4?A2}dpYs?{BZ&Qw*rnOiXW6O*mCmVUOvGdLGrym4?1{xm{01j9C!WYgT&u?#`pQf zp94%!cL4@>?(hD6MTaF=5~4!kEu4qeICmbP7bN**SyWbAm;EXz{Z-eF$@DiPVS{8g z5muH`@1r>e7Ash^7=>OG zTfv&4vvC9q%$P}L(iHL`h|LdA3f1cNgXSJLIBMoBYEuahq%%92NlLV4E@x7u=9@){ z(l;Vi}yQH zI#Z_%1vBXT%^v9stH)^+e9v53s7R^P#k{amv%<}T#AJ{#og6YtDP{$@(I8_f$QFZh z*ku-6w;1vucuK8c4nB97Cqfs5G{a05h#jWXO$&rDS#>0-sNw+Cr4hvtd9*PjOUl^7 z3pBLJ7Tb=n@%W>TIrbK$kVE$N8xRRuW*l+G9VZb20yyUZl~s0t01Y6hXvaYrV0Xd^ z0{!v-(-svfb*2)UNQD6m0y2=y$ zQ56mygU-WMH)rkAAfu#h1K<&+3KckGqL8jCEQp=#=>Us)dNXDyzLgNlh4eSkqS zD{Z%uI8;2FW}#G-@&Of(P8Dvq>wYs}xsbx47QL0$B;kZ*QfMYAW$0TWAR9WHlwuM{ zcJRSzG-EBR4da7JCC|j#W33hE>f3D-6Zup(JrM%Y$&9dHSNk0zh*UgHTj(~DQfeD3K>^@Z2_K3hA#VIT!UH?ATQg=g~(5xg1;%Q?f%i78J67yoV3cit4+imS#G za}2y^D&?U<=G~7R<>D&C3^~_O&5i0GNqTSiaJ!!WLl<(?v8NtIMDvInnB8E7KjcW-6TUiW|g)9E>4Hjra zui#iehGil+KZzC%Ldd)kKF@?fO5yaXXFbvg>2KT9jKcB|!^;hE05!ZJ4s)o(nsBQs z+8f6_xTm?u{HlmV6bwjO)eKC2f=*DFMIrbgKymRv3V@1}8y-;#{v}QS3|r98ESMpT zP%H>vu!9{hB_tMl&7>R5m?T~Hbp{7MkOqtp91IUYff+er1$pqqD6eHlJudG*v0R2U zsNqNH0aAsrB4i;ANgWry;gB+HFBpKm$?~EWUJ*zCx$NWah4na*%-%+bIK&sqdP>aMcvDAVO|7 zFP0_T8}-O}!kEG|oi-&T$9DBbPwW$OGW1?NO(s5nb&8?&9GA@|7bTzWs-M5oX=Rqw z)PuRGDUI!<1-NW``Bp^>U(KRVZ5V!ob{>WhXEHlAA3up-HyA@Iz9V z!3Vpk$1}WCoHUhb^lpky!9q+1?n4A7Z17X2GEyOqBtS_ZX{E8Lw6ai;UnAsq+0ELL zYgF==$y8LFxz0?gB1r>eA?v~)2(@uKjHMfP*~Pkm6|89K7g^8Rmam!dfougM1c^7& zDGZmS#N{k@bU|+-NhSNh2}}@W-|{W1>Mp_4d}|YR|caH z5#LV^R7tuTPyym8sDYZO*lcG*o2G*mh#gX{@|Ck(ASr$;i(|ax!#p5Vl>Gy?nq`i)G7L&f38% z%v()yRILmT+Z4EB1y?!);&_lx$MGNq(s?eGAlEg=+nkG7jGV+J=i6X$BY45)2yh&$ z1dxNXXvgSagUs&c_XyX>9+DR3~V`L&Er{R@FlT4(gq{LDbr00cr+ z-8+B)ZmQ=3`vk>4)ec(Ag14;u-2QvO2tSY6&t8W7DMRkHprRJw?siuif7cd|6Wuus zioVCR#B&CGO|$>E?Y|0pAO#NM(|~xz&f5TwaXjQD`}itB2ZV0v*SsQ4R1jv#0Ra$g z0XEaqsnT2lI+c=?WkJ}y1lP?=)*T=Mj#a=7lL5-b?G4<&9NfpSN`3fA+;v>uNgwr*+}~Xv;c4IY#o!_JiYLIF%aI@BiH!Nx9OTJU zB*|J$SXAu=g&Sz#fAB!s5E^JKp^RDo$7M93RB1seSzUP;LcXNcCzOp<#Sdrk)d(p; z*UcUl*j^aM-vdU})s4$o6qUN&nuL|X4D5%;b z$rw~!$NfQ>2uOf4pD+qS$gLSiE*&#O!XkiEI7wqP-U>Aa-XZpZ9T`D1Ef%+} z4-Gj|{V_`i;Dqkzzz*aj6&lsm6`ewDgwIIf!ju*PaE~W?Vt?sWryUV7UP5RTR5PqW zdH|n7NdPk-f-c;F0N4Nv$%t2q8$eFT4FrWTRf_7^e6>m`X`KS$Bav+-rL~i>fn|5> z0nyEZ6f9mq{pO~1Lls$}84{;)V#=lXqFtWjFUEm$J|`VOr*ux|a}psK2_|8hoFQ)K zNpk05RvJ55BWcpuJF!m8OBVW*QcSWKabD z{Zn~r+L2~KYyIa z7AnDvO0=3Dm;NSZ_<$(#WQA@G4{;EIVrpR6W>?NaO;Cgucx(RwCsF$5ERYET>egAk ztJtt=tI{jIK94cr>#PnUbUn;?UfYUn+qTV|w*i2*MMpN38%RU}wynsk{7`p{i*o|0 z2yt7+YOIRr$?{ZTlfDEu5olZ?s1uAUiBN=Gq3b{G3<+>)%rf8?S=kj@M26VHJRpgYv1fdOCcIH= z=e~i%s0+10>Z>(r2!LdujBM)8NRFgL87;*GbSab><>4+RsZ!|2ij|q!tr7a}>kaVl zS|Z;*Bk>mRVO9bIH?RY{LIZmr;KUpbcH=`pNsC~f54eCF9IL^qVpQH7c954>E-1xv zKt3)arb%n(DsE}&XIx>bzA@=;dB?-}3fYEhV4$slW~SUR>ZWm{3*B!~4ix9EtEr~{ z@9u&f?-EtK7O>v-ZAs>acUG|#d*=*3T9Ca6LQLzBHlB@EX0cGjmxwUt%^23^BvP@6 zWcV2nM=l(HUh>RzuxmWRqxK&TpVens6T4k;1?T+OcG>;SM6 z$6wAuaV1Kz0@LXN%Na1JWFo>)7egUwwrrPTu$PE&I0{wx4C?&42m2LK97Ap#69N-Z z#DJ=&?l9n_(!z)B@%x@_IdH6}4h_pzYyFmHLX0L+HljkrEzO$6%ogWzK5`@z@b4O! zB~WrDUokf0h&J(`=&vNvKdmXNagiQ{JOD0+N=#-SdVytCWFv&SU=63dV- zf(L9P}`iz;1qWZvN%pIyTkLMBz`=qk_G~sTSSc;;kg_ECQowtzz=6 zV)Qn%g6es*c#_}!vM|Yj@%??{We%$ksNCJWVXSQs>quw==8MS6EASawOPvbIA z0e}l&s^wi!A9TbL^l%_^>8BYMV9h5_2shGq> zWl^iGAe$C4`_pjIsA=kNpu}sIQFSn(6=rKx0B81AA8<8v_C?p}SQi6XcQc;KkP9~! z?b7HMlZ*zmwPd+APQJE$KPx&vXFL`L{ERMumfBrNz$XMxD)vDHr=0kOWo9n6ZyIrF zgWe%aC=_7k*Yz7#U^Rqr)MaaTy&ACWadrYj+h>OWLqNR0b!dxrc!qPYs`n1o5DoZs zYtMH&vp9UKX9k>CGcZ6zR6u;B2VeUhV8qn}n9V$UrL{DHwj#lQ5<)P)|M63A?Uz3I zfDYJ~UG=1t9Z;;*g=2UDd-XMcIEZh#8;v+-2*=R6q=`Egd%O3^gbJO&xNgTc#VXW+ zYrvYvd6U0*1WZ~m?BE4>B#!3`F!^2>w7`8#7Key1=ir2|c1!B1~2w}X2A4oiy|0A3?)|g@+8{;|J$(5yB2iNfcijJ zz`2TRW^|5w2dvK<5OzIxF1wSR1uz`B1yGJ~2kdYTuwv$UZv6G$wUWvq7(_2zaj&RC3 zTIsvb1BKj5)XzW1dFk+N5<-b2yr?!c&v+@s?=`GByR$~L086unQhKaTyLWH1ZPdfG zVM4iwyYlK8*?;oP6~38Z$^ECwQFx>|957Wf}6ZF=-XOgay_++ zx14dry_LP{Td)EYnv1AJ+n+B2q#WF5Kozh=-6wXvJVGCAude{B8>_%x2nI7eX`raH zJ1Efwe49!Xzw0gjfW0{fI6h(lc{BXW%2s~mPpDB*CzOKd z+bik&Y^H;_b_p$?t^Uz=qw6n0A82pv3-0v-1c(eE*!1b9EDfe5H@-Af#OYxcW|*24 z2w(u?fs8=%HB**vmV|CnHX!L?5@pH^6LnPpFkpbkWgoCOe0XaVrBO$C_AG*<VIeXU4 zp15-7)@5=R@7*|J_HI%Y3h=UR)vOgZe3%+yZi^W=W?T$1Fv*jl@nVJRa%Rny`mjo< z;(~=Jc}|43VFN;i2?{i}c3prg0NF4Z5E(ddi6c2x=qzEgq;nq^!-*Hq4W!8(8)bbS zxr0v&5CRtsP@jGxIZ|c@Mrv65Q)Rb5d0YyJpq-}Ko4WuvN7*6hfIDfT)TeL12}1`P zfIxsvLJbwP8jvg4uBv4QwM-IYK1N{Jt%V;VoP|Q=AgtpD457%P3l4jegDyDinyU^v zVCdu)f%lVF4G>@Zz0JFc0C}@qq*DCFb4iO_-W*%_z(ZUuN z-;lwGPMrZKAu5`<1h}Pw>#P}OVCl=RS~p>08Br^Aluq$}S)`S~t|2VPVvXGq*~g6e z(MV>U^(>0eWI!aqDXpcF%Gn0Ejms{*h0o7!!9|nY9nAd>PoW}MKma%|+V%wuf})|j zbPc`NUXvtQZ{K_Oz(QO)8y(ouGNvCD^|LOUA3-N_$xa;CyR%#u{-vHrbq!d6s7%eU_|`G%T0^2&Gn- za%fAZ8;~|QIS^=E1iqbm+;Xix7hSB)l=@As7g%7!dH1?HUwr+2D5pZ3>-Sql2~PFV zq4?uK;ksX&r3*oSp%1$8;2x(_jXjpB(TQFq84#UpHH2|DAc2Ai7}i{B#aL-cBt{yC z1(HXdlihjQ&_N&l=Vt@7padrnIhr+uDRme2cikL_Tr#Pynry9g-+jN{xAsy&=CU^1 zJ@wE|+r51Q<`qBshI*olB%5BE|N82!zn*|qS~x<}lO)Vb0zRK!A_cmCV5(yn!k;0p z4o5`8uC`JDj>V6HQ3n4@VAP!0&T-D!8vztu&q_ByUSJ|V`DT z;SFQJq!Bd4FG(O?)P#}+Pk1CjJ_uA>;PO8B!7nWj0@V@mSH*Jtp=37Woc{nw4lfFj zfTZhLqmH%$qcpH}ZBkl++_a^sQP6^MykH2gHibh(M^&og10L{jryh(ggT)hJzZ{ml zhIs2do=}Pk?-wf{bb>{S|CHR*A_P4;d?F5u(2#^S$;t0|NDLvc$w@SnH@Bb#J~OD_ z5^QtICOR>GEQF*c?q`lujRRvZnw1v0*u^iBLyQG9pfC@ZK%%WKX$+u>RUF972hQ$| zUaR07>A1TZRxFRIl7S#nFt2dy6v?r$(Q>ZLE0 z6-;7&6oEoJ=IKf!sb;RRUEAUs9Oc;34^s1L;)7uLlo-BxRa1~7Bxk>#QqT8%B02aW zRYG_mhC**K(%Pu*A}7;_j<|t(^_<@eM*$M|Rdxz!cDP zQ=I0soDd=r9zlVTC0N`Je|fl+_C|q#_?Ms?nRhHGscdj7KL!*U~jgfz+x3a+29P*X%V* z?V8O>;V81;Tmrn^U`bQz;~?RY>p6sof<>k`l|_xC8xe5qV-0z{ol>?rxbtg+^!Jt< zxItDNiiKwb{{a@uawr6mv!o@X$jPkAR!`(Bk$+#Q+uce54`glR6)f6}weoL^A^u-- zNnGM-gt)oRLUFS~8Z%ef=K?0ZE?-m1T`%2+2E(}nMbOux7KJoc^hsixQYju~Fd(r6 zsE!3p1rqs!lfL7$FFDca(;UT8I1c7D($?Vp_1RAaLLM45OwceTMBo*_U8A#ncj4#KRws~ zURc0K9V&xgebru5DKwB`4mV?b&0By2yJE-L6U24|7Y`(4gT3*O7yH|yXY)Lvl{aPd36S! z&iSS*IRR21P+#}mXo6oI)%alLtBLby1e$R4zROjw_Q{PhIDCN~uwPXhIOR`N*8QM`M-U>la(aFP@2wcOq}wfNr1%ovoXrJmrU3x)YaQ zn19f8loHB1(Kqe*y%T|sJ;c7^pd82f7YNSr1QDxwuuIqm?5e5ld@rXw4*0rc_+IWl zj&J3bFXo^ivk=Lc01r9tj)SI14^HX&u#fRVB$m!C7rxKKgl_1_kGWFNxn2)-|2#ox z)(GIXi-X|Ecl?TB+1P8SRR1p12M>7s6NWM%ongrk?2(j8l z0&EO;1Pc!f%3it$_~On73orRL<{$59Wqu@W`i0 ztWY}!ivK*23wLh&Qec+8?^N)>1kLaa!^pXwiv`UH5#A5}WQPXtFb`o$U(AUXcMk{y zQ3!PcPw<64{-93g8I43GtwC)S^W-m~ZByU_Ta47bt8C|2y%ayfBuY z;3z~9{QRH{O>xB15EbbP6R0J+CLqaTXC3YEcHZ$8_fQx6@UZ}K9)GV7@2&{{-~@ni zKY%PBBv4F<0vO#%&5DS>1m+iGAs8=^547+a=d262PYGs#3Mi+nHmnrI@qf?|4cCwj zLsErOLwX>{dIHV{XE7cHg&w2K9`8|2A_wl8qc1p62)N}>aBgdMa$FMZCatLm<-qei z>jM};6Q&3v!wDNF5{mvPLom{oqEhk#4Fx|ZuC6i_MRF-3#Zq8U_+H{`Qc@+?uK!@N zCCkzPku3_0%nfva7kgp}7OfV3AwB9WgT^upghCd<;!otD6P#ib|F6*#wXrE9WHio^ zkq`$Qsq(nQahO&tD^pA|Fo771<|44mD`ikDSCTA^#}Ci47jw}pd7{=R1QE{QzviH_ zXkjVVK#uIPGk0ee*u+Bm2n*?;E`L(olyWH{sxCBX2B0rhdLybp?EF@&F}X;t$T2c8 zBQq?GDOO{~rqk0v^0A+79aq1=CE?uw<6ShPEHGn8a12@G9HmRvo)WDV$iI%*M zC?j$UnQ}xv;nGqF9X@Ojj1xJN)7^#vKm$}Vn-e4}O_7*MI;k_eG}Es#fJ4M!I39Ej zeqchs$E)gbJ3~k`y)(aBfb9H9HAz!yULr3AF+H>91I)n%|2>5%UJE|;P(IORFbVT6 z7-2rZt04b!e-dZ%{B!)46G+n!BwuASz6&u=iaHl?1}Vty)JqMhk;i(Byk1}>GgOee zlS4u7Lvdg{tx$q;(+6?Nj$WX7KIS}6loLUeK0~Aw_$amBG_@L`IBQ1yd=$9^6i)~3 zD)n?vi$qUBM#eC+Is=Y_mJ~Oz?Z*UWV5SQxf&m>lle9o{L$j2~>JThDpi2=zHq3<1 z&@;+h3RGh>O?hxl>EaPeBoR_oRbixLgylHLkI1-ekv_VBN0cvnD z2X`^DJ`*%O<|rN^3Er|H>Y%o+>NUF+O?MMUc{4u2HC)-0R>oCj%oSFZ3*O=_bkspz z7wui0&RtbDWmmRtK(AgIVBqxi;8eC5lY$>1)maVpV4w9nXR{Uxj1mrMXtN+Y9KnM; z^;5l-F99n}3u?}F zYR46F7spl4HET(hIq?*8GZ%ARz&cD)$(*KTJ>){3MtmZG;+jy}PB+Ic7DELvP3$pm zxz%cTs;9QqMA1ULa)B9sNH;OIY0ZQRl0k{2AS8A&V|$Y@Co*yYgL%V|auMw#HP;oU zmwF3SRyl)n6*j?RY=CdxF)ihRkcM8i}OJ`NBE#UW%@I|s31fv*L+fdW$C zo2EcKMt~o=iOcv1av*6*h)V*9S9jHifbn1i!sbkIZbp-Lc_p`Moma%L7BUIQdarj# zSs{D1_awKg2B!&ZB*}DZ072Sd1h*+1|0n<{ZTAm*)@KLM`exE~d9xR5cP&^(cY7;% zLB$Fj^@T=oMXf-?_#hXASQFSR8C0jYCh7-9^>H7Vd7HOCC%7u3S9&kFNbSJiASmG) zbbxEY2^N5lMnDr9Ky`AV1|)|BfH*M4^dOSTZsX2qC9KO3(m^tWZfC3Jv><8QmvBE7 z2E>C3^tTYYf(o1<94v~2Uns-KxQHo=DhdIK4M2%ssC7bTfFcXpl+88BcBJt!7 z;wbG%5LWe$_?Cr30I0%O4KV3m|K=A^zL<#TM;v-sco`WFa)FS2ZjoJR1h6cLi#P>d zh<7o$aePygqu6RY8D~EEiYZN$D**yhsvS)^C4I-54`3A}Sqm_LqN-v;Xz7SUHD8Q! z3KEQ>)L_l(ARJ(%sdl*y0yf6XNDHn+4J0NQw6&)o+P99%O-I_GHGzf2xXC`nUKST9 z9^?tqA`TMxrKRX_>mb1}Y>+|mFRK|6{8s|5Ii11O!ZO*NH@QdQxme0E4eNO`eAU&4 z5*a^upZj?P4&aV0AgpMtox`fhwujV>0*hE z&tGB*PZAXU>k#Y2O9Fr(l_Ebu6%~M3-D3F4k2}3@61CH<77jS7o)~3*;j!&BR-@OW z(7c}29079dsrBwnmLSgW?mlQc#_F(l=4Ygv;05~o;eyxGdGQe}9b=#Xm%(nLBlR;i zV1DqK9WBx={~*N%w0J*`95qCv3JzO+ehYymr@?|}mXk8Zd$*??`mt7^d@r8(0TMF2B z3vgV+zQNc9Jv9>$vk!g1?~Pv1`AqU|I11HEppRJ(yNA>4B)U1C7$JOmz`R-Jr5QR4 zJqr}wvx%KK6O?#PlA#kQln3?qetS9@5*ZS}d?d^u5yT^t5XZ4gw#+>_--C4D`5m)w zZ(thENe~MHgv!l?ir>&`&hwxn*3zNpT9<=uy&u_4ZzZiq@V$w^;$xSvMfAV5B{l^5 ztpDLU5eDW2k^16bwc{ z7z7!IiP%Nfdll-So4q2;g#mL^hScjFxt1%@Ivf-v8@ngr7%sc&DF(M$2PWzRb(b$@ zXdCoOgMmP$-mVBW&xs}pI2__h7!)_PzbUBYURrgq?(M$dg^$aXKra`uK1OsA@hV#f zFz_3}rpkD;3OEFCTmms*=+7*X!&|Xe0U=&G`~^B=P=J$SS$8``-HF)r0Rl>=aI^3v zTP3JavTk?=b}9u+RVYf9wgIE|%o?<88aZnF2-2g-klsQPLxwWt#g)Z)xqJyTCaXR( z|7%vVi8IWXBN{q*_&{SxlPH92X*qF$fYPA_P7qa-CXX6SQlVN<5ru(L11o@ZRnzQ_ zjU_L*{Jen{4$rkXa7cKP%bCJ(Fr*L+78gz1Bx@z14MAtk7&=PMdSKFp$PXesg8c3I z-~kIM00LOvTET#302pS0DXc+&2p~ieL>2%gii)wSb?klIql1=lofiDK%SVqhT`)q_ z%EfrZ?-au29v$&^AqwOq2$E1pr3G@kwQ}?k6)&KoHTFt=idt(Y@M}$1RP`*00x^y5mqLIKowOK5~eX>{~lds zHNXHBKw#Mm5k%&IVXhsa)`)9Kuw8E7)Fxamo% z1!MFi;|7ueAwXr7X~rZ0oD~w=XAgj)8Z#V#xWF5T;8o5aKD}n*Y%GSOi;J3s$AoXd z8K|6rTKpy#a>Mx$LUWp+(A|!8GE{>bPxz@Fcj0&k9zxqyN8UjYsYi(@?cH}0r1Xt6 zseO`0n%{l^Zi-VcIt>_Ns19hAA%o_Gb%0i>D!{6$JsPz_8V6)p!D@V@VC0uzmT2M) zRNS?j7(acon7PCJn@D3vG`ip$LsM-T*}kmtEH6|5i|Lur8V$@V!7)O96Fq{wsmuf9Fzh zo^%?_@WT)fvH1FYHLLS{uj)eIp6+$h5(N#jlKr+i7>NkL2ZphO+r#6d3BTrlgb1q5gzAtC|f z49*$V!^R8=tuh%w1L|L}N+x?{3#wb4{8;TB$d8DEIH(ZJ10Ub)HmJbSg-rrWT(Mjv zh3WjyOvG{Ao>tMY&p=EV7lQg)EHO zF<|GKFJZ|Be>i{|=3x#X{H#<}u){RC!HX*Ju4op>L$j2T0OH*WL~)zl5>O$&BD@9+ zfQZu4+|abX9Hxjbn!uBE_`PP>E)Q!MA8gX3m_KzP4uN4~{~Xc;w=LEoF^QvJm!3DX zw)}~Ib;QGB8Z&}A&o@vMqJ>+FCg=mA{;jkH8 z$>AR`K+7+&Pbz8Z6}AS%mMo@45gsrX2Ba_r;t;GrKEN2mmOwpdg2_l&D5s3J700JF zi%w5iOcvfrB1srxMuyl!9>@6sE8NtX_RjLq{sj!J10z{%1CexO0z@;u*{|L;R2$2B_NW-a!X*3`#QE9oF z!FU8Z#b`bTe$b?5K?llDeTs9a?h?5Q%wtoXDb#&R6{u7dLR7aP!GmH| zt6Y63L?sGViS|N-)iC8n&#F<@y`*(5Y~c%w zrZ_{U$CKd$KtL8C3=*nCou^LWM%1D{RnZ-`sW_DQU6vHt&j?`k#&3EnE55akPOm-d(?*qKN!MQI7iIp^<2A3w*}PARe^?QmiLO8 zy*?c=Q%!tDPUr>1OH0B3tYE$d7{nQMsNQxzt6f>bSH3kq?TxLv+8y(_$AkXyHG~Y{ zL<_jcM@BLWIojJ~M!<Z|Irmy=viR5jN!vx;t!c9BOd>$H0Vjm%;e2! zWYH{H6&&kt3ABPZ>RjT+qPWj|?)9GoZC^nT+Q+sXGHwSPvQ{R!tq5i^NynixOCMLS zQ+9!sC(%0wWVyn{Lncj)v@W6$Aw*}hfjREbhd+>|Hw_ueK~S)28I@s;y=i8Q6)C}J zzEch`U@eK=`A42WVMWEEOLK6&iT8GwvP7bBQU5$pU<(_z6eqNg4Si@tE4rctZgiuU z&1^l#6*7}{GPJM4ifU8a1y#m&&?p>JKXy5v=ZeDHnv5e^fr~4B&_zCIQy6_4>kV3< z1;M2D1F#_KYI~ttnGo@bE1f9a|4%Q64bFzRxk!NmgIYkYxhry=oCTYAz-h0;y;Oku z`KrO5_^^rXiS5!-OQu!Wzv|9f`%x- zAptD&=ZJV4T5bbvt@xg)Da6lZV(po-dK)%d-A|F!#tsHLMuOAqYU!cYF zuH5?p@KhaO$ctEJA-Hz7x83a)hr8TG=JAem{P@XtG_#-oR%csi!8&xor6K%aA?{Gy zg%|hCeXRon1_Kj1yvlNHRxKRGn_H#dlGQl>8~2C4EdcGjQQBbjhA#je;|CHx>Odo} zhbR-HB7%@lgFsXZH!S9*|5a#cL8rd~Y`qx8Nvo*L=uVcXp>Aa;I!S zHiED8e0e8*)FNEd26#IpF18RB31VST@F5N-dQhMn1yLdMr*`P(1U#4wsP`Cuz&&kX1ehQP0@r!Z;x}_fdw0faSyOep_ZkZzJ?E8F z^zv&Kc!3pHeErpd8@GHU$b890ha8A^ycKDNwIN<3Q&b@MYnt^L&I4#;kgO8D67l1oVhb6-UMl^7Fx7Gk!Hv)7PBMBFa zS>ymmbtl*4|B8HoQ`=zzx={lZ5CaG#k%SZ;3Hf45;EQ*_3`jLS!1#uvqkJJLk|k-3 z$(Up?^9wAQ8hTJtFzFR6^)*SsQm(R?Ai#s#(~Vrw70r+jT_Rkl#)#kcj?WV!snG%f z5KczvNCwbMUM2yOw**>&nWG|^-Pjgxh6k}H0bF+oUZ<5&qB)V%W`k9ct8kVCaf|x$ zCKFSYLO7OZa$d377{O*S7i33|*q7P4TPC@Z>L`s{LN}{%m`sV7 z=i@t#nE)MK1Iau-75OyMw#qop<^c=g$i;F{xZ|Pcz z!hmMtNf>AfwN;%SD2Jq@og;~zFat}@s4iL1HRHJ>!Gu2N$(RconLScC4{{;A6MQ&F zn2`Bp1PE z{{UZzrG_*E1F#D37)YYQA^b*NQt)^QB86t^CKhUdu=p#w2xwAthG$Yaap43Xnh8&L zCKIEUytu4{`6BKIsDYXgACySygs2gvsKsWTGzzJ9D5;bBd~CHsEE!*1nwse|r27~& zP;sCY2#>TN7gDhRUNHmZ7apV+4kc19-$WW#6RZ5fB=dkAqId{9GF;_DT?`YGRN{~3 zbt`_eCCRFl{?mX5l%WDRFwrpr2GKWnx-}_K19Y0LvGFTrMyMK6J0Zleul!m8hjhC5D2|0cp< zNeQcw6f^?xGY4_tLBdc2MzDFjI0D!Kp}%rbnaMl?v>$f>OHEqc-UYIC^;-VbLsEr!Bi-xr8 znujZN44)Nh^BOUP$u(6$beSk~n%Qb`Qz62m6a^{(og)m`u?r=ZEfnMr{?NO0@w?e# zLA`qgSJ1Xt8V#2XI&0I|EE!gCM}&+9>X zv8Lg`4`u?foYy{jP(Rq)H~NdLSs5FSGafEdxX|@0N1->2Yo_QczhQ$XjMKRHt4Reb zgni2|PKQVx(?*|bWChH{1{{Y7tibTvzz#f1N(oR@(5-h8UAOWa*OaP-FufU^nK`w> zb*#d>0K&xEyTSm%E3C)GD-8X%Rc|W{{9vf~D6s_SQn;44IpC*S)4fYJ7f>OALyS~3 zJ9NP*fePs&Bp{L7`pBM;9d&W7hte&VgFzURs6vabUF^l{+I-u||D9vJy2<1NPJo<< zG{J!sn_&YaazJi2xM#wQE%0;4cN`7TFucRd$9Sy5%3K5fU;;G|$jYj=<{OXX_ygUd zu?@PSnf14AF&AyRCVz2|x;U*YtBde9cFC~L0>Pr(T*VwyN#?4>qnoZNArc7uaoo9E zI*MUs+!OTz%4Z^H(oq0%=BIT=%u_`}{5CZ;MLneewvTYk&Wy*(48j|o%v%r*Bj69r zG00m$FC$=%Wf)MJe4+;^v*awNK&6W$+K`7}t(RO_CW?lRywLhv2!{GWS!|cn*~*S; zwEWzz3%qQ!9MB{6#6Y46TEG=?I|4ebHHhpkk0i`-dTzBs|AC2<7-%XC&y3Oj;KC)G z(H%`eX?u@YwMgSJYD}l0=oTJK_^g#oFpkBhe`9Mt&*y7~OdM(uo%R!&QOKZ>Cusfq!(d?qAd1ZljQZJ{@{dk6ENR5~IDY;Z1U zDh|xR(Ho6JL1@}=kj%+U*6th^;$YJGOVTQa7pIqc#RA;)XD~J$Re^$3hmdwRxXBSI zA~^jbIe^ue+0>d4y$JNbtr$R)jK%ishCKGz#TeO=?XDoF)Rw)Y*c8fXT#IVBNOa&h z1|>P>R8Aa#3i%+)RJc1;&<~y$L?Y7vow%A9IZzmi)(NVJA{rzj!dq}oj&yp#yw%IpuNKqjw#|H7;u4HPsE{b1|kun%o+(u4AeZ&Fwe zz&StSCI7a;2{Q#Nsz~-Js{{lmPv|$@A{4S`ISjPT?z=z!f?iDdzqN2G2{f}VJ3r)s z9c}~&YhVeE5D|V*Mba4v@?_|To@hG0<9K)84HpJ%;0G%?3HdG{IF-`37y>BE>BUa) zvWiW?3{d4bu-UY4mL?3#pv=Dz;H*yT1isA641~3A>$y&+Z+k$Smn&g3H<<8$5pJ)> zRi%6jkp*vD5sBMu?8aSly||1(;I7|K1_SL~g+qY{=D>QF#m0cH%8!8Vy^n1BV+w9T<#ol98(JN>6A* zZNN}3?~p3X%IM<`eD7S(`}@A{Ur%zg`G{3Z{9~~{0-^S&7G8?*%(X4|vHf#*&&+lI zbNjFl{J;YW^bO83_@{9jhu;V4FnC&D+!;u1DK^APFSJi?d zF^=Eo!aMJ2rT{@DpSd)4s#KW)|70MFLb)8y!c#}$B2mj^UBqSanlX;o99c3HMv|M5 zfkqD71`HXsY}Q=1g!%HB%$O`?qQqHD=T4p~eewdzO6X9cML`*zWNH$YBRVuV)ey#D zQl(U_`kBOr4b-PyH8}Vx0m_s-&6;7#gqG0SKZKYqgE1ED4X<=@*lqen@0>Zi{6^d5 zFPcAp-R9vtoR9EgfBb|EMC@!IzkOgvNO=4WRhn)RHar0Ppas5in5v}9@VSDGW-eU6 z9)NH~I4Thol(RGAwF?CoRz=WME5!(9&6ey2ktQFCDY@BMfDDJTK^KWl@w0&M~2Y%vZ96;|L-LmMDyave*JIy!KM0uU!C(X0XB% z8;G&RGJ|Zg=PrZfn`CBTh8fQ)6yOJZ&>}{i)Ou)uz!$XI%7zHQqs=zjkW&EzTSjOI zg9*f$?z`%m7>^3)pqnQxglPEA9D+2#ql8Slt7J^^(yC>Zh@?=@5%z{j1*89@5#=M1 z=*ww8{1$~~CPsULG|~YPq%_b6j|n5Mr`X8qK~Ev1YAXsW42!wqdb8jH2vSv59D6<) zfke@QLFi%F}`ZHKa?!2CdozCzBrq;j_<* zaw2M@0R6xe4n%iywBbh|ZWusG1)Q`}ODUaWK~6!v7{d5Q{i;G!^I)Ji0|GEv1s`FZ zLJ9%E;-CpRPys{56@iJ!xpe0&N)WN5P zQZbnUfdmi$LEAeLs*-@VQrp^VKA>xtOy0yz?#*Osn?kx5Xu#ID?sPd@Wqk4-_wKV# zjDw5%G|H%O|4EYq>5P#i8T`J8ALkF^041KhVo%#M5UHhbMf}sMJV#tqkR1?9jk69| zH2?q%SoMTv?hyn7nO_#I+59lL_|Gj~1+y;fS(1sBBP<9rReHK*)mfm9S(cE16lo z#=@8Bh6XUi70!CcD`F8(Xf&*0I5c5`emFxim)U~SobVhzEXP@&(UXRzwM1uefK7BW zQOs1R31_sc1(wv8wZ2%tU#e?EQ1~T~qWCr!76d%yc?i0uL>>X+VKc}iARNzNk&6kE zVt~Smd=Aw>SES^R=ZqjaA6LOZW)3MC)WHTh_&ErDaFHW)B8%bKrWfgMG8gLau z|4qJV7s5a&Lc@_m9Y*63qP)TtobkL6pmGGl@B%9-TFcFlkr1xDO9wpg0Ux|i1Qn&i zA*e|ntC>_>SxBihsp*-L62YXl!-Er8TG0X8GA>fUX+$cD%QTMYL^iD{IS=^FpVTov zB2gS3)w##1>T#WbZ08^qsliTd43YJmP{=IQNPT)qeBr^3?Bwt=0lW{ULcrwi3hFyp z)Q}nepy7)!S_2d0q7WA4!3dr+oEo44dbtzJF@bOiD8P`UI~eJ$d@#VCp0o###VixB z`2#p&7Oq2$OJzfl*~b=Dw4hBYQj_|prFs*bb%ZJxq~aa~{=}+^B1TpFh}Eo?|Faic zEkYPL70$1M^_~G_orkg%$=H{D-AHH-|8ir6ViZm^g@m5GKprcQXGmP~D3KFEQvoB^zfXQ8CS%;>`}Wte zRW>kZFB#b^*VIo7c5Ny?5+63iR<;t(ux>AmVWv_}!x+<4288R>d3tKA|0Fh38cuA3 z<)XC@tEt0t^_x4J-C)MM+F3Zdv1mp6>@O;8^h1O}Q!7L>nnT*o7uz=i>y1Urpdob4 zHsIDfFrs=+rY}ijg~U4yDas5#h|f zo!ErcoaQ#adBYvXO9qH7PY#E;5+4bxSjTD$HI$WADh70|KU-*m%J2(`zA>>r$(Nle zbeGJ$OG7YPd%BJSt3>y=v8jYV}G{vfk5>e|(kg{d*M;IAn|SzQFTg>>!< z(=vMTFwom?#*>Pb3toHL5T-$dgRM@9r8C(l4|B)6$ze%MNZN*^|8~bdmTgj%nA?v` zodQtjBdoPx3gU(`xevXIFYwstH;zV%+AY`{{|m@x8fC2_v(M+T22kJX z#e7Dla%=&^C|vWIM6^o?C~go|kEhnR?nBwxJ`e+U*#&-Dxl=zLu8<$Qw#$~rno)l8 z%UO&)cdpj7q150850?ud6zc_q$f0kixB-i$#flRGbaIz_FE^G5B!p3l9uYkW?M{I( zQZU|JKmum%n*}TVIwUECV*-P%x-FqQmE+jr9<^RVTiD)qI&?kO(~0}F!(;D}asU*2 zJ5bzRmk;-5pbvCX{+v*;fYRnhd&Xw`hZV0Ck?>vQe+PVj|In{&Zx-J0lW4hPixAN9 zLkaQs%#!+8m8veP3aGV~KcTBGql=Y`3I~!10adt?g(|Yvd%aPr0STyqP5X?XksZHc zq~8#MCQyg(@RQt7h^(QD5gCtzND727xDliQ5V3>gLx?}1KFPuWbbvlFlMRV`tY*Nz zQd0=D1Dkn(G1)qU4+}rX13CIyEpt*fKQg)XW3!Y~xekgy3Q4?aYrHaxFUW%eIMD)+ z>OwD+DKHGF02Dw&(+eDEx*Tf=H@J(3QYcOEu|SKr*1H1Mo4tGEt63-nda#lBL?l8)fW!lziUM7j zL{m(~Aq=Qfgvjqptv^Y>2#UqQo5gCYHe2kvEc_r`q&Wnj!iAH%5fCaAC^5q7!c|Ge zPT;;~Y{oCxfE&XDgfhTfNRbVI#t0NW)yu|F|1+Pz;*vYmfadr@!TPZ&sT%srmvxlE z_!$!p#0P+@C``x)gxD7sD2qe5GFp(1H9^6z3@|cDt4Gv9h@cypsfD4*1Uc}jhLkOa zl*rVQ3MKRjSG+B5;E$2x%f39Iyn8eFv!`B!iX|b;TVx;iVGhnAcQfoqHdt2r~rY81IvRb4yq)< zZ`lo7XiFkMI31L}eAG3!gf*|U$KYeZ?y-ExqQxvL_Y}9hRB(TSZuAo z1kCLO%ws6D$~;W#ae`k|fMP_+lq{@;|Fg!^JOpMeL!0RlmZ~~U5Vbf+24TpvJd6WM zbB=?YKyQ4>+FK>rn*~%FIAHR?ABeUkAc&~s!T4&M6>!k~>9 zIDbQ?`uYwYbk2x0(O!c{W78%yW5QRAnC#R@ztm3dXp=tO z5EFrlHZB-bFnv7yIzE{xQy3#HP(X$60315#0{yxrI~@Zd5WMowZZ9_gqLi@r|E|7yW=(~t~RBejOScR-!wNO3ntT4SSBnYfEI4Zi_FI`nu z6=ha4bJjU?4?eOj>Z~wiP}D@l)B5A=Zxs*G4V0&}vMeGS_nzqI116am^Jo zY*aV!v>6g@~N#u$^y#>Ce;7%vJnG++%oI^Z;C3|E8w)}TXJ0pwL#wHaPT zR);j!i6up2CBhFZ6!tKnC8SpC%+_H0*bVcK7$w=_`oN2=*!Wxr>w(ysQR}ZBdVP6d8rWvNhY0E!m88Kb6azw*7=Uvo;9P+Kl}Kt*zU; zWm%TB+qdQ0y!8{jHINphTfD{E!o^s)?VzA2uCxtVkj2OfTE&WF6uq=gYCzkwy&^6uAE#0`yTE^vE*J3uw4cp3vMaTWS%-vkt<=SSuT}2&T z)$Lv11>Us{-r=>jYc*bAv)zKx-L9Qh*o7F$$=%1@-Q-=~>7~)%WnI<92;x27?CoCf z1z*-BTi10$HdDVs|Bc-WqujluUg>q;@qORM-A?S~-tS%A?JeFJ)nAYe-~J`p_AOtq zH97Q^-SveD04Cq~ZCnJF-}(h#(B0qrm0(XbJ1*H{Fk>}d<35gFHg;nT?g=<<-t;Y09k$~-2IL>c z<2`odMNZ^4{|@9gCge(HVi1nx9?s-OZsA8RVo2WPNj6_}qGU^!<4c}gkp*Sz4P{RD zRhH!mi(ygL;y5IQWc%e{#$aDo!dbRu8V1{1&gBoDTtl5+ zWB%h_{^d?~$U<&r>-6HW zmF9X@XLDxddoJco{$^%gW@QHFEiPaYw`Cw$K5NKR+mrDb<+=t1>oTn1=(7H3p$ z;(}i1h}LI>u3e7)XGsRUi3aG3wrI#92XB~X!82w-)@XxH=zES_mj>yBZfTMe>5&fS zlJ3|V|Gr&&>3}xmCBA5%wq2sO>7YjFkcR4Zj_BJ;>TX`@ zoaX9h)+3Cj=BW1Qs-|kPHtI+T<{9Q`rB>^B-s*t{>$nDLxen`?E@iXE>$X;Dy=H5- zUgEJXYNB>(y3Xp4F6_MSYj5`J#h%@nCTy~PY{V98!!B!suI!T2>cZ}7wiar_R$duC z>$#@v!zS(0{$`}k>{e9l)lTirW^6BZw#XiA$d&Bc-s{me?bA-|-Ck>^?rhn1?7Ft? z+n(&)=IzVo>(_p3Oa|`R7VS{Zt)DjT;#O_sp6=t;>2OYNM0W1x7H)pFY~sdk&VFvp z|88xiCh7D3>S>1T?{@F-7H{gd?$Dm^-p1+d=4zd2LX}%-?|$!%zV7-S?*OlE}K)vZSniQi8Ai)-&S!O4{rGmaR{ex5`S^~25}kp>aE@73ewYY$GRgEN^DgZgSTCa?N(_39oRM z@L$iK@-h!>_}1||-|{>+Z;RgZLWN~jKJgk)ahT?DGav9YH*_cG?l-4r1*h{tN1yU6 z=jI;(Za|U0*K$@@38Y8R-oS$f z5h_f`u;D_75Fbj6IFVvKix?|vyqJ-r$B!W8i43{1WJic2NvcfAa%D?+Fk{Me`EsUB zlQ?PSEO@e~PMb#1%%ZQQzn zrw&eBG-%$%egBS~TzT{7sD~p)4xPF6<=2H*mu~tvb?w`oPybF=o4a}J;?EoY&c1#4 z_a)n}UWQytGmPw{*rjTW_ z+2)&T#wq8ObfRfzcXN6PXPA7Bxo4g_)~RNZcTOnipKcoJ=b`H?+2^BxM#|`86k_+G3`lR_ZB$p8PEy+UTpWy8p_mtg)(krmd*vIw-D)@>*-D zzS0V;uEeHlY_GFAnyj+F-sjt}!m4^~eHtTJ);D#$`wRMtfE{^A3 z8|<+-g4gZ4-^R;ly7blyslDFdo3Fm;PRg&k#b&GRy7Ho9@WJIIjPSzYc4Mx?`%a6^ z#1vO-@x>TttntPXbKLR2ABPMwz}UJgtH|Mg!?3~*v)oL=%)~77%rw)oip@CZtTQUH z@a*%?o%Ag9(4Q1-^wCHsoirmyGwt-#q9pxG(#sTG_0?EseGS%Icm2#Y)P!w~$2RtB zw!WrHRWPYygT2kyZ_}bR*EQozvoguxt+L*GJO9&n-%|L100(#q0fqt)1vDTj z5O}}^CNP20AQzQ@Cn0LIj)2JPUm_A0v*is=8Q0UrICN2qT$Ip+C`@7XvNsDYpnwDu zh#>_?P=Xr5(1tI>p$>02!yfu@1wRZT5C4ZK#3B;Wh(45|5|`M+(tYlUP>iAm|F;MQ zV$g~Y%;GlE_7H9f$r=F^gBZ;~MkjXRjAh&(fj0`kjG5WPDFEVRD*QjC|pFqp~(E$o_uq7JN_z5;FW1 zMDNHo-q00;P^-sBhzPtdfO zG?l1MYf(}2nX;a1uqQm{sY(gzbDznI=vB?xiA$&U2%~|%JK?N!*eUZXDvVg09#EbT)_**wL)^wnp_hU z*SXI%!f~TZ-Ribgt=NSwBB?vl;eu4K+$5n4j62L4+*Y}MGjA@W>je`^H?~$VVH_!2 z#;EnezLxbvX7{VWUeK$RuB?@4cUfAN#^9+uNMJlFjiYhBu!TRMd83pKy&H`9m^BOQOLD7c< zU%3)gK64$OFjaMiFaK~N%!i!kJm+RyI0{zXu%4ZufC&g7#8$wuich@a7gM*;F>Y&$ z5j|rYJG#-u{cDc>C<7ixbiI1SG&yxj-os87w@3Dylgr38DX00sS=O?Dy(};ifjL%U zF0h%={56R|gUpOr11??r2Q9#v&cK%Qo$JtI zVA&u>HW_Ma1!lswqPaM1ktbRR^CI)pc$_a3#;^!h#3CE!DuyenjOtWxc^RyB3!^X+ zU@66-c$veseR%!vCc}ZGqeE+k!gW;%@m)5i8{N8g|sHW}7Umyx{)kyVb7@daq<*ye61h z!F>RRtn8phU!s%Y4HtGH^Stn$!ZfDT#4*S!dtHrsRtZ6MiC2^b8};Fc z*?Q7Mb5!hy?mE{o@8}*z+K24T1jHfkW=uZ4aK_f@6vhB$lu?TpJaUl{0SI3Eh17Nu94Jg1BZ0d5|>L8|8`@RQBYe)YQ_jjQCBFJe#B+G~k|VfwqkSyAv-y^I{pAbbj! zc#0=_c?MPpI8J31atDA2!sJ|D$9>qhc@)TbY~_Gi6%6ACT``1y1@(Sr1cD;iZe`FvVB#^rkU3M3g9=4# zwg5tNHivmQb$l3v3HXPe5QyWoT+AhH9k+Fg7=cXqeZ^Q@jED$RC_j=&i3B!@oKNP|f&^S~yypEkkR8Gez`%)FhK*wwDA3Y( zH7GCGAPQ;3Gwv3Sn0RX<2@Ji*iiUGefzyL}mU=g3JmFxN-2?ISdnb=Un`iAr1*_8SdJEBl_MFEQg%rQ zSPOXB&u&qIL(6D1r!C zlz_PmBw1^Od5Q3*m65<-Na>VM36*~$m5;KKY2;w2c$I@ml0zea5B6q9c2LU~R%uWU z76v{onOm|5mT%BpaW)Uiczt;ZP2)!av8i3whgY%*n|4J_Kt@QpkSHAKs2ytlAei9q-jn~@QC(f11F#Y@YtWarCWT{i0BoF9;SgRX9W{b zeX@xNWfcqy+5nYSLxOmq4N!dz=m`)oeGuvlI~GXJ08IvJQwu7aA8G_8Iu1Ed0u802 z)+eGy5RiuyN35Ake+f^rBUD;AJPKK5ScweTxt;EYS=bn!w=$j#!k8EdnJbfjxyFhq zAP9nE2ed#jI@6v4S3V~RaR!)S7oeYB+NB_H0=7_}xY&*j#hT@`i*zZfLUMh)S{&GVl)zk2Xg3OaEP62m`R;Ek4FYX zFVLm=xu3Y|KC_yD)^mh_q^4?F0&cpfAhZIz7zJ|r2KnGjGs>DVV5}di2hFMm)q1Uj zAOj<$0()8l4me)eIs}56p};l-eq?snvjjD|S+8ISr%4K`$LS{^sfkqcIYqT@-S#_nkaJ6uscs6?CGkOE~l`jKAQ6N~z=B+Sk4k|zZ ztZ4ur3qk?lfFqj^4QR3!kPkqh0L{v>Y5=mfFaRP{vIL+7DF17+Dxk6lptGzQ0Op_z z05G%E8URsnvoXZ8Y5)MqH+6ZCQCEiub~L6{Sfe_sqw$(|t$;w4HE>>YUr?2hhFPlP zLY^3Dp5&5|%rJb6=&C%Ej(sqn#1vV7!&N#3XLg2w=A({IM_X-BeaQB=*){+Pkg-6Z zw1taj0H6YS@LP#%3jiI` zgLqiARU(vj&g-iQxC(SoG(mWpAFELc0JsBy0lsysiT`^7WOV^#Rk#LVz85iMCyAx7 z34>6btcRl#tYz>7RKh#98ceoTp^8B!&P9p2|U0nP`*BVzv>HK@7upoAin^7#60}N>8k~)JH$<3#Nn&H_{pDk zhG%OFP-Aqp&gh-_%DWXj!J3u9@W+`s5_{?|PcrA3WV1+)b*8);X#nQRQ#q-6eWXm9p%OD-R z)5y!jd#ck?gP4f6N)=D)*v9QQRmZ%8%PtEv5PEOG8?mbQ?go}2j(EVOFhEpo7DM`z)^hx%}N9? zfYugWyBh7qMzeYiER-QF(sNzc_f*oZXVPYSFJI`(U-(aI?Sba^qz^@2X9r)-I}V$c zz=X^I-1?t4v&X#limz4K$_v(i@YVC+*khfuWc|M?iveYQ)>bW=5`Z@bkT($!iwtm= z1UwF{z1ep#4^KMQN=?P%V-EGn)@_}|f0;8bl)wsQly*(mwQS4#>ScP}%PFnWz}$(i z%znlMrwz4o##BsNwU-WS!s#=XK>sb&rI$FF#8WKUYgK&S4`&V!*9>FI-csP+@Eyg8%+{XRjBptTP&D{TD zyc6@alZt-oRXtnP%+s@W6Fb?x1m2d-PTb9zD1O=9Bi?PsK51JAn>kP|-kt%MIkRBn zIDXme^SeCm(dp?h7Z%77u!jWh+YavIAf4P3zT_Eu94EYmhoy&KtG8 z8sd~4)Fi$*CBEaLG)1zMnEzwuP3nydcIE(7MI4Rc1rF!DT}A70}4y9YXI-szhj z2cB+<8B|a`PTaYho(7rfyENBoSXwb{OE68>e*WjnLXAx>=wBG+Y<5015MkgGYCQ;+ zMo6*j&E?_nv&Sx3YRgl1Y}r{#>6+tIvQsph4hu~xTAkzRZ!UbC{^@UiqodB_rCz(% z&U)QVqk=uxCzNNaEURxYI1N71fiUZ|9xb?j@3!WJy8fh>w#-+x=m0P50*}2O-jZBh zXZdj6G`WlD@avL3=|L0Are*E^MD5=m!+U@O-ahS@zVVNs?YkR-uRiYN?m9HUJSz(V z;9Qcc&Uz;=Z23I|?EhYAi*;e`N6Yxuh4s$YBZKeyZs1UW!pDc@Gxf6@EAUY#itME!@D?UQt+?V3f7#j2G{#f4sS`WD00s|0JRtA!v0UvEU+ra3blffSZ>#1& zU+TL{3lz&}u)7?8@t#wu;9mB!JUXZVN8Qi_ z8PpB3F9oy2-3YD>L5{;qyJdJUMzNq+%^+jqrwf?ioDAIZ7o77tPxvN{_|<4{0d<#- zAMgRM(@_ZUYX1qh6-LRFnT*}t`P>8T+ztq5@13Pj38Sz6WAE)|@A|{@z+43|9)s;=nKph>kUUE0(s)2Gmu(Mc#% zin)BX5SbI%a-3JNVa2Y9^0g%8cb=xwLJc={Z= zGo`TKMTY6faT|#ttVMC$yn3u^)2lsjm@F#wi?ioPps9v#2-!5{)1@>^P6(x2NWN9O ztTA)PrT;TqDP$8GXzR`onF||oQ6ikk@#6oX1Qw?$4;Y(v=LGJtTYl?izBj{NK361)mB7N4v<1x zK|V2>uw@))CXx>m3=6_>xH2Z235?~Eq#?&27&8b12?!9dI_NIg?n(=?)N%&hmZ;*( zXR={0Ju=NRimAQaQ&Y|OI)cF?i_%agAG4gnk4`%G%kLzx;A#O1g8&8eFHHK#ql&r; zQ~xWi8yho>LhQnNG|&4!32`1apnw8MFxEN>l2sOkWs}87%rVR4i5>N#g=$SU*}(Bl z;K-V@rC{#FlhUwG1qUmFe)De7z%X&)qb?Y*Yb6;JbCeMZvl2-ce5T+`->xpjbg^0o zX=PK+WSnte6h(y*VI+BwSb{A0&1Z#*xQ)}|7myPH)&%9?&Q>>ks3XHV>X;3NMnDua zS5+qhHh^G<6}DMtOUPMgovUDmyz->g=}c>_wFO&f;A=D2?oyERPJG54cQ3QNdjF%& zv^4GxP?AXM!-gh5PB5?M*iG5qI@D-}H?~0GtQIvc^et0C89InH%S_bNA%rr07&Xca zw%cQ0flxyRD1B8N0tmEtLyHCoH?+_tJsK{PJXcYymaHCE%;Q}{|0IPdXFfmy#&5=% z=M{F=Icf2N4*Hz)h9=rfqmhRFJz}PE@^=cqt$H4NqL!M@d$h5$Ppo^@`XDH-OA{>u zRXCwu?7OU6ip8K%`&)ihjzrJf;(NryG&GL`mBU8dbPyxaAabMOMEtvyZr=KM|0z#T zeF6Xm(0^~r*dIi~h>BQl4a>-iNxtEsaO9&TauUeMLU%zZ1q)aMR9yg9*Z(YH>`rJS zyj``@^pq3oj(72)h343&f=e6`7Z_I-wxw#0B>Oi0i`rreb8A0hzS2Hzs(1VKvASPl*!f8P=cPiXn zd>|5$COk0{ZPWq{zk&{V+{1Xs<3=8!QKv|`)uCH*PF-q9v)nOu-S zqTs;%m?Is<6Xm#)XN>gm&^}yo5G#S`C1j}~mIPHvFK@{-47>#q7I2;j42MMefx&QT zImjkamr^g*k4Sh3hd4sgux2J9lY_ZtPpiYm)&(}a!0 zPz_lDWFUEv5HS2oN*hdLH}Tml8ysz){Uobc0m`N@0fQG}>{d`u8l73zP@xNLXw>ZS zlS)Lvl;MI6y0oB?=lxKsdZp?Q2BJ)JgkxPGox(^^A|NJwApe&}pqDR8ww=>$Bmya5 zoMmI$%=y_AH^RW_CN2x4DEO2*_uM9HJ8M;rTu?<_P+K^;(4@U+)~fr{tRWAn$gd7n zpJe6dSpRuefUfln+_{pD4%L7sf+ZMp(grbpB1%zmgLVa-OSmT5L%;@h2D4B`*9wcd z84~YL!C~5}WKc!klHfuWWzEeFpxMM7ClsXktTR8!hbPQ}v@u13j-Jy~2Ml1f0!-x1 z2pBq;l|i;NDbCAy#ia*NECdxvnMG{IEDCUtk->GABaItL;=(S4@Ih{S(z@IqsPTXZ zm1wwzd!9<ULxXjX-Fx?UOR;< zjkCYQ8G#sbA=+fjW5AZ#Mag<&3j`||b!%p`eI0DX5Ev?C@I;a!ewhpa11hR`h6!&? ziXptT(IjyDZ6bL_;zmC5#EWKeC8aXl7RO?__^OhkX#C0P&MC)in1l-um4J7tgs(f| zs}IF1hblS7P-ofP(km*>$xfp-xBoU`nE-@?+SBRwZekPLgpw-omX)($ zRBNz(HN4K?=K>uI>nyt3oU00S1L`dAQ&s&mW2k5>{<~D8=r=700HR9?0eHa2(y;y^x7d;}v^yvu zppzLm8Fq3y-XlG1;;KS3G@Im0OV0d`Lo$?rmT9tIxr431 zgkn^pK42x-dke?|MriaUROmQgyM)y8vkbsCQ~;*vunxFUfdw-`L_qvDh~P)hPU7ep`=|IwKOkdsjZz=?E8k4dR_+JcqKsguTHb665kkb&u1J6afmkZ8%PO1nQm zhe(9RIG~fiqB)63r&S}Wgy4a0d^QsNxk0qdK_o}b%uMEUK}W1ZPz;H$aIBNCBG0L` zDj^H+dmPrZB3g=#!MmR52rrN!GRM@-;H)#=l)V0$JP57DkBlDyGy=5jn@_+Dw@d=% zbjihth{{omU11EW1JQ(B5)U0tbpXMqg1Wl+#tn9?8B0u}V~_zDs;NBn1Kmb2Y@0O&g)OPXA;RBb77L+>FyQ^ipC|QTRcEDue?`2-C^qQ*-LZJW97%0*6dMDq!p> zi|8o=h*GU0mgaC zr%brTDI83BjR(tuRna_Tqq#*00W4rv%g9q=jj#!;yj$hd!LdkHfD8fbhTs?h{0Rv@ zm?khvj{oj7R%P7;JcW#Z#aNH6PmI_Rb)dLQcq%ws$(ziJRBM1mWltV;Bg|ApXqnkY zBUhV!v>_!{n&hx6p@`;Mlv0Ecl9(4H5)3oADs%c1e_1Sl?65D5(_vJGT7ZNu>xA|z zgqi}d&x6I+$Pi(WzE0={m$Q(KbXZ8}hFnNnxr7aw6I)4u1@h7yxhw=t7{FwJ0^Jl? z{z_J6?OQpmxQxNlYW)DVp{#q%I@8(FZvCoAMMRo?+;A0>w7OZ?0fwB_SqrRDIW(ed zLkX-KqD#C)UyXxaWdR|og^tM0CKCyZBv!`c$iKP`eFYFY*jLfQ0|9(FJ?ce4;GDkV zMgN-nJYcN>ebrYRRn}w0j4$|Dzir;ySlWtQ+OtH>^=w(-dRcIFT*yVS?d@Lgjh)NI z+@1A+4R8SR6-HoMBl-F(r-2(Mn89FJ)1(ETB{Tzp3x-+I*u1Su zJZe8Kq}!JZU}WQ8_2~q=?bVl4wy~|fOw)^VdB*1LTQVBU49?&_$Q+d+0?c`!h>I8z zyr$q|x8S?pmu=kTlb!F4VRej(@Xg#gy zHdQEFjl9kfFg;?Dm<|pVMexu%_Si*8CkSW@(|br)8Qjqu!9|_2|2brDEj|~1;Ysa@ z$dzG7hL5lLk{|Ths-npp&fF*XHUOhrDrk_aunXr|g&;a&A_!X)Ak@TSV()~z=iHT0 zh+^~#kEM-j?0R)E22a-84(+pww!p20{0!D+eg#_U}NH7*80sNy5#)4Bg zh2_0tNr>i<#D_famF+YjC&SIbeSyMA(!|juBS2&pCd5USSxBb0Wn|Yjd!tLvWDBec z?`r_^O1eV&fxE?nDd;j!Fy*ktfF3d>i=B^y_%ZB&VmW0$U4Y&215sVJ0{<`zhc8p( zZ%~aizKzV&=&?;xFsy@OOW-#a<2d#sI?mwa>{yn@<~%JgUwNue9@Ih|TvS9ivKh_F zbTf4Js+eVG?rmolM3kP6=hw4gdS-(6g}IQb0RxG-_N@V47HGv{0!u+rj{M(fibQ0M z(=Vd}A%5s1ZOk``y2eaGsUF)femPjE!Zto@GFFXWc)6+Q+Pif*i5);+4FLw4<7lo^ zS&riLiLN4hj@x@D+>0BzSOIBMT%LZ^>=joSrrh0em!d{wK~z^v)?}r2<0`gjGB|4l z&V{8WN49qcirSwGD^P zu#*Av<$V=|JX(gx!fR+|-j!}?mp0au;R@77N|R0LqFMsQDdfZz#1ee$Zd&J{<|o>j z?4W++Aoab=P93h&xumYon4{a)rf+qtZB;|K5SNbqlr%?#@Y>P4tfJ&o<$9Hgb3Sn0DVGQeGd&@4!3 zZNpfFQV@o-B_ma6KaGwC0dqlNn2Jp(vy+)}(xuoz&@*hF!v7BsY%;6^^~se_a2XLd z;q%ohHDdr27cm(pZ=o#5b$ADNsE2q6bb1(cyHcn^Z-#l5@r&q#W=I-Zumf+_&cZ;^ zv}^53Zy`&^vp{yq5xK{;;SKvk^gKP7J)Q1B0!Gsv;8X8`>E?todv!OMgD*(VSFd$j z9|c9g^;@r1S#R|;$J1Z0LS_J%5C*eWJKkW7n>M0z2c#KicUkP6VRJmO+eyc!a3pN! z1ykw+RtTBAN_6o?oo1KUDqwVUI369MB|- zHHUXy*YyhB^?x6DfmeiJpY^Wx_0`CSTB?Aj8g@jtfd6DDIqjA{QB3in8@dB%+2L#N z6KLE$N4}^b>O_~5ir!R_4rjb*z1GgeQ1chx&mRdghhpows@kwN+swhkl7^h31PmV%>K? zqV#2Um*1fFl<}WtkC1-~4(x7||Gl{XsgqYYcnt(&%tAArVR*DR<6T6hnm7?)2Ie1vlWbYQsn&4B+7cHY~@U`Pbo=U{qn z|CGt(zO9k0jswnrATSX6&kubu9DUFS2t`=73?w)(&?Y#WUP&s53- zf;pKYMam;0dxU*jl}ubWCSq6%>LzT^YS-al8QR5Z6!74`hbsyt&|tw(x0D-q=p3}; zd7n^%fpN!U7S3SySpGa&0u2rr#FHN%VL?kEX`-mnlL?V6bK~YQ!xYvlRjK;-^Y7m^ zKY;&jl}8#Dq;;TLYq7N;T^+<_7laOG5Lg9xEx`s~f(2#+dvc`Y9dw@^0+|#eG=d6> zdZ7Zv2SK>_T_=3VDB~2>As3=(!Dv#*AtPKS+C1TW^UW%y@R1K};z$RHkVEcPoN!SR z*PL_AQTbem2Jt3QmLRq$0eWDbH~(gs7wGkdF7l~4A2fXsgWpt7`8Q{P0>W8mRraX? z*a|=;_@{y{xh22=4+x+^gAzuVsDp?`c-LLpbqOLAS&T4?GCO**X{H}e_7SIsjUyR| zrRE43eILXz+G|3tq8nt8y&7aLPPU?ul)mL!B_+pUIcX9Jl?q~ICQ*ly2x5lG(y~hw zmZma>No!7<#e8*VoY~TeC$>a!JCs-!sAmfzfSMbqOLH-*=%bRl3v8Djv@nHE4cNQW zOy&8+6A3&q!y*{NP9y;%ka0Ri2#Ju$WHPhPBE@T-@ZqE;w&r^A99Mp!@vmGmb_#a^ z3ycNGB3GnW6M&UDCI%{8*#FZEd@jKuV1EUI%e6;oyED(geKgl)A#HC27+jX+2P2Y3LLE;eJem}Eyap>ItFYaq ztEE7BH^#&rkzB_fgGNfoBa_^n5}6>msTS(BmpFTQq)EsfD-aRhln8@t zl+!TAMmJycv*(?E9{>92V~I;H5by=I1Epm_PJ^2464*ULWaM%3QG-m77L(o8q$WV{ z1W4NVo{YpU63atX3aCK0Z`_7qO@hR#l%ym`T(5--IZh;q&^PF8Z&HBc4kqHWDO*fn z2#p&T`w*cQ`N{8ckMPC)@RvXRDG`5qDo_v6v?m3iZh&Ypive55uGlHCdyKhArpVQd zjGTog2AUuREtsZ!Ac``JlhIHW^ z$`+2Rd9j9#i9=%YSP&ozk&2}mf(1;+COlLsex5YqGLlG*CN7bGoU_w?s6>(q$Sm6a7;QVyklaD)D&fu9XDS06c*uG7U7A zm$33_u>Vu&-@s6cXmrzhD;4QTL0Z>_Y*VE(a*^`qT#$ ze(A#)#fVzel1o0ih{a^0IjNNivtGsQNdk^$))U}?s&M^P9<3)`kn-)W$Kfk=L;AhH z($Em*e4*PuDBkgY5L!I?DiVjPS^g+7wdnW@eB-O3`O0iVS+j3MpBm9Y4M3_caFcx4 zDBuAjsS52p@H=xTF;Gari}bsiP0DJ_1G&?pE7_%ISo$1E_{KOQl@j0hs=X2S3ZWYg zXaDyc+1*#PqK@OOl`hPyqZz|E#?qQ`Cr*S}^=`JP0K$iWU66nw6S>GnPA&+HEL9?x zm9tgG*Fo9I45ZvY?QYCNo5jDfx}prj%X@(9D>ym) zP3_IOIeRT+c1i5T!L^jdhSh?rs1U*$pnwl75TP!ta7P^t!O&JPbj1`sJssiUo;+S} z6FbYsB_FxdSZ08ybth`5f;zdl1u~Tfj0sK?858QzGH~Ur;XEA996_9Lf+UQ^vs^HP z)r?Ow+k7M8DZp(9(Vht(j)^qN>>HUDwP z#PKMElWy2-FWqC!3N@8>U;}*ZAn$q8d)_iz4}9^|9rffGp+9 ziMF&ubPC=qM>e(MkrRegnp)u8&~3vuje<_-UmNBUZ^HDt?{w6w?6}=<3JVkolZHx6 zy+gS^*XrJ)#Rkcx0`{hFeA#0QMr0tw&{07j018ZG+KbmP>l&|TxuFw#`TxfFqX#yTfAfHZ)^BRIh}A(+H@3BXChF@i*=l0Pw&_3Z{`i=O@Dh7g1x} zxCC^WTSaiWz2y?l&L{awt$R{gqZC(#7rb{(-0(g3h9QrHg`bTzhF{l!3OhgT-qhhF1Xg&yfiH+@%@Zk41)9e@X>8P=H#?Lp0h*wPNX4x0_mo#96Ip;JnP zO-;xhU?@Pwam_H%ohH21i&$UCMZ}5pK_j$CR)G~(aoIU-6SG~P3O1nj!Q4xIUn>xn z$xKY55t^D2iTvTgqKRA3twQ--8|dMm{RyG|F`W^H1utMypZMQWRsUQ?VI3_E0Ds+{ z>r|luCLmy;+~2VvI+=%B7~jwwfVfZy-Ce@#AfCs($2NxT-;5_odm^F)-{c0xRn*^K#Q;$7B0q>VBa2=pbU*-7@CnzoPgYI+)e0S z@Td)SFkc5gga)iYVPrzW425Ss3spz}b-_X(=>{~u4J3vGru32e09UxuIKQOEBMW=ZrR<`gDkCZ-?0 z&`>62QIe8+Ip$Gj1ux{mUu4oJiWwDY=BTw+Sx6;SrvG9>0oDtK2o5}4>%ry3X`I_Z z(J21XkQKlMl_hEgo8AcHTDBzu=!qw0WPC7#1B475Y``x}96cUq04|La2tziZ;N57U zOgfu)IA9r|WymZhb}nZ1Z08FxhbwFYC062Nj;Ca5Vp3M-WqP7hZYC*`Vm)$!3#1~1 zyyR7K$>KEz@aR-*7SnP;RFjQa0n|%|Fc%0e241;{1^`-Q9E#K}PDiZ6*mb1f5vOq? zr-fdqFfD@~4qjhQNpy10@>ORYW+#c3sFgHkc#5ZrlILWWr)BPhda@Q;J(U!)MNPyf z0tUf30V0oXMzO%$mpqMUvZw6z4k})TM?zoTZU4X&kP9KWrA4UbJ@B4%9)SZy0u|up zxloW63=)SD5|jjHB+=w`ny8ub9g3!?I<6>-wrJ}0-xOsCk#;7(_1A=)BGkx%idZC{ z_Nmdik32CFyaXw6p^@zAq8d~sT~?onVFpe`fB^h}lVW6hpo5#h4;E~IreZ0EYAKn? z=5%~%hbrEEjwzXP$C+|x9+BuRq-oBm>HW1S&}@t7)M{Z2MXiEM9+VNC@+W~>35Kou0bW4;UZi-E0QK@L@;X=1c0PQse{r(6vPHW zI0_WVK&So*zW^wiSwSg;fUiWKyecVlng42;rs}HZ>)*NRtFmdVYDGHW13lOPk|o!t zm70=;+J45;kg^WKRspc3nt<^TlNl_;>JWi29CziXM!i73u!|H)&u+l;9j^0@f*^)y(Hia1>TBO6 z?a~^hP?f_y;DeBP8r3RnuNDxLJ^w5*po7E;SRaHfUuf;HrU5mzK_6(Wm(dEcc4et~ zYHV^GSt7~Ysw_n4sXRsMq=u`^B8LZXYT(vgRPn@+nuQjW!iLtNs6r$-dEd~Y(}+Iq znM&?>GVOUzLM3#s_kOP_c(291lPlHN1cq##f+oNToNfK7-b>D2;M*OO7HFvRKJRlv z$@DU=<8C43nke?JV|j|+d7jP+NRhqe#bx5!?kvdX4gec03fKMHd_jtk$?1j!*Hy|- z!lDZhU)fXYp6=0rP8}f1LI2+{0+0g;K!dVmJ@~?)%WNms6laDd?ZR*Yw8TBCqI@>WwdV01KUZfs0GWizl?C!z z@ER3}W(JRxL^f+&JTdF4ZWS!*4Mg&l=774wQWi_pa$z#hPRG1%U&(c_y;>(3gK#=# zuPGlz37@b~tny5yaAraPE}}19_FJTeW&z%Ej*{k{Va5W`0p9*Z`RhZ4ppEU#TCLRA!^OF>0y80_&C>hdmk24an5M<}!$ z`Q}2u3d$xkG+_@jrysdI^UF%}Zw8mSTJZs3Tp5A07*arS!Lbq4!75sh&`w+Q7RO9_ zFsjCMJD(^~o^q_-^BU*0PLG8v`;gU2(9Rx{90!20u*;D;7ZSH(mVj|TYAOw*V)CJZ zQ%f|<(OIJ=Xc%EMCMeD&sRKFyr!`ZtM}J{dmG3dB4zDH%>rO_>7MpXrHHgAAJI^%W zlycMBG%C-tPSoFzQYUqN%A7WFRAL)o2aHXl1k6-Z z^)ln8leVm$82?xlumLX+D=Kg_TpWT&S8!Sz05Ov^1OW4KF>hukah_$&_Q>@Ix3d}J z^(pDKwWu*p$G~5&z(D;pYp1U%+A={ic4IHMGaaCFFSZ77nWbD9@Z5pO0Ha+&!~>%p z8;XU2%^P8W^%o$D6OVPSt*tkIA-MV_HL&#PT#X% zQ-Be?GEo0?!w$*;ge{&HG;WV|Rx|Umo<)gy2*YtVc!$ z129xGwEb;zuI5>%QO`n>L~EaXGqqy;=&AaiebZGd^pF^})Hk2^u{ctQY`6<-WC!+xe)%(1s7xD|518Avw? zj$kiRP8UTM9>fUe#YlV8A#_X4nXEuu48TOLLGC7TR!X#>9!o@vHC}q9{x$$=^K1HkvW-ZpOI_niN$j`uiU0|O}VyT7{$!1Lxsf@{EMZ1R>_!ogb=u)?u-mS=hG*wKu_ z%a-Y^lMIRIVsLCdvX#m1X0C}HAsf!s4=x<)G zdWLN<_t7`KR|d}?reSKQoa1}G@87UKE#_`6)!J!)QR`V}EvP}P=#DNKJu`;na3xqO z*#cFawMg0`!rZ|GT42DqR#OBV0&q6D-koMwWd|=VA{c)FKeh?9BVlqiP&H|06s<3T>k*tSaNvdaWNFvJqFQ3w!fKxvE^6gGS~CIH0=5`f6? zf$@(uRhDG2VYv#ZOMwHs@cj8QjcNd2t2`mJQU%Kny!(_`*&;l|K6WkCZyJ zD!(9M>szf^O&Z-)3I{dK1J^%wrunQPapaKZM`wk|>1UMiA8gRPkff{YR5djZ!#F76;$bKrK zIUz*Epf%Twe2qu~Z0O?;4sLTG$lanOY6GJp3grvs7R=b+Q5uZ{8ua{(Knpx~v! zUK7tSHeb@>7c{mUrHg#V8ON48z!`H08JrnN8A38z0l!R|0mY6)`+%mKMq1gbp-49o zq?>R$>4cd_Hb)b>g z4tDev*ba6LcGw+^mGZ~uA~=#+3$n<~0})V}1zKc*ytYfzMp-2TRj2~V^raiunIWj6p98PK)Y zUvnnbpq`KY8EBn!1a<%fiaweHIzG^e2{fw27V0TY)AkGG8e$G2Z|AtNTt2-9rA&%` zFmXk9-6YSvFU9Q#M@&BE-E*>UDL@J}TMUw@<@`4Uq#A4<_+d7>T*O-6_IbTGHB0mQxWLaY_yC`F~JLV!U)M5KaC^HK!7CkG;64=GAP-}%zlG#>l_4M`2(B=A5F_5hL>r{oK8bY$B%1#qQXfqy6*Gw85kDB;6L`2u zCV29bKjDdh@W2QpQt1IiTu^zE*vcgWj)_g=Lm~8lAanfbPF0j2{YbS%E~dj{m&;rj zHCIgksQ>~291PhAlg2fECs#m{VCw=&$Doyj1$l&_3u^#{>qr1o)vD(1vR1V@{SJ{$ z_?i)jhYu*65J8zp*NqU;!46Ik4iLOZJ_X4WBw)Z!_G;x(a*{}0fC2^sRpKj|Xu?@q zAs6uT1Q}ojzbo<*EWYgGMzz7wjtcXO!xSTHEWi@e zO{MIVg@IDZJtd{X5f#eNSo+{_6?I0eDA2xO?D7#>B+D;7I#-YyQ>1v+t6u=PNXm4k znTKKHVB6Hvm$L3!a}>fUa|g6$*voXTArc=Bp@P-3v@~cB>+aFXz)`h~-tWQBJ9%$)_D#oe=ZiTBW zwowiH*>$dU#otFUM9jPn(^U$fKnN`sSi%ZcbTu=q1mUPCCWQnO&7hZhLvk_2o;Go5 zfIzd}v=Ihj;7ORxEVwYpS*&?BcC#DWHy{5%hcLy~pO7@-W(;BopkT(UmBdOU78{g5 z=9ZvGsV++J!_x*ol&=1S!u7r)po|KrGN29?s47ErvQU0{sP z;2B2{V!i2gpL+9@ha%qLy%5Y3z?k=vS^%mx`Q5L!{Ogea9-*^cVnljkkx)-nCIk-H z@&U9kk3%frlM8T1K1kYyrEOtpd8yc(DHa7z;fA1fBua%l{M)k*x47{86N%>oD&}Ss z#ne@_qA$lNcPI+dk!I4Q(M##PO*#r@3s0Rt(}!~0BcP$S6DA@98pkjxex#|OYF2Y3 z*l-F6<;cefd?U^&&k3kk25>*c{1N}9mSR)nJLx31hep#T0h=rP%L$m>Y#oH`@H96lUck` zSF{QcN9Xpo6TWk$ZiD4PHJnPa&yZ4Z6-WW);9U5v`X{GBNAEf?F;(eT%?)>|K328j{v^RqR@iUaX_X)D(}+7~w6b54?O&w!!1jKE zhDR{sPQHl8I*N}(l<)5F&chl|xIj$X1g+cP=MoZO<^r$!#0|!>@4B|Hz4{6fz7O6Q z09W*lybg<=b|&=hNx%Pg<)?xukhG@tTo3G`MC>S!_QZq)p~L^y&W$u62lHat1TZhq z1p$-6F%(bY~yXDRh7UB*4Rfims&s00rY{ zf?&{r?oSQ(i~gz#_Fm7Y#H=H5BM8x92?Rg{SVQP$a0wZ)q0lOTMoiF5>~W@1BCF9F zuW=L6t0O2NnMz<3J8}$3F&xEl9I<6fh>GE2hxO8t29aP5Fli*Rj4Ccn5vT5U^r}+8?w0nY5+iaVo02TTC<{Aqe+(rA!7nj#%mIWW zBt>$ZR5A{$=?zzs3k=aZR*N^{2NPCt75h>CbR>ytvf~(Xtn6+mmGQ&&k9a`h4+O-} ztk4o$ti}H-QYym;Bl84FZj5xmaVtl1{=D)evnDL(Fav-rPi{c;ZX$##lcyfDEo0#m zr)9>}039wtC`)q@i4s8gxAfGRn^~Agn!tvMU|a4UN+zB@Qwc1Hdlg_BK;3 z+men@0bpRxw&sBl5K=ppk2{%fD)uPa>H|~;;TBRs`Yh)XEhIf_a|^LcZ!)JuPZT53 zXgB|%N%rKEyj;MqhDVV+Lb*2LF%DF!zLLsvZ5LKD;8?0m{7IUyQvP&woVZ{KcyG=w zg9_;~LRD@;^=>^@fl8|sVme{wAf!W?@yADq!(*q7bK!FH&mPOv4lQSO(kQPhzvJ+9GltQy~DrU}6 z$MeOUa&o@3qwIt|kDnS;lvUwVVKJ+3*Kw-@;FCWTb2Osa~pzgM8 zjWJ(PNB#3FFX9byF$$DZ9nA>z3YAcw6hfbrQJF|XlQm-`l~QjGx@eOelFkNx&hh`K z_1$<5-qvLD^esB-ja6OsT3vwNX0I&`no7!;{wvy;I9bgGjMiOclPkI zwaJ_=z3i%tCj#d*H5OkA{eB7nF&*4LUUg%l*Bv_Lq)QgB}{9XIwd9n5J9cP;Vd0CIs3s34+J zpcAOT8S`$)7eAU)V+qQH`7k&38 z-^ft%2;fK-BZA^rXE2~WC13<5pb)441tO{t+DdwaetQ` z5mfaki+Ecm1z^{>Jht0fCWbDUcxxbXuUC(fLBu#1d`{DQzjs5)bA12Hlu~UoMAO%W z)faW?mSN?$b?CPi(ui83cY+VfaA2VCrXYbIXbh%E1o%s;tc(s4F5!A+%aW89*^j7V z4G9rAce%D}@x_}sm^&nCtu(s({_bM10a~9La3DF=Ti14ZRBsa$kJkmLVlixN^=1cMgWBIAaWrgg;qg?QvifD z0gR7VD2SJM6^;syE%{bfjx|>nw$zTPkA+(pW%lNUtKer{&3*q_=Qo4JhIub?yNLyA zR|2rEWpu%M9XPRYc@{M3kc|rw;~+s)!Tnxul1WHOGGe|gU_ti9kZwSg*vSdhBYOow zkNOS~$!P;(4a&M9m91rwpU7~8jc`KAC?(e>QUITj2eiUN6>e-lJLmQy)h z7f&jgM9_2v+=tCt%Pf78jU)ig92ctXw*eXjfibJ%^1)^NL>-ElUd36T2o;?82%|N6 zqZ{UB#gu$QTBJwXt>wZIcw?nmnucuze`%>tM2v#Bt$P1WmN5(AePRx%!{HUC>Imc0ED2;pda1H+oTF27;JlcX+o6&5I%7X_rHLJjV;+ z2AMKJgnD(feSiuKphd4+roG8&m%u?|#YPM+l1ga0KdF+)%5oK1<=*VQp2&R6Cscwf zbN5xgIXbP^x~|xoe_R-^P865G61O2c%mH5DY(cfGHdb`yj^x&gQaC%%Cke3996Kex5>7|H!0wiSqm4esiYitfyXCBkOFq&h+; z=BE(>62#zA;sYqBK&#?!4RcpWG1Iya-~zny$B`hwF#!bRb4j$395`UKA<&C`nZgtJ zs#6w~5LK}z7ta&KpDm>h24N}gHOO0T$jft5*BZay8p%!J7?vD6n$2&8LrDi^!0oKg z>TJpbe2>y&35+5*nZTz7ZEJUd2!vr0*Q#g4{9IK_)-hJYtGh*G7{$HGg8(4$s$d>z zN8SKS)uy|f*J`2*XDa$g2_nog7+f;A7YF}(I4GY*GISj4{v!^ISAIAt5gcR;mI4wM z0o=0+5VGh&7TrUWnb9*5eNB{fCqc>o`vdSRO!8ng)2@Blcf>v1b^p0RBWct#`bvag z5vFLjJE3VfVX3;f+g9PtgxWa5E;vKneQl;i2*7!1#=7a+hwpbOS2r$+1- z8-d~Nk$fVNlM4NeU{9BqJ1K~a=zB-3&Q0IlRzIfydX$t&LhR0Q*LS9x25F)X8uZT;8es;Zh z5gz`46+*Z%%hq%3FwY!IZ-mzqF9ZKv^k|3Ofu5k{3q+T5af%tCiwPfmD2or?Lma4u zoPE3TU6+!vZk4{R(7^x_2wmt$ndh(coDYGPnL!w`lv!0fW}WiU-L^y_eG(>p$&r`p z&8&0MgzIVL?O<(>Swc2+Xdd!mkv=hjc8!w+|L;N|!ufY>Sy06VH}2=&)kx6pbsf}g zakN0r*MkV~H329n%bSQgi@yPgjUVPE0I;jO8MQYT7vI{`Uuq5K&UsoGV4x0SIp=Xc z8Hnrt)!s@=f#?Aulp#yYwmq9>Enzl=3%zN32$5UEh!l}oyoeE)MqV5{;$r17B*;D^ zE0rwi2+5Hm8ajLsa$-poHHH7)tTQLN-dKqRtJV>#oN3Y|4pN9(%5v!nM+HX@wJRlkd6@+ilSo}MY`9CL&JQS1i1=XJ z^1*`;dxdA9(eCCNj0K+3zsceegq-ecK77IzD#f>}An9L|CF?2-x++qX zUJbjJaiVx~70Wj~7(Ttok0;x^FKH<&ORIJxa99w)Kn0x_Z28ZmMj8TYMqq#!M3EGK z0}P1#XDw#+!tO9Kk^rh?pbb1x%%&LQDZV)1p=@ z-KB+LiS3kAjsfr?g$5OBFvBb=WYUOa15ELuIl**eS7;Z8Mg(z%C{q)2ht&02Ya3|c z%pvwMp~aLiO3>UC#pO~1ZDc%gCLL@t;RYI1IM%C_Zd+xE5cqU}ev74=ss%x< zfZ&1$v^WJS3wZkEkGZC179MJ~z^7`RM0nMR9NbYBA_!^{W*cf40HBglRH(=t)_ULr z6*k_OV~#wwMD70?Qqkbj11%iq1WmL=@FZYS=0e1l-$)@ry?(hi?^;|^!4l~%(y ztDK3#a8|I%TP@E$5pEr536je!kURlNpHk$>EFQBs2T3V{a(Byj7(q(%q9#j9X-1cV zlxfSDnskgxodyPBA2y6CZMQlzkbtVLWuKQl4)f<)};%XZfK;WM;^W-6;gqmQ0SlVhMW@1s<(u}x*3QAI$V4HK*CneTr0kV*v4Co3omYoMhpUwNQv_q&4; z>BE*dwJ(X`GZtU;pe&sPhynQt8z!P48X1M?H*tuQpv>?^$mmKfP(V&yXkbJE`NuRC zs-FK}0JFi#$p!~B$X^Ja;K0Zo@Qp+4f*|^Wz!BgPYCgbYoWKySFjDM-UEG5gNB2R2 zcn3VKDEF_<6t)vU8SyI0If=w>GL=oI+ z%Pp9fMQ1iMoFO9T9IJx2a*8or>^$Tn2jR$g%5xL+l%y|0SjkI@P@kIIWC}ako=Hri zh4#ZtA7s$Od8E>YpLC@QXjg^(*nm{U17Z_|_`;wFsU^|trHnj!gI=oBoZ;l9IC=lM zF);X!zxz0%8DxJ!!lf@s0A#Fw6i`)0EoE{5vBQ7kiwOw*7{}HmWWc7S~E9ZVXJMm z5{TQ%6tKh59GlPSR?1SB@}y)38rA=TS{9*G zBY+M+%hmL{z!;etmuXS^sZplQwZ3g*eS!HF;hyxRbKvD{*#eecS%H6g$l`D3GY3)( zcfkxUu5k%l+vOU9xy^m9Bz|#E?Vb>mBz!D)Bm2~VLRG4m$;`5Tc;3sXIK7oJi3=Do z(Iu?_zH1$&eVXuE*y1;~ZltYV_4{L+s-_UcG~bj8v6kB2Ho^LJaD$ypQsqjR!WFJ( z$TFNB#e&^4)x}sPRd5JU%R9t3mqMwr5DH1i2#EpD z=VXhH|GJ0#N>we?qUl$PIi=n776%7r304NtWGLITdN@rkgr`j9Dl7lC8r01)cD3B) z7K_-apm^#p#CyzRA!ur5R;ldRby4po~0#n)K1pz(I&NEA6sov|*+s5CQGB2z8C-QOiEIB!b$OkS+i z=XA?KW2i_$rPaM|cDH-ep$2t!?S1brry_RXMKuhKn9wx9m=*u%O)pk?fo6l#5@>RM zZO(bUJ5NYmKP5;N0R;3taO@H^@+;sxpn%KWx&0FkaG~7omJeFgU3qxOM{ce#KSqyY z+rKd~yabP1EX=&78PcsS-j?nGuk;THOt*0(EajoQ+Z9+Yy65Q)dXXhv=`9;Ei7l&! zsGlmC=>2R&FC#0llIpZXc}l`+;zgzV3 z{(R`QJ9^SXRNplo^NE_FS?VQ#Jwt%Ak_XR6G`8?Hw>SSz7C?P{z%vstD>*?E(sFTZ zrv*chdTl^8yzwAf69xVD26P7)K#+k5Wd(YtJmvyF)t7QxaAYXJ8@6a^)eFR)KMpF%t=p7_4OgYNQ2G0T>@R6rKQg5z>4hm*6=t5Y7rJ&7Sg3^sNPu3*9}#5(2ax|Cv9uEypiE&%ZoODY_3xb2w54%m0gK%oz(wyd83PZpa#02P_Tj%SQwHugepA5RSh|d zu#yk?P=O~vkrugZ(m0YsxB)JCk`&O9tHmG`pdW#`S_aS}xE72zwjb#PB83McHi#|5 zRtulNapVL!vtWn~kZ3EoFGLwQ!<2~SG>JPXW0|NiykZv4A(hPqidU(GMh9wMd4F0d zS@?EpG&h#kSA0!z6Fek5HH46;;)S|+6nZcZ3RO!f*-Uj5k-Jy{Gx!I>mI+fK8YFp( z3UG^_;gQ=pjVc)dilLa`xfJ-}m=RMQP_%6uV3WYIejNyQFV_NtLji}DON3`WQZNJ& zvm2tp2d3$GeWP3ybWgQEW4CT)3r+;Gl<5z%4xC1dw=E*}0t^sT0Cco`ATK3~7fC&?+}E z0C#z4Kd74VVw37=n#|Cjp-};iQ!t3epFn9AJV^!I7l}APRg_p7aDqrt1qRE3F3n+} z7c`Ic*b7#fj}Tg$Ybtsx^j+b#R=ja9hFLx!x-)k=k;h~&^?{VVC?UD`cPp@w-GT^3 zIX}U70}lcO4IqI80FvB!OoIvl0m6JWU>K}If+U%2IC6H!bY#CVV513sVxXG5Qim+T z0en&?kF!zZNGG$9F}<>+L{R@7F-itT5MX`LrHB*>u!=c$qJBO^rWLlJ4yvXmG>Q_c zthc#zPIaMvI#p9)AFR@r!r34fMPI5FS3uw)dGIwS+C6%(HwtxDHnABoU^IHV09L@L zsS+`Km;>p<4DfocY7h(DSsg4zFV3fp&F6U7BP=?JCi&@;ODaG`W~q|bpW~;m#57Do z=_XFenxP{#R5Gltd8YJctZ9m@6UvWm`jy@_S>gq)ctdstg0ciD0g`c6lXf8{=dCc- zmPz6j7np;35ChI)0#^VD3h9AX;5a5=0@Bx!Lc6m&I}4QxwCVB&D%b-wJFvjfuMPu! z=yEjq5}zc1jmRUDl8FB%>o~Ac3KwD*IFWN%_`{{m=Bp62F^-X*7n@kc`e~@MtaTf* zU-_&hYi?vxB`z2sE_kcxr?(5XJwcNotOWoJ@F9iD7eo{cC)f}D;4e*c1^mztmODTp z61kRJl&eswnvf?5PO;idQ{Jg8|euXLB?)0UYA^5}!$yL?)CAOB#AW zl!x{>x~c?K$}v0#1HzD+%mTM@%Xyy0vHZlQAKR?V8oed~r~PNW_qn)Q&;Z<%fxQbw zw5zR8#%!Uovcb3)fP{%P(GMoT4_$I@V-dNL+a-G%0+!3U;^4WSsX2r&5A*sGht>cw z$}0j22mY!HPzwL6Xd$r4#-ER4u!>Ux;wqZCfCsv(8tD=-!oi;~z(Cu_nppxIpHl~6 zM6CF9tjB7%9xJ_{B!#D^w=kfTZ*a1l$uD2=A$(wY6)C=#s8c01nJKWdmJ6S}!M^;! zxni+J>}#~l!JT9jx-&sPJOCCNQlzMBz?wh?s_D8w;*GtceBfJG04jDCEXMpvpsRo` z*D(Tq(h0trysp`V&8w!*`@GXjYAjT}FQmtK!cv7AulmUqhPSxoG`Ko^Q}Kn94fq2O zTe`CG8#S;GO3V-F%L5Dq$zqWLm&t2&q-=~3B4R4WQ#*2|sU^zO#XNYWLqv`M$}cbB z05p)Vrdj{Br0Yi=b3kHhMkK7DCOo%uOvfs$bRtVTe!vDUe7!Eg$4bc}W-|WzL##i4!yH(91ILoj;;1XDyn{c0 znWLN=2b4uRILc1oKx+}gBWDDXR|9*}&n8U;Bix!(`CJ*xybx;82R)&7?2mbT%q90Y zea!y?EI_&u{KeKws+xI`F7$(A!-{MG6mq8wFW|WIqZ;H2nYnNUfHx0~d?wZeKG&%Q za~aqE8#TP52d6qlbwJ8pG=w31!8O1pXH}((OP0iBn1s={kDb8~q%jlhK>_X5l;GJ= zJu)Xe)gSxIr#QXGP|U_G) zmKhCvGq?{U;0c%854kWl#|^4*y&9>dJT>6g2inABG~3x-IdQ;e&*t6SEue5RgUg)B z#pDCzECC!$-pL&&X#nj*bzf?5-fc33a93V3+U5BreZ%Iyr5yBo)S1^uAld30~&ZXb`eRTc3 zl?jcT0WQsFAmw2&r}Js$+AT+ldWhLgKIo&~i`;|mBOuO#AU41Ty?wiki{X@=E+!D- z&fVdFC%-1%;UlmQ1u(Qr`xWCEIAb1>D?W_m+Z(ki&gjFisWCA5X@c0wIF5JA-8a8H z{)Vvf(!h;4474X`JgggpVWQ9r#2f|_=F~~9nizdJsB!jZsj*7-4cRK9ycy#o=j3lK9g-6Nm9DUZN!$FoblV?$Gs2xP(Zzs zG{=M+<$wvzpbkwO>~-1~jZVS39y}iB8hr9>fw+rh$-#F;Ojf%BVR!CyP2ljOOlmNf zEx-b}G0T<7NRHs?84GXA0O}DoPo!SZ&MOPHS=t_9I^gQDPVoW->Lm*y>78ug z+R>EJ@$+ek%R|VnJo0Pq!EZj|&AwZ~aJl=y59#gUZs#u^={^5pvL)j29I7pbJ+LM= z6>oV&PO6JWk@LH=05aelU403&fFN=}83UQB6pR9efYx@fOn!GR04x8(cM|Xm@AdR% z34S192+v_9BPple@GCsMscmNyU-1SCX6Gpg@>*axg=qR`j`jFX}{9KPsAlGc#} zV6p7Gq4E`1ahGev5BR-L*rYBu=_n$A54ZQG zG6l&Q1y=9z-!r?q4HX_QKRm)KnA3@LaHd~xm7g9TZUIldPxh9S@Tgwe!tCUHRb#LI z_HQ5e5pVf*MHEFb6bE=k3=r~Jt*yvTC17C!DS!BLdDqR(_!7{qW73^}L-|@I+mHA2 zbrhWy3^^{x`5;h>LUJ34qflpl=21~=MnjYUk+0bwJ5^j{v4H<6pO!X-Sm2qbj8-2z z2W|WjsHRq~T6iGUiL=PjGFhoi;zG#_nX#4EME&v;296<`hS>Dl#`2jpp3t)W1R7Lm zPijPW7DJj;=~AXCSv`dswU5-OlX|f^l2wBa4Q%{atrS))ShAC1Zbh3`!-5e&)PPH) zVuf6~3G-o6!68iu2OA#zkkbT82ONd{9^PO$G2+AFz;XOmW8a7o`}kcBHV>cXewX=4 zm>iCyr*BUr^zADknJrA1(AnurGSKL*4OOcX`N^o;g zh-a|zLNtnwB(!9#Xi*`dLJp}!Y;8NbLf^gxfuv(bWUT*TC4D(<+0u_COLO+rw1l%- zQT|5#{~zk8rUq20K&qxXkgBn;(n^CczA9@h39Xt>t+g&>iy?6Gn!o}GI`lA)c?hvV z0X+EXE05M}VT`aAU+l}7Z*)=S8$K$l?6Q4i+$^6fJi{@Z?s6&ZwA2RZW*}&qi|v9; zqN4$X1P^;ErO7BjNg!M3Se+sTj(lx~%WR>X1qD9B(V~LhGU;;43)bXAqu(&F zWg#we0|Gpbq<~c*C@&I76N#n>$qYCAVDG(44LoX+V;osD;;(XgbW%wzHi}Y>f#H}y z1T$Uuq*1xLDMC=m`br3{NX_t#a4VRA0RRGk8G^U|7;gaw1aJU|ENx)fRumi4L=Bh6 zA(JwG_UWf-r%hOTvVHK##R@zKab}ePa4-^xKhx-=wNfjn4Q43sfOR8&%Zv-R3JCK0 z+bx@EBDWV_5RM8&NJF8_8VwpxqGd0^fL{N9eQrh~8V`{ojwhH9V`4%Vj*mh%Bu2PB zLnGz9^NKg$ICO?V9vu{?Ou4GTBEI_cWKppSdITe|R-kqR2C&`c5p#Zk=je=kz(t{9 zlwAg&jc_LE%9wV#X&ot(;EF41hyeu~o$VRtgTU7LhAwM{SuhNyD9=s_TDaTp+E!RL zm?^fLLi{BJ17b+Iy~ZR8{k4dqePz;Z!L%|#2(Zb^BVBkLLOBiQtO= zI#>H<(E_#+42xH*)3O8+0lnP8j8HgW0wFhs1x`+Lr8CPh&L9(+NUmcV#2_FASwRDl z4rFH`nWs!vLbI%lWg1#xs!RmKobgNmSVRb^ZZ*8~dFKtMQr=`d;R#@o#tKEGObw(* zJzZ!mS5k=ACld$7DEfd0xAY=%1mO`p0Ff`aMCuo^N z4{Q^1+WbHdLfX;9*wS&-JZ2D7x>8(z6s6VtB}}#XQedXkrgrG2!tTfvv&@GJE+A)4 zE44X{p);L@WalAO=M{Ly6IAoO;SkPMyAz;EF134MKdF!b8vd?U!^2^&s%cP)8dRYT ztwBQ@TEx@fXap2Uu^+U9Pkjp1K2q?qN9D<8OUF~rhH$Aa(4!)~YqArfP;a%UtIwi?)p2N5 z1hL&xm!Nyt;Es|bCP1bHs8(RPNgz24&c$F7!{EI&i=>RupD?ex$s}wcip%^1Eq-93 zz#{V^Mq|WA3DW{OoGMSu-EfSvU^GO0<1UDmY39Cb65`B-}cP zsM1;YAaZ3H5bbDPLD~fgII2@!tt2s`J=jinlbR%O(W<7(F(ufy4z1#EP6LH8tI`U) zm9Ui@aw-E`IG6wPa|<*6%i$SxBExCW#5_`rD>1}HnzPL76X>VEKGBw>eFmFgGI3+U zc1fFOz^P+3m#7|lMF>CULOIck*$bX`$V^VElU?+nC}(=Uo~CC#Q_G=9Zl@1r8d_Mv z9J{LRh9k*>Y!uTmaFxVgjDEjfi{zZId&O z%WR_q!{FU2fE1Yu;_^X+i*n}<>9)Zv7E&r>QdY`Pv0SI2Y-xL&S9JR!>gRTPYLAJf zy{axkmpZ-hC?~wO#$&BM_37><;JUfGuI*yHRS1d7&ewC5K%J&Wy?u<*qIIa67jCqQ z28STne7OG;u%&RmjLx|g`})Ge0O}w!MH)pG_`pq#YJ;a*buLmk!}T3&QprvW z`NVtPkH?`CU3|dABQBX(X$6q8yyOS26N-{!1?q+TTd_gnff-^}b5z3SH%}bTvw#k| z9o--pVM1R-y5+G{T!|!oAvCUDk$TNwF`y90M1Jl9YW-Ky3ZX|9wnPM`MmFrvxtB*n zO)<@$J)IU)``SC`L8ZRk%5sPGt}!H>H!uXn*Dqi!_0V^WXFOLM9|9z(M>QuDUO86a zA+rA$0EwzGTa^MEaf}#$HLID=20H(_S1j;Ci_!pEwit_x$d^VaHwIt^t1F24p}lXo zrJlPBinzRb;l4hIJvU0gJ{co5nHGTn0patC^ysm%6Txfhx1A!j6GT2uOTL0@FYiLQ zp(7cfx((BkJ`ZpLwMiQ)D61!FfCiYn+3AiT;JNOzm7(i3YB~UAGrx`_fs*?TCaRh{ zsSGd31!|hB>Zk#-!mlmChdb!8if96S*fTEdyxTZ2N)v(zpg^Rv0X8s*#0fnIWQIP_ zzu5whmckASTYsmh@y}y1W~MkRgj%0RIrp21~7rY zw%V89xG#f>fTD97)(eL*qydH46Bj55R}rlHqYX0*C}qilN4tmJ(TJKpTjY6oG4TH!mbXax}zq#58{kxOM-`pm&5v zNQ}ov)T6hnM7pa--7=qeag%+tj!}%ZycShg6gS(rs29pUib93=*{jX7g-DwpoNOuEcq1S~ z%*0g85wtgS6f#95HKClc$*c-ZI~k&+%m|tTtw2gk^vur$P0&=dI&p&g8cR1DKRZD` z-xw^bOv)CKGX^U^zVNNRDKL`=oH0x&qv$!bj`=r9DbDGc4ZkNQp%h0*T}1V;gi z;@b&sV8>EBGRfpo_4F>wjIVKuQ9eS^ODqEV6ivWTCa7G}Ce@|*0Z9LpQvVc#$5Eyw zZNLzSgJ1hKNO%Gf-A7?kG$`<}!WaPrQYI^n!zpFc6Kzu?^-$-O)2%Dg4wF+E-Ovq{ zgZThE-}BQ!v8=}|&;G#CwCmABWxl8YC-!92qC}ZL6{kJLFbn^jR18zjI*6ZT+EYB; zqfUL%A+1t04M*o3)9r+UFZD1IRa5NzC{-2JQ58N=^-fRo)LQLS=nJPl{nJl~APGsB zUj^1iCAg$j+O-8-v}N1Dbz7`8+^p@}y6xJ#J(K`3 zT1e4axh31frChfa+{>LD%WYT3qB4!GT$JV8#bw+@#9Q;!+W_%fcum~TWnIm6-PeWP z*k#?-P28asU83FF^nzL1o!s<%+rX9G;00d4UDn_A-Q5jcl~t$Pg-nS>UdcV#*g8YdEned7-oyRg_!Zygt>0`=Q~>T>{e4~q?qBw8UlQ1HsBo9 zUBeyV365X~rrrbhM;V3TQBBtWoP2ncKSow`#_r>Ba z-r{O_*-1Ry97b9xX5+E- z<2Y7dI1XfjJBKOG<1xlyBd%YJ{bND?VnY8eKiUbB?bAD)trr318Xo5=ATn+|5uwcnw3*NFb< zcxK@NF6ybS>5@k3oYv_~ZV6!)kdk$3mnLLEjcK8->YFZWqekgIPU)Rir{!a5u!d`( zmSU!+>Z?X;vu10p&S16H(R*fU&xK;S7HmjfYlj}{y7ueBzU#bR=NfhE?d52wj_aU4 z>%*?-#5U^6#%k1kYsnsL%GPYV4sFI(?9nc5jSkAdp61VnXV5n7%U*1mPU}qWY|nmb z*3RdOChep+ZPDgy-?r$@@38ZI_V8Ma)5*|djP@%(y3?W7|NO9sri5A~o z#K^ItMSLGacHB5p9!ZlZPnPs3GGj`WFki}?39lwin>S{zN<(Vk4pHYL`!XXBckt2XRc zx^?l&&C9oM)wXY^-qlOE@L92g4SQA0*so#0I1%ri9IrCv#Y7WpZhRRt=g%%3ZwBqT zbm!BgMWG*|SyWe(ahy?%TdY0}oF8xAES)3nL%Sy!r9u&~+Do z9)0;=>(rfR|IDs?ZH?NuhyRye{W|vX=)IRuFP^=7@Z{mQSAD+z{ORk3e}69@fB*RR zab(|s{Owm@eh3<1X+g}`sW~o8e#|_iOPMb zB7-ch=;Bo%o){vEGL~rLjV5l`;)`en#^aCN#8@Ma2@>hfksgki1LQ->ImnX8J?*oMRsCI=bdcw zsi%|@25R1zgqG>2p?U7v=c0@@`sbrv&ROWBlpboSqex<^sHU7k>ZzcWN}5!th?bh^ zI;bY9>Z+Z#%BqQ<#{ViRprD#cs;#eHx@oSi%IYhvzy>=gu1xa!tFg%Ls_e1ME-UP; z#p)_;ug+4-rjN~L`{lE@PD|{!*21c-wSi$f?z!E@>20mywwr3Q@Wv~zxt^LEB)vPK zE3Upb-Ak*#BCb2{y8;u8Bfe)Mtgtr=zshjK6I4tn$h%m+W%OFvmQx%sWPm@67h%Y%#a#PHS<)-JBV;$03J9^wG);opdrU z&*CyIC9eXt)Kk-fO4V0qt##JRbP|i#Uw;y|*kqS&_Sv8Og0|X>q}_-pX0zQ(+jF~Z zH`!h1t@qx`T>rC8-+%ip_~6!LLvzK5!pSPZTjh8r>^?yXus|H>yac9dnvQa-uCUb%TfF8i|F3_@4yEi$0NcQZ~XC= zz~YGV%v0|C+{;j|ci_|$&doK8tE;h+&%iP~@!tbthWO!!PkZX6Yo3faCOYwn^D9IqrZ`1BpnwDrm;e?@Z~#|8sK3Q7UQq^I zEI|av7)BLPV2fu&V;a4<#x}a~jcJ5q9Opz(f3Rlg)&t zG^a^OK>{%l*TiNvvH6QdRtsFrxyB;2Ny#T@@{*ZMXCt(hKR*1S5cYFMD3#d0Lg+%C z_ERMvG@<_nT{b|Bw8Z5tjY-gA7WAO7H0UwcxX^~4F^^(gqdO-`(Te_$q8H6*2)}oR zN?x<09t9~ky}7OJB`SKuGp9;B=sVxBw1X<$-z7??KV?KQe)1HhJe8OT`5Q ztfCZv7}7Pk^{pa}t0S;SQl2?-LBNRQNjU;W~@ zv;PgSfICY8_SW*Xu?1y&^$^yhhSv&9)I$z(F<&KC*tYbPrxiK6hUuCDZXhnMaYsB? zyOsuH400|x`Pw$>`oYE2g|QqkIfd*3@WunAVq$m9<7P-$3qaT>A;hc7q8b1I4VZ6N z@he~Z&bP_$?Js|)T;D3wx4-`NY?cAcWh0b;0IYQ_k|XE|36Bz#ny3L4B3y$}QUU+M zctEo!;#^sIs=2qn-RLBH5xrD=IK*{c*Na~qoI3@ymC*zO(OZy~HFn`u5>i@M6-BesQ2VtWD)EcRGkx z^a!I%X-E^pa3`g7BCu=YIA^-koyJ6{MNQ-)qdL_`U3IH1YinB9y5Ik%GOu;LYl3gq ztM@&?WMSK2RnFPr&{hVV$GSgezuBkJ9v<9)u`*!zIoog(G)N0=V&51{Ua|j(q!42f zV{@PEXXm_`nOkbcDasyytyz zWz&mcj7wd}5H735Pk9Isa$IrO{;;-x=TUDLOirhQxw!A}HAuVsT6O?oTB}&`Q{H^j znfNZf>#Ykubalx=ANs9Z&1x(wUGPX>y5bcNcp3cN2=jEJZgCrV-pX^M%P@2?WHGmp z3mThd2j{xh&T_Y(dr>U!;M`}rzYO19(|ISvckAs_DzKKTXDonbIeW@%ua2e{Gw zy?Eg3I<~FFJ@RL7Y`s@>mD-h>!_7+lQSyP3h10w%0L|gh1I45YB8UH?+eQoOGTK+f z2qD{R?)ul~e8`S{{yu?VfB^oTvS_yN?vGCVqz|6ouSRtIXM6;>WfX7)!nRe+Gy^{% zTokld=@fwy=ztxTJrOeufA$NX=LgsWevJfk$uc*@vrdiXXBN0BjG%h#heWTJ5AO$m zJV#{zL=!>R_=2^-ZuK^Q`GyFcCIdd$ zZ$F55b%=*SxMjL9hXFM~5SWAagZb1>+JF{os6$YVxzhdlU$sJM#%){1+WYwfdX!5~Yu^ngf6h=bUK zAJsR9GlrapQjF+`cZDK`rgoF~h2-!uCIb!2$c&j-W1I+HoHj)~=7ONMcaKGTX66Mp z$Z2%=Wa*WPtmu2=Xl21yO9g0nGN6W+SJD=UgBFh;B?@$vefJoX*Q1XsB#gs&lMeqQ3I_QMVW^Nk$#EgKeyoQF z7e`wyXpIy}Q9*TPqk~?RbzT~obm8c4sHlhJ_=>|vd|4I@)wqAOq(6e-K`fbzV;Ki? zwUB5DlRxPToVS77urvE2D*-u|BNKBeB?X&^jBDv}*QPsoxi$Xt2ANyt$A zRg#@~cwy$1R5qPUWu54lovKBL0QFQufSpZc0$Ts%38q+hJJt-qrIMp5n(NtNY`L4K ziJHjJXK+Pmt%+g-X(qBsko?&UEn=5(;75PiR~_etVaI6d=S)LMlsSfxp=JXRdO;@; zWW*_o!+BVVRbS*;lEHRaWd;EVc!0rBRL;O!CEB4V+ILZR046Y^>v%@9H~}B3qO_G{ zBif>x6{GGL3|D}nA&R3E5R0&b(AfXW&r+KFa%bA#-#6;f} zU#0k%F;GwK;{;qOsPixca=3N#;0b@KsEPl&j^iL(g?gxM;DSa#sg0_qgG!!;8mX7c zsI!QvkLm8_F!!mw%r8P!sw>5d()mS4tMMkr-&ZMBLw@-;W zY$X1gD$1f3Ufq3!;;i2{3C5 z3Rk+RtGPkOx}aMCusgNuMF>9EPe@h)<~0BXU;ubjnARwdnZ!H^+nx!_s_Xf-ZsQ1a zi>oIHm!EQC|Jjq=n=>C)85R4W^UE7!P=+m6j}07P|aFFODr z5CLPBzh0%ks5<~@FuE&Xx$8xLo2vr+o4@`Gz^Il643N1Ekh!P}r<+T&3eW_v`?564 zy8+MyHoyQL%m92uweKUAs5-FhDMRIyw(VKHX&VR;E3xvzrQMsi+&f+P0B>rFl3>M{ z4py=WNOk#SeWUh#siZudl#QKs0Tp0C5PH860HFt5f0LELRJ_0f?7#(_#c>+NR3HEe zyuk#3#Z91aT7Uss9KkDqWHifwOgy2O&|_3!3{5b^*tG~^Lpq(<4B^1QPTT?&5XupN zze}vYt4mog%Lg#421oV>4P3?$9LokY!L(ddx$Ck9fXjG5#%7!bziYs=9L$~!0!#1$ z#SEbdl$d0@yl9)p?YYO!+@!pCz1R!Gmq>;rz%E|NO-9d(a1<3@ShXl+m*R}F>(j`f8pOz)%+}aW zkkAS36pj4Dpltj=Ra*fkV6`d009U&N5g@JsV6HWL31vM1Ml-eIpwK=Z%=Q()JsO$A}?%6vTwXx$AFz{vwJ3*WE+ zwjd4>@CT=j2fYonP%sY+2;9VNd*+(gsDJ_AkO#_r385+2>d6SVnRiVLmP&}&iH+TL z+t@qJEI$7|!xtNdl3m9ntWqfePv`teVP$nZn$8KG%+@`iqFr;O4VLx|YNJ-kLd7&` z(1!C`3rk=$Ho)KPJkcFnKgxOFo;$Bkg>N(enWmW(8xebZe#C#V@CczM;_%>Wbv))M zj@b3c-7JoNFAmdbnQuj4aOjW@>4F~J%6rpL6QQH*N2$aF&|Qd?PSYX9XK0@0^|HNee(Ew_ zP8@`nL>W5rbA#%;zMVAPA8VoQ7MN9Soc+T)l0G}dlX6Dv>oQH)yk52%n5wC|ws)N6 z%&z4%JzN3XfS_Dy5;)nGh6)K<*)BEHEqvyqPU@z9>XwMjnCRx_^qXq-KSe2|;M?wx zYuibUKNU&{eakt7N$cOhG)XPKe4guzyE~0Oy)vilDHsVT^*b`HV#JQgfsX7AuYm+l zXlUE)6c6flEZK-2?dHD6f56F~Hq;Un1qpgxXUOd`Bi%IVQZd8R31uJl&-I|`x`^PKKERbFIc8?9) zu}#wmRNaL-fBAy zDX;Q1@%wi_{4W#wAkO!Fzgva5{P)hU?hbDdPFPGNO7eU)B3?=c#P5#3{a*j0`M4tn zY|E;T5cQj2wp-w&l0SllWc6pu{(+7LwW<1ekmaCYO5ZG8ffRvj%LMfg5Sj7-fxtio zAxwSfJYgcH;X^sfBzm%i$cUOZS}aoZG|yb7Pf*N^Bv~@huWHt~tt8jdt1nbeDSDON>A`E8{(YbzqE$X{)@8iab z^{N6B$MO$mO!;)Vd>LQWDGV8gO=GK57L$wt{a}>vl8p!)X1th%iirQihY-yLYlg^H zl&cvr<}tgJkQ~uQQ=jRzZJNrNIi*k6sjm8Un?bWfgGSUO3@P98am>{0tXA|<&tFBK z>8cH|<(E)#TM<*UG!>+L6{yE|ITh53tB<$Zx=X;hq@cqt1{JfcF~9`d1dhrk+v}6S zic-x%$y!-#ug)+_Dhkd@97#N=fWVE0OIpMawG~z4qM^EOv$3e$a*Ie0HGoq=g@O(c zl0CG3NbX3IntKj9>Y9wsN$a$ea>}2)D~TbyRyYU|_0s$7%W`x%51h2xgDtiZWK+%Vy%n=3=z{5kCt-AcODf7@9Pln=*!=nWlQxIuR zHwo+`iU{_bV%H_ig(6G38exm8wH8SV+j&q)Y*N5B6zt!?76bS}zZ`wV(mg;lgu+EM zp-7D*jnvdPDpr)O%{x+s$dXuzkaSj(Z)Avp2n-lNfglTjfCLIAkU3_UYeoUcm_fiK z8)akidD)+PPIgLLpnXPKh*Blt+A+7K>K?Jqszr^g(CYtKX>#W+>rL1~YWJcZLp!%F zHtkKrBaZU9V?nejawe9#Uis9qgc)sRV1gU%OkfWe_FI*`df-6_B_tq150BKS*2deS zKwNFc7nm=PPiRne#=vgtWEQn7tCkNz%4K}?INm~efB*tunPr!0c3n#fN{}6P*>~pI zXWZdAM<=0&o{l@CxkK7=(Pk^HX{Wi>#>+R*yK02ujfBEnun3Gd5fKh>{0$z()|%`! zu}a7y{05yGp*+ebZ0=A0>g;a#-M`d*hggj?eh4FP0SF!~SY0+>$pBFPBSN(j04nRr z3KnI60*W|CwtRd;4{L(aLwukF8~s57Dk}g1aMk|+A@S;UH=AATOgKB+r0?MaT){0CWIoSqs%}!Bo16~ z27N%{;&k)~t0luUtB7C|1V}k4=JvIs^-5oC&ZBvR!Njj}qzD)lODt(XRurh-X0h4<;jTa3>a1e~dAdG*B=@Yx) zgBHCk4nmL+8m;M&@PtDg-9Q*ow@@n$rd6$$q-r_Qi7v=lutIKtCXp=m8nHCNtY#fa z9zCem4z_iwm4qu@6KdVM=2g4f<;6~aZ~@A`2?Bv_Peu!SSX6Su6;()Ix40CskC^l$#38GvqKAlJ##p9G zys#V(aGOYi@W9(71%y!?E7?l$3Bu5(8F7%|{BZcW9=5UnL@48Yo_MQ(K?4S_o|~ttPqE84IW~-T(A5!#OuCeTV;1+x8~T= z`^L17O}%Lvrc_0y0s{Y#EXqMt}IxD1klS zuK=Ngvsgmop!)`)UO1*2_CZf|K?g>L0^56_45795ja-oW5Yes`RJ&c{5CT2AR%N2c zx#~F#&LVgp_vC+YgM@4Qh*@AI3BEKEXGv3X+=Dd z2i9qw>s{AUJmvpN*RXp-_y;9>@wwZJ88HL7vIz&r?WBvre#Dj(d3)XIKBV;dW*n|o zik;H5_x_d${Fx=$3=DNBblhQ|yey5ZW!|Sn7+(3x*lC-u{op|~aGsUVdE4)QfBXoQ zspGIFa0(Rw003wMrLX`5d?gGRKwt?l)zhjSnE}P?mu+hzKvBN|EUq89fIZ7Sw&Fb; z3%-QfF|bQP<2$}r0RM4f2S)xKjZ5@C^lowZ)>c@S~G%qK|ij z3->6QhQNFaQ{U z;UIuCB#!?}ND~rEGh!386N>{u$`$ii8%|NDdFrP)axO<&8A#f?aw0(!a)CVfJr$fQ zMI63PTEW|yrjRKLeF~3>nZD`sLA1j)?W3nw>%RCJzbD{>svr(6ImM;y=pT$q>KsF!pMufF8o4L69$?4za-GAVLTj@x(Wkm!zX$!D2lNK$dxl= zoC!oAE_p*D=#%|gw5Zy!16Y}YBgEaqH+?ff9E19ou2pbD9ut#`<74Quoe62#0ypXY*VB|ua zYrFq_fv_!{5IQ8WIq{DQ7=Ss^qLAF9T(HJP!T?*5x(TGf+87=Ah>bY>01oJ@W+XBV zd#-Tgy{{ujpBy{AO2-qE3z(Xv@!G6v93MASN@O}FKdA@{c&$O|JbFZ{v<`oAq~$*tNEb11p62*L%}9NtI) zt?~dVusw65OV#@&Cg6eZQ#2Cefjba~{}Lwm&^kWbNlEL;pDah9R0*N1kn$M377-4o zLcM!Cy!Yl8|lqmd^7*G zWXmleG&g{Ri*$;dTZK}POKjT$y8HyIQ=13~B^Sd>Sggd?pfbV~fmoQW&^fIU=!D+T zJC8KH{o5>C%RSF4M9QS9aYVsIyv)qpOz@B=!O{h=*@#;KuYxPM(|j_F(Vq9p7OE5^ zXMmHfi=@tU20c*Aj*QTvf)xrSPKnG7$ScmjfCI~WpgK4z=bX!y3;{-10Vl`@yc8f` zg8^oUftLEqP%4dwn#dE&4E@|d{Y#?G+NUuHf;mY|aS}n@gEV}bwD~+zM)aMcfkgYP z10aI3{}QduB8$qZQZ1R%{4i0cG!BVKj=(fgEZV&6Sqt`>N8ut&brdvRWV8Pa4TW25 zQ~D8)2W1qyNf@~-#*UfKXupf#11=wQ%u{ff4>J?N0wCv%= zJ%gkXB+|+}H6+bc%p8h;3(7F~)H*DL-S|PS0WANhL3xbFiID+s+@7bqL66C&T*4qm ztCs6g4hbchTvdfpV9V!>$cm&8VenHy=tGF<2HC1JQn)`f?M_f-tpz$dxd{TT8iZbj zJVYgtc)~yf1Az^cNm`{bZ>u-yqD(}@)J@&gB|XRIu+M9h)A#tmRuxc(v_0D!9|ZtC zy3@S>JRTgu4YO&B0zrkhsI$7sRfMI(IoQQr_*G!t$WRG~EB&^)TLjpYS!4&xfDQ==p(r&K-Ra+T%6lJ4D1(3u-JHj+3cNJO+8`U>E4?iU$iBh7A7$d z*nk**M*cZN6yOzRf|;h;rIU05PjJu+8deezlkKZtS=kh4;9TT&R#J3OFBq&ENKUuu zm)O$XRgkUPU|=oAVg}xiE1oCVl^{weg*b!YlN>Cedg1?JY+L&aSq*+;0T$lN5a2|+ z+W|}&(bKy24BWsqVH7@6#8qK-70N|ZW5<15H2{M*7cK|bn}y#9549;bPCl^#{JG~6L=xuX*Fo1m24wFAO0?V2VDn%XF63<% z11mVmmD}M4GD7GfxgRb=7_j6~lSbj1hAl!byNRO&c#)F0fPXC7I`t|Lp>4L=5+$l ze8tHL0$gYgP-&j#6b@u$5#&wl1C)vBfs$Mq>t&GUCnosjLNK;Gp^xuN26us;b1oot zhLdI>p8mQ^s=*Cg>`p<@10c+SZ*WW!6X1ip$mRTEVZfkO7HDw*#y#ljS)OW000W2q z2Bz^-#@ywPe&Z#^V7LZmw|&&JF=rv_(zezkKe;*<5Fa1`sF^nDe5+>pRB4y~Bxrz* zw+g8L={t~qY_0O)Tv+5$rjN>Nfyz$e5tue~{)BMA048{&oAc^$aF;^GAhq>mi5V9> zfeNUmY9_*BQBBZvK4@Alg+b%uIKypNm@WSb<6XIMv3;g35`~IGh+5`l;|q@KHJ0lz zXulO$iYu(Cx^_J2?cG*`K8EbZ5EN`a_G9j)=ELR=OsG3PZW&+UW)o}O1m!>7jj}U* z;>ljl3?c%lCEhIV-BnPgBaqsPCSo3T?Y6rXs4#-r&H+dO6vyPE8sWh9Zaa!k!(M5`hD4tz?-Xb3Nu^1f z*3Q~nZ5_W~-OYt=fF55i#=CW6jx1fdMu9kcigim)QX7H&{lY_F;7dVcCJ^9mSY@(4 z&V89`30lP|e`pR@?uzbkq>f0$G|&G-j#LByJ$-Fva(#0+U-9tm-oj2&mIhy9z`Yp{ zH$3v-46yN=PRwi`^zVy5kW~T!fs{sYj5<83g&p%zOkFs-NKSMJ5|#Ab{q1hJzkp5# zV3g=9CkpbEi-t|`RiLn8eFRij1i;8#&XwyTPwp`%oEj0XcM;DtUvoWc^F4-H6$kI| zexYVr2g9O=VmEeVPxg*lc5OhJJ+C*-)3#=S8&IgVP)dvIaS_9!h{C(Dv-UfR~V%>u@7@7An3f-f!}7KZ|?pb2P+_W!I9{D zr-XTj1CH!>PJsG-pL(jd_b^}Mx1ROoO&nuG>pNHq>wQ$l?E>{J>5xP~1mvJz&t6_5 zTqUb>7eB8oo7`NP+_}5Hx>u*W-+P%pc?g(zxjWZxQKj*KyMa?DDVvD*oyb6-XfP;d z#nwFec=rn?1zEp*yalTc|DLyCOGQ9VuvYr2kNVLs{i_#!sa;cF-9mTLbA8ly#LtA( zGjk_pIu^)1v=2JfiP`^v0`_#JhX~Sp;lF#m4}82Y$$b@mzZa5}Kb-^8G{WbhX6Jhs z5T7vNqAd7jL+huXdw9jf`s5yZ=FOr)Nc!+!1dMbBX7~$yZv`)BdU!8=_?Q2G&)}>N zjHOqE`geMJ2Z&~vDlo{DZ`G1!Q80ke1@04(AZW@c5kW+aCM^dDNH9PEK!5-qHxhs_ zp@B)1DpQ_J$$%9!ZscUrtm$ng&YU`TYU^3`YXPnyIW91u zfT8!6-%(OfVPVCIrMzO=Y=4s{e4jnp&!M_LW$DfW?$hU{#m~UBS>IL0`dn7#s)8a5jrYJq!W|4n3&2Vq_k$_~M7bMKYom zQ&0hk7<-gbiZF(m#>9@0bkfOdj|kIAH>E5_8W~*oO2(!p6xRwTqlTFTsT9th9W+kU%$^p?|SXO7*ZWCQ5fkkb; zscmN+loOynYLU8@N5@yN4uGEc6$ z!h-*-vBu`s+HcHqD#>aJ>t-#*6^j{pyzR<+9)@q= zt0>kSG5YVo0egKg!eNWuab$#{FaZfIr4Zn@Ytmc7*(8);1T#`ZtFs)smhAC3kks)o z9=J+DS}uIFY#6S;>``0H#VQ+0UN$Fu%9uJMv4+V$J@qXmOKw&k!kYX%d6v8lU&<1Ozz^e$Td zw=+(pP{$9RRzLVRs|362W>mENurRMR-gx9Vr%!fpu2rt{i5)h9mM6=l_jXj@3UvR6 z3SwqU1PuVl1vm5}cCAa7>tM&a*~QBic`8(S-k=ObEQEI!%$L4^_qE{>uY<<>kf@fh zDiJo}0?>orNK&Xm3ov9}SUn|hVxVT7&MRKu4HALAwq#!=M ztzr<%n2}J)~`+Ff1 zz9>L3nxc#cM56+umd5{7usLru5A)>ckve+7j;Y##9*dVp5T(J7Z?xqgPhy}s7IGvi za7bD<`JAwM4~T{d*%Jz}iW(e7M?DIhIFyvmTropQwDFCU@X0vefRYrPh!s9gg~}y@ z5Q^18(H2BlzgR*b23L$i{mh}rDR{A>VI<=MUkA)!GOaypBov|Ml?Do;Cyu*2Y44bM z$G@CGnlAkiL={pmmn3s{6S&$6>BWadUUQt$ROJ_tREw6`vu1q(WwGvQM8>6W5j`8u z;V1(&OvRWIt;HXt0LEOB^`gEsphnZG)?vo0qqP5fASR-69+Ikw zrgqgGGo5*`Cqnh7FAX3FJ|_YM2%xUJlfen^bpjTq@H`N3P`60egjXFQVOwz6Cd&t# z$QjY7FzYO6KY2u;_LWmoy%HcGcM2V$0#n9O23Hx<1pQ5rVc8q&E}xJ&jAj(AzeM0# zIr>pd@M5O7tKbPOs1lNF2Tpp0RA1I$$`@v#y4A(rb*(F{mzsz=K{G5eO*+VhUNh)PacUux3FiM-@2H1sv9KP8vm_O-&Ysa=vOskT)VFbqBUV*y1A`~wB2 zV8LB5C2xZZ+zG#qJ??nNIx?)`4X=yC=#US_=<>$oUJ#`fIOP9W7V<1wFylX;V9F<$ zfsdkzn8r2EkS^{*+WqS0yK_Wacynsv3-E!1Jhtpok!l;xst>-QK94vVe2s zqn)9B6H1Jg39{9FhZ0Nr8phXV%h)VSkx6ovI12vx{+8!n1zC$ zOU-Ym-~A*e!&|@njVdfoF#{1fqYzYZ0v~GNZ!P>0;IOoU9sphlghT8Kh|4xM%>3J1 z-{UOh_%+6lt#M;>JlMisZk`DeZEfUw+4K`Ru>0^~j3FXSI5ULgq}XLJNz`g4`F6+I z;=AVbIm|`#X}a}0N=eY0N%H2jKie(kcMtStr+$SSE(Y5MgE0l`6nNokAijVPoa>sX z!P;)P$cWqe*1~kFHk<ko>FITE0c~}o79TQLYImo2iiJMfLyBCWgS;F zMp0lHxCkA{J1d~11~y5YIr7a!j!RrX54z++o#g-9bXMu9d{VuuRc|TTduRP6?bF9F z1|a6bMnmSV5Qs(YZ*Iz5?hqKg#1Gxb&IB?UFi5JH$MGmNPuy*go%6_ve__SMLGD(h`P0At z_D9jsk%jdA3x{T&Ej<{p(ubs8E}fIs(TMEr)a)^b;n-g6Nzzd5UM%#U?@?HV`5y6Q zATcZrT(!ZxV1i5~U-GR5Aze-MX^_CUg$k}9uj!KM0oY<#8(^);0u&aTU`1Eupb06{ zd0YpjalnHiRJGw(sQq8fgxV41ALzv(|84(5Wq}r=IbHWD(W&ta>){@0;GV+(oYsMr z7G~Y;#h#b3Sglo{*|ilKYM>g`4%Hl!Q;-ffIp4lC7r{K2@Z6E`;1P__SG)D0A7&09 z`i00z8vEs76r3NP&DEkoSBe1G6F%Y*WC0^^juYWlSy99i7D?y9+~+aj%WWDe-5Y;Z zq2}ZgJXw$2EEp>E+ZW46R{%wZeF4V?yudXPfvQ2n%Be#$%*0S0T4k-qkf@CJB;`^zrBil- zQtp&gX4}g|p_QcGRAr8;9V7yd46J=6CY3^&LFDiKo0 z!-mA;9Jm22gv0BNVZ)shXcGSoXoBX5#KF>`7XD#nR%)eCaVBMWC2Yp#;lQTJpjw1E zBrXD9MW*F$247sE-!UNy=A~ouh2Y%~$ONXG*pzU_AFV$m4YUG+zP!3af;6=rDRIlA3v_5mt_3)m)$hD6A)Y zZovjDf*=v8kTTy!$iT+v-)2oJY;1y(suEf4r<0z@BMd^PP8X+sYN)D$rS>6$R>Lh~ zX>O9`SxTgFt)-m(SX+i^v3*6EBBv0XsUEQiU+HA0BqL$nAV;u-_?2K+0qb5l*>LP+ zhSXr5X3>t)!&*oO79^?_G%BMupSC`#W*w>$erq3MYMI>UsFtg_p6iq(XsWU*yJDcb zwkj_c7yHo_^O*mXTW&8PMiANXsEwH7Ftic?|UuQro|)YUe{XG*b9k=bQt zLgBLBPgaNk2tun|SSCX3#SJ)ss$uJpZY%jkD!9gs(xIk7Nh!LnEPkHqy1wkHzH0`y z3r?J4I-=vv+H4vWDdT|~>JSQ2#28t$*km@tj44B!9SuE95D1c}d2XM<-etlntjLxi zA#Q+O9wnc8Cn3m`17N^%1S+A{ja~o@3}^(%YHN9*MAhQpGT?>g8KK;g$t7ZTTZgZQsW1FIAkgv55<0obxVj&t65Xj$An~+vM6p0`N<)%H-0rTpt+a zsNul;B%~VD7Tu+-fk>9o{wt=G117A9|Miu>`~d5=t{P};Wgu!rGy?51Un)N8RA^J% z)@67Y@jfI?2rgJWuLVP|i7C_co)~&gE?V?~_BOAD+^pcX zf%b*3@r-O0I7GUJ+1`t^Rrd9iZ*EIUWfNE8Ctg zxAOl$5zegYh0en&LG>V;)PNb}fJm01fQ0p!P8A z;$z1KF}B8$`5nOjrVtA(gKilx%|Ij5Eie@0?SF-G14{`Nm!%ayD3=;vc`DxO3o`by((MyZsT4bKDNFF5z$*u;B^)h}HU9td zkeV?f@3LGXtTJ{Dnx+@Rz9}#Vvp`E89`N%GGhLs46J0o~BHynaXaEvL)s!HuPBf7D3^4W{%yP(0|!EAN{tJ2!F$Tr-N5-gAzCVQI1 zu7-P6Ws?f8Sf>`LinLi*u}!SCYq#}lp9?EsB)ztz)!N;G$n;&)^j?3cZ!5-X`6^;p z8n!H=aUb_s1vR=g31|7K7Bxy_3u>NzSfyd@HD~kqVMY2iDqd|BwRV~+{6Fpzu_a@&x7Jr^2jC zH*8yEAAGYcg`aI-HC7{vT^sd)19*QMEGUi*fX}5}`aq5lLC^ruxoQ8crg>-X$%&8? z)7mAU1BrkmQ}~rtZ6fY84iaN|m`7ExfG`Aa{%u}xtJ`V~&^enpdM90qKcsuJxLUvS zY@3&t>JD2VF&WeKO!M}R`}Ix#IDoIpe`qfr6wM;cCHEmT*Y+e6gw-*ZD8_t1JmP>v z4=HlV;Sp=MQy(}djzVgOD-xCY`+RbmleD^`Dr;Zhil_Ryyttf~-#pj%1;MmPfFzDT zieUITuG@5gV{OOwx`F@t9y+%gY=z>jf%LGOyOk{rA+z$V$+B0-_6Z?kH7birb+xYm z4yx^r=xXeq?6Gtx$};e%`{A0icAKmEf3$jXh1;ygx*5wdhlc-=c3EQSj#_~-Oo8sX z9*(0kLw4jIcK@-PQ7x;X@4CU?K!syBqk8k&Hozov_9?-LGUuatqj@?%aH;F2mcF@r zr=g4eBD?c+V9hw)dG3wlHCr^nPprIIh}egLAnPKAP1Atum8Z@78|)%>jDas-!-#Zw zSF?jVf+sn;;lRLt4(Hx!XA1xnOuO?n)Hfr_&8e+cOT22Go>i8(#e?#RPf2>0J6dx* zy0fKP@O47O^+qu>s%Y{Ys_+z#J=m_(h_z#lsysndE?{;*E z4mZIC>W;nu)-t`*KYhCoan-*_m;<2JZ~cjfw5evisk8sLM{&GQJO|2Mum!6~PXLv1zihAn07$iq@D9?fky}%d#a6bTK*6$Wg zf!Qj&r3aZdqe5b^h`@6?AY^^zTYf2!iuQNHuP76B__PE*o1f z`=6Wu_`xKwf&2>vP$C{gG)bzlob7*Z3IRkKHEC?k!$5_ZgdRv124VOx$&Ekd2;~WI zkzg}A9k6_vL+PWWTaxA+@dyzI#|;~_*vix<&zpfV-PL(WM}`5N1b7zk`Ew^wp+ixu zVv@395=E3+kqimzkrFUcsaCaG#toXRd$t1V`ZfPjC$VE!ty%^N>M2s9wpE+P4Q^a( zbKho@dlxUcVtb7d)27ZHK71e&6gGSq0pi097VlJW0CHr>L4N`OaM(sATlDgE_WU^? zF~crgjIM&oT+CP*PI_Dl2h$~GKrS8_Fj-kXMDq+HREQFIlxB!54gerrqlc0qM>Pj& z68K678lqf^Iozcumrp!y5W`|81qegQmj_zHXj14OJKkuGG$MQ?F}Tre-a_#y@v2qjE~LRna;u#yWk?4u7GF0cTIKUVng1P?2K zV7$px6w0#O5+E#t7_+d0LmK5cjTTI*AdUY=(srx@$SA7&V7=D9(eZ-XT(pe=At&lB zA^FscXrh5G3gEPI;!w^t*G5_{It^BuK_D5NEE5$D+;ad1DR$CA3ntB*^QRC)SdRxf z^z4F&I%t65&pJ@~!_Ob+F!YH;6%B9^GxpnWPiB&-CA&$(=_Zl13UrG>1RL}euLqGC zmDFcUJrz|6NktMw;wB*RF%&6Vk;U3FaAAiI#C(9)UVTjr*j;~3EJlZl0FG59m24nM z8~(6`13aRf5;!i3Rq56+c4!ZzT_o`qOq6sT^N*RzjcFh?dHn$e$t<7~UY&qg=D*j7 z5Nw=e?wF?{I6dhMoLcV4WQ9xa7})gcAMRqD9$g_lcIp@|zGnS9d&J01Dcl~qQW z<<324xkC#mXu}g3-c?}f$ZSrUHJ^V4+GnAmHul)shYorHU!Jlrrk# zi)7BZtGgAJOmjJ$se)dKAYuPjva7dE24SXJ4vl46YI~OIv{>Qae5r``-|$t-&=d|E zmKfr04PjytK@^q3x=OR_*x`?*;q-Fk5a0kYr#V$|?sF<*MGQ(5gVMb!RyvVW(v(#} z23U({uZ!L7IM_kkeK2;@5#b1xLpj$cbcBGl+i_Jm?Id@ zUWBc`BFecCSt;YC zk1cNrAS&>|l3x@l7|j>VVTLS?1SDWGmH9?qV4`Jd$mBCo1VNiJi<&{%TBG zkPCRiM`hv!I34v6B0!4&(-yF%pSH70@N8it=lOtoYRpMPxCbMCfH>`_%UAA!=t+8& z&Yvlvs=E?v?L-ice0?Je6}9LF1~4!&{yWK&N^9EPa^xw5Jvcwd-MF@)$YR_(U#5 zGMu5)=r#iBzz2xla_ScLFg4!BFDp}uKsUaK4=GqhJrRgv8XD1pjY71qSoGH}ouLZ` zdLfL!e1tHIiPFS=7rfzBY-2^0G7h0gZZh4GG}%j8_gWUSJ@x6%WVu;YW&(^Vpy2Kf zfPsU!Bm##p?P*j03z65bwza$E+Y&*$q$)0?O&hVVBBDB04 z&^9;=mV7f{2OvGqeK}%Aj$kWc5Zay3OL7XeLnJzZt)z`_gkch<1x@CFG zJE?_kc=O-?FcTP&+&c5H(X8fbEn%QKtDh8kU{yjWKn?u*z&g8;4>cHoxfk97jH_%= z*a}ByV(xHqpBr5m|J9d)72_dpZ0VG84jtsE(Y)D5?*pnPWBHDP^>`R0BxZ>-0Q(Fk zUPNRMk>)z7u>(47*VatfVr&N3viNf2>)isI%w|4wfzxa}2YiA!%z#1&egRE~GNB3R zb(0Z=YB!y%^R-P-?+Ty^Imn5;2NE4(q2)^AMVs5NWB_cs2g~m5o)obK9925-RSw1M z4(hsl`qXQ-y35e40$Bph)@>1W-MJt(Y!h{@L5Ph{b0F7BV?YY}&<%?l8{>=IxUoC# z@h8atB*hz8geF`zrWarx@Qe{1**}~vSUnEhG;oI{Im~lf*0*|@T}r~?`>GK`E$qnH?pe)Q#0a$%f>j;*);MPP(SfW&2J?Wv6KvvQ|* zpa>PXLu(}JhoH*9gm3u9Y|oAlDUe10KtTBtAl3YX{2C42EN|V^&$}{@r8Y-%HpB;g5Cv&!b&@Q;_z&46XTb*TPoRkYnoxwvQq5}KXCe_sYFPqTp~*i(If&kP#1Q|1|Tm6<&X}`4-aba4sme(P|6Q| z?2Q01y{$c9OQ26?sHvv^vcL=< zAOvFK>=3WZ5&{)fkriEW4qMFJh)O%u?TgYc54&p*LuCi!v9a_G0#v{WbYTMYNFSk2 z$qq3Qg;91+%W8^o5(_fpR*3jGjtl7kY#adDPT&%tz}ZkITH2w0na8p}= zk!U39S@@(CMRPRC&n)?aJ{m`t^oL_;ku_g(L11$x|Ij7&?ILrkGWK%G3Q0ME&nSm7 z*nG1+g_0)HU^umgB_2*V#mzmtX$_2mal$D_L^CvtCi1FtE2^k*NCZcYaB?WEJ8khn zxuLu^4Hs1grebqED@#Ms^ET(ng8~xc;*&Rv@;5Uv1K#Ei>N6$(WG*%aR05F`6s&D- zQc4KxdKq+d2B-tdqV0%0}vFlF3w7?N~CD>OVG4iQerj!dl|U#HD3 zYzeQFGZW=X<%+-#Y&W}9!9ajZ-*Z0?=@Z65B~$SYvTHGM)K2YV zM|t!#pCy0Jr4c-)Eh}x)U{XoTtFbub2NCrL=@H3{tTv+#5!c1jHpZTINno-K8ATM> z!c=xP(&w4H>j3z$iK&?Rw>@lK^PPkGeN$ih!cfpSivNC(wBGsjT< zBK4r*fPht4gZ17z4I18{)ZXh+qs||RWdJb>Z-O&g*KJ4tM)YbzRPIXjsnE1V&eRYu z!wScAKhcRurVZwz^HqB;Rt!N4+P6LeepbBVjQ#FS1mbSCOt3LuO^*;q#_W>hP~I2yrp`@Bcod8K}--Ip+7Um4(I>J_!?KVWN_v z3TxCSKSNeOUsPma)nwTr?|Bh-2YGRyPljQ$coWAPuA{EW_Z8L!9VSfD zhBkcJ6sj`pd}~)~K&k-vZUo9A!fdEeN5g zLp*SZiMS{G=)#DVOg z*pB&^UfYk4{TPT{Anlk{CkI&(Q$U3O%PEmjKnj}pvpS)=WSe0$KO~!oFC9*H|2=;9|sB zmLEBc^+1+eIUJ5_gjeAnN#LL}lC<#GcTaY3H@a{M#F%$DnU$GKWn^>cT}xG&n6b;4?^gP-;R6LGg|Q!PeKCgk zj>yYg1*>bE4sgd;`B0G5C_TyO@Y#BCgy&Yzls0e2)XlU6?=QOx3s>UhVQ;=g_T z$Ax>Y=h_s6{J48^?Sc#iyUYlbJThK8IcvN>EYXrK^rgnb9&;-Fx|x7nqmyvAuy)goBk#2Rtx;9)pg@&bL(Tei>v z7|7d4(RZekxpx8oF6(Zb=g6~)3QSsrY{;O^Dfia&cF_Bb&)B^c;qJa6-0L|4H#Xxi z*gkn6PEwqtRb0BOYN+F^G`f%)z0p&dz`<)9I-y*5O^96Ou()UGmj&Svgkf;3V!B~S z57OX^YVHF67(;24ujXn_NMy#r#3yb-Y-6semYhQI9|6OcT;U~Kn~7t?vGUWexbiE! z-M3NVtf`t5Sy;mG-g~9seB@`70;gdh8DQB`ZOi4;v3(a{3f5PQk9x9gIf^A9_GRn` zA{Y|M;a=mFeG;0TuPeec%|Y*Z z0s>@81Rb5`uCFJuPe{msH)_&V)t#$vJtDW!{b~5}Zzs(we7fD8`B;bVF+w5%Lc@cB zGpaNJAznww^bu&L4pG5CZ1KvIyGfFi&?3yswmy+ zbxqndYh=r2!=}xf+O%ldo^`txncTT_f!Q^7mzQ2#eEaTY)d#TPz?BLMHZ{o-lEoE8 z*l3a_k1m`i4h$fG;g13XCxArx965B9&S@YJFu-|*4A0G)E&2RHiA0KUupGH<@!~9J zMg@(y**o`_BZh_x@zF&|9F7}2G-raG2#*jqED9XRr$B)1+z)Ivu$^g=i3FoyqA^f_ zh|vx$K&)P|F%gY0LcA=5M~fzw5}=R_F$I4fS6+YZIGA6C zrPAYJiZM1@L<})-Sp}ylU`rlqH0flNQigUxX-Qt7T>~>L;J^o6#&<()xD{a$2^+Q% zjxHQErAeE@5qE@RnV18a4GeIAXJ%?V`GO6?U55vF+%1WkWdYc}G~hyCaZwbK?hB$=~y*=0gX4uFCGE%P*! zi3N9_TfujCs*p(7K+S4@kJm3FesAoh9|`Xa1!y!YmRWc>Ykr=0|KTyfC8>AI(OaQ9N zcydFq_3%ayYB`a)9Z^VIj z8tIJ#{KKzN%_3BTYXR#}zzLrk0stCT897?cH6X-PQgml4I$%hWkx1TfCYBqaOvgiz zvBeuNWkTWnphJ3WK_`f&s_9rVDWrtKRG5-V52&Oqg zViieTL=%mP%vTJ{E56uX6rUMBDn=8FS1d*rwMdw29V3*^NnZvkpr9y>V;n$WV*QM>BjAhcnsOdugIvqEl!I%G2sY{Q!hnUKA z5HzhRv6u-(Xi8D1(>zNx@p8=|VG~PLQVu7#69L>(CP(YR8~E7g&NT@`vlwCc5#d_A3C zkMmHL263>2ovC4M>O^Pqbf?lZ%TI&a7i(pcTI($87Z-8~)hIPy(6B@O>Q{wHq*E}T zZH+n11tmPxvr7jVO}hEfTd09oN@4aTe{+QzZS8=8$ZR?Lg?|IAFQiaGd0}fDmb}v9nW%^`&{QnxS7_J z?sPBwwF;9DyJPc%cGc(I$HlWS38-I*$vX|hfY!9^)q-l@3n`3&(!RKr1V8DUP}|NG z$HDcZ)OHIY*fx~F!bNbAkDTD-?)bqFj<8n(5naYEmby3ss+F%C)Gd~865IT6m%z!I z5xd~TV@7ecjIm<(wb;F7h3^9V^jaP#fxdFqv2X7i$W7o`p;F7if@%O@kA~Hsr#P^^ zkWAceE_l(%b+B?->SPH^8oI+!tdys0MeAPwSIb+r>=(hj0?XNn6=g25sq1$RV?4FQ z!9Yha*1XyUgys*gO>|i!R$u-u_zP~>vwaI}L70?GCzDC3jqzIOBr99dd#xCgU)tnJ zQ`*vFsWyhGsA(SoiNingG}PJ%;Gh7Xn}M@ zIE71~4-&SUaG+6S8`{>0_Fs}8{p@H<`xR%d zhFGrraZF?1 zPvgwP34%S(H>@y|Hi42EM2XcxE*&@jLz$Ew5D^RNpkY%3n&C{0jo)N<6e^Ya-4bt?`(vwYd@O65+bH+)mm?!$HCDmgCa#2uuVnb7 z*K^{k0|(I86$1)rCo;|$Cq5U-asZD*4*mNvB0LZJ%Rh-0^qL42D9FH8^MDha*pC(R zBM45=6daL266aNf@L#7V2Dz921pwxNszn{f#bCe}e9$(0IHh4fb!i!xe3eFbb_arq z5e9kpX@9^v;$jQt0s??XUgwk~3~)E(f*M)HZ?$G|JCFituvzLx0W7c?SY>)}W*u48 z1~Q;@?N@{X*anTzKln2_Ptt^$)do{Y3m7#AeU<{Fgesa~g^EyP{!|ieC59&<0l5(Y zBQl1;U?9F1SB+98gj8qCPWVZ$8j_ ztd)%_Kmqdvia=8Wtnqmf_y8=BjcsHepojtGv>7MhiVe~e3*-_T5qmrq2mr(ZM6`=P zVI4?wizv}(aRmdTRCW~TOGktXcC=C#C<(;pOvz_q$;g4rsE239e9`!a@a7x_(=KrI zj~1XXPd;C*2WMjUmL*xYY@2`R?rhU(QqhGk>NlFMi30KQUg~IWW}W)ZK4Typl}!$2h#%_ zpTGnpS&S!%k}By~+SZH|W|%J-jfmlC1(ARAuo^%RK{#nQ>Jm5W@>!0^M%b7K`G9|? zWm4!!lt?Lo&7mHw695#zk6eOyD6j(O@{|FP8UtWDq!|(oFd6`GQf+b|S%@GENsBNb zJ$JAiVu=$sQ!3C?18Ip81j0IIk~@E8N)tH??}(SEu!WPsa);oT$B2x{=u?DQn1?Bz zCbwlVDU*%z1}rxk;Mg8z7MYdF0Oo?31F0H%Fb|H$Nb+T5GzkUkNF5PyI8y*L9Z8$= z*#Knqh_KoJIWd!u526WK zSOZOCGZAt+b19uxs9bpo9BJ}Kof1rK_m_ESZ7HdeEa{`zw1<@C2$>_3-31!-(-383 zKXQ{U@VTTy_l@)@8FrIiLU4oq;uGM&I^76^Q%E64w@}`)pkPFsOM(JaDV18HHlX+b z@gkr6_%qPcd7eOkV$yZH7=aUUhT$Ls{ed7eL88=QqMS1V>gEN+Gg1yCod&{EOOq9R zfQIRJ2$F=Oa~PQ2sZ$*ooj@{ZpW5exAG-_nz(I+a|)dNdFLcmg|DS~ION zg)1Qc0D^KFb-7+`wjd61nuSw_U=#o;N&qZS2lsdyB!DIC7oiIh0lccL2TpaA?W&u}aHhog8vYk$NGB^;|jXaUr*( z_9~dlr>S(9n29-A-S!LUlUK93tUmyi2Wu!icnRnsnxJWa-nTI30*(!!H|a`FM;VGJ zT8|yWk2R1b0DvF&csrJ{0;x!vDf*sSXRQnLJNHqi8_8XqypKoJC)OI0~PcRLG(A5(Y*o~Wvrl>;VVx4F;~9c#B%5Cbz2g-S>W zCfkY*2u5GmkT8odg`l`hV}`pSw6O@Nd72=?iV`cTt>a3T_{goRB%?f$rK{jHMdY;d zidgiDulFjL%1E`ViEuz;AgvCCSVNV{-ry9=V9WQ$<=mOqY|Bn~(Q*_s=kAOm*G z5BBw22m&I-^`V#>4z`)I zCCaBPFs`{UQ01yex+yyNK?1aLv`Jf+RMDTlLOhd7sgY*7Q7gc#+mijNMTo)wWmuaj zxqDSxST~WQ2OC?zS63kDxmNbXk|Wt81fqYaKOE zS6~9i`&J=U2>cMe$P2?EpvQaraAg8CqSG2V3^e)z#CFv;}`kOjZE@H5UC!Q;{5vJE?c@ z2UvLnzafSImYg1q17whZ4v54b&CMsRiPpKObgheNZ3F=0t+^Ar6f(N%Jg;;(z>Rg& z1Z=gPX3sr+)=%J~x;ep=!Nw`Tz+;dVR*GLC)C{B&wrpTg=23plfC0Cb3q@CcZ`IKH zG7d=5(EinV^arjUae1Vtt!drO2R9)B_7s0&CILp4biJ}NJFez`ggo!pfrG2CX!n(QeBtog14TQh{9FtE{1<{=pqCFGGj1SY)X2f`2b z9Skje5}OXu5+J=sz_(X7w=YD$X%GeFP(tKk zLfb1G}UEQ-j*nAn$8c>J5DXU=?@*z)Jy=h*#H~TB(`u5 zp!A)6b!d17y}f;b?oC!e8jVOSK`+Y?wm-=BCNbtwfyZ62GwGn0G8qdZBX z&t!{=eZ26w0PcHjtp(CsT;R#1Sfn7D#{f}}=o zZ#j6;Gx_aClEJJRkIhlIZ>{empvU~s>Eq!4!u_!6`~XH+FW)(lpr|5Mh0OE?;f`C< zdV*lIt@?NNlQ!XksLMQkk3lY>sM5R}=`3f*p7T4u^VuYIAL(^cjxTTrtx0PZoZXBr^(l0y!lv(6 zUw)ja82hFgL$1y)Gh>2*utMldpXFy)ZDeF72u%oZuS-2tom_k6q z^X@nY_=)`ff%v35_jd49jc8yrIYmO}WnS;jAWtPu?OW0tV2xY)~T~fiz(O zuWv8`r7s)YLh-3p3Oj5OQDX7Mx7(=k#koMjkc2bOwxLWk)>eBG8YiI?ZOGZI%q`2` zl;g6GEqi&6IETXUgSjrbWDYavq??X9Dmq$WEDI)>b526;J3#?F6uOBElv663k?Y25HCugghgJv;he+wJL!ohMHmu zt|(+ntWa%e=9@S1pa2N}6#JUhyFy?@VW%1|%p^xX>W~64GkyUMGG}X>Y#34eI0i{2 zKMQS2DXDCY+ik1FGTdjsJo8IQiV#zWr&l?2;LAcK&>W8e-9o|^Dv z4n!VVfmu+%tE&UV>9>MeUnq-;{y-A339D%R>M0aju*F3is;XkAOf;gRjyRrP<1-<% zgm#iLt=%IKOR$Zy+pfJW4O-jA28Qh0zVveH-ohNSOm>?!(>WpDZPU$p6{x@h0{{pB z0=)D1YSfwp_yFMl7X%)7J%sgGdLEiYBB>wAooQI5eMH*F1ZN1V1)NoGf=B?on8~ym zX5zh|s0bohQ7Wy3Ui}a8@&>Po43{osw zK_%HI#&Dxd$Ht{PGUSe7Bgw4U;M!|##}~hAvCl6%eR0*+(l@rDJLc^(;}#BHpzO8< z?+W%_%5T8;O~IZXKsZmC%zlCrm;*+j2Q5Tl2v&%R1VC~*$yHD#{E&|XtZ)S*G%Xl_ z8qatF$Q@eL1U)OOlkZHH0)gma6|1Yx2{05u!2k#Ycu0c`ZzQ|Ac%nbCV++9YaEajg zFc(0Oz!H@Ip_dB%Dj9^4o@NqrJ+(=uF<0~6X2M54^qH@GT?FIV*ar${k#TM5B9r{k zv_|CA&wlu0fZyV%KRpF5bqgRI0(`dw1ROAd84JN18rXy@jAK#?+lMDC7!!UZYJr8Q zK_tgCxVi={_zA=`OH*Q2%ncUVc zHYVqN(*cUS@<+$K`7elf9N+`?C^A1LFo`8{f)nON6GSSKVg0yI21mdV6G#$-A+%&c z^037J00EK+=IJCK6zaq}NbsPb3c(K!vCvsm6b2DRC=sZDLyAtppaHOLOF7UT= z_A8yWb3r>1r$8#CMGG4|DM&V`5+5S!r27y?Bw1im^+>ck!7Hdmg0j|wt~3Qq2q9b7 zy3&=(b#55_fl5Kj1G~;OuYV1p4<6dd!4?xpFujuYv`JGgG?r`Pf)-A9x>Ish(=lp- z4w?n#i*%rr7(qI@N${cX9(njFNFdB zQi7{96@b(_FuoGAxII|wpo(i;B_NkQ&JDqF1P3Wz~yW=p4mrL+vbZkb~Jb!a%hB146w{dk`v!jDT6NLlay}usU>bgBuL1 zLu;T=4suW!SD-Bpov`3|S^=)(iR2&$iQVcB7Yxl!uE0K=ICqAa#DTr2b-Bx!Ahr0r zMzUHMkkZ&@Le`n*4IBDmsy_Ad^m^>ofo8$iT^4iG6yH252kr|26_l2#s1;y1^C|*( zAy8lo)TJuvDFI;^G0P{NWthV(1&mP4i1QKzc?uhy8;lqtDfN^ID*^{j!dVXg93E~L zns^63sZ|71sE|e`rD86;)Vtsfix3(s;~L*MTsXFIqcl3jMjluZ$2v2*$cU4a;4CWD;vxLrS=TH1oVXk!FZM zVWe%59YzEdo;1+q5(HItu~+KsZU~sTu1rIPPO`GxC-K%N&$*rA8HElUv*3?vYnAH|ZL4RaYgUsb(Pvhxw z<~TA-2xB-G{bYC~k6323EO^Z1W-Wg^rdDj%t_ddL_C)lI$yj~`v~Mo|`LO|GVrP%^ zMoZ*lc6rd!bC+>iLuNyGmB8-mNe$j$%rO^td~_dksj`>Ncc(L;>97DD(MYzRk_VYW zIbGmItLksGSAFn^B-{%7z{7$Qxn)RYF5^41_^*V45MOhApA+!7M~mkcIoK#4W&jF9 zoU?4=z%3tpaN)Tl>>zs5S3T%Q zk+FEu;0JeGDO=KL5~N3iZl*WA$3TsGr%k_m@S@ApNzP8!g`de+@-nRhma5sSIsy`y z9U>ak1ceGQ;a>sth>Y{U4HLoakT5;?wXq|+vV%29$S{X1vm1EEKlQ7X=lTQtQx3NP0=eL`@wfy3 zV>~4?8aJ6j%zLQQOF-1iD57IM@`^XGu?81YLkqk>a>2Ud$bi&4JsBGgq&C zM7u?lfKth@M{G3*5QK|MJ4)O*h$zPvprIUO0Cd<3q!7aT8-X#QfucDYv9N;{*(4=% zkqhYyL7OZ43kw{f8SJ8gT-=MkxV#zRBWweQlbD@hTr^>vF$Q$N#EK?1OuA%rLwmzO zmHaW>b1iB7z<;X)A7UjcFeMF{l&A1Riovr&e8!CbNw})|##y2XMl6KvdoT%UH5%-v zi|Ij*(~1VrnO$l^9dacVuqc2$JX%1So+!WP3rKUYk)zn1ePNHVv<1VnfnupP4m-Sa zs)C})gio-cCNiCX@yNUs$<-@Hd5bKQ44ae`Oqg29^}@hv$+r#s$pM?mdAcE(xw{r* z92$a@o*YM?Y&C^L6eA$Y=et3Ts{smtly;QF&^$*^NHJgw&3%HDG~LSI@w}aS%!SlEvr%%*!9%4`8Uuw$ytL#^1!d3CVK`&hNC*UO>$8WC-#+PaTcM zXl$!ri9sP9(g5AhaWqdP<)l?0(j(=v0u9g*iUU;=P)m%6Ce^-(pq``{rs+)27R6J|2!nHrz@tkI7@fcxol)>4 z)EnJS%*YHh&BI0gQ6*(F!Gfjc)R0#HDGZXbRFK-yOx4s)_0dd4mPwt|)Pw{pz@Z0o zg0RZaN@c-PZNEqbRl|rb;%Kf}y{pX9R7G{uAmCA2t<@yeQeJ)2TTRnioz62b)-^a* zWK~vWT?7PV7TK7$8#_kv8B{@y!nCD?Yw)q_pZ-DB2;jhY~I z*uyH!zLZ$&BpV4dNo%E1mb_ND*;r;YqmKnyknOik+g5I6gij!&#P|en9sI69%tU%o8)3ZI=XVQj?9ZWc+x2P4=xRu+l zt=h1?Td>tzvBg`y^;@#VSf{18HC$U9J4v&JTa6Xmdt2PD?c1MqT#o(Q$)#Mt72Cii z)TX6dwT%|SH7~&Y)5E2`#BJQt9bMDK*vD1f%3WQ{WnIF2UDst=G1^?t_1vTl-DxEa z&E;Lx#oYCY++&K})|K7iHQwJv+_`np)2+SRJ;}n=-Q1;J(xu+I%~;?qTj4!k;@#fv zMc(h7-P#?JXe}??1zooP^#-t^K=Jk6@LgK)busRh-{PfT?}gv{U0(KWUI>I<^OfG+ zecPqe-|KZ>>#bkdo!_v{-UDV}{O#QUeqb95ROj_e|HaM-7U1f2;P+kN4sPHN{#ygy zUOaflt0cPO>zF!cQVX2wnX5nBDzTpve;S(-g+cn1MMaJo6-W}HA z7&hV>mf;*$;vz0#6XxOmeO@5eT_FZv&RyarzT)7;VI($GCYH3!-C^_+V@;ufaju;{07>Cw??C#$%)vT{YI=HlE`&7GXFB&vW2nFa~5lzT#~8 zV>>os^HpK=jS@Nk=HvFQ<3KLtB$nMiUSvAfWJZp?Pxj;q9VAf@D9%g42<|dv7Yer{1K4*2-W_BhAWnShm zCg*t$=Xw_B-Zfp7!ab{%L_0YM9>N?Bw66&gm&W+CLrV zXbx(oZfcr_>aT9(qqgax7Hh(V=cHC!w0`D;2J5zdYETAgw$#@uzBX!$R%@c=>Qa_#r~d1)PHez-Y{4Gv^7-Yi*6PFFXR220#cpiK9^a(p zU!1n;$j)rbCT(!OY|Nfz&ED+PF6zey?UGJ9XmaDz-s#ddUDaM~+;(lz4rra8Xt%X% z+pg@w_GqQk?c#20_3doNer?~b>uDzL+Ro~;#b>EL?(4oz&aUZE{_Jf|<-4A4wGMBk z&TZ`FPUm>u?hAemB3WzphBPf6@8}k9`8MzJ-tGJL;nr3H?guJ2G*Z)w%;0{7a0_2<3MXq4_w30| zY7`%C0H$@@38ZJQ zpuvIh>K#P5u%SbS4--a&IFX{fcM3CB+-R|*M~xgoiX2I@WXO{!Ob-%p`DlOkQJG^f*>IHO9PO7*DKrcAF&+=_LnQKDeG zRu!AIEZL`M$EqbKRjXOIXW_>ED);SNyK&Rz)ysEm-MfGT3;w&e?_j)!6BkxY_^{)! zkRQ`+yjJpK#+ET_Mx41b=f#^nhZdc+a^%mbNmD+}_3~(puT{I&=r=WN*{pG|Zfm>s zZFZ)0_pU9x`0nArW6xHOd@ym}zo9obp1ZnqsLZER2aP@W_3hfXC;!h*zI=Gi-nq9E zZ$0|=_VCA>pYMJ-{qE`6zdx=&o&5U%=7*Jl1Qw`Xe?B8niYSk;53u_YsnB~HanjUF23B8xk&m}7c0 zzKA1`Hx}9BkQ?&%7|5EZW*SCRF+vKmrlB4 zr8;AN38t86wkhYCW^ObmooTuW8JlqG>1UjG_8F+4coxd%gy*f<;-Q27>1d;bLh2)x zAK7%LqH6Lu6r+G<3LU4QO8Tgqk(Mgzsbo%ys-~>26Y8j*zW=%^th1V0Yn`~>+G(t< z(t4e(vI48Cu*9P0-Hf^JI;^kE@*1qG&=$MwsM1zDZL-QX+v~R4M*Hly;9eUgvepub zt+AM9`)0T6I_qw@@Rl2|m@txSq`df^3njPgs@w0o-U2)?z4n5uO~IietniHY5}dET zUGkQ$xc`{mjkVUxJnePVRCg`* z*xOtawb^JNowhaDtj%_jFUJitO)VP=8r&;AAWcy zOV8r;<56qfiR4>3i|}O^@Afs^Adk1)luj%#M%MVwd+pT6QN8uHLt-^5kgtMt zGR`PI&OGCnPrf+i7k<9u?6)uZ`&nx){~>qKPlX`$Pl12_`uC6j>;98M>jKz6IS3Gd z227v=7YK#yy>508j3DlAhdt_%0uZMNLdoA zr0B#du7HXZh+-C>$i*#MK#N}lqZh?E#xjP{j2Aeg8o!vvHePWCU4)|p5lBY|N^piA zgrOeyNQqnc;$A(H?@|XV$W*rg8 zM?w}8hRXnCAe;4!>M`L7&}5_{6DiGTvhI;fOrItz8Od=Jf{2(DVkecT1{Qp>0jrEb zDu=Mnbk_2nw2Y@b=b6h`ZV{GQ45co!c>ljY;NhPD4JaQ3O3;G-lA0Psq#hM|NDcNv znYD7>udcB=F+}sCi8SCet67KmUE-r4<-<4D7fvApk()!5Bsuv&&V9-Nm84{6JKaf7 zSJqUW{In@NcdFBUqSL1(K!g!{s7WOf^{7To>Ofh#QYF%Fkqm`uR5SQch{|kWz;Nad zIBL2Du3@A4gCr`{phIw+^n4{P;z>)YR82531{4TD0Z>T>e&#@?GsUSo^-4>-@^z;s z_$viWFj#yNwy>@YY+z-1*ai%C2qh>$0r(k=w3gJemsP6|C#l&(G!UwOKKnKfj4dN%oeAZY?Sy+G>Rr48Z^ZIDi%O zpoa+r;IZpmt`WRoh3Qh)x{AeXbhVq^?r!(H-VLvK$GcbOLRYS&q$FK=yIjr+w-99b zW-j7ONz6XAntl+4e)n737}_?W4#ky4=C!GSEn^Xg{A?e9Cx`sfcD4_u!&V*X&wz$i zt>)8a9zNNGH`Icu!(ByECm;jlDmT6d2*6&Yd)*ehIJ#G$D|TZ{W9-iO#^B}diubzX z97i{<=LG;|^$>#~Zc-B{z=IK~P{bODR13ikt`m;vi<-%CT zFSap`7ky|%Gup>Gj`Ur*dw?M8(-Upb!&WVacZk56zySC+4 zb9oXD?k`kLJLZX0j77;2TRdMSI7Fl0k$fMaH87P!XwwxG~KV=ObitOP(ZK0u7}tO zTj;~pg%)t$074Ho=HY(1cH!RWxGS3GHK%!@7h7m_npD)94ZLQD4s^k5Lc(iErj2~j z^rvHx)>nO=!!v~S`oUT0&x%C@?*R5rD`V_rm-g95Zd4iEvy>@^>yB%jcf8*n-L5}( z>sgsf3MkjFB1dZOw`q96hl3AU7P!c8(TiVO!tr7ju$?B)u~<+~bXMa!lZcV=jUN&8 zcLw=5o63Dd_ubt!&SL*y?f!bWznJylPJiqrEq><4*o!gf-y7xl+4_Uk{;WHo{tGxw zQ@E_)z9wscAPT1yc{fvLSjRMP_6(DEF#f>^4Wx1GqioEEePu^_#I<%#W=hHxK0icv z>9=ws7=kZHe(U#oLl90cmjV&c0$a6F)f9tZ7F9CXP`toA$U=O@S9})q3sZM%4cC0w zb8K423J%CJCesKJXnlf%efe;KAJ$gX=4a)F2;`G}cLjphHG(Fnek6!~uJ>{k_-s2? zgJvjIFnERtr+8i0hAG1e2G@f>*cZqWgx}H&L#TjiRx(G}htQyeX~u+~XMr3>bRtG= z+_zi@ut`@q4(k8Jg^;L)C0KWqh<7n2Q|9x8FEw3M00k+KhH5y2qNq^CAbfs!iaN-H zr1c90_%D&t9(98<&R09QrDmy6aV8URA)|}bXF!vK2@}XvgP47J(1=f%2ApPy+?R-C zpj^$?XWSNvUs#FdM~#zMjq7BLYS&Kohf$$8ilP_`)WC~w*oVBpco0Wb4dX8sw=K;e zET52vwup;tLXNmd4ERV3)rV}m6-0X03?Ij7QJ{9)Cxz~ZVtUYQo4AFDAdOzQde|6| zus4aBxQ(3XiTdY_-w2MgHjeg)iUpUBryzChSY|`=j^Kiaaj=Bahl;#NY?`-gQ%F{k zM2N|Vfgk@BM4m)w37Kay6=Mw5+PqmB@gO$frE};vy&0D=5h*W64_aD35!%i}sk07?*z%C{WC04mSC2X!mF9!(we0 zdJE~0<7Si-`IqR130LR|Ov#YRa0#9jl~Or_d~=mp`IyPWYF+7uEreQ<=Wu07mTaMt za^^UdU^MR-Z~zCB=U8l?mTZT(XPqQCw!k>%P*MNrrnFUr7m(yLpL# zxqeKUPO?{fK{c3rIhdXBf{MAAF!Tuo=a|s>m<1OJQ>j5-X_>c{nS7$12XmItNH4DfH1XaME?kRf67=?AYL+8V3)y0U}CrS-T0Yuhz%{W~LP-G4S)zaFaZ$ieM3N?0vc3VM40?(1QL1$gM|nm>Y%}pXArtq zn^ibUsF6|iLIB2`i*SojfNEULR#&!Ug;tgbifV?K8g(H=gT50Aqfo94H9Rp|sat8I2!}ysGdEx< zs@XXsp=z(>aH?hXY*PSIM0hq!gPKMag#}p)rZffb>7MT?QizzE6_^PGU~L4z0%*mo z1|XLfU{(*B4+>C>5ReA>00e36X2&8@PrU>I2{WCf_VX7ss$VCt7`uM0W4cM z27s&1_^}=f1SX5IIjam6`-H5yc7--qljNrwrJ~X_uH(9*<;tS#DvqOxHmcZAHkhL{ zlP~#7E@gSD;DcwsH-YAvr6>hXYI}5Omkb8$NgmZ!-b9mfC1?{%u~P~Fd0-2A%ePIS z3_=FD3SbTcFu2I|4JrTt%@75On+AOg06@?Tfty@JI{=m23=p8WGN8AA@VJpXxr;cl z%Af#>YXO`qWGBD?)}{d8M*+FYv``s?FQl$gJ3<>|uIwtAuoJJEIq>w5yID`ZWOT=84J=ykqnkYef!xhwF#O$5LGOTXibTtI*U6l(wq@I}rz zm~{}Y|Mj#V1dizXz%u90YD(W^Si?VEC4_tX!(o7SM8hmw!y#pii8`fKO28HH#1;QQzKlF*a>WOaENFiq z!0ijgv+BPp5WhO?$Oh24ogB#lzz6tS$wR!u#&yG1Y{*N1n0>~Yf|CZ%^ufp3oroRd;x^;T;u$pqMLImHVdk&&Y_$FdK<({6v}DdXMOK)|-Zf&)zeTP)A=ECJuV03V>x86W}=O#@p+f3p0v-l>>poYA*D#%hd% zyWGZLdx|Xc4Ai-js1VJ4ums)cl*fF+Q8u@OsDiNB2OIxJ2<9N0k0Z#PM`!Zqf^swt z2G9iazyUAdu`5f!Mm@3-FtP#gYyuz-m(bL+V7Yv&L!lfF<4d@Bpw&rD)JdI2N*&f~ zSEd=82@vfAXMIr>{l$R{q_L~faJ|uH?7QXoyFn6*12dK(Eo_c(#^D;IY$bF<1=Hzs zWr&LbAkfWA0M#7Fq{8s9kb|0&lf1J!4?*44DG=5yDAr}&)RPR=3lI)`+YML^J_~RO z;&9qSO$9&O+BzJxR|MNDiw79+Y-=so(uAI#XvJ6-*S<}I8r{(z{n0vFFTZRu2&T0f ziIJOBc$oxmcJ_C@r3sy=(zDtGa>RLWsnb)4o|^x|Kcn`yv-*2~n+GTG4Va(;uXzEY zodWG`+9(}FR@x0IAk7_yxF1ZiWt!hn2-(!8Jz^Evl5IbfeV*z`f$G@} z`badWTH`u2tBC2Ds7=$|kUusq<2LS3KrXgJ%F1J9O>d3SKR^rM(3+{s0FjFcQGj?4 zzL>!s2ogTwcdf5ni`T=~Obzwd_(y7mU6CPf!a|qgw#wa*z2X&S#j3V>83l+XUFUP| z*26^Qp>tc*x#xZkfq%Y|VvdG{j>e~o-9-PL$wd>(i`wW6mE=*GOj$|cl#YbHEYiSy zNFBc6LE2ATt@i_ z86{efUdwGQ(LOz4Gf>O_v(opsz{<_h!mZ_%E+3bU>Czr>4W{YHyMIdhQJH9r+)h5@ zB)z8Y$~gm!-V@)o>gMx9>#(&u5GXzD-scC#JMRub>Q0rp?mPCr2E3l@z25J>zMN&S z=@Wg1V=QeKNX*&%ln!2f0Xb!$_azfIi~x;>lpV zwvc@Q>EH8%?zt{;y)*B(b6{KWe6s&jd1qPkjrt4rKDGY7^Mzi6FBCeWXUMJY*Nsr0 zer!(7s>Q51P@e_dZ0PW8EYcG{?dG_ZAVYS%0zfpxq#KXK{(BEJcjL zn)0g7>NO-fZh$*DfA1vp3Bs@k4*&;oUpsk!M>3!5@$UDAzy!C0y#C(cJ)dB(um?*n z%?N7&ZD-#V-{_x^52Rk9x*55YQQhcowX@_+Zr&FVpsoLD_>68*lP^vLS)0uo%_qP4Nsi^!@cA^u{-Qtiq>pv6 zoC#mg@n4^P+Uoja7jnuA5c33@Q)8papFHNkEmU~0(VasOC3d11v5&<_iyktn!*LL- zMoSbB~q#$!O_@qQOK8cWM+g}6J)7YI7!Cg$t<5hJY9GWU4rjtxD`X6+_8nQ zQ5G{%qw4&zOw+*=5h|Qm!t4(duwl83p}}Zeux5yw$w<@YjT$|069SbZRO($of!Hj& zHjUd~z~lr6COp`1;dF@=FJ`Rnur|ldAVZs6`EliKGVj8WqEMNxd7qCCB<)$@&w;g8 zs3D!HZ5~@`n}J(G1<(JVD6=HT&QswaY_+=38Wc1UBW~l3aMpp+eCG0%BVVdaX$TIO zI5cg>d~W9ED_Fm%i^qxmW6ykOsXVbOHT_u?cfG>C169oZF*}#m6+wi?4@Q0vYrs!F z%Oi>?xH`dt3luPDuC>^c5I?;5@+%v}F3WJk$0F0vL(3@BCbkemT*eQc#A<=51c5v2 z#nOxlO_|ctTWz%7w5ouE8Hk(WwzghN&>+{?f(xmRlCx+Qr}(HS5?YXRDiQs5s;;~2 zxKnDn?{peZ6*RV^#Tob}WUq;2(3&Ma{A|D^El>ocaJn+;jBYIdd>C-g`fBsjfB^=8 zKmiDHv!#g!Flhfl(HfQ-&MXR6`Ll`;@X!>mAipWm9!^RdiZy%?XCSiJ}u)<@~|9 zOLEmUfv4DY3kY?Cm3i1?gUP49NCfhv)mS>ZNi&_yGwJ7Q%e!`^@3h@E38tBTLalkU z$TQ!35mNte>IHt|@7+5Yp{TjAY_reauv%hZfj|XKw15K)cDsZX;FddXHzVfI;Y}^o z+wO@euFGOMF|KhO8CONk<5*EO2WF}-Zmvm8Y$%Hc32b|7h$d{+JV|kKMI?=Kf?7lw zT$^E*x}(90Ly0e|ytCWwvh`LZtbb6$nPwOym4ieTg*EqlLl7bBcJVBmrEWsQtq#t< z5m{KURV@1t3HS|kZGhjNdwaPtx$thhIo-Q&Px~h3L@iI2`P9MhDctaTv^kCNarBr! za~{EJ{A5&ki#&e_tmb7g+nlY!F`LhHqYvR~)n}Gb2bEX^56gMV9R@Ly*$oA0uY*_8 zw1xkQufd={BP3Mz21T$F_KpKS2*C^f0H+ahO(ALkos<6X6D`Q46Hu7i)L7So>RFF% zC?ulx;3lxRdGCAR3!nI$I6m_|kXM%o5)e}Mn~crReeYYhg+7QXORB!nqxS!p z>0FnyK!l$5xi`ol7@r`*M+`*>sSrY)UNR&GqvZy&+=n?fX+<=sQ^P(;AOk^+Wdn|A zN!?+xKc(8Fo(!6oWkhXJG_sm7n-H2T#L%FsJj3evX}wv}@`MIcqAqjkQC{+`m!V3M zL!~)Pn^`k4_&_FV>@kfr@PbT)TV_MX_{_$wph3<&nUHSxf^uE#HNltw{)m8}At1tv z_*zL<;1ZRTkpN~r7@A60azWIYOE=+IMNP6AOnrU;ZT+Oc+{)^<0IrY+nZv}*ga?L$ zE;6c7NYBY?(IU5^@}Ov#WGi1u)*=Fwqagh#VM8iXia`Syi>+uERN&AhIBEY##Z-u3 zkjYYe$b%flr~+(K8pg%Viaqxl)gAKiptSrahY0d)3W9nuaH;`-5Rm5Q^mIW+LEt_; z>X&~^Voxw3l0C^tO-O)iRUFifdO>t0q3HHpot9OYX-#DoYLh3phCyCsnahOGwJf!& zQj!!^!bM>TShNbZtR3BDVRd=f!y+S6XZTO=Om)RVWf3jr&;~K;k=ZT6;~eC$LucQl zre=zEU86+>LEr*W+s(ss0x?-oF#rXnx+T08a7`W#bX&e+;9pZ4m~XEH&*0w4dCuU& z6NJG|lqC0xVRe9C?VGTTs$iqz)oWNzv4^m==Uz9(E|0t5pMAA0qw4=nFMGo`V8g~2 zv5Bpi3gzgUqUv|Ql==&R&oL<%uojK`4Q>8{+Ten`;WGjPU6yIo0>HtRjsX>(DB0LY zG^Aih2Z-kjsEgGTUlpED6NmFwaX>(OYYR2B1qa3&faaDEV3L-!2q`KNd}TK*EGgt| z#x?>v;9&}2o9G&yqUS`}MyDnu7{UVUHYdL~d{RDNt*879lHOUeC^Zf4c4iJ&R6^L5 zP@@ZWRfQs=~s2j-F=|b{yVXAOMes;hmU2Md9djX)y91{Ql0B-*W4hZyK*}40-2VOAUB6pq% zA&AA|N|mM;0uN1Uki&R)owez58_@!q$h>(3 zLqnLxZhQ>h1T~xGA9J97=29v$>iVda3i28rnh@z>I|SjpF;J(~GM#dwghZ$hOlT?z zT%_s<1?yP7J%EIF>p-rWH>}Ho0Q`UzkpK+Hgr+b603ZMqw1oj!!4?cJ59lpE>koPf z9T2F1g@6d@lQ4Y|FSXjbw6U@CvOA5lq`ccZ@iV`W>O1md!d}p>&Re8X;}PLdGhaKS zouQ*NkQ@dvg8SP-a|n*8QLz0Xw1`u+6o{MLNCW@4d%b+2mlQ~kpcA?{h`8C{BSl+2 zAsC6h3caV>LqYJssuKopak@>4I!UD0i5GAT#}3TDc-sY407}){0(I;>=G(6sh=BfC zh!j`=t63AH`~Y)fo}#QuA?u-1yq(xG64fX|gY3Rac}1~YNRW%dn=CgFx*{NP3)B0Y zNt{0~^g>zT4K6IIfO875U>|TA7`Bjs1CXNzO0kukMW9l@2ds-8$((LHOAPczs8cj{ zBgAfyw`V~|qO3~hYsP$7f(Y;m(1<8zfV^vq$|{Pes?4qe=_m!IBKdtcW@FgG&ZsOWq(9(DIDcBqlY% z1b$J)*r2IIy3Ero%4a!CO%qHzaZEdD%M0c?9KKfOW83>?nn+dph^bE=R~AY!DERPy_b^06ZBK~QRPVHo zLp_P6$u3Dygv*=*>_~<#d;yzUi>&}fR#n6nc&>d}R#lW$2&L7qv{ehm)sQnwd@PHR zxh^jZB0p0QgNVyuWf1=cIUA28rL6!00_>j(!{XECDd%qpzu5-G6{z> zJBwOir=dhwbT!YMIM*WOPDaz$HiVCTmC9(7fk}kN?GxC$BUpo-P^B%{rHzI5c-hL5u^pG zECUh1uuHJCeFcsqF^~rMmteS-Dl9%)%81trP+C8p(CrW71|MB$sxBG zP+xpMGFb&Z(Odr%(2cfT#CDwpGFq~Lv#*(Fag_}`%-}fxaG0BCeFoRBI z2CjNd_^n+>dsQ;UUx7W{C4A!lJy`R5-IJ~2O{7{)xC#uYffXGA6kyFU5*PVou@x#1 z67U2I_O1T|oT($r!1L9FLc0R_gkTyDhA{TgNEHStVBx)GvF)t`5q6$Z09(NI)`jzy z6^qnD|?NgzxQ~O|Z#5fL<`*zzi=0G(-!N!2;9Z$ZbKImUjE{sy9HDI{dtIaK|jnXSV*`=7}EZ$-~ z;J^PrEF3M9^%?$jX(0jg)?HH-O9&OhD;Eq>&z%m4(Nft z>VigC`HQ((>bF*g<$H93H2Gp)u7N|S$5r5BUUp_OCISw;u%5}BDLCG}-Cijf3pei0 zXTC&`B@;5}UD=JjEMAC0Fyv|ehC!ZG6?UGLU|~H@lMkHf6aEHsJj!u?WOgQM&Bg>b z`iU?`l#qmH`Q@t5B^{mOWUBULgnSsRCTLSuT6i|Ogx;~U(GeN6;?ffiE>^p1j645i zr0g1y;Lg)&6oX+8bj^}w8lXN+&_D_n2-&|bX?OeRZ{T98duB2*#OnxW6aENtueoko^MCf=XUY|U=x7Wl5ba;myQ?*+XC`UL_>ya2x%(|z-2)vi_4^#<2Y)7E`$ z#aa+grn}yfZ>d$g!xO^>hhVgTa7QfE-BDw1OzuQHW_Uxq=O*8?U~3PLH=qbEHhuyV zX6eR$k9SjKAmhNxR`1DOXBc1aBIOnHjY$5G?-6pjup3<=+V5B;T_m#VrQP4wCX4}R z-6{K@fst*m#wY>?>nbc6*YZ;<-yd=!)OmR$cuIo2mhpCOp~Q0|6i$~q6YT#&5SO|= zH0^GMG_hM5)(vZ(Ol)Z|SO?J+lCfP;oqE-+Bw2%IyHDOr-Vgor**S0-r7i=?7t z;KRy<127MBhE6UY<$=*^*Im~W9ad+c%+_E30#(rFVBZB!Kz7vw%3nuzMF4aj9)v)b z~YjK+KpOmBR}`Upa*qd_e*bgP2Y`9;j$b_ZAudW zL6IC}3+U5G4@@tI_A!kXEJql)){*cjH&iEcz^+XYc0nI>B?tnv>m2_^oLq;G%Aj`k zWY73Rg;$iwc#QvekpK8-mv(ADXB_ILLji@8r&k1xm-5owf+(Th>%9ZmWO7F#-DK^? z7-h#`sXo}1iDkJQ9D1RLIWZMemt*u!_vaT}!P>UOnu4Fh3H4A%t=(0t@Vv~#eXeX6KO1^?P@301Nt7ur zN+y`Yd`kp)Y>)}1QSVenfC_j3*ax_;z^)t)d$E`DS5AcbnRx%n9i7i;Sv_=nICj~P zpZkR`ez>>$7zguqGXA=Mepn!g9g)o`D2M5ij6}HgA4qd8_^YOe)q*_m0|0@ijdaMj z3_BwFqF;ZZX9M;(wgz#~Ah$ThM*#abw#dQ;bvK-!00>M81~MQBf}E{9&4LAE#$yhg zEfBF#Orj(RkBlCAeDJ`bjv7sM9!flQ@uZNHDp#WHfoBy-i!DPm66fsZxSVx3?Si8! z*|%;?AS(LO^C%^xJV!BY>Qqsvi83p0+GO(rSmyWLDCLgpSE95rGbZ@zrX^eAgO)5OQgs!n1ky((2| zmN=nXRF#SYslOro3A0KoP%ZTof?82^3Msd!5>a;8Nwl6PWI=I}4YVvo3k@f^5e68; zspMNuW!+9NbQ6`&DWHe!fQ!xT2 z6;h1w1rMcg2UJl0g%i{~;`FzQP5CWiV1kKuRTWcF#rY_R7kv^-h;X?TVHuU86aojG zdirSudu5pkL(1et7BgUBlMSn#;rObnH2T;ot+T3j9FT>{U;$sY`C25fyYU8Ga8EW_ zY_Yv1n*p=Wb;uH=5&A#{9xw$@Z6EoHzb?=jT0|v@=$q0Lj{b{+R0uYBaQS=b(4qtC1)tDf#4; zt6zB*W-8Wx^O_H~IOCfiz$*r66S4pQr7O|VT_M~92`F@@fU#?xGv*WzgeTe%g%JKsZiqlcw$c(8CajQuVhY4Eu0g%)(U5Q0 z%bxZKX91Df0thM)jQxVxEAvH8a_c*q<%D;_gqR9^^dl1aenkTBeee%fGzW+Lcdi81 zfDhj&Ac3A%wQo4^PhZT3ARrh)L^Y@=mWvYwdk4p){OffMK}7e0Cbu-iD1pmsVTS_A zLKw<$hSjs-3_FIy!5u~+YWN}Ogy=XTD$9sRd}DT^@Iht3h%J11Vv#@@#a^j}ET#aeG7;1WR;%*rn#*q{$A;u~FvnZHPJwu}4@ zLOuf_j3%(dH0Xgv3mKjY*TzCYMsIq1Gvpz`S*t~UjC(TpWB6th$=OiSl3aipJjXV| z(mhfvE7;sUX(P&#eR6~_LLmyt60i|$QzI;V(-%FtALK=aQC-OLMaD907cCaX@(D_+nF-3Xtytg+V~5{^GMf6!n65g45No7LD9} z)hMGVD^b#_)&!vylVtzX;85Xk2P?W}l{&=0sK5aZKAb6~4}D=TTq@3(!c<47sVQO; zYq!_N6UjOu~k0LH01?deW^NB~kc0jNPm z&l?BzAA21IPE9Fm1sx>`N_=6h%Cw+YO(I7nXqKY^dPNqxpsOj=33+td^Ap(m z%CyQ>>Y~IZCh`A>Q=HXqcl7=W1OEaqXRJmo2IWnU@pHv(QM7&3(_UT zF2!sdHY=PpykP~sahSnnkbg9hlRa;-fTij)06&q47-o07+r@_|T|kgUGn&zgZZxBh zeB`_UfYH=k%aiAkX)}NVX|@-j4NW~v`wqakSfLFq&U+v_ zAI^a^xD)^I?Jh@J(n7?|qFn%r8E3lPoL;S^ulc<${E{m%O9NZ}>Bm*qFG+XNnW^Me zw@gS4wrKWhZ*7e`q+oay1&VWqeQh?1La5Hd4H&VpqRN{%fx4@x0Sj0uG-dFii&HdR z0CUENX+=;ObT{xgZ6wL4`KWmIvGW;E$4e!Y5o5Mm3zQ`}(bMDQ@<; zXXoOWDt3s?pl3~hyuWp-MacpGYAxh~TD9mR9!#Esm7iEcE`R*SvK{kjP`>hzx9O#$ z5aj>O^P6NyN*S@31f5ubtmjvf&cL@cp*LqoEzN+gRig?4*mFI>J5=r0b>`SiP%#te ziEHkH|1aUQosMfgKJSuWw&h>BDp^^0u5HP znB<6s{L5)N`NSfl<%oR)bGp}z*6f%?UYDZh+j-g0H;Zlk|Njf1)LSH^-EO&6-921{;2rRYK`pcg z`4tTc;D-@(-}v?2;l+o^i64qN9*m{mj;vq%v7b#H4ju`a<|$a`1q-?r5&t2}5;Fgx zZy?da;aMBu8NnQsN1#Y*sTPqvk#hXo1^Aw8?EtR8DM=QMq$E6%A z6@oQ#QYR4FA|_%YSzdNEA`t#am_cI56quO(%4AVmL2RNEBFheV;wR402HyXp2gZRu zW<&zsPwnlHU`UtOkr~s>nt3E+SE$8%h{>vq*JY4OH-x|lf|wvoN=4F4Hy%PpVq|=D zLKt=AMTR7?4Hrp*#~o_QhkSy&1lrXAnoD*^CzOIWej_+W9?X$rIWE=&2;pe~#y_qX z`RK)S#N!RH1U-@+v`APPepS(30;*XN38)ivg_#fGPbwamNL>O-l^V@(M8YlQ20UYl zEkFoR*}!dx1WY6v=m`Sd5hs;jinS#)k=Qt_La5Q*oKYX#q2!0)AhM0oN&==}V$Ey? zRyBy@P43_~{UBrJ(NgE6t7A*iAhNXm7=qQb8RTTggEW=P-pqg4>hwdd`hG>b7=-+G_RW5%P)(kR?!g zmc31>8xiO;>eT`WKoq3P8aviSDYMCT#JY0-mm@uYTu?%A9z07sYypea)hjDd?n9<|)!24M3oKh3vE% zT}eRPq?uXl(LU*%3lQtN@o~sR_x=I`ly@T*wyqh66mo zs_Lt+>{Qau2DQWl+C6QA-74`REY;F#!#=Dcat1!Ym+4s*(e2!lb!``NECl2z(NzH^ zphLTD8k2D?+NNy?sa}?`>&LE;1eD^^uG+>nzytVz6_SSqdg}WWEzkBWJpirH;_J~O ztrr@O-VVXID9XPATLFfM!B(wyD(szZL$Cg-IOYF?HjJCNJx92eZn;U4{ZQJ{`O|-D zn(Ml5GbBK6{Kjt#fD2Fubs&P=&g~K;&K}KQYQh`i!ke!AAra;xMu;MTiU)860PFCc z)1B(nY|0@xLCkVpPO)QbG%fN?t|*3@)E+G6Qtj0`Y}QKbcZn|THrmpW&-yMJ#|9+= z5(L^J-P+2_`i@)LjzutRsgOidsr`;>SDKcK}_-bx2l`qy7R*VvZI_G6ax;6h<%Fb;7b4OmK!2wgK>(PsAmPZBhs72rh z$V~9=RWRa~9hhG40vbjEG(c36ag)XXntJc7vR%S;=$oQ&3M1zf-U16NCXKu>IRb0Q z%pVa-B9n@1(lRT&t!HqAUb4K84cy$ffi01dOTQ$SvmmPhw6DKmV*AEk3#1yBXys?# z7FW!sQ(meAWS-$x!am;X1B{xOQmWG4kCK9Ln*J+jqTLBc3a-vE9d9n@wy@{wF)&}7 zPIeC=RAoR0auk;jx~*bL94fRTvOF>it$i!FKF%du^Q2P25*niem(^_UNvJLX1*?HP z*ntL^jRPct^@=MbIp8aQ8RSZ2_jdnkUJh3->+&vJZ7=_F#PVsy22 z>P96CfQ~-v%5>tQVp1_KKo%5_D@)ppqD8S_aq|Ukf!5}4;OrimDnk5MQR2V?G(wot zM$Rl?0AQVgIWA#nU9ZHTgVi$YVPL=(>@81iGyM)e-!UFbMlc66X~e)qx3hBiMetf8 z5;m(shk-Lk^F#kA6Dumt+Epebo3w4UAr{_saC5pI$n!0)foAn?C}{MOv)W2zQ3eMj ztg}k%XI+?P__Q#q1*B{mi6evZk^ZWy>g zTRZe+YcysX98PODdPMb(4lP)hVZ=3vH?IJ;U{zxfSro8=9pHmwkc(T#Kos>9Uo0T* zQR?=tG1UInO4;ReaAY?CcP5=p z1%`KkF*XSoY(Pf*t=}4CSl`ZgtjM%1mL0VCj-3!&2ZyL?us=c;euGYPGcAVVwDFy| z8*dma+%s?o_FxmXKO_G)lDijULE==s>|?9bCF+HBQ})bmx0O>hhkxRDLp8NVfw1_~ zBAm02k2QhP4J|^Y?wRU5sKWi7b z3w44g`C%`)V+VCc%+@|Bwh-FP{IGH{qfA26L2SYrBzSNQ|H+P=!P&g`TC zq?Sy}vcI%hhw1#%rtpb#`)zAW_uT%FY#aeQc>o@CJyg47@t5*S9qJ zN)9wU*~O8A)bcXLX|L<@ufJR|DE9f06T`zu#yTBsqi1>+A$+-4el0z`sCJIwW^35z z?#hv;e6X2G0LFjywU7H$l=lN4+=q9u*v|mb#x~0nSXA*70o?YxgL|+IIbg?q&Qs9N zBRF!m3ZhNo|4nlL1x6$HuyX^EAo~is^>F>(?)@Sw^FoA6Rivcr6Y^A+?_*^7F@Y5$yp+SinC>r!YQl(3o7Emw%0DuG)eA=*M%iK<_TX!-8 z@zeiB&kR1JU`3QBR)#NY2^xZf>5{T20SFki>$F)uMDylhn8HlLksv*g3^rW&Mw4c? z-b4iu#ay#99p$ z-MNj*Smon$k{fIHR0b{ZuyDf5lP`p5(aa#|38_=Jn7JfO?b~H`Csz|ZPVv0#lsA7K zeU-2mE<~}x9)tz<3TijCZ`ZNW3}3_EiA4)QQ2G#%nP%)*4?#AnSa3kJ)bh+YEEY?{ z0=gI&pf07f$U}|1e)uae2$@5yk7hWeD*(mNp`(M!6xjr`Q8epJ#LzTw;DH&^h=Tt| zHFoTg1{-Cw%SWYnI}%9(HVol7(BgmsN+O^j0tPDU=;V$lv-r@;AEIMIj5WTT4!SVA znMJiUlL==PkRBN+JMLKWPEM6>LT0>Y@^q;sJo&V#5BY#NYXbJ_Q>YB#m^o04kR8;l&r9z-OFsX8B_tVr2-1FK`O^ zfh|MDtWtjQIAUI8`lg@(eywm^BJr(8! zP+SIFVS*B@&;w%n7`WJCXc=|v;vjFN)Yp$i9{FP(LcX<#lt=Vns!&6XAkj=?JtWl{ zgS%N}lOfb~GhQvT@mF$(-S^l?Kio}Yjj0d@ju}S@qZ?)Bam}Y_;+UUJ5>$@&BN=3pb@ugglylzvR#iWJd4Lu^iPgzPJ^*c1oK@d>R~a+&S>7!~ zenIGcOZ$U^NF|U`f~1j7Qj2Lj8)zFXoJppMZ85KiW}|9>9YUYzb8U3*iV!$eR>Ff-*1q zf)C8Fr51JpRxPN&67WEwOpVV#Hkg?Ux8#T}M1)LW(v)maqYNWDBQpLj5v2b!pm3}cL@@P00#ND@ zYmOKUB{DH@PjugR5H`OmnoVa zbzqVN+h{YkJk52xiX+gHhX_t$!jrmChnLRe%qjSC2D`k{65F-}U>@R|P{@)ufzUPZ z9S1pFc!Cfq*-i&a08MhyV?!KLPfL1mLwe8!60?%6B);nrO`Ov}M+wDJni8S#KQDhguyZZ~$A!`vLz$4nvw+h-;}stmJdZ^g%p~ z%M(b9rYD^TN@+Mzl%qT)6&*&_z`;e8!qOsBUTGu&yfr9jOru84xJF+spm)DJKqDoP zh-8GQt|>hxK`O|)&?twTG~JPFLa>Tl1Y;JtW6lN$QdyKDr%CiGPU<)i4t`FFsfK}> z3e-S1c23ovmdxj4j~9ayI1{pE3+OfeM9_k21GfVmt66!=3mFv3HxyD2#u^~ewsuh} zAE2dNFN)I1E)s!%SPT$8_c9Fl#j$-oX<$`)vud$anTE~TO=VUTy5XQp!*n1LT-sik zqSK6V=tdmenFJ2dVlVjsZ2;?=JjleNN>VKYRa5^f5g~Trs%^+>YgZXtuYxtKP|R&^ zcbiZu!X=fp;n#vx2~oBh7b*cr?f{p2qaMgVDO{>BSbdhjB03wAft?8vOIcF0vW>?8rLNgddXIMw#(xl{}{-y@l1GE*8wSo&jT3oQV1Rp2^qg3A8BDi z01d4L2GFs9z~K=&Pp~B_WT`At-e`S!%>(~>68N47ZUY%utJP?>iOp`F@UrzJ#dH*| zT?xg8`4CNv>E%JMkRt$x`Qw54lDV62l>5D8zKAqw;B3d;GOiZgZ$F79!xrxeDcwVY+s$NB4PyZ`_C;|3O@$VaYbfG&f!%3it3TP_c5?c^&nSN!5@ zo^#wd3IQM}3s%Ba`Enn{#^l%wD6BTYefN8^Q$F~?m-5YqPW+|B z5_85oUgtZ%SXD5g4!!{W@;C4Kf;J2UVdXRsQ#W4b) zL63$mfZf^rx)7xSRKR0miV)!DZ4&SV%ufRJuV3(Czld-6@Xq+^iG@OH!TN4n(#I7- ziC}Dt@GNXy%4!6w#{^HXdY%sIFadG@uy14F3PZrcDjGyU79{r6BSFF<2gByYGN-+I zaJ`1Z5Im#mP7l*ws^7S7$Q(e#P{iBN!T{jv=3FTN6@UQKu7k|)P|St2cmx#?zyYm^ z?j-Q;JTBN0OcRvP!7KsgK+p~AM9x|+&IqUbSkTU#$#GoJQT~j?c&ZP@sIGp{xo+gJ zAR_+c?+8^d-!9|+qEHGq=vee`3iT`i3sC>oj-e=IgJ@v^^pBIaYy!#f;3`n>F3_XW zQ2CH;wrZ>Sq9_hUkhd_bHtuJCv<>o@tGN!*u5v~Ri*O5U<@8XGu#j*O6)`vh4tE5g zR61k{{I3Od0{~&@ot8C?Jp_+{v3z!MMw<3dpse~}bR5e@q;TU4w|P@$Yo%A^KGFP4jjcH|ovAqWLcvApnMA}=VH;37lO z49Vo+SfGeBkP$)>9bFNuN^**Niwe4yKyf+$W`;@tjO?VW8#bgITHqTOv&+Cy%t+BmXlON#sGt6Y0|V~^ zyHdAaF)YRM+O(z?L$k*CXDwwC{rIt^j4>F6aW0LCF-lGdMPx!EtJ5$if=u(4G@uMl z$H-!B4bFuY+;1u|G9!P=*MdpiR)gBOGBe*0Ec3)OW3e^N^6Vx`#Ec>UN;5Yxsx5Ew zExQvnRVFW)@1>0(WLd6fY6E_jDHCMAAFVtpq(=-?ZJyXW@;9@vy z^Bx_57V-hdvI@5U%*{UU(=m$^3}TNUHwY_0GC(o&D+5RHhJi;B)FgRpKnABnTPYro zYY%a8a`;kHq6|1XkFEyGXTWpQ#B)41=L9U|XUH=YeTd;yPP)%Hx)J3Th zM%~~EQfz$Mbj1K_BF@o3HIwCf^lwN|ETIigg%qv!Y*CJMLP-V$1Y{Fm^D`O4LEMuU zHPk|fu}V3|5mf~Xglq#sECAZmOK)=x_+SP6V_(Vv6S(Y5iLW^IGZC<)c2Gf8)8|I3 z!Bq*APU*x>eUv1_GAyUZCP1^eC^14EBU1R#qW0rAudOtwEeev;2o7X58TAk|^gC;^ z0Cw&}N5fM8p^H696jI;GOh=WCs!~6{H4ZWYRmas$&_+&MktJr8N67(OS`6J7Bnm1a zUeSUEX^>%vb^EN7QF=qsrUE{(;pKoXB5$u_1ntn+?MWAvCT-GLiBY9U!_I(UT5$k( zW+)UhRZ%gOTUo_h!PR50!)^qo@X)nD!GjwP59X{d24nF0ilWAT2UIR(=SF2Jyv_T- zZ!2z2{ZpQCYn-X#WJ6>oR(>ywnaPkV+)siL>6mRj+CI!+Hf#ICZ{JN*HSQLavM+E$X5G+!spT!AJ734 z5X5ooXx4x*VV`w#8rE(DOABhChO(3qd#De(G#FXPB7;_N(KKqS7HzZ#abq=BpK~33 zp#xpPdUz`Zed1-4cOKJH2J3a4)}y>?&*p&EcHO8Li8kVjHg&);N(0x@@a3^s#Cwwt zc}`~*xSxOqifx6{uhafO2LGSPDV_Z#%e3o`8h^ zx3w_|7uYccx5gBnrPmd3B!3rq2fDIUoH358`LC2O@HJsrTMVN#&fsMsD z7gAtbsCR|gw|!su7GwcO`vhy}7jSS`hxK#>Gk|}87(;+{Eiqt=Cm;e=VQb)c8yf%; zMj!^9SOXCFa<(8Y;*&KyBjbDzXsL{Mob(1@AdW9!bb|~z)kAl zht^4sdnF6lbe!xiJ^2914wH>rU`)k)2|~m1LJWPn~&(Tau6oK!|5)h&3T%kf#QKNzF2eqbT|GM8JY$BbqQ^Bbi|r3=uO{ zNHNCLDSV>=0Kl5}$sFEbWehr&*|;M%0u{qyss8N*zM=9=Z#jPGs@(>KT__Q%q@M9W zbhbL1f54+UA*!TcBRqJa@sdxvc4npGz zYJmbqz&22O15_Ke;aIW1xe4;2wXyxwqGpt=syoiyN6`n1)Msx&64g zOVAQN1Er_g0rn$?{*ME&GK`nnliHZWN4R>Mn3FQ_sefa<&3XU-#?%6A#{$^9uBada znh?fg97}dG3-IAIG z6V)$|4t-buO{@j6Ydwf?LtfgWFm^W19Ty}J4M`*zRXpcUX zlOCdaJEg$0`VL`NJ%Sqe%+{;t7Nyvx}`0PRFNM4|TGRs=C+3^8*H99GpPo zS0#@^{4A3ekO7NOFMa)uN3FM?YvE%J2<2GN@%D&sO%T3kZICTYrna8m1Y`pShTr$A z9zErtUDBm}68xarJz#5Qjo{5r9%W9ivx)Bz6&Q^voQ+MX=3u8wVZ*hGHs`$1c z)_{BBPxv8x-Gm`wO|WO3;64sE;!QdM?sxaNkvqBnNnYCT`WGNoM;Z!T+~Ywp#0srZ z0V|H_jG{gAX?630L(BmKQmb{IPZ1!Gq5z!A0Y1z}Az z-YOymnGc`--qeT*lnSt;rm-W7^LySIprG_Tp^L{M-hCOel8@`hI2Sy4;-kbN*xu`x zc#FTGgVE&JlfmM#r(BO5!JA#V`F_%aR6uh0$;p#;&SLOe2}5dq3lkqmGKBYBD@9R* zOAY9y#L4qNlD;1MvB-YEfc}yJ0?UnoM+h3kac~0en8QbE9+87J|K1DBPiJ_NKm4Ry(y?uEwN!|m3m7m)T>_Co>8-g zH!n7B-ui0G_BSwJ!Gw_+K8zTcF~z(ZJH`e1F{{axD_637`52PT8al8%aYxIWW<*yg zh{zz#1p*gT0C^oHcA7M=L9d=@0k#@t=49L8(s^JQfdz~A@aarzrx9y)1q+8K1j3a^ zBz>7K2230u)D;rp4&GpAjeH6`bcp)2frn$aAlw++MaT$uR9Luwa(Y50&td+YYJ?`@ zM5j|zF%2kzOu0}sLW43L(t-#g2v`IE9y|TeAPiW|QphT^2$Mxy|B>|sS|+Z=;a70Q zbxSeZ5M!5KGWu0mje^C9V`7r+2nvsVV1}6^oCU{H1{J)30dA<}fk_mZ5CUbCOirmr z6c<=25o-tFCIJE;cmSqKI@|yUaR;@q)KAWc;elD6XtIiQ`WbRvi1Pq(i47ES5CKP3 z)*NunAdgu|bssSA;JOOMRQ$ROj9VI4{)>3C_CCrMO%vD#8 zIMPThIfL0KEsWVR#v_oId7DcAxS9>=nFHoJ0Mec|&GVE35HR3wyznxxLX#?}CQ$|O z-s@$#WByyFam8);Y9T8bMJiAV4N7O9&LN@&1)}LurA=*^qX!TH&%rT53>{j4x)c$> zXlG>-@yQZt2)Tg<^r_KCrr2~UX3bw_#l(S4_2ff_Gf5W}ff-tnoKwq5Km=A5O7(=R zQ$2Aq7+!(J))P{2D9F{})WWE8&O)2*jDU@8?X_f&y=~gvg3ER#EEnf*NQ|5y5hmtf zd|RRksJABsHb9K$ZBJ5x$^seri%`I4HkV5$SuC8$f8%uWWpO8~mFb@oRB7dtyyf9u z#{-YxQ0WkzM{=VBQKAI@xhi3*g>1}30I+^6;p{*P;vgkKg+w@j>X)--g%JrwW3Utp z99YN)3C!=Bv|A&T>4wzdfViR7&DNn7*O1uiRw=5CPf3d}ihVXT>93Erj_$u7S=-+R zIc^a|p6dX8E07FMsEQ^a5f|8Cfbu#LCi0MmX1GgJGWdhx5I3j&XsrvCn@$Mmz#&yd zkUpeR!vJ7o3knC=5*rypLoWKe@!iscg!%70&TW*?xhlKs;9&1Trwf7<3JVyYWd3r#rkaQ=GcA5f`78AeoK1?glXCa?k{)PMvH;8VVw)TBEI zA(ImgUB@>6W*j__Fc~YTW)_z4Pwv2hPGg#B>qx4>oTgHzz@$#kzQfazjuZ{myQX#q zQPggR6RCS(s#Bj83Q3qxTkEW3g=Aa2( zC_~L)VrD7O?9cOpD!pEuMzePcRl>!%8D4XS0t~A&ufSj zs~EIV9qs$pyT10iNVRdSF;`tz&n|^E7-O{oFZ2uK{ki}d@ub&U^>n}nJ~GD$7Jz~m zykW==w!sLLJIjDbTl$>8F}0SANYfrj533b<6zE2Ske_0-CeDmge>>} z0DMD8fm4!`X$?pD6k#SQ1iOpsGJE-?X-+en*?i8_N<+PMrn8;jX=8B3w_JI&k}k%| z3PH<9$nlep1&%y`w%%dMypGK!P6%Q_y|u0p#V&XWuYyW)qJxzn zY~af1=5Wo&oNjjGJiPFZx6jMhVppK^#kE$azV~fodFr`e!lTWvX-k`f3!5_k#J~Nb_ z7%mua78a)2s&#p-HoAfHrCciZ)Fy{Q4oBSXp8tHN^QQMQ-b{2_w^7#IdNb0KuJon< zJK+2D@xcu~WZx>B#|*Eb!&_FlmFpA-Q53r#6Z|fV5CZK)NysjxG7dU)oiwkSoCAi0 z2X+;o8ONW_6qKJxb$ozhOUcl#hC=cNle*^1tM1Wux<%ssmCQ0PH<~A}X3Jxq^D;g< zu0LPLUGw?_sZqlUJkduX>;Jy0U;ROoJX{QQbC2Rfp`DmM+<-z1^JLU&rxDWAsSTCHMsx=GiPqxHxv-qAC+|#PxKtA(r!E_ zdFWSpK<8rXmw7v=gY2hOKNv1S2zqc7P-fz9A|V^mX94@7fBRKisPQ+Xu_c(`3HW4z zjAt`ff`NIkFBDJ$5b#0Ervf@OIQI7jb#R4XVFXn&DOZRq6{7|lI2{jAFEIc!aVS$H zv~g~deS-yNI+O_1<$V&MAIQOFFF}Yi$XD)WgEy#tK6r1Mmkq-}iIv!cmKbZYAcR9m zgaIXgyvAn>mO`0lLU3EC&=0(mS{_VTOlpH*tuyFand+XNMRJ(LsKzR)dW=Z#l?= zGBOQegBO^1iRQS8>4<0ZbU#G!0>%Z8_yIAhxHo%}CHGhxel!4nqbL|K1Gf}%ugF7u zvIZc~FZj~}|JX#fQ$+cgFj{NUC9v^~geoL_iRsF9w7$eb9h(m2)+jY3nBcVz9R!?s6$Vz;*q20QPZ~ z5wU~@@C0XB6ZUZ_a`KUUNHjT9c+e;&5pWbdu~DCf0^Ok$R=_G(Kms)|6fg-`KcG?@ zqK%B$b4(Bi(pZ3Gk$&<9j@&Yikw}htfsRFKlt^h~e!z|%)&foNB>1NqR^l$Q_%8FJ zH%xdRR;d86F%KSic52y`yIEjiIS?D*WKs7niD)jkXBrC7Gq6{jE)-xDWKVq(F(5&g zh?tN>^b>d~LyXBNi@+w^#Q|#p6-iMDBLqXv2OSi!Ab4stkz!dBxwoR=HiAntk}?!{_lXMK`7g{7 zE74&T5`z`!f~`y6NdDsrwFXEIh=;t zp%aG^U`c>G@dh#=0ESXGWKeuS@K!U>CvCZsV#yjsfripiks;%03b3tW*-<4imp20x zNJBUelYO-!h_6zfD`AK*3O?70L|gO zYFf3brEK%1K_Uua>YE5jts*0=wUrxk<%+(!n_~3_d9XiC&>CQf2xL$~v-MW?r=WVU ztzD8kW2u0%@SM3*1O3Q+U`GY~XaS&*h%1;eR`DN`nguKfD?d|{eF+ue=8Tj&70%NX zMDv&;p)@71QR}G*Jb9{D@fBKAap9N$GN1L0LdTy9+nKgzs|_ofrqERqdxX>4g|&fC zaSI#D(<#$tYF()mZvwLPG7=c+DmFI2#3RZBY_GXMFw};7 zrCk-m1%bRH46wG$Q4F%7$Yaa1SIF7io7*U)HIT`Uw}?rR!uk`D=a{Z)u;)e0#%#>U zyfy;bT5PMw{*tP;_|B#>0tG<3`-;ul{E8xA0~gmPg$vGAK`T0Bede4Y`>+o+un#P~ z(&j93;T#^Pe9y34jZq`eQ(!w`^hELLp0vErRamG+9G@|muJc+VW8nm70nAwYsw~#f z4sC0b_@6&uq~#W5Q;~AiA76z8R3bxjcBow_@rr7m~<6;nYSj(={Lr zAL0bR%+i7|6)Kf?5{L?N;Me)8DLGvOjCdgJiWEFm)KLp2lv!P50L~PNFz)&sILyj7 z(VjT%#=<<+u!^OjEso}>)n|Lhok-Eq){a&~Fh6V7IZH5sDW@5j-pnX8 z*|`DEC?qJEu=uzn=xOb&5bK-+RDsf?O5>ZV&ilQ(Mgb=k+}@vE)$yIMq&?rp3&LN0 zKfBp8W#Pn29?Sb}ASVtM$s%_;2^N8^LmoRD9Qqp*`mtgcy&)ja=h`4K?Zf8V54dgN z4)T(Hr;)8KeP;r4a$V(449@0!6@0ByI*Cg>GXhQBHBQ_D_QdChpsH0I0_cj;r)&jV zB%{CVMV(a!l(*1+@Z)8>!9&h@pNARuja>7D61plY87=B9o;zK8(CRGeH_jb;=Zm=o z&!Efyc3-Ygk81)_Vd4DHxWVwlJAvyl{nFv;zPBEn4RmOz@3?vs^n=|L`MXUoA}x^JA$2J!As zlpgQ0*H<;(GN)1$Gp%y{9-dcNo$#pT3Jw;pG1+3tAx}NHw~onYJ`O+J>l>}Z8j$Q#xv-eVd&cISa2kSjNy z435s7-y{ljag>+G2CjzgS8a5miRr6V-|5)QN3H{5AnyVN^?ne-$cm7rlAwL7kXax9 z@IG*b`2OieVyrZNmQ0!)7QPSsfcDR!>t%lSBjD1)9fK}goJ@*Hf8|M9p7je5cqAP~ zc&|Uqp$xiG?H(~eb?*dQbK~(*`1z2KvWzPOqWGCm^K{?Yj&OOK_-5peX5}^X`L?h| zU)pz;YeHD$&Ma}1uC`Dg^`x0@1!ffn4Y%qPdO!XO&CRfK346nmz*Wr`Q z4_zYR31ceC8341;KY5Vx&DSTOfFw@%8i4<;73TsIrzbNYvxgNiFZ2g5wdGB~_||NZ zykT7aBtRWry0MiD9vB+~^$m)oQfl7`Q5^^7bop8pExjQ10D%RLp*@2r5$v`9%^0$0 z*&IHEXpI{+iV-a~^o46!v5g!(eq0DrBP)_6O`b$qsTZ3gEp=#s$;OWlB*WLthOJmreK45S$Hke&SqxMV^i@*}1APRq zX$&GlrXNzsHo%iFO=cGt%rR?_iw^}V`Ox*LV#h|73zlq&nQm%h?v z&oDV_+Q+%GNLH&tsqD^!C;_98fv9;niy*4LLg0c1BDmtOD+&3^MV#Zlv0|>X;$jP* zwyuavg1Ig%DM6NN&bf@@qfQ!AEb$|dFtl@X zCWSCE@4WKpypugWdEpa2KjXVelKtqL2|q*g3*)5-U*tfBH_QQI0t+Ukbb>VU!32l| zMd<4fr6%kT!mYjn=fV>;?SPVN+Ne zSRf5dZp#QixeD$TX-97#N)22WDm`?|p4iy+bo)+~_4{%gqK+*vYpj3?m zA^sx-zksMB4J{y3iT3=2FXq4nGXBfV`rt5E243o+>qRa;0TgkAF=d0ooYA zqlH>(F$)7*G)q=nX7NzXtllC>y2;awE5zx1Xw+5(bH?H0I4O+Abhgr>aLcHr(=|iR z=-BlaBj~Jy6A*%y*@)je?ZgwjclZ7`V1Q2|cs@3CLO9|7o7zm2;Sq{bhEfIq2tWY{ zAh?u}Ci##+fEPKw!v>H;7I`aOl;Ojuc~0DrpMLrYhV|9uLiy#cOI`2*Aija74;GnO zz$*|&gSs@JryQDApn=c&4-{^#TG?4EsNjSP(nux>GL9pmQQpKB?lhyC);esd4;;D^ zDbN-}?dj43rhQ_Nfo>*x<+e9(`|qD<-~09E+x@@W`yb#6R;a=ms!fXW$$}U+xyeNi zdXzhj4KR?g2WW0KQ2@bxQu1X22rE_g>X z+wq|aoXZ(iM8JUuV&HN<{9f~X*rXW%iwZUm2$o+{s%o*)J$jKL9fTOZr_#E|%{ByRP4AOGZtKfBG*j{Wmf080|Mmk41e z1vDT66WFk3Fz_*wn;h8!G{MX<$^}b6zy?Ei6$*vWQRDbp(e~j972<^-BlzUva#I5q zjH4U1TONU;fD{zLs0Bsj0QX`gMJdIyh&B4b1dp>zTGgmXy`tJKg9e#D5C{q!D(vXNu zWCR&m!3O{<3!IWV_NFftnloen0>7WOR zfDU*-Md=E`EE7`nFrC4uMpqL89(q(aBh;wzelSK`p7cK$E!s+D+J$LS6AWfl@=o<$g>WC0%zA<>_-phzg4g~mDvBfj!=j5XD3U;TQ_X$DpZ zdi8->XDStvGPbcWU@1!44UlZO_=58Q0<5}po-I9M$Mx+2S~WV zIn{6~fofC}^ai3u41+3|lme~_8I4*0a0WEeAj7^|v=LAsdmTMt4djwmfRgkq2Y{}m z=<2%$RF_tt+rdPczyt2~KvW;#pc2T~SnrnCyxcX*Wx=~%t~i#w?0v6#E2}<~oVHCe z;c0_@x}MTXqrak6Pip@(z(1z*wKm}+!yHveimkwLx|OFE@Cll(1Lqg2AycHtcuXi`ZPk*S+&aV}0A@ zh@Bk6SCvJ>egXVn{Vf^50v7N&ix39v z3xGMdmD7}jx%^(?Zjhgz7z>=s0#;9`biaF|GHFtMx9voo;QZ>1L&aar($f zo)aGXM@OLw+{xvHvH%N~>P@p6PzUTaRA&9)2Lf%(wbOC0N2Qv)ur)7iCaExTv5y)U z8`#KB0mnVwaXVM7=W`3D_3f?eg(%KX*ZzV%v?2uo7=58VDUe);7+U zpp1IoTfb&vmnit%aF24q4N4Am!41yFq3S!~Nv(OmH*E(im7SfTlBy-0cmcLcE)TWn z(xx!3aY_DqA3-a`CKZ%sl6!nu#BR1QM5&jSj{vT?iA+n${*)%Lz!RXjco7mO z)=P&?=4*owo@-_1z9C8HDz~5?pu-&4R*4F#;2KUnmJZ1>H<4w((@hKE-}()Bng&kr z@NBkkdP1#$kAZ}Rqm$}X*RpI`-1fCQv5Cy2!i2GVX5V4&L?_CH5xla|16YCM8y&$< zJYxb<7NBc{|H;Y!;Z90VWU?FLqI?CEfeq4EiR_hl$|ZKbl{;sDZIFOF3A<15)xnkV~@~h_+~>8D+Bq_B)OjgEl?jK^u%S48R8i zBr+QL1I{bArNfo~!@qi?BwE0SF8K`yY=-t44rcfRCTs$3p@QuTlLlP6ut_##+L|2c zseUs%43sY<%PjY4vT4A>JG7fmGq|rC7%V6l`pB{jKt2=<6)GqnD0wIq2q7Oyfm+iN z+~BC`qrSTTnxcbpg6-3byCXSRp&2(Lzw7h96lgi)s=@`?apbStzF&F)io}*(bPF}Ope}kSk@S(HaHgt5$vLdLI^3Ip`(1N*I_yuSyLqIT9s0k@V9GBN!!56hD>I$ElPk zZ-Fakdo(M$r6{a!LF*1ZfAh?8#@_`upodaRbV){As z%FWiir@i!nl$^t!x;mEhCOym(&2ojpBuwlKzU$=9;VV7_!b$K9Pw>nFNj##I!a*wk zIL$3lqJ2C{dHl8ZoTW?*Ir(I^DRKbMfR+1HBFf}U`wRv==p8XAOJAa>o|6O4_$}rr zGf|-dMi>pi8l(2=w8Vg-m5Kz{{IhlQjo|nz%-A9B`p_S>OmyjC?4g{DWI_$jmIG^w`gj|AH6~VZJMii0z8`~fZWj@ zg@pC8DaM$w!|FDh%Yw*i&MT$SOC!;q5-%*xE_9LuC>XD2JTD?ZpDGaE4bm&(gTJyb;hU5;Nm zG!DuzClIk46b1#s0VYrmDu4vX6xHowQsp458d#1HkWoM*v>>q2R`t?X-K##DRalkO zE+x|-rLj4W!aMb*p!9Y`^yiP^;kgKqmGrv`yQ$Eljyh zTEm@KiPhR?rCY~6+P{s)yro>b$)AfL+CH?~z>QqOW!%F>+|XUyiv8TjZQRl|-P7${ z4^+{vt=!ken~B(5zvbN3tzFf{-O&}@&_$}$^0$i-QHte&6V8kHDA>KrC!~w-t=XvXvJRb zeO~hA-s0`v;{{sG9be+T-RK2g>y0}01z`0J-~k5U^VQ$xb=?HE!|D89y-D7@;a~ZU z;0cCb|0UoHUR(^;Sp%kE2`1m4tzY}CPTFnO1MXm%tY8iPUlmT=^yT0eo?j21TD zw&4ceTiKoBCU)W{2IF&x;x3NgAtquYuHhx~-XtdDGtOa%o#Lz&<1x;NDt2Q#-nvk` z;w)ZcvAtt2{$uF%+%^{E#>L}6wp%^U<0IBvmtGQTFC`W@mF& z<#_hsdQM|~R%h@fhevj2ZvNweerI?lXiSdgbI#{`QRsyx<_AXRb?)Yvyy%JkXNpE> zjn?Fp7PvXa=$6(~d=}SxC3tyXKU z?rO14=)aWePR?es7UH{oG8p!1vo2_`cI%f$>9+Rkut7yfsIFW%)@#cyZMf#^%~oyBp6o3S?bfE@ zvaV{xKJAE3ZO69awf=0_?&#OP=eZVNtHx#1zGb7n?bT-O%_eHzhV0)q?5)-8%s%Ym z-s$5;?&L=8!CvmR{$X04?(YWg>c;NzzHY11ItJ#&F%=bvG9fxftM`z%sh5{e*cCPURNAcPIZyvvL7k}{p z_vk1e@hLy>#IEo}PV%)r@&@t*p~HvqATFFJQJ%$m7%OT_NU@{GjTu9V97(d|$CDsaUW5p*rOTBi zU&@>bGiJ@3G;MN3sj?@}pE`qr#0S);P@G4NDn-h)Y0jrmqe^8;v}n?uOS3xFYL)9% zs#v>vow_wC*|X_#iVeGVtlO__+rlk7H?G~gb@S?_E48ZJy?_G)_WM`xTf=-23r730 z@L|V~BP*6Xxw2)(Y9ZsLe0eBg&F>O-4lR1L>C$vTZ$8ah_1V^_GjrxFdv?Uzt4ZUA zy*syScCmR2A0C{z@s90u`~F@0cyrs&U-x#d96I&r*Rf~ETz&cU?qI!({~u4jFmUbQ zw_8`wzP$VO=-bC%%)S11`S;(a*T4UK+WGwfDByth6^I{y2L_m5f(AC|;DgCAD4~J` zM!2AV6+)=th8T9(+qLiyyG zc3s)!m}Z$-Wtd=o87G`{7Fp(-b>5lho_g{r=bV4CiKk*}W?878hxUo+nlv%GXrL9c z1gWBeCi*C)mtvZ!qkwigX{KUoDyo}kmddH6qYk?1sYrSXVNa=s|5_=kq}FOHt+=kL zE32;d>g%qv#u}xt)#Yj|vAia$EU=$Sr)sdwntJQB$41NSiaGv=?XthBxNUK~5^627 z)Rt?mh2K^?uBGX2>rIdEw#)6j-FjOux%jGkuA11et1r6p2CS{S{Ss`ji|`7p@WS*q zTbq0TMx1Xs6r*GD#T6Udjg%W_TyVz)hs=z~B$qt$$tL%Ua>^_toK49G$1F3$GSPgm ztS8cJbD`f<9FEUGH>1qYL)&ulEJu&L3N1`G?ex>Jpt6kAR7Y*~Cs$*gwboo?{mUnj zc-;siVt)-b*=DCb_Ss*zjkeltPu*I*Mk;Di_M^2{~w zOrTW~r{y=(edAqs*K#j?^wTF3tvNV7i^KWnLz}~SCP-UuI@PGBu6ie|x9qFo|A_L;3lGcl(6=r9^pi{e_w|FviPgH+ zJik2hfPf#q_~PsC`qHSJo=h^Jk26pE2G|4v z67Yah073#6xB>>I017H_AOt74ga=lT1Qev;11-qG7i92*Aj}{gpfJJ`Ht&SrOQ8w_ z!NL``FnpAFVJJ2+!y3M@7YTD%zP#tc^7(LW>T4R(|C~`jP0%kFMy!P{kjO+OdZLJ= zyB`%$K!OQav5F<2A{J%9MJ;;Kixj|O7{N%!F_zJcSw!O))hNa_y77%yjH4VaFoOvu z0(f}LqaIP%!#eo-V{7q8gG7;te2sB}UpO}WZarm>Y-jHN4KDN9<`(w4V;r7qJrM+*i}fB~GO zAb=@MVGhukdL$np<8X!-G7^x3yxGK>M2+Vy@(I|CBneBXO(+~9h?fXwA?oMJO)}Ab zoQxkI3L#ESsDYO(kbx>&X#`Z3AP20(q-%&|>%_@YKFH6A=q%^Z z*r^LphSHt!oIn94kV*=$GL`lW>M8$8RDiP5j71HoPuVEYU53%8G0^D&sz`=lSk;`G zlqpu5m_$v`v=;f>rbbgr*0P#aqlVyU!@{+_XVr=puV7|2QTjVO{LT-(J8SwXi3&Aj zB6Re7pH+!CI+=Qs5DQgk8UfG%GPHpV5SS`He_BtVR+gwx^(SRF%h}GFK(n9ytY$Oo z*?%6O04As>1p*-1He6K?S=H)bV~a$?{}Ohv-w3N*G3tok_7<;fI4fE$LCB6Kq;bIb z-duavhu_-OxxOW99roJN{^9nvpd;NI{HjUqCU&7Dh`|6D3tKW2AOQZfLOmVp1onEt zz3R1WYAYMxE6mrv_oc59@T=d_&iB6nF70_^Fi*y^fC5#mFa7jTgCrWk!Jg2=gUj1q zn7Z?by0F3>%QrVsblADV#po-}QAkkXrE$`VP)K)6!aJPpxxS6@Bp4xwle+j1!4x3U zKKZ^L=i_~ZNP`v*dg1S~a|k>Q0C^p|)BLvg$?t`)e))@ED)-mE>aFjS`zzne`q#6k zC9o<1KmZ^hIK$&SA^_9e;9U4G{~7CKq7~L>1R~ae3K-TcBk5*}RP=eqAPzAss-ZnH zeUwtkWii<*0b@Y3;?a+WF*aWp(L!kQ$C%FakMkg8?i%^X3TOZTmTZJ3PZ`Qt-tu}+ zy=pDHI@PM?vaE&LjkNy}nf#Kp3Ru{+IRfeY5&1p3RyP=5nDgnq$Y6AEa z)>n}7tXUmv%i`PB`wqCkb8zc?DwzNW=q|F|yzmPz7>SmK_?n$j;fPZ_5svimGvYOF zX-|UMz%A>bu|2M~PLotC|F$tGK=Is4gqvMpvoT-Oy>3kh`OS#U`9h5-Y9-%Wya@hx zs|Rl7{R+L+QSP^v2kvDqm!JUtT&)36n@RRMT+W1rtvlm9XE`tU*d+u8$2(*US4_Lw zFC=uKO%5%1q}+Tgw}Z=xkt@4q;hXrwaT05u=}dImt2%FIh8u8z2K*fAM~Ah*i*9s& zGkxD*mhaP}&g<@;Q?{7Lux$rRg*|iX8NTTAF+3b{7Ls`F@;H^jB*T-lr1&za@Y`diy=u$2PM|`!GfcKXmSPCr>w+gZajTbf|PbXnZ!ud{i`neAsPDC}>Lf3DR&lPPi?_0BOH4 zh2XOd;-@RA|A$=WH-hTNFDx@QQUD4nh*;{@cX=2Eo)&{@cx3P8hCwK11xSE%n2P$g zgU0t>cW8$MmWP*>gnjsjfcS?v)G{2042ZZ0RM;zQ*Eqc(S7gJ59HWcGfP!NPQ4!a7 znP7A8^ogO^QdvfdKNbvd7=3ftcx-rzs#tsnxO9r=Z?AZCvG|6w_(8RJgtpj@W)KP0 zU^s-R4fOafz1V?`h+=LvFlO?FmJ)k(aDK)JhR{HF1xXAD8GAQ&ZZOq??xqD22Z~^o zd(~J|rI>Vx@QtX(gW%YMt$1ZuCX4l?KnS#s?l@NNXbjYFkT27cEQv54M?Q?GIF1-3 z{-`m-|Hw?!HIViAk_ZWU(Dq`>$UowAhBpV1*k^IY_GH#L4?@U|+sKM_7>DAR16A3P z%okq-7GVFiiJO*<@|1JzXObt0l4coh8<&=9>5_mmIpAk{`B;Tnm}39(35~RqT4RVl z8I(aO1-&zdl}0f>Hkk5sV|U#3I>wt#6t^L=?HzZmZnLMh?r5Qpf_^(WCO=NOWfpmyGiYB37FDXHUibEM%{iUWnOdZE1=ksT%xRqj7M%zd zM%x)$M*rZQHK>(CKu($PnV}g;D`b|>)Sj_$h`e~3D~XyZH$H*j3$5vz`^Yl`W0&2Q zJ}xFQP!peV2X`!0eV%D%Hqf9BY6DSF3!X@v68TcxsA)b$aU~E8*vX-o=?M_fe`h9q z&cH<)Iu21qaSfnJ5Wu2ymZ2uPfX#Gqskxy zfB#CTKsp5D#F>aHgncTgLr_zast0`P2^*Se%3z+J6n|n>i%Pnr%GIQCMGC60LZo?{ z$UvX>IR+w@i~Pxtb9pNQ!=)>tHBK}@NaLF z+oG#cGYg)>n13)bl;bq$Mte>KgAwOHzj?B}nUp=VMCKp^tTk^W0G!Qm0Ra$y1pg3! z0Kho_keNH14+KC6KC1%xU`6Pv0O)$Nc~G;q5CB610B;bqP+NFId$Uncvu}{Inb5OZ z>#b!LwE=LHo_JcP^`Sth2I>i#Y8jtmx*Gri6S_w5DtVX zpeRVJM#Hk$1wZH+cz>k}H>C-8gm<9WnH-96^LB6o@Ul*O0+kB@Q2+!0paN+?UYkn@ zWJU<%l?evmwC6<#5OBGfySY_+0es-OKRdefhPtM!x|++mu$#04fB^`1a63hK$r-39 zg|_Tjs&3$(@42dhguKQ>QKxE_bK50Y%Aa_f7S+oPn1~PSc4HOPVsB7;oBx+*mp3}b zs)Cd>d(F^y1z-ReZ~;@-yEd=@n_B^_%UJZ=rUsV31E9HIJ6=toWNA>kKyU!_Tfi6a zzZGD)44?)2s|EuW00ZE_1>6KQ`=<4KzzNI%3IM0VH<)Qaya781Ym2-iEUU~bn&A_@ zwko|WEH5;|FxBe_Q=q*zS8gp9Y*`0o5LdFF7(e3xc+KE`=u-;k+ftqir*R4b5wHXd zV89k&z?!?j4Gh3kjJX3wz!VI{o2zU1ivdwg1^KH0om&879Kii+x)M-96x^U4D}U+h zW1~7!#cNF@T*oGC#~B5&5j(w?a=nyTkB;yJov^)rX-9jT!y`*mS^t-2w-rSdV+$73 z1k6grosfx9M8Oph0hz4HoP5Ez%fJ{=#<>f@5=^^%K)`u`0k(^|y0*Hwdr%krx?+rQ zqH6%CYs(4{!78u-9N+>i00O_9tZdwAs8C1j$%hY%$9KHU%&g1_n>~EI!ut5fJxR@i zLs5UwcVlVDx5aSb@V(wlM8qao;+(j`31g>^wlmJOeGj%kmtdU%RyQEXn)Z z4EfBy-=N9;YyjWz0ui7LCvX4(&9t4{4GH}R0$?-^Jqr;G(JvUm%Fwk-+|MY0(dGaF z7$6QzECCK6&mu6=K0pLCaMCCZ!o(YM+~my7?9$7u!qF_v!vAuyh;x(ItWo9Gphj2& zsL;(1^>E+&!>&b6L_M<)%DHGbdzIX*=DSi3l+9l|4-0??RL!+O5Vb;^z(XqmU467* z{RazBqyvy!OuMw~Q`X@y0FeB%=CAIl<-eaE5z z&4ozQ(p+M9qBA&+v4N~nFu;dldR=Wmvgg!mM(uaApsX2;0kp|b=c9%p>pI-aIt3IC z5s(M{d;2_rGklEp23k)DM31`}WwW9re2?0$A#eLi> zpazSo+}%ryNNUGW&<*0mpl!SafB4ea?b35AeluN#e*gT0k?_)S3_*I($lvh8R;9@E zGq|)_K%R(hX_}w{IdcLU+VD+%=p4k%od&zj-?V^dwh*DSAjtvV-*i?Ew0X|WFn8nDEUS+s1xX4tn>3V|rFC{oHlD1RaCAhIh$|nB9T>(t|yZhn?cxZPO}F&A#ZK z4>&<>Hn=Q!r`??1iOW*--PF^#I`&OLX^N(;hYH2`m!nPOhFs)um*k?2ph*7Y+%(>P zh{9&+;Q{{972sF@&DT_p-7>x6VD5z&YoEu9wiYy33)GNI~ZUtv~hM z&Uj>F;eA5vM^Sv<=ZgTK-iAB?>gT2KNQ3U&8~@(rkc2f3I@0j`f{)$d&Wq@yN!W^g z*n4cv_4wV@Tum88QX!N;)@9W7GnnK|*>FBYd}TChI_FK^I==I_u|Dg9zC4A#JI_Pw zN;>G~W9z=|JOLS3yH3Z6PU$FI=w+_gyT#0xxC(Y&f#xF!i5}CIUf3ue=8iMwH{Fxl zFyfOIOcEugfo1CC-eX_|zDgv-=j~&iO*9&7sIHD%+e5D;7O{{p-AA zmqhIE!Vc`Ofb51|>-%2tk1z(uv&~7E@X!kj$D82C+P>4h*XU65h>-5{myyYh?thT+!?p3# z>+#eSGa)bXY>%2Ro=w5l@;+wXLL)SYAY>{Je<}~S$BEhje)EVs)J?$i@$U0K&+jS} zJv{ILh)?MKPVmmd^pa2Y+$8TuYUpGr<I-!px1BWX* zMsuWc{LI(B?XutEZQu4d9jxF7_dsfUD_`!O|2cYp+Xz+RtWV5;)%D%w-h?mj%s=ag z??*fY2LliJ0{`dKf9p)I1mF|h^Z!%GvZ?aLtWBM8OsMJxRxV`io8^8b`lN5d8-wGJ zx?S6a{O)t*DxJbJ8T+&E5@-()>d0v$II!SBgbAT7yhg{&!zVv}@X0sNoW+V6F<#t= zQKPviC6smK__4_!ly{n0T$$#=9W|N8m29@+5KNpmQ|ioFYEdFNQ0feQMt zfvHcgCan-I>ShX;`7t}U+y6Yohi#GMJ(6TdXKGcPz|h3_B|<{+q{Nh|d6NoL-o9zv z##7BkJ#+57BR%_1DM7{m@>=!jAUIv7+WlFS{n?LW(f==QgOw( z9HB;<7d`W)HPhgsvBuU^Ge}3*dc2V#xd@X8MHRI0WTujMt7x*0v;cx8AV7!!1`ep4 zVyNX{%A=3SmXPAD#~O>GGN7{4?h&^lJcJ_H@rGq!_#tPd`@)qS)oqilG=%$ur@L=yTx+pKXA(p zpYT3$dQI}SYX7lMbkkKA&)WV}CD8h`24i8e^XoZ9Dy!t6NF*y5OaumomZRXL0vUUi za4s3jw#p_In6PE22z7t}0;u?6jyo24@sJyLykyf*u2u3KXKi_9gr0dhX10o?8A*IX zzj;Y$ki_T>j=p(JZJ?#)G7A^zfP2}*yi|H=>6#$~E;ZUTmv`NDiKNa)f)=XyAiq|= zZV2yev3anguas<1OIY1vP_ZvAqy{R?;DY8PrIN5aFXH{mLoC{miYv$eTfM;-C;;Qb z6HnD~#__L&A(Sas`G3s$NOQB1!za4K2<5PEL!aA-=tifFVyQ26t8<^QhQtgu$q6ta zh!VE2<^L#$7%zkqVUw{M)(qdE&{vP(UEgNWomnuiE6k(efsn$L8MY!0!e|pJ?f|zC z08xAPOVto7v;;WC#X=B^M?Tb`0AkJJThNJ9CK9nh>Xj{i6{8r(hFC-}l8hKJq8w-@ z=RaIoE`a>=6)p6(r7tCiF}mo@JsfzmfuO^5dYB*-MzAjyscB8LNP`6W)k3&*ZiLYK z!I;Dp1AKgn4tk(YW5!{p>ZoBU{@NY9&~u)4L5vB)U{4O?6TT%FQUa@#7!e)U0Bm_t zU==t45MVbpB|z+K(UD+*z?TnA80~L7;6YioII#kJQJG;hRTUI+MrcM;ngE2&8eO)A zEdQ9vIAgNnh=Rczc-ZldqT@yyO2-jQ^l>j0wBR-&L%}FyFoS$?7=sqH0x2Q}0s(`< zhOA=&XJl$-w#Y+Z{8>;mEN>4q3}qLD(TPL6OQB5IT_&oOF(V;yXq;?mYH5~1Uk zxJ+mkmXVBVHLX91#DE_4B`{(Z^C8Ea-!c`)%#A^gR?jk;PK5vCLQy8RnNukd1r~(NfRm5~+MNLjP@9 z+zY=}*~OaT^roA&fE{+VQ!_@hr`7zaP=~srS|TT@bTplb3L;LY+NctEfs6S}7ZTPz zE1xmhWfkEx6SOrbW_LgY(hNx<8DwGzZ?I@CBZNHz&}B*!P=LWmL07%PpcE|81P-q| zuMHqJm4BVC3_Fs-a}C4`>C228qrNQU6N7gHd551CvH5oi6oS zX@yD^z+gb5&w(Xtku;KZVB$0Ops+l+aSmdIb@u6RcO7IL6?;uMFqH~>??0i--EHGoKyW?@xeysAmLhCGvh-V~kwxUa=ZKG_mx_u?@^(BT40H;?qH^yvv8BLD z#sd50@_)~}7&DK#e+C|7vz>irFsGRf*=!73i30(pQk6d{#Ip;$?Sce&o3?w_W1syf zR^bwrLmI}aI~yaBbfX(3ffcC_>hQ#l)gn{)v_)E#5EJKEHyWD$3jZ8baUenPAsnC{ zM>je_>P>@zI2XleP5Enaiq*8&(~@pxJ`jtZCHI9huxm4xC5VXOSAC7Sjwci|Kf`5A zv&McVfyrFpWapfXy}+gcyXq)flE_5H;*W{CodBdG-P;>>t%uL8DJU<|eT$fJyqPe9 zCIqkOw?s_KGXXP#F@bi75X?jmZg2qMW(0-!bS@4Bl}*=zL@M||2vofQ0kEJk1pq(* zWU%+X|J_6RUAJNt>ke^r&>YSICUKDZ1H`v$a!jG_Rl?XvY*9D`n6K|GoyQ+k zX;>~A-o4CV#>7Kj-<0Oz(e__AwUtjQMZM&qU8{*Eb$Bd6F#w9yr{|t8;A4qFgdAP^3mhe|26v4N0TTfXl?JA>1{?Q^?j zCXO1JdkbJ9n-_$gE!2y(`4k!p zk-fvbnd2-w2r4tQxdUXtG-JaygsIa*4AjvLZ9^wL3Z8tC!-~18!HCA%$s}xB6WoXo zpP)5P35P+PfHo-uSa1T{3kLg$xJ!XY90SE3Y`cbII8UTUIG8n2i~v+95!9=IOE{7~ zSf$+ohq=k6AR0R{2myX1sqnxMpi8px^MUE$1OMcEv$yJ^Vf3#rL&jr7Lt6PPHDt!G zfT(+t0**-{K7z)|yELPMlXPOqVFN~Pc@!9+2p7ZzY%>COJb)-NA6OudP^iSB(Uw>s zDnPI(maL(8BslVcxP^i{AN;`}yhKt^gi+keZpe)lYso566Ux9MA~-7xdB_54n84VJ zh@_Cnqk|drNlmB)xvNQz>_`A3!({_Wk`%o)B*#TT6L645f?_VilqGLdm2cx18!!Tb zLLm&C06mC96rxN-G(R}OJOYTPZyHO3+s6yp$_kkfDU?U1s6-vZ#68f&sal0`)HHpp zm#stv3P3R%c%hyt#o9tWlIpiEp+zu|iT`gvguTg-M&UIjI}F+plK_OdEyK(6(#tgB zO9m96zl2Ca=m>yeN5d?xEqfITGL`1Y!(kJ-)EUgn@U?#HHVCMm+ibnBdr0|wN>idf zc!UbTR0XNz!AxvBNLUIUSRUosLD94qP!Nf3$N*|15VK;IPk6<#iG(&|1_z0dSzCs) z@inE=gH-e*ha7_-B9iqYwgYfZB7#oNQ#R_nxdtRXz`VDaIFg=e165=>O3I`Vgvl6) zs^+K`kt-2#(FH)FO(3uXw}HG>`k5$lKf)-`(4?16GXyEcO6x;SxWmM>3x^coK}dKg zP=d#;jL;&jgD88INRX*mk-kYh3jeZcGAe~jD$F|FQMe5R1H{rflEt@tV4Q?dFTcV zu-Dn~Cyf!2nBbn6J64JJx?` z(`;%Ver+-=m?0z(SXtf5lTe|u8P_|#S;;HRJyfn${J&(vOE9`vj9tJMZQE!|v+GpV z0%c2IWZCTr!R2xS8|8z!?Aw2#y6Ynolzh^w%e50+)2uB}5d*bO_#+Y1#6tKa9x@D5 z9RXo@+!FjWq^(M~bG>o(#LEpwdz4owm|7{d4}eWv)h$rSdmGAHRb2ATmn_i|MZbz2 zMzvj5w%y&0bz30$SpT^lLM~i@yDP;l$)p>-z0Qrp{g{P?;i3GPr?bSREYMsb&9v23 zNQK)zKBxhrlUJhpv@{W4ncYNjxZH4%AW)-OQ+0!r(iz+fj$mrlRMjD^#M%B(;SzPuN28I!a<}H-=<>61G z0#Ks@D-efk};4sQv2HwjD{$emLRM(Y1 zu%+NK_B9fSqyJp!;1qy_6p(;b2qu8six5@@(w*KZtQiSu;^oCiXMrTjSjZkuhRY4r z62zNvfG8e_SN28PQkw}KUfe@4ghp&XyGrCBqFkHBU;7<}PHy6?#fDu%wM1;ceL@s7 zhQCljrXmTMR@34X?cy@*Vg;sUor8`w6X2DdEZ9xuUdF}4pn$m~fkF_3K(n%~$p4acHCd{XGX7=xD`HL9CoRop-K(#Jb6f%eVU&VCLh#wWA&MUeKl*yg{AB_Z zNH|T*g(BYOEnoqf)r9fw1ey)qaF`Gb&4pz6>GYsnQUd6ph68g(|KSOLc}w76+M3!!7W+}kbLFJoQk`$=Hlon?q_U<0cHi$1c7=H-kATaM-f ztosrt(b7Lwoq6WXM9@+psFQmGXh-$qO?EG=H3K956k-s~*pY#wIkQ5s$G76(rZ0i#t@tCrQOR_*+i1EjK7mH0X?#3c^;MYG+n zFFb3T;^GD7Z8B=n{y19Ki5%59VJJvd=tiJf-pqVz#pn{TT2@T<0zm>y%@%jmiT zkpP#N0pBtL_aXyVzyq&k-|ZC*lxtbEXuPNTJlHj$P>P}L3Qt52ML5?|7&wR{r9B@w<)PIz_whNX?;uay zE*Mn+<$)IFk1ZEK>nq>DSecf7 z4jqSc^>q3M$~B+Ly!D>ea92G87BVr(AT%HNg3pwML(ktHAf{5L0vtGYW|!4w2XRsx zg=kM_DBXo>pLT4Q_H3VaAV>77?e?n;cTk2)Ct|KFGZahi1P}G^BP6T%={qDh+u&BO z01H%y-tBcrjqo7`SC^aFs)RJKgD4PqE$_BhSxhT$@QZ=>q|38eA00fRzyH{Hjqw~h zP>2BkR^t1R7D@dr3b}SlSoBd$_Gj1V#I-(cC$<0Nb9UW!m*;kG$60U}*jr`dCc-zP zYwS9ev3drd8&$8uVR#drcaAAW>AZKOKQniz2)J&ghEKYN2Z1VodT`U_+rIh+CmaEw zJz9Erh!+m7&z?RIkPP~Zc0r2zUT|ocDRF+N`MwM zYp3>>C-%Tkd6>`NtW4c|-n3hAip2+7cBO-O(9$VM6V|H?giZ#8>RIGK1Z+j7zFR+H zW2S^Z0Ha^qTnU|{BW?i>d#a~BrCa^DUiI2uKeh(}fiIn}uLx)W75}QA$y=uj#qV~v zeDLwJovTCqATaj2euBIy`NXZt^+khdkMY5;e!}O3>%U(guj*L+aPR+q@E`0G0m`mk z@ZI3=o-G;)&~&XQQ5wT~inRe&IC^^*)C?1NHW2tq_;))1h)Mznvg$+7P8Ayofml#* z;333_1|||%7ePoP4DjF?rklL()1HuVT9HHAlPOcPe9UMF4Lj5U z<^`H7a^~!EfC3dOMw12-Lk;Ra(6nJ=&F!@}*W_Z;o}KQt?cBI!XXD))wr}3Hfa53Y6j+s4Z@F-owjD!_jU_$8P`*($yJOzuDv@rUp9J+Ot0Q8ZYsq%x+ z|DfIn{t)TWqZ;RlN)1oI0Eb%42*chu%fwN~EDeS;%UTB}6V6$?;B^;-BDRosl47(7`5SP$8M)h$ zMG|KXkxQ;doN?MYq#Q(@Ij2!{(|ObZbzFK05=$+C3I9?{;W0Lth(ke9OD#XFIU$_M z;9(z~AH1mvo3-#E#(X077mO~!%pxEY6VAhxIIBRC%%O-vveh@@IJ4Ma8D_{~U>dTR zCVH=+ncf~P<_DE~o0%$ssi?h?7+3SSYK}5PGTCI3wbrJjleFggBr#E1FxnexG&k&( z#RhP!b6I}5EVI{jsRMObahje8BXm;;9(q`@EuPhSE5fRJ^w^^_-Hfmzx=BdzXRCZ{ zu&6VTe$$CjaXx5oq^&Gv>7{+?J4cAq`t)zQSpmX?RFrv(>ZzM?@Q07&1v+sv<%j|b zGSke8tF0aHs_Vza<-w2)YBb}CM8r0SEV9mS+5fDx0@pXKeKj_4GtN2N+zktwDtTP2T6Wg+u)xEq! zQ!!h9IeLvT1|!=v>iqeXq37J^sH76DixyG{BE{1F@nKY=Rm&;0>|P18h=sXlt^0;w z5F9Xw`~ALOTRxd_k}~9#9BgW8JO*ppF)GX_!egBAi!plx?lJcv_eyx-#*O@QxKa2F z`jiww9yzl!WB#>Mh(WyL$s|yDe{%Q5aVQ{{RKLRj-9&xL)=^w!QB0 z4SXB>9+ZwlKD4RLarb*ju_nU4XSwfv94un|@*y!qoFoqgm_X>PM3E^@Q3lFTj1rR< znM_npT}(@0IF?o~+reQ`unUL(jMAu5R8T_}ypS8QhP4oUVj1{>-v$e&sWLo*5D`Ng z1core!<=j!;~>NxgeJWoxDanK)ZPqfSVLUV3WtHq!{L6`qaB^^hw8hC0lG9DA{w!j z`%%N8RP_dDM8N<()1N1ab(mKflK(uCp`24xHw6H#?nVBC0VKpC45vvj9IEgMB$D|G zw(#y2c5x;&;h2}+@cJ84zlEEOcZw zs3A`oMzWqZY|=dgrv@ABa5OyYB>D~jO3-L#i1e|Hi%iKW0PZtxQ+y?IM3=wxE$3H@ zyTAzN@dqRxp^G?B%13?jQAU;JLD2}+*0{Ezy+|{Tni|EwG?l?=cA6=j zX?nidDWT{~FRw_XXqvzWD!45LhvDQRg{nCi8WpKYrDr~kB(j900TbvL89zmL)hH^2 zksq<^iRuuGJ4n_bwnfElaf^pFl$B-<=o~~W$}@^Ul%l5Lsz*7RO;C;Hq;LFdS_11= zz4)$o&_yiq6r0_=EEAZ1EQHa(%ZD}?ja!pFZ?^EDUMWB!5Rur1G>!^dd;ZOD9~%yS z$KxX>s`jd&d4Y}!B9n5U1Rv~S27-}Sj*2n3!Q+9Ip7evwBWeb?6}@Q6qR_*+=Ewoc zT`po(;am*z^}5%kuK!Fi6}GQwm${p0#ezx-#u?MtfQphs9p^b;`qmIg_(*M;*lLj> zzvMeaehHE->_gBU3nNHAVPvoC0EJGpRqw7qC90uO9s=F26Z8OM2BVSSqOj7=woJ53%O1- zmIg2l6U)(rUH?JXAthc09Sbn&!qk~64Mj{@%F^bp7|!t9LW>RN=k69WbxUgj6@q33 zLDhf?%5y^M^#9Q9T##YhU$JIubo^+f2JSW}F}0YAZ0UK22|PX1^rr9U>6gd^8l)aI zCNgbWOehf%<}qvrQ3)sr=Yzjyh9y8(vYaT?1d9wO#0~Dsp~BL6&TuB%G4HIku}MNH zq=>egb%Jt||G8^ldam~PFC8Ch}DzJ??${AKxGvhL6*}L{FS;z@wq9T7` z(@hRt2>@IvI<`Mlxc@yt`sAkdE_Mys&L=FhOaA;~U&!Nl4te11O!(+xw=lx~)Z#5u zK-67cQsX`^&9i-e^{@AJhXy)=gZ}v&j$ZWNbL#j?W(%Y52q6%tb;@u??9E=Y5TN9E zn73d>hdf^(z=s*MPzL-~@-@V9Srn}i8REE15TF5zFvVegAP9Dp2x^~L6dM+RmE!eP z@n~Q70b88~h2n)D2fkdlO;+RO0xwY>6l|LnWZrmL7M`4&{mCDX(O*8@9}*f4Kg9sr z=>MK}Foeoj3CmQSC>daNv`=z<*s-A?7lw`JOaufdL}>VohMgH*>5mH70&*o+iV;Q- zAlTE{$sf$&C3J#q1<)gGpqv#J_xX;>sUS-Inn&eb;;onrmO=GEUbm@E)tpznWLqsf zqM}6JB$m`FFxn8#Uy*2xx@F=JW=s-hQX2dR+@zLiL}3(aL6c~5EL*%djVDJ5x}pQ1{AX0M8cgJ+D#U$ z5N~~z8>*GJ#R`rH$%5rq#T>!{KqD9|1HSx7Bz{9NspC1;7#pP=@a(}MiV6oZK^(-S zSEbvxrI&Afy&5WHu?>$tau_ z2;W0or2f2}Sl(L!5Rzh)mfUI7-FXdlI8P>|fdFtoOqiX?%!4DOBwrxdA_52t>{4K& zz+e_8VIC$4lt5wzrWQCuO}?440Y)B?qb)wljn%<0(Tip}&1QCHCnytJ!2be#H5w=4 zlRPRVQ>Lc<gV>z!^kf`>@DCYNhuLTfius9u!i;on>UvT5Muu zT3(Tq1~;1d0}HM63mixU}UbcmPTfh%mI06on@$stg~nQS?dX^j!@D z;$0&llx2jOg{K;fGPzN4n1s#M@d?6!Ccy&wQzj?4C+dQ;34YA(f~qp=cBk!3T6Fi;gFq zx}KeuCkM7v31-MRqKON2;Eo2VpbqNu5h;=?>7gFwlJ+MsG-}c{os(J^OITHu3RG5s zK*m+6PHGlIz7U%c;&Mf26}~8KX5bn|%Zm0Lh&mk*uqk#rk2@%Ygk@?cAO{uIsfz+p zj1mao>1pBhA!U|h@dPTN7VD69R-)o3q85%)J)Mr!kA^wnaYe*{0@1|%&u!MsJ1LCT zEh95F5FFBsm=`dA>K}Wz+9@004xF8wEyYAnG1n%l>psLd7UokBVq6@+NbTuiRn7(s zfE;w*tL6bc;9=Ql01m86zoH`gkduq*0iRINo^~$cvHz#ACal7a?$n;#D_HH-o@UU| zgJQ&+mv$}e{-S_RW$+CdMrnhOsqJuXM3Pm(X{nQY&~4pjk#L6Px$Z5%LRpZU%ob%_ zb@|nn;)tHu>*IFA6GU#=paKe5uG*>16CE$j7Q*#9E%`F-)ig%vMs4XbAp70|>OQO{ zN~|4#3B1v-{jyd4;#D)D^v!zrIG{`0FEQ6S+Ib{Mq+yT(i>h{tsPX*VLHcmuX0elcFzXX<_1#9)F@A?wz z>Ao-4;tHcC;W_LuC_?Corrsn}LDr?#*y2xC4gY`(JjE5V*>|9Aci5iSDTIchhF47s zfHht2by5V|soKpUsFB?r6mDH$upfaZ2K1~d-GgM2Ri-9F3D9X(g<%6=ZfKMtaqX%c zEv;{+Sa%+5j*4y#)1oinaQrT-5A)|!4T0$a0q(VykFj&dcFvMFEKD8cC!Kd-xTiG!*^MjYcBC+HSbN3C)%RL<4xim%@7VZnl6 z8>Vj{3vvygl=~tp{2ub6E}f(D!Xp0)c`?KsODe8DNAT5VRlZ^?wqljyQaFATb&MJK zRX z3NsB4GA*8*AuIE1Bywsh9ny8IKH{^~T>zQYY5&%x5=gEUl9D&`=GRDXd9`WtfLoi2 zWTryrnZBSavq^P=fVj4)Nzj6PaJ2T`^CUkJT*2mBiLZo3mu>3BUf^<+GVD#^5_5DWALvJebRjY*;sO)<38q<}wIHmQD=wGE zy{mM(bX|Hd^UU4BW$M0`@ND`sS)S!i`?IbhO-IQQ_O)xD_CN0j?Kel6^(^OA0l&;n%u}o%LWjAy7 zUAgOi;L>!X!MfIo`^W`quPVpxt8obNK*(JqoQF;0b(;0pt(j$xXaN+tp1^wUut6XA z#V{ZX_i&3QGUsqoE;Cd&a||?cRkdDokEIH~-V_?-H+qcGW8s*@edRzpjJEB`2je>QK1sA;w8vAm=kd$)!d8(+|tki=obEhJNq&4dXadKXr}^xNEwV5d$M-6R1EIFs631gl{%D z%S>-kcNV(e3OHE>XY&zA`PCe*N1rOBi{JtbtsIBO6Q#~YL6IFl1fHS?&UPP-ccGgj z+l?R9Ip~HFBDbm|H;+>l6&-k`cG{4)h>>%00Ab9K2#K!G;oV*ZpVb_mG>A(dO?@yz zB;W*~SB{m78D{(sO;?U!J8d@6LEvud-O$MZ1dSYT`ZRCYUkf6LG!{{Vda#Q6a1%G2 zgPv3CIDT%d$%s%XM*pi-Po3P@02@HX6Jx8hql7Zhgh}?S+#)+ixyQ{cW&h&yu3O9`pQUCZ-=N$75n;IENofNIFv8<7ikboYmVk*1vO3{a8oYbH{hF9(1`_(|1v-`JN54EzV~n5K}UY zQ8L`l;#F-jCv%dld!xn%-Rphc$8M(8{Jz0N1?zmN)#c$wpwDlS8pO8Q>xnL?faFJ{ z0)X8~Jj^O^cc{9lMi2t5+);0zeKv{yck#~>T>}vS3>`HNIe$Hi_esZvdpEh6d=xce z0|Xc*0|^cs^hj00U(2ppbI9#s#A*^DR7Oqnwa7^taW000037$8U>vI?ed^vvP=hOQh^qkTr2Gzqkitu$&tWjJ#6sv;~~ zYzegxp#LV%lmrymYG!sEhd;9xU=32_EfTpxa0r#UQ$U`gOm@gsBox7of z2EG?CvLW@4&=y94dLCPWMhIo>t3lSf(Y0tK8 zyPzB0yLpfCT?bWg;j4(3LU`)nwuTh5HFthd8uN@CkrA_gz2h|P+oMX#fU!Qm_fL zBItw}2T+UaFu54HBR&0K>!Xamj`(YiQB2gzKpk#C%z+2UYYaxJ^b5kv>GZ0~BUZiG!3q4Dk|7-4f9f zie$nuOOs60JZhOwUP;Ij<*1Pkx;pLL=)3H&%X3d)c=?m5^&V62r9yQi08yLbGf73o z>~j=9NF_yJ(!>(Wa4s4+PzkI~@oE5r1uB4T!U_$83sVs*fSI+U={6@(MBJwl>gR7 zBUKeuhDD^+(@r0}qpcf6_~a7{9d0<)j!7hzk~s|Pva%L!#Wh#`7U)&rmJQ(X<4C5> z=Z`}&QR59}oz0{L@|qDRLTc?8It^{I-FC2~H#rD6L14?`Nt8dC;!Qx}l=EG9;EfkL zJn^m9-mt}v;e--m==Wa;E@;m|L?veM(=hHk7-3c~RhZ+AH(p@Z7WuZ&hlN+Y(Bl+a z{g_x;vwHQ>7jcD>V#FlqVnLUG9c_aW3wvz|4&_*4hF(?Bq!~VzkYMy7rvRPY&x3L^ z%-GRT4F;&uQl=lgiDWx0l zyYtrDwCwA{e#P#^-@fou7e5iOk+*_$iWqO5O4M;pcdxzrX=-35{52V)2NO3rwKRhJ zn~Tm;3j(nwfOPp$Zx~1r4~zf`K6@7KCKwCj_-;AwdQR}R<_!-Lk6s`=9`g1@0i2aE zfA-Un0IF9vNo7xPFeG22M1VjE;HQQ*l;I4^ceoOvPjRjaUwyQ&rUC}T6P}<25$a&1 zBvN4otBV#DY$gai1V;;0yrLGlm_%B0B zJmjAEpil@;eiD>y>B|DDR54BV@O(gYOZsS~h9k8h3NeTbK7KH!B=W%keXGw^9+89} zu?TRF3M2OTvc4spiI2ANC5CW_7CF?$jc|520crNq0^%04&VU_rRyUOrFwVijlxOBN-}@n_zL@wxG)ki)bev1S_3o(ho7SGOnQmBs$S;7fZ&*NlV07*55+pxxv(;Cq5UT?O+y6E z@o%AXNXThpv!Xz>CV_PL#1q`;OB#Ic60smDGwH&ZOV}b3&>U<_2^-Uy;<2VisNe;6 z3fT@yR!`%!?AY>91(-wuT2)Yx+B%!mn8Z^j4PqfZ3IDvG%i4v4uSFBI8qro7md-p@ zZBQnZfz8^2ajy6@ANqQWBK7H(hB@>Ab1CBhR3z_bgSb#g(PN8YW|Rq0&}Qq#MjtCsu;VSSVVRfBaby#xiEyk2=hWn$`V`9X{TejrL}e?(t1GuU3uG{n z2KwcwC!KmhfIKwSEM+MvLJG=t`mjjI;4~yMEFFArs6~_@)-8ug1Xfm(v7na086zmOp)`=Y18CS+3X!S;OD2IvjX7(z z>D`qO$e|o~u(zl`Eovc`8fS<_^3K8XDvl`Z?>5SMtw1%{IV%ZDdH z`I)g9K97)MBg_!Dfzg8?8LnVO1TgC;$>HKdd{C;zIq5hqqalU27S=5mWbQAblf z5MW4EfV&q~OU#~!-WH&W2r#{Le)rqogr%l6&pU8WsM?!PPWT7Ev+$0CAmaN`c7wvs zAc^zZJ&DT6PAcFN21rRA9_Ki*Z4veaIR8c{NP0H+z#w1&?{m^y`ei8xr|m%d+2%Lj zZA%O476;Ivs`1V=fL+pVv|3VfQ$f1&_9deLNJ)t*h5G3KExW=}UEo&F>A|(`YOaTn zXjQP*(5%Gp`~dtgBkjzH3&%3Ipyn$d+A$^4k7)< zAQoPey!}{mX%6CD;U(ViOLpbweO4eKP@kG#o4jRQGa}j%qCJ-X{6|gdIGVQ&6C60! zyHjP?PnNQnOun#Vo$$*%dcyZ8>EaMzz(&gf^&?VT$iebP&@jo;Dv-|(5AhVwhd5A0 zL?FO^ZK|r^?}CoM)Q0E~Bm6EvJpbekNeJN%MDUh!i2g{g^j-?}^ly)@&X0b_|NajE z1F*7$iU?VO2r+>e8cA)2z!ui6WfoAl5=IKA5UO}c`l5>QGO*9I&jT0FM{$3hM2><{1`!|I%Y2Ll0vSgrLGP6&r^ zsE}}-RNxA10e=K$?ZDyzLrVg&kbDM%`zTQhwXegv@V5jlRX#yYE}=n|AW24wM$j;q zL?tw?Du7%>{Z5btUjV!ufB>T6SzrJUZSV$7&-&=d6SV2bj%@XOA%hUE)x2ZA1~8}$ zO%2vV2Kq(qqEQ;*Bc4L8!2h0#3bV%w6VLE6@m08x6aC@~8}BfxBns4u7RG}XT7*oR zAs@7gZz4bp@rwo`(CB8Y2{sLK_$CF?01@~q7qPGYc#)%UfCHq!m2{Bcl#KO&u%`;q z5G@M{(MBC3ug+F(fUYqcqbj#hg&VuEBRg>e!%;;%F|LH*<;IK$GQbR8U_8wh>|rL6BZ-TBHpEDj$tW>o zARDM4*J>@!VzH2doBtZ}2Z3;$ATk;4Vl!?+sXmMHxDqb!DI_(JB(sm(!g7W}u?a5F zFqCEAX6`P>XK#M8hH7%U2*)lz^V9T@Aa|oC%jRV6pbjo55Ct*5X!8fB@-Rmz?|QGl z;7MW}(^3jVI3tsMz)=g;hb(iB2)ZKqG?Ss$qB&91Gda>I@T-9A2p{mkG)Ip+6-YH# zvt~4J#vpe_3|Bp2W)oFI?juNk7CJEb!(SJMrE13uxivEam;YSS3KfjUA|M9ZlN z^Ti0Q+Y>y3st*7K z5i!HR`s+oF6gZ`%j-cc^vkXnnZ8#~D6DBj2jMMNWXGq5pE;PVMPZCM_OvG~Nx^PlT zb&*PAM}CN7I95S60r5O-a}b^5OTRSkpn*nDv`jlSY>$RxC#>XJ$bTG}EF$Ck#Ii z4OHlMR9*jp6jwP`IhE-PJ*?lbwg(EqO!(m467;JqgR5Ml6S`I)7wBujHf+U~Y|GYc z4f9e{Axx984-#N*{SG4=P-k1wWTt2yS(l4u05p@dM+4VzpO#0%AT4coI&%SvzM)qR z0Sa7WC2OM#pnxx+rgrK87h_CuVPJ9$=m&(Ca`W|aGuL_zVRM`H^@NZZyEjZh_j|)P ze08OCR}KoJFj6XD6_lk$s2~>#LEoN53*mFE5vaaSpK9hX?&w+5VK7D&f^ z=l7H1R}-Wl7v6VB3@A+&cm$r8fZ;aiF1JhYQTb@w*jnxfxn2l?$B~GSc8)$d)2dZJ$HoDxP)23 z1!@gN+gI2lg)8)|@yBbbm~ zIe|&11}@k;zf+8thJ%Yyd%K3RNK}o3c~eigbmbV6>R1yX0E_!k3jVkd_Nfk__=f*| zR06W9G@==ax+)QtVfz%S0&J0aQo=E8!e>ejH0KwJA9#R?qk)ql+juAnbSD@+Ekhb% zlOe&24Xb@1=&)*Fa$A6%t4Vd1_kdNI(xx|zb6Ju;eNDiIOFA^~NC$t>504@oxS4-rT8aPK-~iNV z0`#E8uro6+3UbpBu_5iEBOnFh4}gYh_8-PX%gb2pmJJLQSqM+q%J+szYS6$F_Stc;ovyohJ5S2oI+Fx)T(k*Q%K}M9E^T6I^y*ueY-uI=uhKd!kAE7%~L4 z>4~vXfCPRz#$lWVn8n6tJOt*LMVNYT?i-4c$bAP`T+-E>+ekR+rdHEWNUIH4Lk__K zQU%fF{B{M~Zb#-!gTjE(cfYbDGux?GG7A(^YEdDtmGw*db)l6;%}cy<&p17`8njVd z&b!(VaCeCGDRpmX0g`QXQ0AK2J2GmbZ$`5R_eaImDR&60+q9O5teYU=@}D7W`Ebd>y=h>die=yve(q zx|+0ixMWJ>0`4Gci3B~)ise`^P5G8H#%wK4$Q}9c1hm`IeFPS8r`A`&uI_XLSo(pR zWEld;Buz4^z&(20wsDepRKK#X%2PLvd?_0w5Tm7QKdpp~fIsm5EXf&ZL%CeTIYK3ZN(jY-yeC ziV4;ipWQh&9bIanU;3Ya^|%{PdKuWI>}*REYAv<?o9|E18bzH zQ7c>v3A2)=7^=w`lo3(2JRR`^MpP+Fp*Sm)qL4ExFv?2Fg*GjuC##a-X^Us=*;#m! zX{u|N58P6(RE3iCYumMFgso*ejMy7F#oD+vehfKrV`POXTLxzAGOx|JIJ5HHr}O90 zq)Sgah9ot|3^YKzXaeK{&H)4rM9Fg^O(t8rF(LBZTa5}8QA`?8p`d|<*sPm-;K^`d zLg+ZC<`VnuNeJv{@f@sZXyK1b&7h!KB4L608EJ0wX)$AZLJa~uf*eWPfJu}plfc=~ zQE`~iAb?8?=ms1wAM|vQ1at)T0TF09GYMlN<_WI_(fWHFwBW?Cgt zMyc9rEvc6u1GGh9PBWgQae@V6jwzd%w*i6X0kl<6+XZqA@SQ~uJp>09(%mVR9(h89 zLO26OrQMz3x%OvqK&a6ka)i8LP82|-Hb!X?oEP!5Ko;WY0aY$8KBZlTPkkWPR78YuRCl(D;9!fns>q@iG1lqEj0M%mqmFN( zCCnjx{TNtg)EFitHcCoXB)3a839kQWQ%XthxTsMlCx9(wal)2emWL^(#z6ptEsm;j zX&!vsre3bf@$J3yoe1*smx>@jsa2N2_WfYQ1x&ae-6i_LC z`SEvaq+uxG+;dcjDua6qs0y=Jh46r@I8&tO;!p|S3g}rM9r%?5T72`~0dSEZRt`*% zSS*alG9gSZ&o1^2$$*jSLJouT8@|HbmYgjE z&67q%4oxR(0QZS(Itv$02eAJ_>Hc9a;O5X$`KqiwSPRP5{pJiaz_!J+%^BY1iPR*b z0&$BT5>0H@P9wY5*4_2kMb(7o=;J1Tg^g{t-JcD<_uoD#Zrh`kYsw_QybZSp74)HK zg)78sLlvwvW|IphQQQn6^B_04PV}pNbSR)0FhU)V#ere2gGeL}*uWwTY6gf1hP?m) z0euw*0$$44;rLXCtaRZAb(mENPlynXsmgKnJ5)1fm%_x&Nf32#oDYUp3$plNLxGY4 zk91ZQ9S&t>Zs7`6;SO@DZVv2+48jSdoz3l}_eBTpW7sH1(&5+N0 zp+TRN&c&+d#4Q4N7!Lo1(x9dS44?v4r~(_vl_(pqiCzS0Q~~;Ts4E$c2fq13AN(b{ zfc;C6(IKImEVj1`tSJw?v4h4KBtwZj4Ou3;(-hibJR-9mePJXo6J z7{h$oLst_Fh=}$1gs3Oj)%7z%! zc5H1cSKnGYjzV;ooD8W*_i$H}8bYr?V&+R>3Ru9(RE#q%A5GbI&66pVBkO`6PyhGR zC=f$`E~so}^?{~yRD!bVY*#zIQH*Mbswz<=u62Jw- z_$g&AYXkpe;IXoOw80;lIzTH4KqMR>ZFE$dRjRIazP6piZe0tV2@$N1S^$q}Ef?SW z!WEIPm1}T?OWXt>cfp2GF1Y>*P3T%{u+p6_VjYpqfW|}x-mHQFD&b+vMzXS+B^Nq7 zyHtLK)&Z5`6e1OSS{Tc>O7x|%TJagF)b5s{N3jAyoFKYYLPrvN5by^HJmB7D^uP#K zFoP+9QU_PVr4Wwrgfmm&wTW{|)qQGC@i2rF48+5mrR?5DjAd4ER;izb6HXC;raQ>w zwMbqhSZf?$eJVo0L`jhm>jK)`DpJr&F0@v~PuvxCDrO}t+ zFGv4@0v%=mFTNYeNV>up_KPfv+x+Ga%yX+WV8k7=FiL;6`bCUbLTeJ2+FJHj$V2f# zE_tnM4=_O&nuwf0p7Ire{dZ1;HguBf{9E59+GI^$^ppD<%}7HyrWCfY8M~}aZgO}W z#ZmK!Lrqjx*yN|_L`N{pi)yEO*@k|AvPPBhj0989HMyZ(jhjFWqe=`ryyE~W)*?!XxyrK#t6c2R5YJx z#D_Q%;N;11}%>jfbYY_{i=oqW*SZ4P)d)y>dWm6#z` zk`?5j0dnLwzGbKkrWbD>YjA)<)wy~`Y&8?AM?~6PKPOwL{tDy3Mi0*Fry5YfMZ(XT zLN;Mv1~vO3G{Dhp2iypvOaE3K-F4svKhh_2FXk|(@-)65UC&Tj=8Gp+Viy~3;n?05 zc>}>w!F$f1(gvw7R6qiJ0|eQ-_n^7Kcjze~1EHsT@{>3s5Ib)W1BlRis*?W+72|+( z@Bnl20P~bVm4JI5FoBEWfNIBfJVr#xh6BX3afh=|()VuFmjZS%Tge7;hc;Ix$7qA+ zXj{a6Eayz*Q-g&?b35304{&pw6L$Rr0^S9G=aeO;GBLEl8wMaqPPJOYp@CWgbp;?h zojwb3Cdb6+B)M?X~{1+q7!WN5cZHcIe z-UyD2C<~3KKIJ51@MIqPl0%(nhD~Q2n23q>BPK_PPFJUnK)?k1w_AUwcU~riAWPqVyqo@jTcFWh&Qw+xS=EsFUKzljBH^>H}`>$8h;z73`=% z6A+K_NGAEAiAzV6n2>n{Wn>p+T~7E^_0b*y009`tSO}RWTj~F0KIoO^^;V@40l6Vb zNk?pNXGAPXGXVwzv}Ojqgf1y~l8w_w4Ujb?;uB<05IL}p4d4YS(oyEIhlzlPiWWs! z_a`$Mg9~PpHwk{)b{IPuem#kqKRF5K$Qtg^Q^OG(W-^LHI4>okka?3OlgKDUse39Q zmHX&O1BH%Fz%C;KP;fJr7!;N?VH^w6l?bUnJE1N<*f;uPiU-jmv(hUw^M*bVeFO0o zf{=6yK>`Iq0h^*1oCcCyHIhY%6vZb9e7PZMv5X%@S4|X?`sF;=$Y7SR42&q5kT#i= zX_;wOpISzK-^Q5$7lwKyj|X6yW=5I=+BZ+<8fX%P6lnjI9ybH2=ZcmXRcrVHIAw|q zpqseIn_)6XzL`_NX`#Cr0Sk~E2@n7swT4mAb-{=@IwE~_aDg{ajLJbGBY+g!*_UWh z1D{x{$1?7yk=1(E20Hf%M zaRiT;NRR!r0#=%eg?2Fz0|X(Xm3p8C&`D1c*a82CK|9oybyEcafR7mZ04jh%^`nq@ zDoGbGnreg=SWyEbi56t=hflDFIC7f}f;DO?D^!pcA?Z4?^;Wd(By zQgnblYk?QaaHQ;sT<@uk@_C3Yb(xEppNK_9-rf_r<3h)L$Adu~1Isjk`WtWTpcpCz_K_mJJH%D|jq&h146US(b zB?F@yS*T>;mz&a2BdG^0KsW%1qvA;i#1;b`SppUG9ziOIa}gIgk^-I3b%l^cG8mck z2~AJ>s@Dy#yt~V-tM2Nun_By0Ox>bC~;|v0ZrI> zbyhDAxTX5RJ73bKBbz7+NQO|AC?&fppCXH%zzV{NffI8tE<2veV{8`LF<9UL0?PvM zffZq)ArWDhNMo}}d!u1-JtN>_A+)KvaIgDvB2eozHk+twCbv}wRLC4D@&~5whQ8GaMdtzCLDg7dkE?|Z16HOlv!<{4Ttn_`r;cf6oSdA zjOy#S0!xx8(svb6j7Bgt@3F4pi7SU1M9L#eEaJqF8cP1EFi>z7kWizp%fN&9z!B`K z3p>FToIc0E2GX&+y4x!HdKb#kG7WGd*SWK8(2peiwiv5bz9v*rzy(_ui37KF=2Qjk zfr7f&vJv^iX>qeTOP)n+xhlAa#CxNSOmtK69a2omk$N>nWWY$Y$<1Ow4SZax3dSub z#$)`G=ZI5kY?yNlv^xCEyEx8X$W!I3Pe;I7dTuG@TtlLbZm9&#@{5 zZAo^pE9mkGq_l5RITw3a19L~sHu?~ZT%JY@$tuurVSKa@u3lS{GhzESxI>oqO1RK%E>nFz8P>ot0GNaZt;7aXUhgVRN zQgb~+!xf+u&ad0ReqfUbOS`B%Q@3k=>}=C%3{N6J1cf@sNbJOQea`~vKs!P?$~NpSRZ~>$>4xgP=`4H7}Vj?XQvRUB0x*#w_mF<@c?Y|uiU?K|yL+N;OfX_VSl zFm>5`ztCAhwAlaAT#`Jzs46o$qwReNf`A9Lq9RQ5;0``Z6AmiQjYR7)qsWb^3Y;F3 zixtOAIDUeiezMnuhke#f%B9?m+YRI0EvuDa*oeJh*-phFCAYm=$k}hijYpZ{pM|N~5n83pA6;KD*2T|!|APXE+J_+Zb z53Q~bs;>VIElWx2z~#PH0hxXvkSw`pnTooCbw>08z^w($U@!tJq3Ia2z1JZs$Oj#n z+q5?hQ&AN(dI-=W1=V;(j?fE;^`4LZ=Shm)fGsvN9pf^tK4rTRj9!lG?g)%N6DnDK z3<+zzF1fv)ieL&_{!+{WWt;xQm0y?vo`VU{AcSFgF#}&QF5rV4K~(y{>aQ-G3!e@z z#J#}WnfLDNXfYM_=%2UVy(zt~P>8(XAng`p6h&@MOJ@r?`^~z*O?4FCrcJ~NY(?6> z?Xrqx?f&h074EaUB;;OcHN|CTeAqe=26r^J>^`M8-6|rli>Ua=2AA)4JYd#K4zqr@ zjMe`PkI@VQz3{jU5Ag8tGR#;BfFuK71q{sqFadu;^Ulnp9GAc@BF0S>dGhwcr+P$t zxxP8CLQb|e+hoxqED*BrV@a=(59N`bDPAp^DO2rEWC<4N*3vE6&5c6h_=f(IMSt{3 z@A>8E*F?X{!x@*uF7Uy@h3COWvy4eME%EFJm05o!R_!^U{q_1B_VCc^WdHT_Ko9Jv z4QU@C41o9Ueoj+C=Mmr=Z4UwJ3y*>bFI_0*0%8e+S}pM8BGDimQmk5S z)^R#HPLSok2K601TEmSC%rcRLR(1{bt)G}778si!k`5G|F z*Cze)Gsle~&XW9W1O1Gewb9v3pDs-rwd&QPi@i1mTe56kv};3toV${4-i}QjV{=3} z$RcbmAK#5!`J~J}ozq+}LIxpoKrCFxP5}Z#Oe!L1!8q{`Ch{9JenkI#CCUGvw(hA6 z8K8YIbReK3U;_`|etblox3r}am;7VOmaDExpsA-?2xlq@Y>A?*3bsln1wJZ#kgTOL zumOm93<3)S>d1S@DGxs+%)+;-IIs$H8XAlW!{VUvlU{I>1;=VC6OB1PZV5w@&OU?9 zwbWW;4ap;IW3ovcdxNsK;D*y63_q4jE=%QtQ_new^iamT2_P^)0qnSY=nn~KLgq@e#P5Hq9K8*|jL zNY@to5so1pVQfhwlT?yf)nbFz$!Kq*HcD%WrLs8Yu=H)YAB7;U%P+%Z-~t8!2mk_g z(G+n_mO8xS1?SS6cRi87k{by4okLwb~WOrmRR?ZnfOfzb>P@s~L4G zH~?LzDp13&G#3!y-6i@EHBOJ#!{nee7}~c8;`(hP7_mJ-ppyS{a?p);y6@VCGAv~DKhTR@>sR7HCIhA;6MdJnz&%og*;&$i0G052lXcQCPXmSONMA3 z&1ughnPl)dtY;8GiF-s$L}uUcBHukNKe;;#G}?Bo0xT)i(1 zq^eA+8c-k&w-=1t0o54~*WP)3$fF7%95)AG9QOa9Aedf+4@e$Bq7I?sVd1)zbsR?v zp6t&k3bKJ5W{|mH)B**ZQ&psXAfmzKA{cxa!Gz{A!WmqsgCR2A$C@ygKb){2Fc@FF zEVPP`b%AHYJHujl12n5$34BL$!_THQJt8KFdPhWJ*82Z&J>GCl2Uglv_po$^`2A;m z=PMT$)z_V;z%6dI3g6x66%sBWLT`HN8|9So!NAQ!4hN)@IeeUF$+y}K>~jKV+f=$p1=??B7IEc4;aaZMz-LPS!l`w@vzBF7NQQ9gaRgE z7|I{qkO@Q}UeJsun{8!=3qW*U5{pR0A<9yT#mmyQoR|S0O3@VF8e>iFVhb#qDOBle z$W|uyMOxWSj2$@xLI&~%LJ+|s#K}V*hF}CeWdV*Hk|q!W^i6oFfp!a^7r5+n5gU@y zn(JI=DM|SR5x^58kenwBGZM&pvQwV&yeA_0DNz4E=8!R}bP{ub^U8vXA~e#19@J)O zOWWj+G=ueH^Jb=XXt_@?bLoaO3x9TGO0aKYCqp;&wWaD1W=9YKm%z|Uvw3r zwDI8&m-0#>HVKK!lI29Hw#tiQG+P_>QAdySQ8=UO_TYZ@Ng)CRdDH0DUt;l z<$wwbsY468)&+$$7O_MPr~MG3P7kPH3Wa^Fd0NUpFkIxRL)}7WKP%YLYBs9z?5k%3 z>c_Hv^)`+GpO3--NV5K_mbOvrG@eD#X|?~At-#7CNZ<-rtr5$6%<>Z*-m6vbcF~4;j-Xe*5yJB5F>A{ggRK-5>>OJRg@@wdIR`| z;Gd?&Z$=h5i$BbOv_zo7C2-b>2vBva#<}G|VH+3ljnxNaedui^oZBM?E5X%T;%|YA zg|`kD!N(;QUF#|W=AP}jZL6Qh+(p&EdZYn_)YBkaHxg8Mf(+jsj7;U=T`+m#2ODqz z6Vj_Wh|Nuf{hjX?V9qnvNG}!3v_nGzF7X4TbjYo@UE5y zt<6njV7pPV&bFbk`bv!+FwBjFZDYoK z>rg)u)ftzPgsfN<}0S(-Ky1c35HCmf$5hidWfcY%%d!kh6D@mWMyr-+?EB>-e`}%vo>Icj z;VOIR4wiW!l;+i%Q&@UBfA+PRm^}^)cH2aci_ybf2&PBe^^U#lyWto0c};zgg7Erw zD?-Q`*iJBf&}<@X47|)u)m1g{T_sD0kGi{E0WN`93tSYSZAmsCQ^i0@7XS9Ri?=R8 zX}jA|rPzoi9$y!CkvabeT)Gvw#f>ZPHn%ZP&>(U?t)934@p$g@CTlw17Cd=mS-YzOgVl7U`VB^FESegAr1n2B94D3%`p~ywaJ0D2Ry< zQI&$?f;A|*`J=xw&^$Q1xmn_dXmc&=;XiBfzi;|8B#6173#>-c0(u*rt6K&1fi#gB zCh+PLbqgj9u)P=%gb=zA78nT$tT(p6i{Q%zIvl&+^D(o_i>VOA5PAXTJCGh+ly1O? z{rLfWFpw6xlCsZ zjFCLF0&%W*!Fs3(q~0qO|Fj}wPRh$l%* ziibFfI|w;NDF?S1A#iv_vx7l+L5FWR9se@2HF6W-k;V3Ew#?f+XtTm*$vgH5=VQJ8TC4(@j$g& z!Lm2%8Xy0li%GnnHuYKkx%7%Q%2a7_7 z!mVfn%VnuZgd$7evM5|S%i`(YRk9G0#DO~7`RCg@;VZ{ixO-`w}i&FNI0to^r^l*WvJORR@OnQPQ zBtXs}>vd&zCu|^7+!Q?SNOs{bC0jhIM)`2-};BikHQO)pLPuRRqCE&{b1kl@b&mfUPG$>HVn2cMD zs4LXP)KE*`1Wr{FP6>TbwPefVyv__wr}|tZPC6_P?W7PbN(?1YRwEAK05IHJq{5;n z;b;U(g3+&_PBQ>ePV!HUsDs0jf*oBXGkC8X1=9Q^j{gi&&KlAO1D7Nyv*cXO*eorL zKvK#|QX*w2vRa-ib;2w~j74ahm&DVXM zS97&6h*DM06VuVK$Z;)Ke?3=pMc8Y>*LaoJhJ{#%jaYuwSBj-ji!D=w( zlf_8kRN0t)*`qbuq@CG}UD}#WSey;Lj{Q<%*jb)^S}iP(c-eq6`c3=NZ z-SYh<#aUnn4&VaD;QBRS1pd4WZr}la;1I@J3JzZ5WnU2X;IYi$6~@I4{$B7!;nI!a z8SYmR4qg%#-Qd-U;icgU7GD--VXM_(7`EWXE!iUe+lq?dBz6THUSJ+3S05%_59VGV zcHtpj4bZ*c9(G{fl4AbtU+mQy3U=ZsPGRV^;vG@rE9PJ=KH!1nV*OR&B#PrY_Np@e z;u$tv&`Vx#O=IT8;@fp&875>o=Hn^0V{OIbME+tg*4!%o;{^Y1<4HaeH>Tf1He*L_ zZH_T>(y+hZ1HYL?||HfCGiWf#uoK;~s4 zhUGd7XKU8uao*-`MrU;P=5^lSD-7m$HfL;x=W0IZbcR~nHQ?F(W_FflVkYN*j%RJI z=Yh88NQUNpZeGV7<_`wwh9>BThG`)Z)@YBW=YalbYrf@?hUtjTWrVKiXnq!k7Uwf|Xp{e?+L)ecnFeZW}H@O zmPX-@_UWaDXpBB+(Ewlk^I4ESYNSr;p#DmcX6mLs=cm48k%nX6+G&5TYFMu6t;T9) z?q`ywXnxLUi?(SanO>^4YO)6CoI~rp*6KHIYq#F(zUFIbGwQOoYn{g6o%?IO24vZ7 z<&I6`l;-4FF6=r^YsF^l4o>W%-c7*XUcnCIoHlI2w(GOD?6gMQ(x&9kMr6qz?5W;q z&o1ql=IqQC>Y+9w)h1+-9c$5EX|7&o*`95a_G>wVY|y@_-LC6QKJ2Wv?Y53>gZAdw z;Mq$yU*lFY*XHew7HHGX=K`+i&8F?Hp5&iyW=VRc?&?jm?AC7ePVVQv?BGW2x9;A! z?#+`;@76}{$wpO{hVPc{Z?<-C0RQi>4s42yT$R4>_BLYt4Q-eHZ;tlu059;Dp779x zV*Flj?6z+z&hNVA?)a|l3J=o@$M6hi+T5020*3GomoPF`<$@7#@(%A1KV0(;2mm4Z z1O*fT`~WNf0000$0YCu&2>$@@3A9JB;6Q^26DnMIkYPiI5F<*Q_z+)3ixlf+)X1@; zM|mJae!RG`3^L&zt7OSq-pqMz&UlwYR~G$OvuDzv zQL9ePdbR7Oo$;D}O}q8!+qh%b9<6(~ZQr?jBW6duI8WZff%l#LTe-IpMO<}Jw0OcOdd)`b{q$RC93geagoBYx=HQyHe{;fg47 zxZr>$R#zg7B+h6fd=cJ=LI+Lju_&jUdJ-q>)SpS)`IrLYZWZPD;sR zlUTMGrIlP(Ipvl6dAa0~U{XnDmK1J@o|?F!StfgD!s%w5Z_;UIopaimXNO|4>1Tio z?TM$JgA!`!o$e&6=%R==TIi#Y_9l;q+&Yir>1tw z>8Yx=>ME-iivL=weX-7pDy*sIy6UdHzKW}_Zq_QRmcJ5dtgWCXtLvt{0_*Iv(Ecc_ zH`Go$?6t_2h3&1;cFS$E;D+k0xYkOWVzuR>>ms@1ZhI-8*(xhtFM{+ zPS`It`(C*2zycTi@3{xN`)<5tnwxH>_eM#)Cmde)|IZF}jsr>#5bZwun~?{_2uJm16bT}R$>9Iw3I zbR^Hb^Up{B$n(=zZ~gVyXMeruo&)ZEFNK4@`1s@tyD;WFJ%ha&fW*PR`|!iBx7xSY zeznuA>*9JYRO`aOGquo!IOf5+*<}I`RG0z~6u3YcG@*eEWPt=Hh=K}Qa0ODhpbAp3 zK@O6jgBA!O2t7yv5`J)mCcHojU$8??~9)frKrP0jE#1;tHu5_@xK7_uZv#H1p&dKiA$?0mUbH@{?JZA}G5+$|+niiC~D%5utK4d_hgC-1-qRi^Ttf5^n^{0BgK`cHuMbY<#j+5b*k z(y;*&RD&+z_(op(@}R^_s6vs+(1s2Zmvcm<7SKr1ioT(r7|m!qov=|nbabP2D5pB3 z7(|ke)P2ABCOEq#PBD-(ly=CWNY?;^ESxfH@l-=FVA)Fjd4h}i#GfuKn#vqRGyoLf z=n&ZGz#$;?se5#&1C!d&I8q?0dK{!NOJD-31{0vZY(N2UiHOt%L!W5fXF1UhD%&O826;Y9a)1`x~rD{wGV>R;b4nfSSg$`o>oktP6Now?yl~D zY0yF%`1x2k0${Ra(1R5MfLWzl;k~uJLVVK;-_goabh%E)mYFb)4!w5C_Vh-FIVfb=Um+eg5X4RStxW-Qu;sddWkBi(N-qlYJnOAcT z#8)b!ZMrd?M2v|`SX2H%niIumK6cDw9%DkFKzM9LWl%!NmY}jmK&pNHTi^Ne_sRE# z@@D({-vC?r$_38yXb*4z69_cH2bd!wJTYNc)<6xpEkq4$sR!OlIR6}lumS+jyh;rR zmkWznTqD4?RW0RKySmC)7I3Ub+uG1t003icz?f|-VUF@480vj1 zZ{M5S4)cMda3;1{flJCf^O+dPfcCRxOx(WBh2oqp0Qh!Qc+akNoafTWmaGT~_ zthv|^FLpARy@thMG8>ZavmdA(=rjD{9Of%DzS?wXME?+`p#K2HjAH=}RkfJhSVRSy zZNa*9`?%BOwP-HDtmMtMSKs=!@~ZPq<$d$J-&wBpRCC?tVLLdVb#}9|C7y5q0+k9V zRql_Xt?_D`cp57GR=i%m=t-Mh8~IWPBWTD@oF-O)Hov*2g*@>?mT;gy$G6e-zGS|u z7U@n^x*G}Z*4^q>x4xaTOMwBenYgU$!`)^!g2volz^~k>a5T$b6$`f8?rIp-V%#}B z^PIOk6BV>VYsH#(KYEtQdnfvoom%wS#~$!xeJ?;M0MOajz=j!QU;=|L1?^H3#~!7) zBRVE6Y4}2X9shXPOK#|q`|1;y=i1rRZlRFY!JVLgcmK`fo`>+-!t@qRfdPQL#;7CW z^+Ct*&{KV1C&!n+^S?dqJG;*-*qQ|mFak1^1wSxT23SL+^im24O5W5jz~Bmw<_MCc zb;Y7d)@L!~V0qKi2-;T(ybvqRfOa7`Ezsb7VH1AhH+uO{esgzv6y;3$RRRsLQ)3i= z^p}6~hl75nf1DLNTGj#u7=#Hpga=rJ*n~|MS2RWQXoeGcUpIZ!XMy2@eLB>E#K13T zmxWr04Ybe*Fz0p|wmPH7fim(TVa;Sea*JGMxLkoC^M;MD02Z@uTE`dm5*d}(eLULG$fsW!0 ziO7MA_=qD&i!-A+f|WZn=M3uB44J3~K^8!q*lzU&iaaKY7A9Y%_=Yj4ciotZ_l5{` z7+}EnihHPsCPh*WIC<>Yj$3$(-E<9U!YVW}f}9tODVK7z*fG+kIgkblo`+b;Cr>J< ziS7n|{4{&kxMSETUx=`c+_+k(2#%-~W$-tSykuP-W{&6Bkt7ufkQO=aIFj)AKGHyF z65CG`K8esZ9=MN?*pJc{SOY0@H~%SmwFHYEnT}TJY|wTy{%DN2Ksbw_POnf68#5{^X@T~Lfh`$IW^;x7 z7?S@eX`Q!dxARPAG=i>U4ihsEKh~2z`D+Z>kP|?I&d@+jc8WGgk$rb`5;+9jcvZ>( zl>(MJ^Uw?w$C0vll^6V;nGpwM2b#-mU z;EwK6mMZxpXGtZ@(2^N+!TyS0JNks0=+wgO&M;p7RQ=&`IeC3r;#j@bhT!DUu`8g!O5XP#7#P zQgZz{Az_LQadT`OWIKIOGBX1<`DkOIXG^o0M<+0+a!RMOiAt8ZI%?%!kJ)1@&<2yK z2P+7uWHy-@`d~b|Q&AwOYX8WhfGSWqnubmT1d6Jt;;2iJ`hJGWqpb6z9lAS9+amXQWN*0u+haGy#jC?U6>2QsDzqc-@%Zp<)@ zZ}~OQ#ZGT0jUN?2b^5G%GzBO~RQLm&3UEjVpg(>p1B04|dT9ZZ8JNmIspBe+f0_?L zhOS!RtvtDc7BH#K_^$I{0dJtL_NoRHz^)L0m-5=LkJum89^tu;2!+ z>Y93%mR30oU8=E#*oa^Hu_F5o&$u;})^^QGE9ovgPBeLnHtaeCliM4DJ zSF=u+TZzl8F8?UFql2dd3Ty)l(7(64K?95d zQ6Rbu;JO45x}@8?rz-$U90l)N0jWE`Dv-NVY`;c~2eR9=4q62rl(OMft+B*H5}dpj z97PZ-w`E+$edsVA+{WCOF2G6+=hO%Rn-Ap>JisSUW&lkX|_Nw zm}m=BFvD|+Heml~et)n83=jd3TmcdQ$-J9Do_om)G+BQj#R0s@3IMwda0!x&37AU= z3@`whfB{Ub$~<|=jqJ*y`=!0g=qh3hV+)06}+Fm44HnXMD`a zoXlx_%ow}DY|O#l;(6d^g&;tNx z_uSC4P|p(mYXv~jRDcgH@Bty9(Ys6pOyJR=L<4>$!Q*y^%6!r&ozm^Ij(!NuZVV*S zOoDR!K0^O^fGDsEzV%kg$Fe>psPpt-Q&RypAk+#>2!kq+>s&tyIyN?z0`%N84=qoq zEDll4v{S&eGho&EfB`YOyXeXRUmezI(6hG;v<85-WPR4jKmbr3)HD~$^IFr$#h!RL zJnIQPBdyYUt=FyqGnRue&s>2ZOoDVk(=}CuDR9Vp+y=^EdQ#)lzh#>=7L zNv(`$bHe+K0hy2tpBxUJ{S6?{*?%yvNQ*(}Dg`deqD=s_Hee141l!`!$Z$3TH2VUS zsoS=I+r919$?yWRFrl9vjW$5nhMm`SZ8f`nNaqZY+qBoxebOxLynsELgYAHYeau0a zK{fvk1#-5>j1^Sl&0$E5lb4;E!icAuEt|oUK&d%VH%6J3250*`-|j}=-|%C)*$t{R zamgUEmv9FmxLxC=-v^#p3)c;cItvZ%iU-S{6580O`@83N2X?63%e>+BX$pRA-5^dh z+0D0p3k-Ap-2$3HzlOMnOWwR0b1<&nj(bMx9GmV9d>V2NE($u`=T(0G~ ze#T!Yv)SuiolfnIJwUV63}FF#+#f?EgHeR0{KzLliqCl%wp6t`Rg@YWz zTmJ0p1b5hejbL2u^Tg?zrl)p%li0qB;MJ#NX2(s?<|<$AbB^u_YdwUs1vmfT^0hAS zFAwk2!)-S&x2k^c!@Ra=K)6h<0=k*uUp~@*S>T=M1b>Q&6z=89?C1|a_0DULTR!nu zFY%mY*f%_SKixmtPT!R3@ja%eZKq0Tdr_Vq_7C|}DIe}DpRDs^?ma~BP6`V+umSYG z^E*#&ch2|R^9wv*g#!-Zv98{G`124;@2^ea7jUf?PD4kJ^lWd)LHPy^9(9d8O3ZAZ z_=)9IzwB0T^;!Rp7Vk-dpb2qj?PNcgV=soSzs)%{@Mgb2@f7yowFGb<_bd-L?~|~D zPeg;ju&e3@t^V?U4*1BAyu$B3S6x7c-|gc-h43RbP|rL;@3Ia2yAS_9`A1L41lQmQ zj|k^XKn@<=5&Pkw|LE*e`tJV>r(aWiz>w6wSgp@#E3k^37Q-M95Xz?DA($pyIc2uY zExFJqjlzWH#xa9(N03E~hyHYO*eK(tMUmu48Ceomsgq0|rF^xiB_%juLjGEbl4TY; zGj-NH`O4-XH*tiT{_aR9QlZ_(8)V z4JK*1sAU4hoGnCe)T}rW1qcueHqx{~H_u!jrDC`aCLBl9HEW5vEoR&}ony$6ACsdz zSzTq!mo;bB2F+GyY@kJt_PlIO;j*&?#;B_6=xf*#@lBL%)bIaXWq$KDv%w(3g|bZs zX3SH(IKwD-Y{^iw7p`89A1$JyPUT~yPE+b=A4$_|&_$01wQ?!G&Q9+r&94N+v*%Bm zXx0<$=FHijz*@U>ff#Iwej31P=BoYJVlOC}2=jq028r0}2U_M5&MpH62*5W8fbb-t z7ceO6LlC3t;xJ%J%*GqeFq16B6knNmEGWy2)^*-E0etUL|X4AoYZ?y z6i3kko=nDTQDH z3Np=WQ%+yN#WTxV6qT_TOC8PB7(bISG#XDM6u=e!=#k6XX0y#UA%WaWwyF*ZQN-bl z!vus18bPi(x?cFNq%1F*4p+avG^$H*b_#E(GxwO&TA@k-Z$CHDtTWEii$AlXxNez&_WG8kR|EnCI`yoFFoFM z=gV?v&SYnu^vvl{@XBN9URb&Uw>_W4{Z2hF7F_?ZScwS1kOQtcsOz?_Mfjkhy|Q2s zYB26Nh>w1!`i2h__A}c-y6J`@UI_@$TklIN)%5QYG7Wrix*UcW;)z{6b=8bnHSJpc zSWaO(tA;e^o{&XG$EhGOITDDue`Wa+OnP+zEeSA~+1)ghV!2<~?;dc#H@ZnAi)Ct= zeZce%!BV}~wN+a9q#rbc>8PbH1*Axx5c~N`8|Yj5=`AJV`6g7Ycdap+MrAiyQPCOv zOfb69YP;zWc>PGL2V8LY=_(wWXAW09V#O<7obkp>+qjNdY~&Bx3E=9SajVzh19O?P z4aI8XjCXMj3ss=^0M2&I6th>0pf0jJ`< z#9nAvM!2H!3jWL@drKIL*dRqcOLYoUle!*R1XVY>yfO~jKSCBfAasj0AZ z82CUBxi|s5WJriZEMgJ$a+)hXq7nHR7S_b1g%eny8Tr726=TUhEo#w=U?k=QSdm6D zCi8x1Bn>h?afU2lqzQF_OOkkGxpe>7gMkYqMjnU(n+Q&1f)rd?UhLwbA4H*MfTRT? z6Qz|DJRw#+djkaWv;<*kNIv)~jt*5)6?|3=hUDQSAk>nGRWM@}$Q!5`Tvp4mEZ~O? zkYYtwDJfSbXLIONLPxzfi(Kv!Dhj;-5O&i7jy{hqdE;9!yI4$^_F;d@MBFl2#mxGZ zs22~qixiu1oCz4@M|7-59c@()ar4zQ$FndS_T)tu*S?^mmXW7uTi z3Aw;bC}3DvS~O??Qtp!!1U-}ty8s*S40ID_{R2qzCQ&XjELrfwDE6Ag10;fn3Dyzm zqTaAGWt1hQVp}O*D`m^ zF;%ES>2`-QD2$+E^88~zY3ViH*_IiPoUQ7@nuZK0b}vRatOXLJ1nE7>c?`%U1Jd%s zwHhKQdzdair$q%DzObXl0l@-F>41qcY_h;nX?d%5-ar^MUyMK^Lf3^p^THsa7D{gW zhB;nM$rQ8u#p!0ph>T+F^h1vFt(1aFt#S0}a=bWEQSAYbVHl$lE^uQwE&Nodrl3A{ zp<<4nLx^4Owztm9Ud<@wN?jmnoMpg3Ux@%5-;}`zry#`*sp~?uGNBU*e%mU-vB|kE zXlQnoD0uB#-$^0)0xJJymRhVP(3s?eWdLehD|^U+;lz`AT9{@73NXY=;Wx6CK;~sh z#ozw^ml$hia~hoq&Hum{pM5KRLuB&h+;HrW}BKZ&CP6cwfhH* zoswjLOVEyz9L1+XX02j>PCx=&U~U%xVOOOMr$7u1RYi8FkFSHuyc;AyBWP5V~$_{R{&oNsTlM@>0X9cFm7n zeem2xA=VWiKq0_7umikH3jomcyI>C3QhJ)=9Ie_6Xh|tP@J#I(Fx{|KJhOxSP}$42 z=->3aqH{Uv7l%sl^X6|eu^7S+WB5HbmfOK|wP(7kaBXWZqz=gqK@6e2@YTpbiV*kxK|0Q=&Wexs?6uKg(+z0gSc+#5ppYHfJh38FK?~ zfv^7=NU^xEGoQgT)pM<@c$YxR!z7U{MA<;STY-EVf;!6t*C+%}y0<%QgAExWDd9S2 zU_t3J6zbx%eS$bsvq7Z$2CC6PC+x%*vxt$aG~gQm0|>1v8yo6zK~G!|*;xp?dA>X- zuVIsdWSJCoYN9U$L!T1EG2}d(E5n^bz%>+uHk7Rq@S2v>g9w~MI;0)X;R{7vL{jXU zLt_FPLUVS||jY(U4f2JpKd2W~-@RG(-Pj z#3^B9MTFvr)2otbBm;QK1G^ABg&|1?jEjQ_7T?J=M7Rx?$q*-qfhJlsIUFeT!V%=4 zi+$MxsECp~XdxV9zo9e`PK?JJTqql3zL;1=uINB!XEwA)i46O3|W>`x{-Xxr@X|yS&Ss3r00Os4bxta4?yW zM1-wbB^QuMxTvNBFp^n$z*pq8xTpdyIsgKA2s=}QBWTK;bV`p3$4-3f^%I*_G&$9pWi0Wuk{vX z2uh@Jve`(3ZF~zgV4@drnS)F}N1-qMi%YpIroF8 zH{)O@6^zlqEFHOsl-#s1q^KJYk^nLAfe_dL9z~~(puz_E&$B_Vri2M49fhJqQGxnQ zcoYT|5{Gz!M@Z_1{vW5Pnm|6Yb1fV9K>zGFfRz@LRHKvaG^m&6`d&mnQ02PSFW(+q$11s_?H**5Y96cYMj2tO@8f-CANqx`I zq|YW*tAl%l=abTdkil@^5>vCsNma;mDOJ@}ugz4=9aIEK7=<}VQX&w&6Ue*6lRq{6 z1|Uewj`OyOnn?)A#W)Stydc2S>P0&>KsB?3y>vpWqO+Y;61{M{x?HLY z>YxLKl0oX2zI|L9ZvhJ88C7$AvQ2PDC|yUF*af%YN1{+gNMO5J2vt{;)F;!-9c&46 zxffM6)qh6mvr9+S2^YPkwX)zqAh#+OdP!z_-oeBVcp8 z+_B2iT{T1ePk<$;)xBTVRaL6QgMIOV4_pNz)m36C7`&4_+!fYhx;);+yx-*?ycOQO zoke7m0S{pT12ej!8`v6e~bZjTSc;-)>>r_TAx6 z^#uAY)i`<8wR%-P?!*U8OsG6lHN6BC$k&BLC$2G=Y-CO?4nr^Q;#vmdG~+F6i|&LsR2I4)sCux zK9$EDhRW%is~ZNk{Okluz&=!vM{mgmqzDR?rBX#E&b6zBPF_{=pjjmzzvb}Jt^{cF#3f9zySeyPS)S!N#by6nrf7?{TgV*X=&2qp)KxUbk!Td;WMIu? z9%&T9$_|y>P0r_jZC!t!O5RWaScDM3xPZ$E_zzJsh-)^Jq1z5U#m7%$~-CP?Ge)J zf!Qrd36VIsjLWmWFD_PVjLfOuojCuuo*uZCYxtV%=FMRRE5vlpS6I!!p+JyR-D9iv zLql=`E8wv^iO1^r9q7GWPKR0?z>9{;Xs+x|2=t5sC!Tu4ypn80X&F6P9E? zR^-Q4Y^8%B5?kN3d1M@W8A9_?Ky&PSM@|6cDbHURkMY(`JOa^^ zqZYwCM_z;hrYpYIw(Xn!_Qesk%haL=cX0Gahjd9ND*dVSAt&;;p2lhIgVG5ZgqR&5 zIV#yetU~JGjZFgWDupMmKxhVKPV7We{pHHxY1W19f30&|Us9Uggjd~#HTQL3&vjmR z)nONQ_4edrM*?LJf@F_k-T*PEm<<>9P2o)Fx@vtpFr6N_W zRnk}&ZX7UV8P|_bCU*ZTMTN?icvcPeCCzwYuX8TXa~kwakl#<%He^+s)qTI7Cj3qc z_6_>xyW72xzqxHhx9CNGHhMUzrptMqclUPB&;fou0t4^>my~Vy4Rj+0lEL=|%Fvt# zfsqJsruB^V$(_N-V7KxC|7%T`SuTbTMl(`Gjtx4Yi>X9&z z2baOg>GS-YP@HCSRD^WYE1*l~*QV2rj>q3blzaV}oCAO@fDj-C0*>Wc8=f?yi#bwd z#S1OZSW{SHg9T>cVTw_>VPduzHdh{{sigpAC7O7d04Z`7mS>@X_F@Jteh{OK84##Z zZT|W91Rr>efC_il1#{gWF!G>BkV87xMRGcN#0)}F{tyCn2U*$4c3I^j&Np>f7bGM< zp*R1gNAD%YUXDYNf~J!}l&9lDW}$@^2btK}%s36^xhE(QPM9Hvfeyx@hJY3d4Tx$D zA=inP5fH$mC_>ueiY~$k+HlHgnr50Cq`-_@EQpGS7Gwb!K?`rRF@zRIw5kg;BXBB$ zcnIaA+jhDLa%FR?)DmQ_xoE;=H_?bW<|gLJr_?EPI7{EGK|S&in>y~{6mc{FC&H<7 zO2F+8Y2-03xgauwi6gO8L(QLt3i|G$b-Ak;CWxk?LR*-a(}raIo|u_tl!mo|rAB6Y zaI{SQ$3O!RR6ruc4nJI=0u4tD0TeVylw$-SJx9e>A$UwBm$`P5LO9+PqNSBMA&dXs zQmsInu(a5=*%ctzu5q1QYMd}Zfg|E8?jKFepz%gT+{gtNy{t-VCbMMs z^2^LVJ2R(mCZ2N`BFMoqA~M)XZWLExRf`&o7*`zA+IoJ5)9ud7m(-+BO*QJNC#sfQ zKv>{#WMIc#)}#hhTH_8g>W(n)y>o@tRpVx*!t7-gPuT&IH&_nQ`a+>;t$WS_MU+uy zm*hdW1v$=r%Jy`g_dI=3)S1;L?vl+n6SYSuOe_Wr-$cR6ZqC7f^I0pd^w6wo6ryj zr#R$Ik9foV;s*ixtttu%az_3%*FXPt4uA#}-~dy1x&tm|F|M;f8a~E02l^|565L(V zd}l$9MFn#{t6XEg7CR4uaDhAnM;@dSg%olG236tQ#_HCPWQ338!@Z^imFmzh>~~3{7r#>TgmH2_8%LI41EU=cf)`uC2VXatid`UXCx|8*K}AT9Y_nRANSTm? zVV*CLlbks*XL@v!&ST9)X6YHkmJKbVE*(a+J^J#Okdc&TR5UQo>nM*pftNNWR4w~kw#{OfM2el zq8yayMH$x0+oANBP3Z(Am{F5sq3MzER3|#$xh$a~uB5f;%aLm!UT8adcOnRGjY zea=;{p2?-JcaqB*gp;sq;b1IlRUq*~I3nX=tO*XV>EgaONi_ZBr|MMCJAay4dCt>_ zOAKXtV8Q}aD5$ii4V*|&JCfD177ul(!y3G((1p(SFL_`ET|k7}O5CdwsYwHGd0U0v zrlzS4Z0j#;_p`TZu!LDbhn~>7*Xi=rIfy9YXK7~G24E9nAoSS<>hV`e{_U@IU8^p3X|=Wr#m93Kpx^~FxWNv7 z@Pa97(b!=I01UoBZXM)Y=Mv&!4z9vpgYgDX5Wx+eP3He=>(Bxy7%gJcRIFR2Io=o8 zHEnU^>{8kZ-)CL6e900eIo})PUEGsCZmXGbE;n3(|}hVF*rj0q24_%-IRaxBJB6i#9APUe87 zcSFcWXKEo6dLkNDo`++m@YKDQbab=c#7>kPE@tsvbbaTUl2_z2%h8c)s-Qy>CrE(` zF04TzYCr`(kbn=GfQ}PN%jE)pnbs0A%xZG2>s{wsHLF?c@!ANm7R?n(aE`ML0#;|? zi6jymsF_i9RL3X^l7oFjEWToFyg!)Gxz|CVNzwn?6U$OUvO_*+qs_$Q_?A1YqepgRb&p7}>K4wzPqy|lp&pvLX)rwDb5YF#iCfoX4?1ir z3II)O>H<4Nh;0KR&0EEV=MT&VpS4M2Ce{+(mL7Jw*@m6#R6c;@RnA;$5ac%5r}HN6 zoaa6#%DTOmo*@st=!lv+-P;rdPT2Jx?g7f~Js>hLpu*`=#Qhd)09vC&%JBgV zXfR*%8B7Ij3iMH)M=cSZJ;uw8S48#G0d$|ZJe%`D1ZII=`6+b6b-E~gI$mnqL_mmd_WMw0wmxR6?Wn$hTdRm=+1C758bQ z%gLC(q>+Od+WmlA(h(Lx5R9s%oB+%K14W`9*jZrZOg(l(4FJsuC}q(262%nb1WbW4 z_#q@Ij#VDTni!6PBA&e1_IE!wLK( zUzN)`u;b@-ouimY28RF8G<`?U^%(o{kXCMGLh|8iQeSDhW@(~jTK?i%Zpd4z<=VyN zDJ>pe=H;a9B{NQtUqTiZnA|{4L=&W($8lfxshl;P5o1kN6J*Y>Q~?nt-Le!`2Xx_O zf=UEOyyzM7eV&y^U zqD9(ffZ`@w_8u-Y5DWcienIueP6eP>U4$llE(`K$&H%w+DAQ%92iRlgR6qbU0-w@_HjK+| z+(59N4569`TcPI!=;&01(^Fdg?q_*}y_A zB(DEtmQ`EG+NfyKWC`6_gFq%xMVoQV2+Hl##hUGP{>bQ3>)T>rpv7$=Fp&N{Xy^7|~u7(`+NjZEk_=edM zc*d>~gAM3Qg2V=ARc0hE2=UY|?O1K)$SUgMB+%LEopl(+S}fUigKVg%v|6i!rr_;@ z5VqFsX^LNvA(eDEs@@WB@lq<_&g`HxXG&0s;|tph`_%Ob_`qH6;>AZ{8%ls?oomQskkE67=?)oSSgumMJl%^NYF-XYWO zloy1`FdQeX0qoD5enz$e)tW+)R$T7nMB>7 zXkg0cLAK@tOYxCbF{Wm!1#j{eKdD;`CCg!>8(N$JL?|(J>4cs#g$Czeb_!*(#e-6l zM5#fnSc-B8nIy=^*Y+e3U?zBuEmb%LwBA74zL-*7&ub=h?0B zR`T9jGA3v8l3s8p->};u!T>jDA!{HxJI1i}uqvl=9q!ILx3VkufCFTKYF+=?P}Va& z*Mb5viLf$pL})-|s(>%Ut~TH#Ak(ffFETT;ZQ?a^*X(X1F7fX&L;_24-dZz%LU1-~ zvu$=VM?Yy9L1`E7uz1OhOu{fJr^rY$qX)J#OS`k0Jfa+?-W9T_{?Jfrn zKm%p-2y7mFujCE05EruSlt2{7K(N^*B0F?WUy`(_8%0O4MfdHI8bwBH^kHyvF?95k z-j-6!4=C%=mG&%UAZ;0A#7f7YOXHyvbCU?K^-T{mPUrGE1>TD8u^ga7h?H<84s~Z3 zba!FtGAnZJUaJ_~ai#5;ojzf9DG^m$@>TC`R$p<3aJ5%&aY*YyC?Efx+lcZ9$uO6q za!a@M!Bm=MuYHvIK7f53giLFjh~nW#8q=<(jcdf8a%wc6y&TdDAtDtG9~x;5R4$ zn-R+@kIy<)QZ!nlJo3QA^efi*rC{GE3nHF0b@piXN6xVuYQz5w}_53#RpngJ2v<=#8bH=N<<^bwS3QSK0IxvSKD$y|+%Ac!;NX3L1)V9u^jhCCHU=ZYfxVPCeF^do7sW_-oTZ=VIr>X1Y}<(^Gkln5tA!7wR{TDk38^ z1~_%MD%FWKYGea?%SN!EYcrvH^zx#PxmY@g@RA41%?7)G^nSJ2As9WB7169}^Xv;D zFv5PHln``_phZAC6OpL9Gg}+Zs=K)lUpu>_f^5inp&tJ+S%YS-A5ri2I`4j~HS4?m z74I-0?}Xz8p-%|-x&YALEZ@bbhxJ`YHoQ?kkiC9f8sIDCmQZZ;aEZp@#b%M7?L<&) zz!?NAFqz2F4CT2*06^lsdO zWRVhtPk!YONMil2dS7KO00XH!dY6L_CL5%nZ;ZqDTf6yV^|3%(gpc%m&u4-eT&M!K zlYS4UuOq#_gQD#{zU}?OPLaaXC-1y;HI!DOnBo5{;$D5?Mwl26+SXTqt9reh-K=TQ z2-$19D0Bdy2LhVc4&;KQ3V^Pd8Ne(wg4~<`6F7&;r@X0|YKyQj^JCvjw#nEkIJF$3ux&k_1W83{$2H z0yxepXDcE_Ns36`8YSt-my{DXbZF^8OUx>1)X^NwNoc4Q=?Xe`W63dpfYaAmNgp(ZJbrfYPI#qc2=oHtxl2p zwXK@9Yuxr`;P0nm6x0 zDrNz~AmnVRbfg0!bBXA#K;nZ%rM4M zG?A4)D$z`jH&p11y&B&`DuB`gs38Bh)o$|RtMuN9TH1iQAJuOwbd~GGCN3y&1hsmv>Ts1EjA)wBhpClc1V_4E|x`#?Z~FEuZri%rEEb5<)V~OcUKTS!=U_VCQtVxcIi1gWfM<>BAOt znxKV`Hi9DKzd|d(BVQ{qL~#F-N%H70lnEWK>f`T|ur+=$sU2~!arm6Af0Y0!;#`M6U!O?;T ze(D|{1W3TB2tq4(I0O|6C_*WiYI`R{p&(`;CL~~w22vT*4F6Xq%U}eC)C0psWWB42?$yXF`NLG8hg-c%lxcYmG|AwFTEu&lzu}msXkPr{8%6gLctzlJYrGO17nmj2IS_W<$<~>ec@t*vfKj<{h7{b0 zx;b8RkzdGLHZ@5W(hCQ}9aS+RaL(vfX+naf=!5R4g9z|C&f8AP!$0?Vl;FBjNAWXADZ zy>SyTiIso~xPT7nqyiIA*AlXoO%Y5Rgdvrs8fFM%dQQ7xK2T9k8luw-e`?eieCp13 z##0F}nCJid>PZ2n8V+ N#I=S5>Q~vY#var&f7^nF$am4^=?a1rT~gvkqjTf(+7H zJrKcDHDM-jy%)duWrMSIu7(Jp1D_5+QM4)mQ43Rsa)!y#j#3S=atu>SZt*n!C6ak| zlfx(CP_G-r!9f(e7PMx{1Jqbj0%6Hv3`@0!cAoQp(3C4XPkYpOj&`Y_48~<-`-@Ua zRdG|L>Q%9-jlihSpHA#USk;q;W<)`(Kse}Gnfs($HZ%)q;8(`b@e#sJP!~6;!ef1- zhGsyZqH!XeeKOicViuFJiA^l)8jD_N(c!Mui9~!e`nzy?mIxY2{yK)v(E)NU|1n6~vN*#)wCp5fh&{ z3UTsUQUolUB`~NQ)ul%`^HF23CO5e@$VlMSu%;wAAPf5GuW7=2NAhaaunbhMV%b|Y z^m2e47Fz~1KXHL3m>~o})XZcezyT^PY9KH0?6Vj#*3v2$&9+#?XDOD$pQcuaan2+> z9~|57eJ>LgHsx;l39=E-n7p^K&2R*58Jy^o8Gn1CZUuVfUfbRDcuKL8d}E% zOG6f5lcXRk7nom@QUwUa33muhoI1rA)rrM*dUaU3;-kb%{VyvtI zg);?ECLZb9tQbfEucOOlV3+CykNSYb5tUM8!#StQc5t1YE!$}88MoHvttu5$)#`#f zZQmAjwNxDLqLfC_45$Jo&YkXbrvcGyo=HaXxZVEMYc$frbp+UkJ=JInXdDfA!5>KV z*ue(h0uZnZID!D?o^;d_ACr$b(6WoG0PU}wLBs#RAq*xPC+{7GpEtPG| z%ya3uYHHYH3jDBEhX~eh#>{+IAmU)Bkqr>Yni)qpWc+$nuc#h+TK!*spwCAfPR z&;pyWUAv|kHL&P^MJq?7H3j~)P4t+Bj%Y0Sgir3~&iIbclk86U^dp~A?)f;(@3u~o4F3Nx0UNq#22LQM z>JPB8%q0Y1;Vx%X(Cnf(%>XXpj~XEW1@NclY~&0O0Tb{t4yPBE!Oxrzl^zfRd4USG z>M~%YzvhHm#;u_qD+J9i{7gdiQmxg{5B1Q?)ryJfV(kHR?gVba0V3de01G&jZs}BD z>{@~m+9vFP?YrXe5B%fphK~@9klFAi%Mzs)?qj?zi20hY0Z(ZPr;reAK}IUW6M$d~ z{c1EQYDc>7a~=w!RP7AOaB9r36pbkj=`9C80mwih!n<)0Qsf|{>BzW3C|V~3L_CPR%sHeu&e*5@2fNs139r1xv&dG z@e7?L6-_a{P;m@hCu9u^5Rj z8Gk4VIG|#5A#iAH91Q0Xnh+ABaaN>ot1!U|cZ|zsP>Zy01HBI%!}0S@kQ_m3$o;y79T9=(tTH5RN)Bc~ zTS|*y*fAgj(kFlNQ#vSqGD5OCM(-X2BB9S3QG_Y2(JB8SfR>`NubjjHK+y}oZ~V+s zBuP?%vXUe~^3~)(sAxbeLu#22PW{fy1D^#0_)Aa(2bAcch2D|~XDBW=$xjMoxOi?k zrVTGY1}U*^FIny{|IS4|Oad`scfM{gHS!zf5wwQWlHBqtQ$jK~GmcixF^k{9hM=WzIuajv&05ppT+RC9cPcu8i(-42M04oLv4MI_}VlHBn0h3aRieVVy6Fylf z0QXE1%!xmu&^_D9FagMv=-VhC!}5GahUsX0&an!M6!rUd{x4fIyeNe~n} zFH{7~4qb*SI>AFMB`^P_F{=WPFa1*bC{jgEObzxE;^;5y z{PQ(p1EqG388Ct-2ozmr3%M-xF(GpfvME8QZb5a_A*OQ!DpWZ!lyq*ATZ*wthYv)P zF%n2AgbpDqDvT~Xrae`zRa~?R*P+G?%C3@&4aAD_a&z(q&qeYeidyCZX>jWv??)vq z-R3ao7~l;xZQjn0YNqt`9Mn0UQ@QTIGAHyoQ>;*_G)s?^3{7)O$#W1kwJcJF6v&iC z9dJG)t>^L~=46iDP_^-D?p+1|=N|3FO4aB7BRQ(&&@4m9IBiOovusc>G8J?|J@eVR zp$kBe2O{Alzq3-$l2Sc%05dg2IrS_!wPF82)l65;Hq)W!@Ih6(byXdYRab1>5@6>P za#nfjTd9ShD2NS=^f--G9L+IUcO$@ba{UM{5YA*$EtOe`$u#2*S__a`2bNk_G*q*d z#CV|!EKl&>)X;7FP4R?BN16=RPiGIbjudBT?t|F~yGr-Zpaz`b2Lxe2l&X{3Y;+HS8E>x>ejr1CZ#qa63b-aH zpFm`fv2O1cZ}V1fB~fVQGw=Xccq4GT2v>9ZbNkxLp*$c3E(Qfg00X9=7FfW$bb*7u zfeK<^a3-!_$wgk(7JM-`9XFR>ze5ai;c?%G52jZcoWL2@*9b6X`OsH>Yr%c30DWT@ za^EIUR>3$0GFqW^cO6nM-BTKWH+Y9Pa0Ry#x@dEbx8abc3sGP{44_ZMCItkFM5TxZTk^uw4_X|6raI&l%fprFEVIj|mGJc>SXlsMl79n+bWgvsV>$DhL4{3raOmWdz3z~O(q1TqtIA&+SN~i-BF6c?El5A!58@o}_?lObEXg zcVZY~1X|dQM_E&iPj`I)Pe8E0)?m#vl_NuySDBSD&&w7d)QSah8;S*F8J9UIgJYrv zW&s0CK$_B+m)2m9`$d{Ih+zC^8AvduK)~Gos4fu&2ms&>EC?SkV3<{-UkpHzC19nk z1A5;kp4T>6MAO;EDW*ABrV;F>b-Eoxpn2=yowbySN!gyu^jcrE8u?jx{eW2p`g2}` zg7qMKNucCvpnu7LerHQ^tuhYoZjUSa8%RM(Q6i2P5`K1X0-E7vhNy)=FcGMr%c4^P z&y#1Gk$@(f1?o?0anYf3$o{jsspHKRfc4!rQxtm*&dWKCvS2KPG1>z@(SKXa>~gTh(cH(uF3FS%TZI zRQOLizObt+T9hUZ4xXSJQar^E2sy2;ay97~!6OS;;7PLXmM{Qrxv1eFkKJZG?G7|` z>{t_=!3ZX1yZ;xs102$GmXl40oKGmp;cLmKbv@ByJtgF@LluEpX1SI2%2xtODkGqf zYX_GOf@uy|wmh^>+k!4*2_Ewk;Cyf(VGHo2gK>SDHlT!e75v;g->3yx@Q1utqv0O5 z*gLi|+LdVkDx^9(siL8v9f=Jtf;yLbifyCGeyt;#-{uY`owcxsEtDJ*-on7w!jG*a z4obn@J=F<~8nI~wMsai0mm15%gID+j&5qrIRUM=U{vrYT4Xma!onj?=-l6;4l3m6yTMb0!AcINs4uL+QTg5@oXS_Z z)US>@UM4ihDmr~lckF5h&xB@nFa~f{UoZlq(JP4j=o|bEdkTT=`FO>#042tnF;RX4 zq@v^hp>`-f-rkhmTGU1UGCVbg9q98uk0luuXnuNwQmdulLU1`5X16U^s35Rck~6U? z))%Z>x2p;KHItsA7uSzSH{N}A-cNb&rar0f)5@=2S%UIR^jgf5Im~(Su@wMa(NLa2G>6r!AQ>xmS4_~gg9$kkU z2nkjzQj`KM+lDM6wTad!a?|GRqQ;EbHg@#*ETqJd#(NOBz8 zgG5g#8hpJrj5q_qZEpa4)OC}U1Jc?iW-gh&R~95ib%B_S2oIFE(u?p-w>TdC=rGJ|?q+ieApu$F6G$c9Sb zgZd=UYI8E&aKw;Ev2Qrk@hQwemwR4Hp$y%EsG=g1jQYtaPebYd%9o}ByQP_WwW(zp zL^h*2{POHV6?Pbt4lzHwpy0Wl%#*vUN-y035zl1t$r@(+@Q4so-{kPRg{in=+5pfP zArNBEV#D-sEk{K9e6AlyBy?hJ?b%&$u~QQ;g2u>JGhYwIa;9M}h&1tlN5AU9m|$=l z%oSn^0!-j4_)voWahSPO zK{8t7p(p53j)4H9NwSu9D=;WJxypI+)a_ zWs{KE+>Uq&QYzyDr-H`tfVM*y=%Yf!17d-yQjQ_SQie2W!U~#T%c;dNmkZStu5uYo zh!S(6b=#FiJ>eVMZ10U&U}iNbO3jcibEMhqrb)s7xruK!q0d1)I07A?9=}Nu?)CzjjDQ6lJ0WU%{47<$GLqVELI6e`p z*t;k)!!d)6@)DPHjHpQOh*s?_6On8^sasPD*GOtIJ~36{Oy^3|bh6V9pRAoI4_YAN zv7!nI2-;A4hm51HDygt?%JOR0tp#j`X-=GKN2hpI%Vw1maG(}Lw@Q{WdI+PTS%TBZ zDprY}b+sf7W=Y?QQkBX!uHvLCOzpZlyyg{VdUdB$cM4b`R)7^CD6CQ=5i|f* zo-Cg#fWFX8TpGGwXP;QrvT7E7poL>~AzDWN2yV#8{7D< zG@LEPQoUlWx7&|Vw9+x zT_}Q9y_q#zho@J@E#=7o1TpUy265T3W|*riW-T;d3tRcpR+6@@t4#Il-`MrHoqerc z1$e4EEaKw8Lp81o5?l%59e`EncqIgg_}qdz#RV7s!3+#!uD%fA4nVFD2d)MOKUz6` zQxpLW&TwWwdKPyn{23`OIq^6~7a!?JczRU>su^ zvy{g8jp=@03h77Z_+>o~<&T4F0}v4Z*SJOo+B7Ob*itdr!CEMyED=+MPf_>+X_$xI zKt&V*A!Qz_Nhn!pM54SXVQ+tF#?~_2pyc2M*fsLRu%!2aNc0(TXQJRY{M^j#PIC%( zh(rh|EX+Mq+tlkawD#JX=-G;U(UESUqsxu%-R5`_IL)bl1RP`r_^#7N)`ta%C+a3U zS!bm_wQIj(%)9K&027Ke-b7{VuDX&yc-#OHa+WjtVQAsO$Ojv=GOlKbsx1~50vuqm z>@6YCOmEpjD7yPy3{l}V7Hul8yMs|xdmE344!3-3JJUw%Hr?vBsn2-~;7~%ar&8%% zfs1VPQU#>|RbcF>APmgwwNh~ZMK`EbB7+H9m*dM+!qPaD1sd0~C|SuO&cuB^>mrgx zaKqBWIDUa6kZXbuy`nq3&dbCbe7HEIwWu?+aL35->Y@m2xZ&e+=$hMn+?FU$ea)%! zitv2r_*wVQeH-CRi_g_ZKkQ+ZE+ONoAt+cj`^<%~0zx2-;3${?5Q3t5fhQbGh=7p{ zGO@>5ra=aPL4@fakOAeXU;>Chw(gw-3!KPZH+u$D$v1&=vti(v7_EN38cts88(U}B zK0UY>e{)}$XXiC8zvLf3`N~`V@_x{+PBo7dyu8Zu_#VL03tOQGO`sVou)i!P3L%c= zBZ~23<>F=(Mtg0b6q>RBDG%2a#j+_)5il1QflJX0q7Wikpd~3#2WoQ=Q-F7vwPFXO zecR_NQj$0&XnK2-R*Kg!bh3DjH-3+1e)mOwHz*4^sAD_GGMCp*Ez@o^BLPnkfHD9T z_r?^j!ZVAdlmp#SrLJ4 zg9$Uh8J6(?Kp-T5krNu|1b`C^EdT`q6M_zva%9AIuEBCqGcFIu7NeC+QqyohKtUCR zc;EMe+TkZpa8hx#QV?;2tW$$2L=pAHiJj<)>&JfXM?(&RKP#{oQIQmTv4pGWgbDB? zRna5_(n@LbZ^H!t2ET-hhu0|tM-)KtTMS@^3E+!9_B~j{7d|!sK4X9M;{bNXY;Ms4 zdiZD3HZD^T93?;kH85U(7gguuFUD2_6kr5wu{?31FDz$l(Dq{6L^0H75QZ{|aOHTJ zsEPG>e(Lv$_&8}V6NIbAWdSn|2X%x+D21q^gqc= zMJHg9zo?8^D3JqDjGzKppn@c>7#GdRjUpETBo~5AKy7qaJyT;AH9!Sape89$1KemD zgeZ=57cb;!K~nPu!nS?7Kr1yD2QVi%tua4}CJB}>gY;;RmLz^tnUDHdQ#~ji`5 z@>>9)bZsb+pdu8EQ5q(>Ks>o}(uiVH(kEt=Yp!KO|MD-ZAd|sx2;FEn%^@!`i5$%L zH&8H?qnQHh$W~p@IPR!9f+GkpC}T3nXwo2?P$@E1X`5HsTbRccFF-+A= zyHW_V&>VQsY>2p-fVi0a$#=%q7FSRr3qwfO0ULNA2-T4}yb?N!mY}h@pbUDC4_cL% z*IVrubh`Kf-6GYlAkk%bRv=!T?{ zDz-p@vy^GSnFq+m8cs@??|GylkeYr+S{hXvW}zD#=z*9?rA&ZPIbf68_?gSoCao9$ zjug;rV~{~(&>fcvjlE%@jTBpHN}IT~iEg?{JGM?RR8|v7k$u5>4+56aYJ_gm0+Ka` zQ8%JSaTEq{sY~!v95)V^PyvW)Hw(x|z-Kqf*$j|M8s_pK7YKYy!bRuOejXsCEk^*K z88;FDrB=X`1Ud_(x&bj+pOR4mIEe$2MUL?zle2-PsGteFXOlzvfntz7^8%&4syUcr zrf2$^^SGu_d2Yt~rpVe`%gU@N3ov>>1_~&FF#@x@ajn?OqBP^KnJ^XIk{QX8G}VeB zx>AJeI$g8}t>;1n2)Gn^fDV<)6-jYL=z^FLyCt6qcu6ov{F)0CfRbiXI8Dm`0+@4? z;mCc5>450zuXwNzP^wJb2eH}sq*g!z6^lO3ryXSaK-+N$32~sns#1QSrZ*OOI7oS* zXjA=|vMTGZEL(dJAR~fN0xePqGFP)L)&iAVxz<{dK2b$l#-ei2Do$YzVh0#_bF^Y1 z0(fCU80B*0f_Ao$H(Q}AR!|jAkg z5)>k{>MuFqFVF$8)aPuSx;cu(c()*|m?*M=n|_0fo0g)LhdYsvX(Lo2rVT(oD6kza za=4V+eV99jyEaT~fQ0sAqo<*(VVD%T*E|nW2~MF6Re)^)$O^5dH{pfAOvu3;sHt*(V6AT76sEvOZ}+(Yt-wmK7*huubV%G3e(0yYA-lZ| zD!AUOS1$y^?R!LA47tML93#Mb9?ZvB)ffE40o+zl3xI0Oc!ev-f?mkKnlhL87Ri!4 zmwc3hLsfl|EN#d|Xg%2f5=VD-+wpDeFHpOFM*GYam+5KhX!WH<3-i4CNXH zEJ#a#u)j;p9pQXy+Z-6VHzF#y!1n5%#rwO4$f@tU0qP6SQrmsZA*xp(D^V(?8r!jM zjLWZy!gI`~cZ|n8xNdtqXc&@4-#4>In!T}2PX}!9eqEsBFeO9h@#sdca0&CYZRWqv>H%s z3TVnsJclUxn6%u5D0z;utu)vTi6MM~T0OOQn+b1Ap|1Te&sYP_$F+IV#-lxMjQ5lU zePgJd+Bvq`Ohi?CG}gm$!~H$A{>>?4IsuA^s{Kv>#ACu`*P7N6XagL5kqbzjA?>sf zQqpr>AkZyDf++$Kp%41t66Lcg#88*E5DU^>p-f0%L4jta%YR(Gm zp$(Wo4}B1?EUu%wLA>OM0a{iULNNtHxZ_qf1yAA8mC31NqL_c$4GTa5(mRuMcR%^? zFT66b)oUc8&>VYxN zo89^XiT)xMjmdl}$u!_g5!PgwfDs_Bojcp-d7u^A$p&_Q=l1q~`UVZnP^pJZ>?xTh z%uwSMU`2B#836;0Lt{Ue+G@MeYzM{Z9i0!Fv8$%q?mk0niny98PUK6zgL8%IROMZ= zwd73Rwtzr;2CZ%K7micrC_0Ge-!QSoT2%n(;gIm8}NNz}QbnqpS@TX1H<(9|2rO=)v@f45i z6;F|yG6Rne8snW5<;oO4@53@&Xm5WQ_<#vaM&~bY6zG5mDQ_8-U=Q$s=X0sy_z*l* zkPeu<1nY`sXx2~sE(0m44)+u^4}Pk5(cp)r)~r?wtq0fn#G6!K3vL3}e{be%xfb;D z@5>PItX}FhQSgBv31RQcBWs(me)jqC@HoK+$1w40&-QFEXo4B)8y~H6uerTG-7*W! z6xsK39{DA&00M{dI^_!Wa1V~J?QZz;mn5{JR7-5|c@N_Bn(t=nkA>ed5b~1O>OWMi zbmOKka7?^h0HH;b781(xT>=&Vg^i0UP4bB=RI3&rNEPF(5hcqPu2ifx76TSij3zJnEu+?TRl(T8WUHpsfpXE&Cs!Yj zUL*Dr=CGzG!x&oGl4nf+_B!3ugkSS#(D_5{E5*N*X|JbIrLyruD^ms}Fe^XU@XD|a z$TCX-2NMW@fwbC6qXHnP2&=M&K%s#z$`Yi^uY0r!%p7Pk>>#XW=wfj*vC@GoGBy|! z3zHFTMCUc$Uhph|%|?r&tZ=AM4YSR5Oal)VBuZfc-$JV(0oQD7A|fCUitRWa4~mE& zD&}!3wilK#XP#MdK~0K_#xNrzjh6B5CzvKV(311e3#6Bi-ir^KKKT?cP(kY(DyjVR z(+@%aiU?4^0tHl5!LfuyL_!M&0HA=-EC?*f>&6n`xP~h5u+$LOKnIgba7^(Auv{1? zjUke(lt#w%Nk1G9E>DrVb zy40W+57lhRr$X17P-aDmg5!XXhm^Ge6En}n&I~cQh{LKl>6?#EI|IQ}&-K~^IAA~n z9(1IFi8}PV`EN;;sNfwPA@W#xnIo%Cg9# zp(+AfIo2eDX~spVUedLabB4{Uk049V4P|4Ioq?gzxRofiWV19)1uAM&=8~cNfVPP! zu7f%kYoX}W>OYY7By6cG_yZz@Xq||PE}meUUZ(CPRNpgDO37b?3C4Ti@%9dn?}Hak z7}17x2&2*ehaEi-#J()fqhkXc=Tw;rFAR(1l2J@qWxw9II*Ju-Mc|Ba5NSg(o9CLG zO3X27xr)-4Z~%v?L}2K%qVGTxUfO>cSIl$K{G){ovM|}S7M94GW+)=yk&4o003ws$ zs~h3Cua`){wCmeuf(buju^Vptwww68xd;5)@4VMnAMd*X4;GsR_&5uBTuEo?@E(Q&Ixr(>PVaCJ@UjL{LKvE<`#%&aPy84BB2!u*gBuA$GEZWFbGm z10=FdiIc=+4=`Ctw(&rcn^a;biLgl+f>M;FG+`=DNQWoBDQ;z?Viny)C_(*+ebTt4 z`sDXTqwJD?jG~hr%os6ZYDJ7|WTOHOsYf_6vT{4JLx(hX$D#R4j*^6*+<_*Ss02;6Gm`3@7bSy$PEyLVl=MWUCqHRV95!*5u2dK=rf44j zE)d~7rnn+4xtPm^{*9Lj`z3&;SWJr2>6mMT-b5a^ObbXTFnMrhSa!#VUb*UyikzVy z+R#IjcH#{$%>xZDF@>+lL8UQWDeKbN0UF5Er&O5dPDeOYNP@wgMBTtUjrz_^;`5%5 zY$_;iSWkH#6p8>v7Or7lj0_P85yAR3k>8);J4| zs-qM5=(vRD1J``uq$*26GQ<|PuwlSe3OLhXyY-b$5Z1AX(4+(Epo_<;Bq$(G-gWO;?cX%JRYH*L6#j%=ItX@Qv;Fil=xjA<~<%w&E zl1ab^MGmCW#hl8ZTPi9jqXz0F0Sg3TUJP`@2_VA*BEp6tSHV}vBv4CjB1M zx}dl0#4Q%LSmG1M?QIWLtCz{x+_r9&tJ9Rr@d%>LwM>Y0XrZZuA?-jP2RUQ~A@YzK zpd<5F#HxzzdIT5rtKO*#^Xc zDh-%nKtnYI9rMtqdzk^9q(pPt?VT`~cPG&@I-1dbrhy4VT(IHlEvtn-cXHpP+(mtb zx!-0q&oVayD*&O=l(xWyWN-oogiO;rzT;fAFaqHW`BkE>jeJkNpj4M2HO|R}9%Ogl zQdg%2(=t{Q=;7RGIz$Yj1wnFrX6qH4;2|jE`T@uLu%MT!9l#x%T6OY8e z%m-|T>C`OSWg5*AZ4FCCr~J=e{p}OWP3W=;n!_nR^nMcE;CIKHihoUk^yC}JId@$R zU+3ZSVEb3Sn8gX7w$j1%aPdqi90jNz2p@i&@Vra`H3BKbwA4^NTA-r}g5U@8T&bGil%sNMmbkc?21y2?ue z0^BJe94kxQ)xm(WX-2(w5!lF#078xeGLP~10_3Z4TPQxU37yET)(fuJOR;14t{nTL zsko?3t2?pDK5xk$=Q)5MYXc}F1G*c#;?n>bNQSMG08?YWgDW^4at`8SxVRGzJ&3On znyeiXgi)J_7pej-TL?BNo^OZ+ocoL1=p7(fIh?yT>8Ju8K|*~3B>J<2?G) zAag(n=CK-G0F#+gz%xX{(<`XdBR3LzKrM>Eu!_T|d%FFg9}GM&4s4b4!kyaqy@&I( z2B-mefjB8k!Q$(&?2~}?Y5-RWEE6mQ;qWo(J3fg2vjUTPH7G+bW}%%_kwWxi1`&cJ zW%~^UyO7LV10^&QXweokT)$%r7ncFOP7pg7>c8E?gmS@yI+6eslAcUJg(X-gA4ovc zL%K6$zy{PJ`H;goq{AIYhjv7Eag)781I7CED8*N;#5m39iLV-kt zJ0r+O6uhr`6GzMXfEHjODSRHH8ba+lE!1Ly)dD12s2Z2af~JxKa3GyI6R<-FNIxn( zulp1jf}W6xmozNKiA=_&vn5?}M)85Yjntp}`GSvJE)7&1k)$Tss*QKd#;4gg=qmvo z%mpT!M!r~*gv*-dQ?i6xxJ6VfcATur$^{tzfIRFWqa+G966z^@BuXK`izIM?SSX$> z_$qS(Fa)!X;yEFbGRo~~8>kewwV{JGh{`Go%duR;!s!l+Tt;fh$Y$h7iaIn4T&uVo zEs~_my8Nsay0R`=FPFTsP9Vp+1kAv^0pZh2m6{Dl6tCKV8XJ^=#wyH-z(ktVJ5AWf zc_K@8qDy)?z^Du=G4d*~=`)iaZY`_$oAr%?+ytW_+Kt zw$vYjnuFKu57H#gsE|v+#4{m+Nab8k-rJ4i1hD9w%gp@B8ekjKhyyER z1m}?<=bTC+YEE7A%Iq->(M*E#RLbK21k9>Z&*@Ch4O~yJde6~}IY6^dVw_FT< zpZ?5<{|q0r^b-;pQ1{VIw(L!)yO&2v(6{o>l7LVM-L1@`(Cw_y3dPP0Era`f3gPt7 z;RI2r;3T0F0xU>PqU0XWhndyiFm^8v-TLoM20ZQqy#kq6DQ=D5bzG71KS{(>~=>KLyk&71Ti`)K)^% zI1Ew&RmO`f&^L`!Ila?JeblD^o4`CR)IYt{K+V)XJ=0G0Q8JA{L!DF(R*@yyw4B*|rP!4JUD=p**_RDkq4n9Fwb`4^*;Abfqea=DtyGx>+Nd2`sU29V zE!v|^SV2QtdM(*hRavb4TBfyHBqiE`J%*1s+p@h{u?1VL-CC}7*f)GyRh3w_g;%xx zSy9!8mYCYQU0bFduB?6AxP92d<%WozTfgmDyA{{GjT8OZ+ax^)j*VP9{aeOu+nN-w72dHuO+US*E_$}YS#U=A)-DyzY{skEJ1>l${ zUi+m}0`A`Ut>1}lsQcw#{MBFJ_22Zp-or)UWW~q>u3!t!VEjGV2lim<9bDK27*{pm z%*Ei@rCSF6T@z+q3wBtNMc)XP-~i60NG)OByHC5H8{e zJmDn%;Q`j+CJy2yj^QV6U?Udd;GN*lbz&v9Srp!2IVn;o2IDBMVlkd!D=y+4=Hf0^ zV=@k7AC}_59pXimVKdHN1w~^fZsRqcS2uoRKJH^6_FXy__pf)2};FxL25EVk5=qLmuIWu4j;DXaFJ9r%vUD_Uf-*X}9j^xQ5`VPUn&}>$65{rfzF5-s_^4 zYre+kzZPt({%A+V-Jx#kyvAjxUTnAy?7@cYqlVak001HR1O*fT`v5Ee0000$0YCu& z2>$@>-NPrapuvL(>m`(@aN)p*4kJR8IPslDh8QVotjLjI$Bhj^h8&r(B*~8`MVdUh zvL(EiE@8%$$uc6%n>SVJOu4frPoF)13JnUhC{cPxXF9~Gl&RCEPoEA=O0}w0k5jE0 zjhS^TMX6B1iXBUqY1gw_(|T3A_G{R-Y~hxDJ9j2IuxR7vMJl&1-@Sh6at-{Kuv^0< z>w@J*6RY0GkdHc!E4XQ7#ELQ31*|z<=gXWgeg^Hi@aWS$Gn)=gy0z<~?8dG}P5bp$ z)w65o7OfjI>(V}PTRvR*t!=`xkM}0N9C+~I%3~LA{#ZEm>)5+W-#$HickkcBJO3XK zemD8@=4*0aFCM+|-?y#HcTYckX!iAa=bz8N|M~y>*%u&y1P<8VX6pHuTy**wC?H4e zM0lWs6T+t8g%oD^;5r3Xs3C?RhKS#Y5|a2~hbX3JVuQV{=%I)&Mg?Pv8=hFBi!_#) z;f##kl@5V=|`J+issutytRK}QOj!kBH<&;)B*(H}?Rw<^J zUt($IneL=1V3=hp$)T2T#z`idZPq!aom(O)r;2X&Ip>~#PDf{;c+R*cp?ac8ou6v* zIVhrlMvCL3kQQoaf|N>{XQPTHs%e9aR{ANZnU;EITVtj)s;8TF%BrZLuKxA(c zYM0E}>guk%)~f5TzTRrrrM&{%s;secifppM0^6*xmPX6uvL*V+?6s3(n{AlZJ`1h1 z#~S-;vC6(_?s(c#o9?#UcI$3t-hL~uy7ZpAZn^iyyY9QSdc&`~LjJ3-jPpYIZ&mf? zn=m@|Htev#+V~5xpA%PXaljI1tTDzHw>vPn1%LW6!}ykCGRY|K+h;fKvh3r=*u*UJ z%r4Vx^UX6`gY(Wg!_2YI9S40dyu^k(uE#%L>oUXNpxksbCqo^M)XGfV3^Q0~tu@xG zaE%HrV2Azn*kqTzOee9Vt@hfVwC(oWaK|n8+`Y(M_uX#aefHjb_y4`jGuwcD&ESJG z6FA{I_uS*+FC{Ib<3%@@jx~Q5?)Tomz`eKEoM)ZP=T(CvPUxbK-n28MD>F+bUAOMK z*|3+5_UyFRe#qQ{G=jVCjP&mN@4!bQ{P4stVmuwkBcHtT%s%ENjhj)+o_lYwe`NCOVlYAg?q>ujk>N|hD+poJm3+Ut2Z$Ij(o1V-4tgr6>*m$5_ z0PjGB0T%Fp2yB7^r@+7lN?{2Q1i}L)2n7mOaDpqiUZghR`A-WdAi)H-sKpXwv5PYB;uptAMhj5jjA%?_7u9ISHM;SQUSy*j zxj08QVt|e*xIiBBD8k|m@sGwSR!m^gQ)MY;Lx&O~z`tz1&^kM_#hypF};h<^= zAVL+&(1kYip|=}gH=9Y(ibm3zi4@;76SfTFjbWRTl%yO+I?^?Sbepmh=Rt#k&eD~0 zi7&0967lCwcgC>*6Uf0kwh(ye>Q~#!+8~DY3}hu5f)?4shX0-Qq8B8f9oV|t-a5jqf{5Q-@le3f znznT2_~aY%z+C0B6QMSZ0s;OiSTY#D16)2N%)q-P$=dfbX!0v+QNSGNO3+=tcyHE;l(S}YALZ;U+N0B z0G3tnd9MIoFZ}ep=`F8$^9x?{0(rfMO)_GUyksL(mcE)jV+>lXh0i*J4+Y&{aSKs{ znhsbKPPi%x=z3)ZUvqn@aPWgY4B-e{bpIRYs4#8iM2-wskar*UFe~=FXW!1C4#OSs zFipJTLK`|Cs!M}GWqMG)IvD~0u(1_z`ePn{*V0E0a;Ar@=}qfc$xQ~ePjl?b001DM zo)D~Y0~}-THoA)X)$)vGtzXok_RMSJGn?7$<_SX%WXrO1?<(XlRb1V5w=89nnwnt9#pKu}n=nKGKTGCXYG8arYS>s=S7Fg9zCQ`(2d z1dL$~ZFo1p)B%9NcL}iVPLtc@od5Rej=lZLax*#9=2o|4>x<(FIPua~*l&PgTJL<9 z^5!P?g00Og3Td=!;B5vg`eG~RV%_$e%dW)3FU;%^n-bc}z__6akx(^QJjMqId&#@a za#y3<<={>^%F+IEn72K?2bj6o4@}ino3pfc&%3Rm@W^9(OyC4pVJ|dFK80U6Hcub1 z)TK_0bj70C@~gNywZ3AmH@D;0o&smr*zGuySIFmP`^iV|_D}~@ylONrn7a<$SFZx| zLCmxWrmeWcfR+LdnRuysV1 zekWCiE@*}J_-recb%llrm{5o^;B|@^fm_9Gp5Ta( z*nJz=jGH!zqDN%WsEO;90-V^1o(PJbNMYH~jf+!?*)RSN0yP$M zGS~o%!pDcXIFF4-4EFeVP9dXiI*6S zA2?6{*i*T6joFA++Nh0{bBo^CksdjN+XG?fBZq}2F3Iwa&4fIw&}SSOG4%M7z#wr& zBv%S0h?KX40RM@3pu=&2Rgf#D3kS)J-q(c>36YfsOJVqn^mRZ7MTQsoiJ@pwMR#m4 z*^=J)hN9p+su&2raD(YsRw;9busCsJBaioZmcRg#yGWA_l?h3ybvVg!&zB58NoB}b zXqASA9XNVJ>5w6qU*ZRP`K6TP1x04|lu?G&#}h1QjG4(bUNS^^6SqD+-) z9ZCTlY6QV>St&YY42pdudZJZGn}#)7Qy`u0S3NUiov=Wiak!D%xt)MRj!L%&;Yl## zS%*THq)~#N$237$6M!)jHln#~4Acn%IY0{)1$wlfT&hO^$_I$}d|*mN1$t?tWN1A# z4_8*E^@F10AOl)Pg;6*U14*Yt(10>frvJrKaB8Q18eV?7pnLkJ$rxsM%BHDSKYcoz zZU4Yix`0;@WJzKbNjwNmIjW5~DnmRvR^X$ZX(N(hRitxBJ}vVk=E;sl$OvGX41_>& z$mgC?DmJHynprhtV3$E#imSN_L9b(sp~GUUbYf@<1c+K?j|eqDP^fbH27ej_feHku zvjSB!jPr1x&HAi-x~$d8tabVZMu4agdT45pMu?C^770?NP^p@lm^`=(p>V0{<8MA{ z3~706+faj`D5@W$nP+mUO4>49)=r+}d_mVr2IR198Kwf7PJF6a z(Uy;w z6{`Oxne-Ys`1)Fgps)MtFi2Wk9rHE5h?lR5fOEwxdwo~Y2di(3%N}&U0SNTr$)XNpaRrYzUTY5RDimN`?>MUpQok;`O5(7o4ysW zy4<^^5P+a8Mhhr7yXv}?iYdVoHo-$GwnSPD;(4Tp=ZF4ksxDcY%Ui9tlUrVgSFm?| zujX0cP^-3(U{|^bnbfNZdI1X5p9jnUOYphk+n@70!0OAs_65I0+`mQ)z&&gOK%Bn; zAYJa}zEFI`FE9WfbOKTE!cBm(v)Ku0rI?qrySQt?Vr-pKS${~w!3Joa$ZLSaPzPo( z3)1F5cSdIW_BlOAIzNY8E&Rd^!^be3u$_>uDMdBS&;%fGM>(9s3;z(njBLYyRlt8B z0Dz@gk=(@m+q07!zb`hvjl8}Wpazdj$*ju)xFJJ@CTLsO_$8ylc z7M!k|I>xw+%VTSF8%(?w=D}&Zqd&k5{n!Z$WWsZMN^#0)qNE3UyvGB2HEK}Et~bN4 zBgckpf*hdD3(&e*+{D(31k4hP%}7r+4UJcU58xqr~j9^e275CRz>0ue3I+4KVy4N0CTNMS6- zyxY;ae7js(4B-Pa!IGJP^1(z1%x#QYBc`A8r=Ep?T&vfM1OLWSc}%_j3CmgBfYl5M zdh3J*OJeNT0%;%)K|8c0o3oZhv=N}Q1K`vv%dvP60Ol|YRL!|8bJT<&w0ywT^8mZ) zYqCv{vw3v1E4$4^-~&Wpqh8#*ZR5sh=+SaL*B^bm8r;igY|`bUEozL?%C-o+yVmYE z)Odhbc3aa2Mw`xjKeUOOh^zn{aDuD~$kq%_vpQ%D^bG`X30^G$V|}w^eF0|uzN5_m z=CIJ>unI7X)#cnooBP#W9Rab82O6sdVm;bpo!M}Wn79lI0K`YCS#@+h+{CSzr0B~_ z+K$J&+{hb~bN$j4G){T-4gE#O(#^sSxIhNPt55iPrT_5LmVMd7ByWnA3l3`V|?7mJ>e7%FwTw9(&?mh`>1Nl*m4_M)eF?MBjVzHnh!)=x+PZ%^an-Q2dRn* z1O(#&NLZ*0r7?a?I3DCcu3#uZZ1-#kUn^* zhw%qogA@mIZU$@~*9*4MZQTv89N8EELFF*Zx&I8jx*Sa-UFFJs%sn*Nr{o6I$d3fjxO zhgg@02595%+vLvfMy@-vZauHggUcWcZvSuu?xXJWj_x&oojGshIuFLG{`2R$@OwUD zz?jWhZ15+=1iZ`H{hazXu?l1p%ID#tz2- zX+W00=uHRc%2pVz>zy>N4M4!F!)%cs*jxV%HBifX0135fiuBmHl@0GWpC2U2o!8Bs z;Gv(3j~7-j00nTY@vMLC2I;JCSq^0n5L58{;e)5yJafwCML7{AO(uln5Y}39N03E~ zo%$Tw$kF4cQi~YLsY5c-7+5LexOoK2j4GI~D20N_BxK89aYQcCNyjJ8oGNb$EoF#h zPlX6cc3IPtQND>;n40;)EEAt@O;(N(!&RjjS_J>JhyY^7l4f#X+Jsctu$v}|ty#SX5(17XEruq;^(=4JY{o@_ z8KqA#87bsRC`*0*x{2m%Orvmi;@sZSsrgqy(-RFpx^=L>Q<*l3h^RE_S!!$@C++nS z%e6*Y%ZC8nG7zq2^tx@g3KTei0Sg?8%C!3`9Bi0s5KAmF4mA$VMTkp(Pa@ym&QV2=Nz7_jp#FIBI%%V80`a_9K zG`#EcKtNf{E=UXv7{GuF9=!j{f(lZ=;L!^lec>!h?Es~cJ~9OD!w*GF$J0$s>`c^D zRE%>i--T;DFUZto`NLQ5- z3948TqJq7nx&&`LZ29cMOf=I3Q>FHPOHC~U@HAx0h@uW z2nZm+K?n*!lmtd6pa9^330|S#NHq+VQw~3M7&K5tm6)0o1!HkVC{&HD)n=iUl{Vl$ zrs$3zbxjtNUms%kUM=z(f}ufUZqg#uLLP_8n&(324mPm`tVUBxwKlT?4gUWtEb;SW617DLr*S7EzfY-vI*HmtO@CcDsaw;g;L3 zXB1}m?ua37ZsKM}B@vm5?W8aTjaQ}d@MbNOQCg9ygSuoWQpi{=cb`y;gO*cEV+EPD zD+!g=meUWUDYAM4PnpP+%G$pM0b1ylQV&daqmNd_tsXDX;;MdmY=B4A+USwM3`9^e zPq$im#D`BN!lUb1sD4YQZ*PpNQMxgq4yruJs4)JIZ+^9 zTmEwgsMzHjQ-YBuR&Y9k+^2^FB!Z!ws6B!Gu7(zPTNTGwxASFDeSBkI`%om6D@|z$ zS3%PK@Tb4TL1S@zah{|ehp_{)?+y}xfmqZ~ECljOK2y1a=Ja9?{NSYxs*=K*=&-fC z{N)n`(UJ*k@a;6qwb4Gih@g66|l#CFOlmtOxcyG*t zCmoW&vmj|NkP}=Tm&zbv4YCa}xC94aV!S(kf-IN>CI_%^LP*Ao2%iYYH-^~H5@OXi z1MR33cN0E@s-U2oIvaFoMw*A)={Z-DTS1K$!s6<5Vr~@ zM74c8X-@5N(sO8oH-A(uUt9yr75Hd?uT9WHX4@lXIO8F$DOtHnR;*&Fp)5==z~uDc zKoPiZxDQgGz(x$#5{8utQfQ+vhC&lc{1(L>t!^(v8Zc4X@(I@?GAd62h(Dc@D|P>s z%8!<)w8uSYUvhnrd*2J+4dE@pk+p9!@2jDhy0HS{Vu;3IGhlNVgBOl5?fZ=M)71`G zE4dTgejJ$EA92;f;jEVmbb-LO4e+PGBiU8yz%Ps;s|9(D1}S2^ho2!Cvt@m-I974b z62d_Qvp67I)rLb*?-GXf)Y5m$kXJY0AwNo1^%T4bWhnc~ljdvy_pE$n3&fYpUG^f1 zeEsE^4he#++>K3ThvY#X%Urws-zEj!#)I#awN)-{Z)IS#6BzADAtc)K%NS4$$!x7^>`y(qvigGFSr%;S#86o29AnjlMFp zwM%qmUjD2LAPoGYxW=`}cg^cwpGfhO-7g)W%h#J6%>&>O) zCU!4}tgI%WACQ0q0^nc+03ZOZA%OG`QVW4Hf1&*9;X_%o&=ZgsE17t++332-5F*t< z3;?);bV>LtKbGXheAxW6oW+wDV1mQZJX(u3iHk}KG)RKtn6QK#C zFim*4=SehzXoQE;CX1ngL;$=H8;7mv2C3qUcL@cQ;Ekeq2@_nABg3(I>pfvumEj8q zVSusID>!o@09q-C00005&VhE%hocj)oudnMVLAUJ_^AS9!&%xp4@(AU zsj2H49Xi3NqGJpCI2T!22^y0sg&4tg+bBtEi+N+gngEMRd&FeeH-1|hCHR0lk$~O+ zf=?5O71%^A0D>W`I4D#&RI)(N>MId&y)-}#A2~n7YC9?P9$OQsxC%Lz(!#^TMP$0a zy77ZQIkhe+L(D-UI#ED4Lo;OjJh}*;&50bMqP=`@2G~*q3z#Y=7y=B?57v;29~hb> zqU4Tk_Br?@#jgS5W<6RL~76ncVXpvrJK1b4JY zM-$11a;PZrKpK!P!TJH2vxTuN#b(R`0?A4t5QG0BDkQIg0x5A6Te>*G{Ij$ZJh}wA zk-AHb#LJJ&NRZ@9B9ke;0vS(u8J)|6x;PXF;SKTxrE}4o11J{Ds}U#T08~iHwPG6v z0D}GCI#fup(L~GNToWYgPygJ^b?g(ExJurG$KfjoNFbU=D^1vpO}MZCzIqE|!L37D zMSjdq5d4GN`^E2~5q3&HsPIAG9LTf;g(e^Yf9oCRgiiOm!Yrgt#H-G_*#_)H!sr=3 z)EJ}?6QnZ3E+2IWrc2N7)F5-?3JZ)7XWKrvTLPtQNDfU){Uov#oKjU71qKz(tO3nN z+&V!B0Tq+PD+Nv^XqT$8x6NujhscVJDue%>z#7^C5anD#OIxM+l&-Vc%~w>c?HRmV ztHofl&gq0v(CDI3(KVE-QNOg$$}^VDVN5wpMtMvP8wddHx(mi}4$o}Q*i;woc+#u{ z6o+b1MW|9ntkS1s#AKMko8*JMDh0Dz%>|8yku=CBgSFs1tQHJ{q+tmwoe3PvFr+#N zRRb0fIKx%A)9cAo9-GTf`my#ZJV0$2L6uR;x&}lo#YKggPg0v41kd6)&%xA-vr`++ za}gs^mD?+cwde|spqx%Xq-97@p`bT0-Onf`&0YmfbtF(lkb;X#1~CwZvwQ+dn}-C& zG!m>=g>@(Z6;&*l14(e$8Q37*d6oa7%8Ckb6OwsI;;D$ttFjO)fmwW3A7fFwoYuR! z%fxFb8imqL6`R2t9^av%nJr8pBSOT4lx|#vZs>+Ggt!=dRli7@FM!xqm{pfGwR$ny zN1GR`lbx-b1yiYmi2J(KnM8VV(AZ4aTy4MbZZv5*}qF!X4T{sHi4+0vp7(q5BD{_^Lqg zTE11aWXJ_|vm8dM+(cl4ZxL8Df=?1^xG|F9# z(vaLxgZ+dYd$z$^d1&?H?=Ru96=TA&EQ^Ov{nQ!0{Mlqxw7v0KM5y!>m~ z{N!7{Wxm2a~bWYxn>zT-X1Pb~+t^eQaHUgp73u2U8mWK~zCv{z+F`jrc> z^Vf+`&9a(7(alkD_$-Wrf!!5VnV174cv#+DVC>bSOzl2YI?+fOJN9coX1z69bY7Ij zKfFZ|Hn?8)HMo4tJ`d1d<@h8*$Q8oL1uC!t1$;wgzyktd-KJqc_038@8$R_-p%AXz z7k;q2L$+cR2m1NF)U~vEG^^4LU>`1o$x&j@{RT>kM;3lCuLNQSc39;q;N3+&`y9GX zNC>l8k>Q0kDNLI^_xxhMpMF&{cjw@MVZY&Lk(YuY- z_5qd@Y_Xh$;9eF;Tf(#0 z4TI@_4(QwE-Pfe$gYHlsQUGC1qA@dvGTkVv`7=gc+Zdhzf%7qoE|`oC)Q$A$YTc0^ z1F4ZluV+3&EMZ|kCY%}o8^3Nf3_x8}V%-^*pKYwX`sIjI0^x`1*&6sJX=G>P!Q`X% zV_TglR>Q0V>f90D0J>q-j0e~Ll0_Iw+-s+^Os6Tj>AFu(g5>~HL8f^XE zu|wm<8s`7HMwqweXgBWQnyu>wQQitz7dy`DsJ%v_E(A@$xCo5yG;{*kOi+h1#VE;= zt=SqLhRslDZHKB~CulGi9%{u7V!+evh5m*aoCAAiuCnR|oq*}7^kK6~=T5jDJ+SIm zW^LEDUHyp0GeD&~vF)?XfRUJB`J}?N{_STC?l$h=5$OZtw(I0wzPx^@3Z3LEfo=~^ z-R>U0tNw0E?13KG1w}XlST3@v3*ilnk<3v|3D`RKeN3R_AWBH*aPYlOX5F%?(i=78 z0M_rSOKmAA+W$5L@ec6pv{#!x+hol?1W+3*ckl;qR$qE4>78&2KRFA(aFKpBXf7UD zdn5leXUikZN==(2Evb$!!CjFw@hrw;&=mv%)JCmRnh+mW8(Rje6pt)P&@X_5ZV-kk zW`?3DA%X3LKQ9Fv3yD$HiBkB@C2#To&w}_-qA@*$Qz~P``eo);#{qQDc2HY!g zM>9?-`6F!<0r1Xh*FJHob%sQc06IqTf>s261$P8~aTOQ$a_59|H+P6N_ZQ!4Yj^Sn zrG;pexS{*kWMBx5!YUHr0L8L$e;-*0_a6C6A1>rF;P^FOK$$#9c!n3;Vy}WIczFK- z>KD#voNx=Hdme zx^x#!Q|;B8-}#x)d2}ClfqwS{#&q5VB5-h#AqwT)IF1Dlf!&Efnyo(R`*$@)bzmC! zg6Fk$2>F^LCC=MCu!j)D412HFcrouKj28d~5GWf+c#!XS)IcirI{`JSPl}~i0cG+d zsEUbam;IpjP!)H0A8&Q$Vi>Rv`r#Va0ry=vWlJOXpZ9s5m;CVN;&-RgTUbz3U5K*y z>d9*qTNqlfZOM&$kg5MPCu*z<$NH?-8-mXVZQv1TD|;RZ`>&_{v1h-EXY2nNO8W}V zfSgmK{c#mru>Ir~_5r&D^hg~R7fCvx0R+f^2blf^YAvq7>Y=}CnBRWx=YhM61Swx& z_O@bvHFUXH_b-@y^=EekzVst7gacl2`A2{HuY%8i0U8mAWUztx1PF88mIzUT28<6L zDpnODL12M_hyVmUjQDT@ga#Qka@^>V!HH+uxOppS@+7&GDp#^>IT9MVdsSSpp;_f7 zPCGhxR;ko8CrukSK^!P*^r*!GN&`%!_y7Ua3>nzmp@prRI&}HA=5vwYE7$}!+(p3@ z>};AFTabLg;`XhQAO&~K{lJ8erh!S~!U4p#F5tj%B^+85f-hpeg~b0kYdmgVxJ__& zDS{I2+p4vc=NMA7xwGWYqeheF6k0Oey>3ZK+F}S6rPP0C&ld416Od;4m_3=%O-+<} z`8sh>jgZG-5epm)fH;6@MTiwJMEuxM1$G;0SYaah@;i8x>0FYBX?~hD@7cnuXD<>{ z8yiUHFRi$=seq^Y_v^2qGYSYU^f6RWLiN!MT5`#j7=uilB@Q1wNaz<16-s!af*yQ; zVTQ;V_hAxxG-d@AHPit`U&55KN;eRu*a;-72-8|A!Z6c~DxR5E8jm`1Cfg}Nwi3!5 zw1Ie}3g7I;1VUz9f?yMf?NLTt32i7r2oZ3gz?Rc_sbzCOq!Ir^6i%4=h9_qs0?du= zxk=u5<+*p>dT-iECoy^OBEv@X8CBmy&^g7QhJ_jmf>i!xK~R7+xbOm`Wf8>Sks$J+ zOgJcv@JT1Ac=`t(U0j*m4|m{k>Zx>yKq;k*1#`tJR2;WVGK_r_&Ma}P0@-EX)L|p9 zk#W`|v8bgMD&uiq@hNdX9))xCb8ftC%WRz zS?9Uy()k8}*uAj=p!Eh|UsKHy>OcokY0&S#0SnCUq0u(Wst8c@R%>3lIP45FT6!44 zER8)lqQTO->eweCNMY-(;@H?L$Fl@04vs$VxH8LOw59(VYo;xDFwE6TJFU&vV(A=n z%k{j#9SG`X4$(ymluEnlD(x=2>)H|?e0kVlK@52`(?-2n2b!js=GAPNv{hZ5LBn3i!y0&&Sg;; zA+>dLT@U;<*kVgQHWOVC@qxC~-M;+V&0~H%&19gQLkdzn5rPvaSN}J|QfLAh6(HC8 z%4CTr{xWMXul2ZbkxLHkamhXvmz=VW>3#Mz@x>(n` z_qgs{d9oeV5OKBGS!ZoYdCt9j=C$B;NO&G(5T`=d1XKVBB)G$zpXg(O3yhEvS#T5- zT0I16tM6GRO{W~*5U<7B=vSvaJ(ifEM2e8p;uvK;rgAt|wDRQf@S8e@wyeQR{K zkYM1_U_D38kc$Q!!vRsJIt8w7fy8iK)TVYnPeAa5^Kl~vd$+++eXx#|;(+`lcs1cf z!2r5T;RTZ-16z#iikTo}EDrOR?BR`XrLtisz|cdINdXchn+I$V$1)>^B8kO2j**(> zK2gRj3sE%WLc}z$HV7??Tl8Y_Xh5wnBC!9AWHe*y9C$|9VI&4qY@-|BSQJ5N&|e(v zpdH7C$9%l%ipt1GA3tKBKq^I$!QjecFtwWv8HXW-_+~O9sgPJvfj4`Ag*QKp9G8x`tY;@@GH$LIXLV$&g$EPJCE7v`nZ$VFQdML}{5z zTiz0v?Aj$XFbYBKAYz3c4W{#knZd9CZ&YUp#PCvj$J-#HOc(f41rM;oYSL7Ys@&h~ zw#hyWwWUS1x(xg@1G&-I(}+b~)@tas)5-mlFHEG890vIYLTEvMd^!NQu4xX2z7V2b z1>-WRAq_E(k)mbQWk%DAr(HHQBeVaDW(8xa00R)ycgP#nu-2DM8Y+_my^%@)+z5oD z%(Ob$Ns31`=T{jja;c5;DNsYBzAsRMs71;vWtT;yqi(i|2Yc%K`VtB;ECZohxFQ8l zP}Qt9V+x*w!9%|)(TSebthGcdTKDv*ve<%xat*0S3s3=)UddkudBhSbXFVAicCNTbs!iW_X?s%xST_S@%wM zsiR~T`9`w{H&ky5-utjdD6Sq}vWop*F!vgrJ<0hgvy_IdR z`hc9*&B}MZ>GLc3`9`XLVCU9VY-;5;bw%UAT3J$sCE$& zaDZO~6z_uM8Bfb94t?*vERioA8#v4t)1BU_lK=UNApt@QB#SbtZK&!io9c93_Hu(? z;*Jz=lccsLDGI}qYc?PH%@fJB4Ma?9J8uOOn8S4yxNP|LxC(H^a&Rhpm#BYlb5;MCVQ_@Mnt>P{iBG~_iLsNt?AyVKfqbGyG6Rx6 zTmx?e7DeJSTA0>0=$)?f@U6i0RxJS~x@uxEKQxU4t_4s*+NhQt1$eZuEgzTBG(K~l zqx+uZ9xBf-4|M<2FcH*4XDVz)jfNHc%|sA8RsYI4Zxpb|-kP9#Ij?>(RA~L+Tu0#7 zbD4WQ@i}&4YVFw1e)qNm>g@#c8Uh9yx-&!S1VRaM-<@iReM*21ZD?!C=bGKcTL9(0 zx3Nr}|9s#easzJx1Z&K#{`9Y({pR*`do!JRPNN2?uJI(NgD!NyLkkui-R@Ng0I zoZplT!Yr-e|8NlkPz3H|mjd|KN1+^dVaK2VP*BMq{K-a;_23cJAAHdt5&oS1eMl#f z57Ci=(V_oTEVW2NVMc)6P}Px>_cS0mWucW>#RO8I*Hs`ggrOK}RH7Nd6aj@$XrI;u z3h!jl@01iA9utRv2z#A|9UephDFK!MTXv*Y{g@nfu@EhQ7hi!!&`%9?f!Y+{>fI3QNnrzePl@0`mCfDO z+1`VJ;RTMNE%MR|uu>JxM1ai18nR&#`_*9`Rtp^b&wO+r=iJX=iC_0M zM|Tm2pJ9mo$p$kp)FA!=t~?Ef=~*8fL1&QTX(U0&so)|u2|MzVIH1TR}ESN7#8=r6$Gr^aQV>+Ilw^um6cFlYB&HRTv<251Ug_!IGO_< z+`u5L1{_3!r-W0*;Dt~YC5y}gi*$-mdJ{M;gL!G<`V4|68pKDAfh*!otTc?OZDk`t zh#>qzSQT7C%A)Sw;zI5oTS1U7NgL+48An`D1_|Tu}Q#sOx1E2y;3IGUfqux!`A?e18^`to-CI9fxK8F9M zCX6O&il%9v1uA4B&8Xx32}x9bl0RABW#mF(gyNKO50%YkJE4Nnh2=qx<(-^mEt;iB zV4!ZP0Jx9|+7wowv>oQer3>5u9MYv-;w2rbrZ$qxp79YNf`mk#WFSHu%emZE;3Uy7 zg9D&o$VrGy_8vvh0zwf2z+K#C!bPl{95vx*eij8%7U7S9#($m{sZC`&;^7KD*Av8n zf-We6HYkHWXbM__Z=%Ofh0(O=V+ zCyJ`*hlIcyG=cMpXItLjA7`+--neDC4K?0MPs!qh98tItmU@ zOXF-nr7o$;5nrqvl~J7+b}~s-65*(#s=BUg`c0^q!YaI~RfcNR^LhV*+b|z^b|-M@ zDv7c|afKK1Fv6!?5qu)0Wm*NOe1fDN;lEPjOD*dMVg!zbmpPK-p&|j$#7^b}8&wgh z8W6+P%mP(PC%3YIrC#dEBAOs{Dko*5xw0dbY$?0WZ2PV1n6j#cUX(({iF-Vad2|QI zy~NW1EldoSFdbT$m>4vKSXMM`n}nHpLh4P1EP_nci|rI0?O~zmr`tJfMIKX>9-^=Q zhSUha5?oQES}Qxu4k$f^f2C|7QGnczWD04jihdg2f-204D$S;9-`;||;%v_1o_U0s z#5J6*A#RB7E58Cj-f07!Wt_$x?(En?Gs%Fj(u`dc*lT?W+=Bl_+~Q714v>Ttm61To z@iYP2Ix5kcE6MT!3ZSfY&F#sq0t#T_f41y#ysY%0Yv2BEyV|VaB4kXYSS37k$Ql8v6tAg zg}|upkfv>SsO~!X8CAh8+~{k{HUvxpA!d~uK=E$60`K1j?(h;Xm@)6+?%Dz~t_}WL zt|goF5?d9(V4{e{I=~0t0l{5rWG6tzzj0^yimy`SRCzMfd77W;zC{Ksf=*QA*~V|$ zl0_|ak>BA#?Cz)ibXSZjt5Lx2rn0O@0^PY%%K#HFy8{2N@V@IT&MPa(f-krN6hAQ) zQ!$!4BqpFC+fAfMQA9PB=u~`|(h3_drkUtOOiAxZ!}Se@BgrqzBdIvfZdzOEGD?;ou|zJezOI4LARsSy7s5eu;IA~E_Uu>lXR zoy4k{?vn3uU{2VB`{QcVgfzT0kCR7RsA9(ljK~Tq;LJOtW+{58sU*uVyGH1HCwX!5z&_} zG4XbCta67MJtT-a94W8i^SW;wUZgvBFf6mDv(EprE#qUGo%0BLYh5KD#YOi%R3wuh+_vJ6#^r2Nr&`a{v`vIAB-T|N80cDbH+6rF@wr7x5W-&J^ z`{|gQ`^$vgW1Z>}R@UTOtKtSEzqDH_3nN}1;*wwyo4qNKg z^>7NQVF5gJ|5h_r4^?MZbXSk|`h~S=dq-KTHfx)8TVb(NGmi;%7*rP@bfWT0b6e)> zHWKP_rS!J5=C^+Hb3FFKu2Aws(JXIG?uHohf;ncR8!+hc@pCy-j<=Wn8l~e8=~E$2fgQYdrIJb-vIG zIhukwLLvGx4rJ{8WPphYUr%TeJ*59djFD_Z85-Rx#dlh^GyjR*=AVz)+-DQ3XenV4 zbNFb3H$sj#S(9}ikTeBM)gQ&S4|lN~A&L%=AYRw_nMOb+IQr;-!SEyuZH8rLz)uQqvyi;aTiQC5h5W{T%eZQ+!Sw)aj{^Iw<56-c7?~M}5C9AT z{vrw3D8x4T4+rytw~u~tCjT{+%xe6`2k%CEI*5ZhHb6rkSh3kFZP^#~n6Le1uKfu> zx^sSuD1QzuC>+Ld1)8E=y;@E0{T(|qA{4YKKKm6Hv>;OEx6fzDWZAf$6aB3J0XGh5 zLWKnkG&BGKK*WPC`q)C34`W7+`G^T*^U+0y1U4+rYxbwf z98N1)o{ECcQyv8Y1~_C$-~ypMPJ)$NQ0vf!AVY~3Df*$CEn7<@W-^CO#1SlCBq>rk zYSSpGNVXcqp`rf+SXzi|y~(80tVJ|Qg!G`2;wAyPac$P6dzX)(wmrD=wc2V9)s?W^ zC^38(abh%X_AqYz7?5Mhj3HG%)Ty#C%&=am3T0_qXlvQHl{P(kTI$oL;jK13%vdpC z$e^VwhmTwZk-H5}WbiwoLEsCC-<&CNpp8nHE^geI#$ZT_%+;X0vGwX4Cd~3EgJ^fa z1e2ajN|=S``q92z7mW1;1i*j*n8q=SIu(xAUAM4yT`zQl2O{WWr|y&@M5#Wu*h&fv zZdw6?yTFssxB^g_;=iv1(-4cbaNvP25JMDkFU266EV9MKD2#~~U0gAaL1ZKav}>e6 z4aXc;JIDVua$e&_Hrj|8l9p#;gY8Hpaaj^bOez850t=$R1i}++xUzx?A}kKP;{>27 z0UCItVF^020LcY8Kg1=3LJz70dcN~5d5)QlofPNPU84IMCrlA#II0D{5^Q(Xl^L_e8=5g6pC zV}>7E_<@h}?#Mz4D17~aRx`U8*33mvJkg9u%vpV`{F-i$1nM7tOtH4j+DCdt~jIL2N zZMp5XM{vvi+UwcG?gL#Z5jk~B1>RLTWqQNt818@LwpVU}cd*;;1wJs~OPUa_Rc<1H z0ubU$Ber-B{xV+A5I8y>U{g>*E)cAHr+|y?%<-y&rCm6=B^O=tLFv3UY7SWAy#Dn@cbpEWK#b+ zlw>HC``F|}79fD7z(JbR+@DxwDF?oyI^rPP&fajk@w{LLnW)7)?(nld2(1!BBMj2` z*Eq^Rp+z&mg9fAZtlC*iG+ATb@@DuvvE49wI0OcPphUg*q-`Pc`Q8wTXn^pck9_5$ zTm914J}2UDiSLu56q(2&MCs3e*y-Hjw1`BHRYz)J4All~b-5yP4~=Ooz=BPE`b3O^ zRYqFb>Ip6V)e}SnhXsWpSZ!p>PK)9IZa%9);WQCB3;Ci>=%EnlyjlP1%<&C&8Y_m9 zq$E7!Nl&Y0ayEJ3#SkE1wkoBqtD^MhD8U0JfDII1Ow52p4eG@dNMQ&(gXZNp*0e*l z6_;GG=*2X4gp9gxf@1uXIg+(VHB2$BRB<4tz_Osb8isJ!DNJATbhAv9L1HEHse$lN zPN8C!kVOsV3O(9LvZV8ooN=dAsmeSX=8#EPb&^cFY8ESGg_oND`zq&0Z6L`)f>H2o&grHHn{(GRjkUE7kTL6qdbM% zu#OeNx}8^};1^4^1`c3tQEqbs1_FH4XC}v;O9dE4v_f$gf3#HG7V{b~I^0oj@DqqP zKeXY7uP485{Zk4kcK7g%+RvWU>6U;Sc36zrh0USfh7AB{72(y)YG373rKAm#xO)(}%l zuBksk@hm5rR~BFNyOhvzSL#N$d%2HrJ-b9NgnFH#`6?C>b>m^4V5{(T`gdl~Fqt5_ zUZM~I(Mm5Q@2Y#i(Gm$caQ-O3^L8TOD@Qs!o)t zjUBiI>kyF-Ju=~67=n;;2H6ak$jAr>CumtcUMIg=sZ3obmC5C6 zTxYpl&A^SEI9{hJc>LpS{@`o_TXM4bNr8T9_nVmWVdr23I$?qn14zD8BM1mUI(=6)fR5fb+KsL3P_JeZ7}$ z$*IBh%eW4s5Tl91ZEBb#b-=-L-|sU&m^L_Xs!>M zV&ZY0`1(uuE^hebPLu3O%bZ8SH1LNk@cU$J zk044qQ0j=vkhowB#<&mtLS%_BW&Q3X^UlvIXb|1r@CM~42RUwJ8YN5WFRbkEUN`{% zh!7LdMRNA=V<2S-neek3uJ;V^0HrVitB?|tCkqcuwrC3oK52VM>0ks8meS0(Mlrbb zY7JTGuJ{TDwP*&_t@G3^=ll%*P7anF>|MkP5FdbH(x6>FrBe!VUz&iUP_S?q(Gdxd z3i#mnqOe*P0q!bs0WI+ov2f%P%o_g@q=#al8BitVCT#vJY+qIhxJt1G8I2s{&??e# z4byMjR55>8DDiYr`Z!>l_+-SOPi?gBeRk06zK$f~@tiaP_n6Qak1-JF>k#hi0Bw)r zZVeivQELp08n>+J_D`QYQSlH)6q9Hi&yjJ~aTQZc20Mox4b2AoO6(*@Z_4HmM4}KO zpl87F2N_@rIG}DYU=Ux=N^BwYzUZR>kRZvv5ZC07b`dCpvZntUu?&#K$!cs$pirHjY6_`v?x1oSGqEhX${tBcUM6ht zxY8>YpsfLZ_et%Y%FkIET-(};5wBRQJ zQ#}r2D34{QxbCAguBj}m)_#pKyG9}(Q?_D)tSV%;v=TEj^DCE59Xk^p)e$7catYKy zQ2am#+;Il<#1@v}U!04rG841Xsy2B5Sol&mg|at^@>Uq;%}$FunsE}1Q$L3}@ahMLNi^FX_@$2@XDHFPtv zb0r@1KS*LqCe$>I!cj<2D16wVNVpMa1!e#09t6C6Kn4Qho~@>Ea7u*eLG+;R*`|1?Og)AS-BK0pM~ z?goAcfEcaQQK6_=cp`OTqNy4oQgn|s;S=q;)Ju0}O?2iH#1uw>%_*1XTh7!!*OV1j zbxjEb``k43#A=P8GZaNp|kgdNsRDGY9e*}rgcawMcnx6R|&OP{bJDuc0`$V=qBbhwX_(w;25iwGJHo6 zCiXHy6_UDjk`(Yq&eS$A0t7!6PW+J#{ITQoP|UngE-rx0{OEN&@Z~UJ`KF|)ZjN93 z^(3hiU;`G4xQ-CwKm$}U=V$?u=%75Q6JcwXVWst93Dbm_mUwiBM#fZQrExam!2_2M z46}A$dgJdDDA`7WKB#XVM&~joLi!XDX*KUr^#ePH^#*x%gaCC8pnw*@sAv=PXww!j zA+~9qc1HjHR%)kK8%~uP)ZzGWBk!yaYxU0WycPg{V<4hH8_Yp$$yPXGM^}6GI&YRN zb@owT=TTtcJLNW}cJXfWHZ2o$VM`R@_||U$cT%D@KLwYbJ}88OrcT!EBdTC_(!_T| zcIFO3bKX_uN_EK$Rx0F34ki?D;ph-*@?iCL zxJE5}OLS>p7j|P;eLrJ%OLbgpR7qSxekBqY7=Fp)hnRtnVwiHl3VMf; zA6X!kk)&|b3$)7_8d;LFI2m}N5IVUAx*-Kjt%E(7gh2QgMEE%0H;vUeh3&wNUpXTW zREzaM0+MkAW&tNdM+-)v)4B#_eB zcm%HPEtthZ_QV+kG8k$=6ByYFaB~DWxs!9DlQbcVXTbopF(5s(QA=c$Ng0GgxJ)Z? zm04L;S+xRS`3n(Ci8!*B_Y4n^7?-V}C#XOILO{Db0p9}ICHuXa*XXra~YN7&#njAO)Q2o9#G~so;X~ z;E}z#hB0`%7#XL7N|fWdZ^w9^Pq~ci8RJNmTnRUR%@m(c_yxRA@3zgKvzmN5AfT;a zoRgse1R(|R7>S{v0e*T}@uv<>!Yy_94M1eAmw_?XkP`-`4eG-xsYeagC<_1ZYaBd! zV?JP(F}a~C)`p$M;MNBLY7b|~mI-;8e)#vL>7X)3po&9)1tEc#DK_^kyP?cloJXJ_ z!x*X67nR-jo~K%ka~F`rCk1;;tJ`pHrhu46paE>V5D{2?v@`6wM*szWJF=!30hnPa)nFcGK{FFp zVawMoL4;NQa^Z|xo=tnHnVM=Hvz|~`db;d{*+5{ZXcR?}00x_9$~m0B88L&Qp_?@+ ziW|QSJaok1+_>PbLpo3_U>gq;bpRVsd!;$$$HN=KjwdUZ&|r?_<^li4f+eE>y{B`< z(R)~6vvo~WjE}mMLHM*wHMLdSO!Jx2q~gB;T6lEZS9s~PpfHkYn1*u?ujGKEjJ&u9 zVS}k+U~(cr-K#*%=ClBtv4F)Jy)OqbU>4TG^OhlJ>_h$t=4^fm&(`~!QrgXtcC=-j z#%H{g1NV$0(!McC$EEs&dx5`?z& z)AFJ`&5vx-0cz17mAuD{d0rAIf%NXImEcM1@#+L35YHAJbm582qghLaVfq#?J3N&%x7gh3O; zp_}3$FwJq80~;l`To6M20^*}GAGZ+H>uzfC2+#a5lhDyc4wzws1Gyn1+4438jMp;CIY7Dt=w!; zxOdu_hV3@NVF$s$8|k3aBT?6bn3Ee*xEvIV=noqD%#}mc8^!CIpxS(ZlQBTKp7$sk zeMEr!)NE{gBdWVuh21Jc3WNdRK`r1Z!!brE4lb79gMEarO3PT8&sljAj-Br$R~kwXN7RYSt=dyJ(S{ zMvU4za`dBG9?;gh^>6))HHj8Q-@ATW5vdS!pE7~T&tLj=%y=Ju5eO( zD8Z5Q3t62vz5sDRj=x~xD|Tn!!G!VtkP-ku;s92e5&%$v50ec$1w5#N8Nma~AY6EU zYC~|~(Nw^pk{|-^1k{>x5&57eSKS7+=)RRLA;l~_Bhp&=(c4)e-E2_+r!qHrHz-ep zCBu_eAr$AD3hrK&%%DMrq$v4`I5Fc#Z{$Ia4;k{LdSfwLwyz1lQv9$T>i1l-A_~wn z7A{y#TEGcu>_?y;c|?H~1V$Oqp9(6ZVGDow(ZEyx5qbFqSO`pbBHUcSg%g*Dl#R7l z268Mih8z?MMgtu*%y5PXLR47U11V6km|_baz!YSfxN%oyQzfvO0EbD14=!Hz}&@l}KaA_!^} zqUr~#f=5ljSbq;JAlhgpUMRsK!d1CSPhKMOr3A2&wT24j)S?E06{s-m1^VG?!5SkX zG1y@_rdohwKHde#5f3z?h6>Egu&M(HsDZ2hT$s?fgMAjtAWp3uz{O#kZNX-xt+suD zoVX(>Fs5B*#+D`v-SEL26K10FJb&;uD2#TP(cMpMPbgbN;Sn}$}u0XLMGYrXH>C&G|?@&VpVHl&^JKw4O?n0 zLRlnowF^aIyn1B|A`iX+?r;{^JkJ{_D#G;~IXRnSw2m3TSRr6&fZsXuRE1;O#*o!i}KYY4H9%CK42_1qZNB1c)XZ;!Ht-7~nw&P#};e04Z`z za1IsfbqMQZ5Ko%p+`}9av5A=u8lxLw>O{w=_pq*Yl6oENyhT0+T%Z{)%N^Nr=YmS4 zp$ebDVeg=Z6mV!ESUtOexr8vX427T(yXgU5o`@GHFav43@&FOi@GGeW;8UImA`Qnu z2k>QRTQZ9P2Apz68NSJZu(F%~Sg_T*t>oc|ZcN@R!n6Y|6wO{6C`cCaMuk`{tvIh) z(^>#Fxy3#3UzP&{1wZjGI%KX-2Z@7oq$3PCeb6C}5TT(=NWxEILt~6eq3Tx1LKnWU z1=(ZSI9_(cwb{WK+~5TdZ2-KdV9^uvkYQJ{RYd!^02^`uMHuU*gh4P1ib0E9VSd(u zUO^}mC-4yv7ytlm1>=`uG#(MYX_vEzbDYe26%y@K$Q?p04NeRKGA%TQF|g5AX~Mw> z>~tx7(o+k}ao_@A0G$Pv2@11u;GNX=94^RbZKBtx1p84Yg#CKL-`OqJ0Ln*FkAKC{s*FhR{_1!fQ7^X4=45>+~?7OHMMEnvRZuGmgt zt<2=wEvVX%)Sh*>yfv$D-@1k1%5}H{+pBSp%gKuZHn1lgY+(&+lf;^o5R7%Ksb=Y! z1Z-74!?dMkIrXdm0|-w)C}>q>d@=zbyylsPSmrz5SC-f6mbLVqZxH|&qIO9s1X3l7 z*M8RC`l7Y91E%e74V(nw5_iGvMXn(pObNaQO1aDhrE{MP-Bf6pQsR4|w>ZGc>Xrjl zHb91DwYyap=pzl1YLM;3>kRUm*JJ_!0IP~u1o*n@zkms7J z_83S%CUAid3}po~c*+p2vV0{ST?&UE^!rB z{M{GNP{wYnk@*z!R%k+1$6l>ple;Eg_l=dvAX-|uVockr4tl6gZnUExjo=w2Sjtp( zu$3bW!d_7S7^9p{)P=vik`;0kdUv;JrYhE*ot8k7MGXT7q9WO9DyMRoc z;HiF&Dvx_DWMF$0(DLO3u@4NQE2JPyIhac$XuB7pmLbqXuJ)57P3cNo8q?g?bfeT$ zVN!k?%pql+>q=dt41{=>{i#H&QM}byw^X~dX3A!B`hduQi`P$VtATd{&M^4w2z-8Y zpQ(+7WAhdRAi_ZZ1eu5hpf=Eo@M)8+eQhUCTH74I^erUz?eK6p++OZzhEcgXQAiht zt!!DlamZ#AvxkSdV;K$YeN%j6pbz?C!vliW1z+)WXD2^-ei)+xYu7Mj3RADHW;!^7rc24E~*%p+^- z4U7n~P`v@Is9YePCL%x~^`Mz3T&JfHUydFZM3jSPUs!@bg7ciR>xvY?ZYGoIHnIw# zEIQQ z4!PKrKRFe+JL!l(n?>s>R_Vi5#^H&jj|K$g!2dS!5dr)6MI4e3+@gB!;D6h+1SNz2 z4MNd9?4$@pg9l7eCJo>WoxoqbM{!(dKdliWTHtT{w>h#EX(h;3&F6g22Yq!XebeWH zEx32umvWQ9VSrImESD55qZBScb8Xf`D`j)$5FpARdNbx|a*5JnD0^+=c}8}6PJj+T&?;Vt2v>j)cpyDpkZcMACl|PTy$4Uz_I^(U zK}WVfXH_N1!8E$^Z+&=zC@6Q^268a?SJroZCWn34hjKH>h<>+INkK!|6Ei+|Vk@Np z(=>kLH3I?2b_BKrZ4eBY@G=xYe++;LG^7hrpnt-|3^yeM+aehGHVZ>ADzF&;7z=Q8 z*+K!fuz)mjbVCp~mv9Gu@dNC#Y%PF6YlU_jxFkG~hq0xAVDi=;(+yQ#IKGb3bT9IfEoXkTbU69{~`H8TWblu|s!O zV?MGUD`0V5NHtW$44k+VSpqI;U<8kqkADC+ULl50Bmj)CkPB%<3|9*dCkRjgac{_s zbySU67>fLb7lNdPN5%qBCXL$&IS3^%fhb|#Mu;PqI*?M1=a_xzn2y{x8VH~!xmYSw zlLkw%Qb)NU7H~^R=@>@>dHj}IZM7md$P*wYBdK?bJ|SM>MGIC4J5A94Ae_R5LNYVN zV;>b+6<6^GI(2(hLUsZHlD z3ZGRcFIikF_+T{ojW%hHiKvq@n2vrhXtrWAY5@eqqDxexQb>tAJU9WCwIfi0MP1Ph zyoi;Z=vx`*HmRZj*>q#wA^v?IA=}~rv zLg83_lS!W}NE6y;SgMo-3u#>d@|h~-6uYw&pP^EW;X^gU6)YqFH7411u%mo0WF!be zo4|RSCjg*SX&Jq#G8jq&NTC$B=|sm#fxz(=>Y{;q`6j5~B4|6L*l3vBgH&hA>!#Swp%tj+tqj7N0hW5%r0POv-nLbqvi}df=iAQj;K{ zkut6(L%U;|@;DXuhlw|)TQ3qdo%sUlW@bBMGuL>F4p5;tbTX^L8QAm#!IS_twxPS( zEjfdae5jF5;3igJF#e-Vu%(C9QGC$phIIi6P9SzB*(JI_2K;j)i35UZH48l&B6T1m zBQspGQX7SKNOT~R?&*S(*`zsHnIQEjq?RFEMHMdq6`e8voK3kqmY6#`f}1sibY?mw z^QCTYFg(>{pbPLr;4n8H_NI`5p|?qMf0S+&c!2_$8m{BnRgNTF0F@^Uqa2U`lY(fD@;RS^rK$t_ zq^_EkXnK`Nm1|~{r??}UNjZKCiyBFlu&#-cK+zOQ86l2k3yG?gP=S`|rWqCr0JbP3 zcv=8F1e_7;A3i}D8p#68RvRTra0$jKnhGNGv;pylCf@0Wka{b9NjcK#Z)>83eHkT^ z0Xg={1bVU?f*=SrIo?%H{;Xf-uT0gtNFJC`za8jxacVt9w|U3>P|T-sDNr6zgjNS__?t{CW-^JvcVf?(f~P{ zu*fk)RDc1);hmk30tv-1b)rGmQ45s>2!c5OyCmmnUE8%g>A(+6gHhUjM(CggNd^{- zz8TzjY&)hxIXnKCO;NL^t0;xZFcndt4qho1J2gZ(Se$wpz4S1VD2gQXW1MWF2l{(Y zv;vZ#c#QlLovyK)m&;G9(7$h?2MpS|6fi_qfTMcjMEKjkPmBivb5HI%v{xL!27I+y zD{^1VTzi5e4qu|)19HG#0{!?Rymb&`XXGVZmqQhy8$-*j39PAlO!X<~&p1`Tg?8|yOwh1O$o)>gZ#e=Qs19%M5A!s6`Nt##42DF+( zRdAB*qN#*BBXUE|9;nVW&CZfnl|d`d%(GS%4UJSFC&fWf2jRMygCcrG%LQ%FT#L|8 zU9hdg&>2m^S9G*rQk^6aCIxi=%*l+=8!*yD=du8Y%xHlh*+@5e9EDP9(rJybB28Or zov$dH7KOJ>yV-f4mmXkl<cQ6L&M!@7)0=l&Sv3H6f1^} zM-;igJ?)XtxwEyMg;!uOUlLGjA{lAn-QT@!lI_KmO%Y>kyO`aR!M3|bco=|s)jytv z4e)aTmvw33zjU>!0=af2_qAb6_VmX=0 z*i(?y0v+QsE^gJA4VK-&mT3y6klvd;d4lQ6Z*snkT?l}Q7AeZ zP#)!KkW>?H3vzM)!46sAZ-714MZL^G21gqjw`hN52#RZ2RwD4{Zo<#CecbHIY{l*- z7tKHHy4afDL_BdA?3$ub@-Cm3&y(!RZHd$Yo#dB>T$4TCO>Gg7Zr%$0z`eZQVSp1{ zK;$F}?>`(SZ==KKdajb(0uHxqTJQ;L8v!2dQmQVBhyV>{{*}HXDg-XhK*38GaO?U2 z@%jLTupM?&N-x=H4h#UB;D;V{9-9en5jX=m#rp(s=GRA|TXg z6!K?OfmZ4NJf^K~&6s_q~&{*aWy3`oF0hyW;=p{Ge;urMTI^PSBNq0R@ zTo4vILcdW2P4wh8DM;_pP2coP@ASGGJ5s=rc`vI^?sUTvajGTt6l?o5(lhj63&hY2 z>Hw{RfqCVDu|0zTY*0%O5BC&5Opz$VBhvLpg_{eI3EPS(T1TLla6pW|?~pekwKH4q zEcw0a+CY#}|NM&IFe>=*4O)Jvt8nv{9}BV||Df*&KW|a+N#l)f5}<$X0KrP&z)A!S zrXo21=@^zHIy5|d7>3OxQxq*;oaBd%5Qh*qd<5x0LJBBpOrDIAvI!6yOa5SZ(MV(m zk2`wUq?uFaPLD0{VM?Q|84V8}1<*j(OsTd651b-EFbN(!eN|l;%|LIRwxCyWa7b{e z3Jn)!D;W4BLQ1%o5z^2qd#3CHEq5f;$^`|38gwhO`~{I|o&-F(w2;7)&(;z@z6w(zg*DO4B=)_qhDweEetZ@wkhRm33JC@GOF_e1{pfkUG zZHsL!oABYqi5o{w?b-3=u$>os4qf(a>b$HU#GV~tMTQ+6O2io6PewL=ig>U2R0Wm) zJW4aYClp`)dzf}ZrstX8sEeOfloH*~99!Ch2CAnJVB?;9syfGorkpxTK}`Gu;h_p% zD8~l5%tD}v8B$;?f(n}HBQCQrm?Z+xQoul&3;`?aGAe3FCWaJjQLHgJVl)RF7D)S1 zF`$;1=z?-2dm|UoI+0=#)OuNMyXc6CjTMb_!_t!Hm`kop<-C+GOfjpg^1AB^(k@Mf z!f42&@xp_XB8|{XZvn&*QHCWI_FO^FX7EX)z6BuK@iF}l?FpZy)XAVIh$0+NsRs$g zM?rWR)aik&Frm_&3VVQznHShFF#^U~q(a1Q8Y7h~8%BhIk6JKj;G4xX1R+=dAS9zA zml|fMB}d0VU<{nhG;x9i!I;To5?q@&5;RX%Da|xG9D%Ym*IJuyI#I4H154ko>6S}# z!DMdCDb+nwO*FNeq&q@0stCbR5wXQ97UE zPbi@Pd~^mydt~ass4B%LA9%ci5CnRFN@tx%W|1_X3ub6lx}=c*983)~@B7mNvtMl%8tDG!4vngZmMQHQf&qJS3y*nqTvC*h4C z556)xG94(TUq*VidjC zE>L8LiU}c~L&`%hSJ=yo;z^snEEBu~?5zL4`}cN94`**7ZbU zF6V1fM5dXpD87SaQ3vKbpWEaVuejN%FluOD8MC>@Hck+ZnZwWih7<)!sf{Ke=vfm$}p>FNvrwdV%XU*h?ldiJc4?NCTeo@u_-_vl8Z@rw+o> zhFHEZ4K?||BgN_iDR4BEsbr-Hh@jR+Wg&!T^kr6sNrk?ZNg)7i z7xW4PelCx&YYkVnL^?e`9HV+{I_XGRdd!uw^rhj0=_Di(M)0uBrWMJT`uc^G&$0dScars=pK6kCktuAu48`#0>HM?^i>sUj&*j^695#upwWQVEJ%1)L_-MegN zW7?)7be4-Y4K3x=6V*1HaagbV=~FS;gnhXn3S4NdYj=Ce2CxVQ60j|7A+V=*){_Ql zoQrMGQra1Sgigl22^zq#-JRgT#K?66iBEhJ>u&dv`Ef2sRNUh4ij@nh8R;&SSsRUP z#03%&B5y;77fGdKI4BCx@+_m$=sGeSqX^@_9%wZLglIbeio>IbJvupIT6 z+d`CC5Ah)aNRCG4GpE)jQh?8VP{G_A*bJ8cca9~UT|46D7MIWW@i3ri9Of2(`3KOg zadK~*FQgVKSOsM|NqZHQneWchM9}jtFljvPDm~jZONg^0X2#VLkmz zjRLM<}!?;WPmeCDD`~B6*?n=pIv=3-4C!kjK>CFUeb(_vUnx zPhCDplP8)Rsirjt)IOHQ&u_X4T#0E-r#b`=G*<8@fP65T97xO8&E5dy?#fjwOiSB! zq)f*xiUDx1iyOl29sD99h7)%s2XS93?~HZbeD&j$3F+ zk4N3#>bH+=yy*4Lo8C_Rp1t#(Sx|=>(pN`cenl&}_~oj!?%9)@7cR$!v*2fA)b(-6 zc&CfgL{>7G5&b|D0b+*(Xg~TV$vfyKig}*MK(ZO5ZDZpl2pbw>Roi@3fTm9cocu{qt9+@_rX-T7$px zLemy=?jOR4VBEc|BnU>@ncxf?@qq$575l?AfdUt9@FOU&yV*%XhMLiJGmTMzU?BV* zmpHx>I*JS(j7oz(G@}ZA_KdJ3sVdqs*9>eBgyB@HdzafNE5@*0|6y~xAn>%H0%;fOTej7NWjYh(VpC#iG1H-kK*9$;3+tAPm6-s00FK=s^o1 z0{^vzkNtE=n4h#WRiY<67s|I+)!?4M1#L3?B#MgU$gly)=H!O2shW`@uTm4t1zZTojLdE8wGsG?WZ}*lkOH$SN-K!Q zPT;@}>_$)j@Pfnm43GPWL#&K4t3jdk05hABM+6{A%mMIR!RjLdldwwDh_G^_sFVAI zF7U8W;fzo+EOSzV69`C@@Szs-pI1VOMlh7H!Zx^^%j0yn50#50VlU?;QNSEM6Rl49 zTCy*oFK2Vn7j?GiT7_wWyL7axxRXy70vzn5BbKx%i+h4s?6o27%neY!BhAlSBT^0E z0T>}Dz`#rgMZZyj(k#F{2jzmiI@U9lek1ympsLpzP|60E-fB8zm> zH-%FXHA50j!|Alsz)aE1a;%1EQMKt)7zI=`;Fmjds75SQi<1^7P=b|XGL-~W59k7- zTr*Ao5WXQz9{QvkqU5AG^U^~-nprrs5OdQ-jWH~hgXYmZVC>TGh%UMKP!>xYU`*5H zQZz;d12)a2TqRXig;QO9)nC;-C~>dnRGecv(K|IsWVMZ~0oHw!mp_Hk=t5KGiOKN% zy=w)mZG2R1_0zAAE^1w|`xI9^Km$jq)@zLe>OzfgrGqr1uWDOWZY@VpLpNQ6fECzlsmJr`!sme!Ic?4;qO9gX*64WHh{aP+qs|olCGjv=^wL;H zXatTuxjMLmuT^+2JMKP9*S$M@+lobLH?Ny%t zb=9Bc!l2c~p?wpBeV(KJA}g9YZo$jaqtg;aQHYh=sGVAhmDoJRTFhd@cVWY=m9li3 z*eFYg)%b+6{aUg;!c|0Cvt`?hbz8N4+qnI#tgYC(Rjh=i+p9&uy=6{fESF*>R%10$ zk|f;1rP}yP*0XIwx1HPb@!GGITd|c~#iiWIjatM$dIAza7Sgt<#arUDF+1;YHo6#oX7G zUDZwA$qifEz1`W>+~+Ob;N9D&1zqd)5|Nx<>Mh>WC0^rQ-tc8!=_TLtwOa81Mc>UG zT&eq9-VI&B?cDC=Ufh-6^flk-6<_B)U;W)*M!VnPRp0I1-u880?9Ej;QDP~40hoAmEZ}k;LN>X{ykv@ z7P9%hGy?wMkc8g_mSGrHV5(hV6<%HoPT?N*VIgK<9qtzD_23$QUyn5695&$+e&H4# zVj^Z?DW>8lZek}^LmIwbV_IS@&f+k};wuhfAU=jEM&suo<1q%~5l&(y{^F?1;5KGs zD{fvpPGd4YU}$FLXm&bk=3-*b=4*y8bC%?8hG$kTXIq|UB(^MdUgv6NXLO0* zW7gu7@?&GR=c&!*flgq7&f$H&WNEf$em3C0jpKiIWq?j5gl^(lhUbI+=8mpphHmJG z&Sc*mX@a(Bk2YzI9^PO^GqdemZ@2XqryLk{)SoPU%1Y4(5BN=PgEQmlkS} zuIZwNXiYxqA|vFb)@h#}V3aK>k#hU%fN>U0iiqBd%g?&o5r=%{vTu-@sbwraAr zY3tQ$u8wGW^9ETK>$eW-u$Es{4rH^IX}i8_v_9*!?&{Em>$o0kxz6jd#_J)AX};de zz3%Hl-sG|c7H->SZM}Bqmf~#@-n8EK z<=H0fy~b)THf`f3ZromLwT@Wl=I$&$;>j*nhAwXF4({^*UShY#?$iBl*Y0k}CKJ=6 zYVscM;a=G5rf;0~YrXVpm2PhLMrm)3YFM7{0k`k=)o?s7W^Tskju z5a01P=V=tT@GKW^GUxCox8V@3@}Y(T_!UCdIjwsne!Mqi&Q+wW?67L!n~j__V86tzf@~ z-5QVVRI_N$W?efrt=Nlg$Ht|}ZLZs^YV+#7yO*zBwtn9V9{iVFOuT>tCti%WF;c>e zBTJtAu<&2Slrw9_=UA!b&7easmiw}^=+mf8r(TVjwMEip`?hvXTXyW*ouD!c$#M?D5|5q+Py!Y?fRbyX& zzWw<3=D9<6KOB7g`S9h_&z=7=`TgkG#$JE=0XQ0gI^AbrfB>d5Ab#Z`DB*+(R`?); z1!fqbhW1g2p@*=A=pa-I)@9*|A1-(ygd2hwmWYPI2$_i}u9%{V?(C&bbUf~8V~^Ll z*wlza%4pj)|hwF*{7gs25M)XdOArLpsfQlkzP_QY-7V&?IY^6M_Juj;Fzx%VcF4!#K+s!hNU zM@+5608?D?#fH^Iqs6RpEO5L{(z~z1<&?be$tas_u*d#pj19{ex9sxFG1p8p%p2Q` zv#V{YCo-oxKPyg{-=wUv$>Efo%*)I)-ObZZN8R+)v`}63)wC?bjMiGOvJ5I*gDv*g zv5cK{*=VOtw%TjA&Gs_aaJ>!OXUiS;GuL2qciz(U|D6rf8e1cH;Di@`^GJq&oS=WR zmOD7#c%OZ@+gVF(b=1uyt$ER#bFNG>pBHV*DzlVM`sreu-AU@Jr``JNuzwP}?6fyh zdm^^yjyof~>)v}Lz{?T*@QVmfM;*l*pS<$RH}8BYzbq0xERIY+{U^WlvP|~cyFK^! zdqe(s;^YUsIKetC^T;d3uOEl|?;jt3`@px3I_jW@?hH8dj3bW!0Q_G7`!|kS;Lmid z<3j@fusS?6@DL7!;2k13K`K;m3KtXv2J5iF4tkJ-ApGD5jVA>XhVX>;6r7oQ?gkAdbmbeTiFomg0 zVjA<9$V_GlY{^VwHj|mqtYsV12!#^vPJ`Gq;x)H9NawLI5T696ILArOa-L$6==+u> zA*8o(KH;6AG#)77Sxr*h(Un*v1QvDK09(d@m%h}35q??BhAQ)+ z4vi>8aoNm=R&NDO|%mvoOI+ z*z~3f~Qq`any{a%P zn$>77Q<^BS0#L^))Uuk@tVYeLT60>$k-D{{aD^*6d3K+5k|TOAg(>ftM+Y|i@tv!? z=S}ljK*F~2us<~{EEjv!ToxbzB_Kl^Vo=Md_R_My#Og9B8ceEsR+*o*=w_K|0?l@o z0;Mf2XB%JwrXHXGmHj6w5gQD}|L(80x5ce)6WfWlj?@n_4X$v{5Z6G=RdMD~7GCpe zJLB2G56_LRUxRx;oH{U-JRDlgZ|-lbllwdsYb zeC7LI_|EsfmbGtw_siddQh)#lFaj?R0DuXQ5fP|VY9Vk-hQx}MtVEUVK6~rV>WSjQ z)f?_O<9b}Wvc;|i!Ujt}Sx-<%H@YaEgd^neM^rG-qn<$U012p!Or${zP0#{aaq8VK zidP1bjX?k)pk6C@!N^Bm?-kz5WC1t1$WFfRlcS8^{$BRVRhI9N;oE=$FB8Evcteb9 zTZ1VyQ4ckMq6Kr|1f2ra{|JQ;t50*F)7#>~w`Z`h4o!jK3Q%^G1NUop(GrEp0v+`Uvhy3Z^AvxY@c zI1dNQO(GU)$FR9z`-(0$S$xnW@ zdt=S-e*1gYv<^6yZ>@z~8vp_^1~!}BoQtxCqrnfJGah>VjW*ZWr^~$l<~j zkdCypKj{dRTDx%||F)JjU=dv5nONMD;CRLG#b`dv0|cT~7itjiyz8Cs zRR4R`_pY_RVNGR$Te-gt2=GA-9)sHcn!LSkctGu}b)h0#%@EFovb!CyV#tEe(2cfm z0U_u?|BPVSUi7mc{cUlXn5N~b(~_l@PmFFnbS zUpl~@UTdd9KC(rz+J01D;Cg#sU|HQC+-ly1~A}FZ= zG+*pl6s<3L|4K=wua(EI5oe?Op&|7Nn~x5kwcz{jgCA-hdpD_$Dr!Op5V$bGnfYJ;Q6AMc~Ztq42sPuclrv_?ZKr;wSbL0hk;NisyJn zc!d0xc+GcNjMrM1MpXJ00WI(YR5(*s_*Wuzg?U9>sSq?X<7})KXnBKNcM>jk1r79r zd)Om$KJqY!!!*YOL4s9-HkfHV2Xtp;e>(P5wj_T6h;&F8h)Ot!M<;7d24tLOSykvq zi-?7e|M*sr7ze%pI9|vKA?HHkXFg=|dMR>-X_$VL=!T)lG&Oe!!iRHpXbyL{gWok- zSI~!vC|N})h_k3*h}VRLn2WOJ3F)SY5U7Q~_=v$MTyXO@9%vZ60CL-NNNhKTC1`>m z_6(m$ccM6ob@y%1P>Pt4MQ-MY&9HNxrh}}gR&BOSerSvMHjC+~jzrjmkhg@msA>z9 zPn%|064;2t7>UL8Nn|%Z2cX_Y^;@FD4$6ge0j*-V}>`052^^WnVi&ch;yjYJ)#Y+>og&LNRkVtJ5 z|7nbdgA4(wcE;t5vbQa7r;tP`jV1MvwO2I~nUf%Bkr!!QI5<|R$a8NtgUL{pBAITo zRFWr2giOei%m;AvsA=}5P$kAeG-;DH`CH=Sj}yraJh^)01e9hXF+tgatiq5ESyQ0E zhD_O%9oK$O36+|lQ}U;7R2g__rCnENj+xdBs8)-FD4AfnYL(}F@wk!5r(im$foZ9a zplLUeLk*)jkpUS_mOz)0LWblQjl|H9oiqhrBRfkOn8~VidV@f6iIcE!vSA;1EkIp$?{&jr5v{e2Di}~OTmgP#G`I(@(j~jMj9aoy9|2aO} zvs{amny<&2tvN9X$$qfe1yb-j#NdJy36S=QHiHS3Yn5X$XahjtpFn^KIT)b!w^+B; zKkX%dlvR#&W=o#npkGE=p0J=~uz3~=gl&Wf5!wJt##&^6T8MC=6S{vCN@P~RS|J(( z8mgijdIfwJ4D!`K$^Z_3u$|lq3$4(dB-L$zDV~obkj1qP>1k-~Vl?dOp2s5vKMizaT=#hbf;@-1Z_GG6RM|k+DazSr^p$LOhpEQ|B7jyFaz9q zls5E2-WffR^ayb!2cqet(%@{Zw@5IB#dt%C}qhFY%U-~=mRtU$0!+8I|gX{nolqa-D%w=fDhN~zMb zsXR%VKuTP36{?C;stvO<$QW4TV25}Bt9^MkSMy>HWUF6VR_n^2QP6-MbO#StpjZ@y z=AZzN^;nrxG}nn<`4FHbo2~`NtPPM4wqOA)s{k{q0N0AF`CtLg|GKUxtFnR$v&R~& zJWFIX8wEKV1rwmN54t}G&`Jnbvqc-MJr$y_1zXssiulM$^8~NvxsO1m|00Eb)0GAtYS6W~}X8@AR00WG_YCyj0>%Lw# zxQnZQv-_fqP_>>S3z-h9lyJFvtu#mkbwFVoNl7 zMwMWSNnUt+t2ZS*e5*w}cW_Ohs|?_=7_bBa09oo=#Ql53P0(d8fC2McOY{o_oE#Cdc_74XAdoWBgb zz(jk=HqgWcT)=;z#pi1Pd{D`MK*a)(z?%HOoXp7!a7R_}pZ=K#0@?@qV-A@>18T{& z`^d&^%*$%*%e{=eaV!&d$tS7$fpm~kjy$t6x_#n$) z>SD5UKNicM95BvOxB%w-z=VJS18~dtYXSEQ#hymF@_S>K`^f=1Uif?fvjAYWFu40{ z04=-D?mP=v+>!T71*aTz6L-IaFaYc}03JyL=9~c&P0#0fNWjPiA>G%0ErvQd%*9NkD4o)uL3*yX3p$(+owAWokdfmTGr)W=iU|GdXlT4HMD)1B?jmBvw0@JcE$Iw_#uwvY@L zK*;fJ4l_Uq!AalE8=rHU2db#kId+GkeQ_{&4_&M+`s(MAspTCQqpQ$*e2Y5B0b!EOj`p)y(+%9<^9bs?%6VKL8~yg zEc{}#&<%IcuuYoZ_O#e4z@)FLmLOf_pAf83 zKFRAH<&j{89A4!fJ~+}X;>ui&Wp3SKTip1_$D}97D{j43H&`}oT`*pt<~=(NG=agR zS2P7k#1l_0_2;|encd{)hTaJB|AW_k1>s;`H5FUgQt03k?#ql0=4C79V{W#{v6OX8 z;-q-kc_o1kq+{A8tl72V-8JVprRqoSQ4ZT&g;Yq`^E}rxVuk)b?FT)$zCI(C>m8iv zz@CL2EZtM2WQvAL?xt{Ba_UkQ2 zVwOHr?^)~+XxEF*?B$kY(W~b#rqTae4bndCcI@fP%jwp*?e6-JRP;)kBb81|?lfks zGy0+-tG9Xgyj*(n&zjSq{{Zi|Q}2l0@A}R@nRG)eSL=_^P5ZvkD+X6E0P6dt&c`D-ceE`v-NVj#5SfM%@g=i;%T!1=cmsV4{)PQ?`U@ z^d%HI;Aa-KiH0R0syr!8CBM{tQl(7gH;oEs_SJ?@#-??bJM7&{W>KRVRM5JMEwR1} zfe0FY_+$wm_R@vlY;iQxVtnzn zW@v=X8d0j!3x+H(RN*08I1&;#e0(EP4dHf+D}oB z%dW1A0=#3Wh^WX;zVvX~X-%lyV`{&lmg4LE&BKNkM6$rgJ->q~%5rwiXO>77 z2`3-ixP|Y$o7_+@J~ZtV%g#E>RWp>MFs((4S{p<~m%y5^^;RiC)oUj-`hA2BS{c2U z(FnV0P%b7JsC0q@7G`(?3P^}pVu~qt0TWYQ)Oh1nU5nLKkXdW)Mn`6gH&-KjWhkQ~ zhqI2@dwD!Ik7WBmV2L_7m2yL*j&%vUCbs=6u}o%4hTC765$`<&eVVh)a|4e0Upk|T z{|3z@ver6}+YmYU>pKqXh(QJy@Q`MG{gv7zg9}asyn)GXG?4}*ARvGM7A~Lwh$r@Y z1;GDC!3K;uHvI5aLvHoPkrmI;WMg$rxk$;SmtN zK#PRpg5h|x^G!4-8fZzV6DbUyMVvRFNTF@HmC)-F3=Fs%?}qjMd*TuVkDqYL5I5Xo z8x#L{vyfwhkM$-%i4c>!N98QXvfc!yH=X+$=tP%^G62V;cg!A*l2Oa{vW zga0izKyV_csY6fdP^O@8ZwgYGlkdv&Lc9!*c%4`VBpN}%3MQ;>?SnuAdZ;OfO)rOc zU`~T3gdQ{mNk9Y&i8wgt1Oedz3gweqSM0Vo_3iD4TlAa!=6A7Dy~=)L#Gn2Y$GFwF zs$`9Toue$0i3<|YfCO*M<0x_6S1}4l$ z2nZsk8&5EyS>CV=tI$FVXltP=)P_Cb(P|lN@?A2*6iDhVz=vE6oCUbZsRg+JS82F| z5s$cxYjFrU$l=@zUQivoP|yGJ zNS(n)0U1({QT;}N6oIAOK$i)cK{A6FP=;^dg~lrEjf_*bjNY10YtRmS9*EdagpO2w>t`jI`Ga z6vBw(4Rb-fgD4hPiOgii^kI3ZUo&kAO=SzM_@jF9D2ig+ zQ@rl=h6~vs0$t#N0U!y*H#CSU&QlAIEV>7F8Pq;7#0ws{r4gvKa0?s|lU($wN*`{> zix!RS4?aSnMX`rgA2}%>=XEW)sC1adEM{db`#zd-R==OkDQNvLqAi}Sv8A2J(9nS~ zdDKI-s;!JK&Za=wHr1Rvl0rG!m>4}~j%JCerEfolh--p)xE16>>>3M$O~B!JJ)|C| z8ir!+LRVZq>DO>-VHIIW_n=tdE-k(50Q$-Be$5i z38U1*_A2E2RLNCsv_{We<(T)04^O4Z0X^AMJ& z>ZcCD05L{_fl?@R1G7O&gTQvgU0sB)NoAWtM-;$SB6t$O2e_3Ue-I%ZPxq5xD1;7hng$8n9AhS}KpDyAbt4c-kj))pxuRJH z!IryxY|w^T%+%DvmZ&sm4XJq}b*7g&hymwGOo9!vcCKS3YkG|VABfSUYp$)B=`UGKPe9(hFq0!U!^#37zm*Uj@SbeIl6vI_5 zgb!FmN>!Bg7Kk{GcXf#g1HUZ}7Zh=GBpO;)t*=byXYwu@8`<7qIjY3`@?}+6 z0a74sh2Tbkg-(DA67ch$M_Y3S4tJi7Zp4-tI%Z%pjD>rOPv9165F!X$av5Zjv&wiE z0H?Jn@sNu9a%NZVWyqsZr~h>eTtu*nW=NH5iJ>-i(7_63^<4DN!W4-9 z!V!InyS8bYaKDs}L#Ff77Uq*XhA!z(O7!SbfHq5_{r{cmfD0pfQ#jf;Hj<2*L?gPj zQ$`L^-GJ)I0000;lJu$nm$PRe1Rkh>6kKUU^F^5sf_+Y>17wbpX1sZ>Hp{u)_ikgN z{N(bZK(WPW#?5m1{*m9mG`F#asfdW*!WoO`8V(~GcUcx$`Z+lR5l@IMqM-s#@h#;j zF_G{-64?%*@vU(~k|9W**#mx%0lF5Pxi8oABzEIFKIM}l~^R%!-D0ayL z7F?CdSdJZOGySFPj#wWv6J0iU63!~fEY0cHV$ee%2n1g0kwHz`;kLXd$i z85}}_B;CjvwMmq(*syDuqOpq8I#Pf^8Jsq-JHAzz6BmRytqFq5lY)?v zL--4zNCRyA5ib~m6;OcY;Eh;liQLIYO$rODUe8HyVqhg@}B>Fo7q?H$Z$72*tr^LsA3x}u-2SF&Pd$h`{v@JE@9Ir$K z*#w213ro7dE6z->Rs@qNaxW6Bpp}Twx zzdJyhD0+ec0-q-ff?HD>Ym!X4oJ(QD$h&;J=#`2#a zA(#6+OH$R(QK;2aor<0qJ3-i%?gfJS?GjO{;}SVoX#@b z+U@OL_VwEhaDWs3fC(ru!gHl8L4gOokX%^a8JG&yjLnTO%m3^t|5y-#z11c7Sv}yq z(cuUi)r2P~zVvij{q5iMb=0q`M6xtKo#-Goq(NP;;)P`)Nd{xCWd?4U!<_ks!7GqU^Im3^lpt$kzj9$Z7TILxznucr z23lEV)#JY{m|Or9w=9HL{A59Bo>$a@(?aBL$YksAw-tk_%`OZKbETecO-KU^rf8|C1<%v00KP>1dwjJVr0K{Izg~<;@M`2K@miIOGTk z2Sv`ju9T2*!)XbE-$}k?N_;zP`L{0eL zS=N?uScNSH<5r;Mg?7#K^f?e59V!6D25^=@MtHUEkq*l4AjFm;nrp;T?b<`o zQ!1#10fo?9fW6TE=X^!Y#wO)CpoEB(j--x>XgIz&KIi$W&f^8 ztZM0mA25sV`(H9PTZk5$v~Je9_SiT6=nr0IJJ@SqYd8a-!jwj74sB&X#$;!h;X>eK z=DmV%W{=;*XUwZ+^-fzsaDyYz8lnYjj*=WFxVh5)?o5VjPpj`(F1P}YgVlcIaQ>dx z4k1tT=kF%)fllh&K4|$y=ictx?9rS*K+8Op98Wx6JSx3ybEsb%9>v2?qI$3 zV9C3YDhML#cH4*iYvTc^%ARrczGUiW$0nBRj)XQXc`hLjPj6NQb+! zWP0um=HO^OaXBYL;c9Wcp{%l5R=|z2Lwl+{Lb6!PM<2(*q+1%^$(kPq@*p2#!}G*e zuC|@!fju)k^CMk-l`+g)g;J=>^jvb$tszbbg+mC3*Ad+%Uj<0ZB{fd<~8SNH_yK}|MeI34AdI-cOdpUq6cI*hh$Io<+1YsL$}lr30m-M zw73H>nfBN_vul6fvy4Q)ge1TqzV)rjUtJHdK4}K&0eAlH&CPB#RGR~3RaE_gc%K91 zQ-pX8bsB4{)y8*!kMJ#rbqUwyF>Wp`EnQj{0Tsr;G>1R-Y4|oS*8dh2cIlLnasZ8& z8x~$jrPufuL`CTA-q5t`FXW%(V`Yx}Q!<&wm5D0x5%dde?)%OCo~$7M+fc%@h9D*gQRz@>g% zLWi)ni_U{&y;wFPgYa%Ryn%b=#NZzS`;K|0BB{l*S9^@6+5bg5dnyEhkp_8?hy9+b z1k_NOg;;Uje-1kunmhPau@hk&1iCX#BCsvxvUxn!TdC-5s z%dh;N*ZhDlcZPj`_=o>TpkD@vXyR*j9QIaP=t~@6coUM7W zNIGTp)GMrQkz6IMIcpSKK!6x+1jE*)6Kago34z2e7_>j2L~t-*fdPR5egiLv;o;2W z1{E?-SjEmqi{7M9?`Yj4_UICkanJTlJ9TT*#(V#WiB%d4$`9Hs;GO{doH}eIQFYp8c)|6FUx(q)1005!238Oi#`PeE2`aeR6MSi61s`~< z5=Vrkl(Nb<;gC28Bqo01SSiAcxJP7^*=QMphO~lNXb5VUU>2FQmfH`sG&4^(Ht1Cb zTXGdAK?G4s3FVYA$*|g$QIv%UCVvo6*Is;a3h>)- zzR~9ap7zy815A}H5z`qZ*^q`BptLaIj~j-Ng)Yw2GQtxrev(KXm0q~$6jEfGDOww9 zD27xYLNbC?PoQWRUywk7>NoQU^T??wzF6apkO_8LXM)`#nvcCQLW>)B<)%azPv8~o zuugKYYy~4U`)rk3zG0v_^Bm=+nbnQC(Y9%dmo1v!Mz^K~E6AV*oNo$8X8-^Yho1v_ z-l+k+_2$Wx2%UPGs9R*guxg`INi|L#3w)Iyt9>QOFNX>v$pjCv1`J6uhBep?6)Ca` zON6yrhNEVuOvW(DdyxE=8^|gv7L|5Bx&HtLuoYOXU)Gw_Oi8&;` z1?s{VC(`rTORs(RUi$ROB#YIE7UfcKH3?Tr$*dYNTv@`o$?(zDz9xsZKa_&7 zB!^mqri-e5$ZK_<+G45|J0H$0(Ojmt$Cz!^3a_&uq^t~G2E=zs&88Yd`}wfRHO z@<$Vi=UY4cwWqwyHJ@+Zxwzk<`TwQod8lNrE?}dA-1Y8gNSoKvc5pD+Y41WmXap3N zaEB~3p?%8H-38s`CgjjT4#gtE6G9jlCIqYynmAz!=%z41Xv|`(fP`O8I15NnuX-Tk zUiWf%G8{4t5Z_Zm8jKP^Ldbwiqf1RDj>M(*wSZ4fe9tJ1j25C3~b=1@}Z0*=D`+f%wW1SXdI~6EOJrd0)>p{uzBD?3JL2*94MBs z4ry#2v!Io&Y-qh(SPx}7+@UHS`6(V5;u3Xh#uD2gEh8XN8EU))6@CB%!;z8;rR+r& zId{eUVXsRPIkr{yQHcga$@97Vv;%bXq}2 zD7cMss4#{4;%Y9a9-55wm<0hs8)fDROwh)R!6OyKN|UTFP%Kt=y9y~gmdHfzYLPa= zX(D3Uz&&73l0HmJBZ;+yHyE*t>H)$u|H%g_h;pC=HK_htnNWphwX0rL6Ijbvt~MaT z2Ff9z?{0TXgTc#;o&Qmph2pBw20k$h%NmYyW=YZk7W1iUMTkO(k8AQDn0R*~wORdZHl(WhDv8`Rb?@9ZW$AHih5) znhQ$<+;3F_`~%P)RI6K+t#*b|k9RPb9S-Iegx4Wqe~K?E+9)e^eQT5MQqD?2p~W5S zkg3bzBbXMZu>Z`)R|{rTB`w+u7Ry9xwcZ*>uf~Bu`F43B#Ksf8JKbA**=yM$%k;hR z?IjPr2cs@P!i3VC=98f;!zokQgne8D9b60BpB zYr4(ak2Z`V=Q&?1w1n!zohug%J+PR^Qp1y*-?R|arQrZ(9e{Ik>D(R9ZZJE-n~-_c z8A(H0r#V`TpL+;N&^#gzDpVM#LoI3#=|K>`%G{O_9Hye=!@Em)HLQCQ>sMp&w`mQ;eit*rl2>h9z>T*f`4kRcr(3~TsfA# zq3CHk*Z(25rHuS3ME>15N7a6b0iT-3prXX z4??Hvw((kKX(olVEy>8p^qiK})sV;i=%S+g7UuoiB-j1!c)LLhhOq)~_WME>$dClk zoN6J62X##l&k9fGgOLhu;8yaF6(&iEh{5hB5!+$xWY*IEVlonAro1M&e%G+clJ#=X z-Qu$*P!&8$23vrDG4SPa3(8?gnGG|8Gd@9XWio+c=7DC*Gz$D=-m4>rn|h1P`8wxW zY5zt_9&kMWQM~2NghDuqcqcTqs7|k`!x)_kA#!>s?_I*IZ@u>5x-EEKOYqv}!D{5I zL?zN7;)c83Xm5Hu0`Jq-{hV3NEs<8Aiz{DZt0q7gbWeX1zt`0qI+hZIz8Y(1n$HHV z{4kQ|An>;N_WBsI>nAr?9F3I-A^Gz;3_anxP>a&L?Hf@qObx8U>$zS4)HpA< zz+1{3PTWvk#(YB`p~y#R!P2ErF1Q{7N&zq#NFmT(18&alK_D7R;B7tB3XoM4h?zq*lB zG6SlWp&6bbFQ(xH@`6NVlJM!yq0Ql1$p>+Opfc`ZVP)I($R9NFpeZtoB6c4GAl?@- zK-%>cCOD!qW>!|{z!HF&YWRSK$qX*0R8>%d5z-te;s6rD)fSLnGg8P8A^+LXJ=!!H zj`RHA6{d>Sl^*E@9M)06ib0*)j8iQ1qAp4YdDxFH>K-rzjqicf7x@JDx!nfdk`KU6 z0_gxVI^#QSVtk>2agj(@NXZ~t4KY4dfD9sZRpNJzpO0|B`ZVGbK!cHd*@f8xJ#fH0 ze&VDE98oTt0HVTC9;GlO!cktKh%6f{^ho`cq6!{PBP67U)Z(at8dk1QA`IL^+Fp1t zWJG>dn2i;NU6*wckm~$~GwETIfMgMfBwZ%qVI39`FvCqs(6Pzb8~&H}u_U#j6f;cF z8$ym-yd!CBK+H@N4LCqVFqIw~!*vZ%V9wx8=z-%|7|i+I4?PGYGF1c!zWPo>)Pk6G_J&@;R z^nj;5gfZTz@ln95)QbAq2q~au{LLpInqvM9sgVLAC;lNR0RL!1R>bEND1q`qf0WA* zc!G^}s9G{;on!zTI87OafrJ=mj{%*22#u~?ANKhsmeLrRVIvZaK?k&j5=vHFCBrmz z=MuQ68wy1RoahF)K_S?vNzI{2p+NKjVGy!kx#?rF5GigHDfvaJ4@v5zRw^YiscmMe zlV)m^I+RLSKpLcF3)Y%~hS6HNfDpjtT)rhQ*&`r?qV3?x3nmBlW$7^`gb1`15vUty zNFrFo%v5lI0K5Rsyr??#!Ek1e81R9h;wYikgwrvB%|)a9?MMy@siQ(FrCRD|Evg^h z0;YzgxQ;8icIpHUV;hR9bh4?2vKd}g09)M4PB;zK^#1`;LJlb0Yrdiavte2G{8^s( z#)57nV7ZzC$(0iJYQp9P2Rz%HL}ppI;W|vD$JxaOJbO6;)xVR;Y0;U3oon$bm-+P1x1O~U4{Kqimena z915&1kib%7v=Qw2gxA5&>GT{{6qEz|Knbr!Y;PW`0EXiO_y8U_t)K-~)1H(!6(ORg z?IW2Ym}YCqo-E3KtICqA1EL9-Fi0aj-`b3Ix=UhgO zaS52-Zmb;XnhzMuj?L-9T8MICt)9Mt8mNPs9RETRz`@00Y#V6o9FhRmNN2Q_z$}Eo z$G)c~#^cDEB$CGMx5BNt%`MCNjDooU!>yge1@ExkEb9ENQoxzx`I?;BKpnyzG;mxu zN*)NL$VU~321XpmCI!k34awnCYBWQWq}#vV!03uV)=rM(aP3)ufd*6v9q57S9t7&1 z&eh;f#KB1kbb{-?F17Zrqmr!cre6T(?lu(g%F?ZXCfMCBobcu?@wSJ=^#mrQ1L2Bj z0=ytoBt;dFqp@Lt^oEl#^_qq{D|ZQr5LwIMIuqtC?rn6?6Ci;3>S>{PZioPh6>MPY zu0UTgX#REzD5@OL>K?hs64_OO z@X16R7t;ZKks>&rEE{(2AD8Sx}0co!-&;PR`=7E=*C>TWAXxQW(4eK!PDKNBeX57FQkZ#K5 zup*M}bz$H7#vc%Zqdr^nBxkb#->!LqD>rwuH#hNXO{5x1=||(4IBQM0U}^luGacep zt~uyBhvdPsu$aQrOTS(YFQP*bqIQ92`@-~XY~o^SfK75|i)w8#N8e7r?@ff#3_?Jz zrKho{OM)bF4H4#^GMRWDHdH;0+kF{8bHd&807ipI*a1s{7@v|mPouu?KNxnW_xQTd-g_yHE4&n zNP}vE-fU_YO&p!-TrOj5^K|EZW3SZXiQ{uj!YOot5R>#=6D}qO7-k4{iPw=`@J%pK zegVA<>kXG}ts){?r~m2ckkm35PI-#&5OUUuMsjvjxMq_=G<0?nZ-+)_xP}*lc#BmL zHTDBTbrvHd0RX@UxVDMAH=DmRGtOIC2uGgc41Fssev`%&V2mxufe`r58|rKuIKyGj z=n*&oQV*(_XD>tRpnW%axm^~=U?ONtx%q83XD?}Y@2-|_c$a&5NQ3!@#{>93m{E1uOixZjP3^*tf zD)z?amUEZ1p;Z?&zsOqDcO&7=j0YX1Q+b73G=}S9r)%=3L+4~(SKyvEsUKqrrh4oU zlm|7}DaH?@A^%FHXv(I{`*OWBjJx^`ECJW> zfc7c5P$7!W{j2v3v9@o!C1?3qGBKw!plxN<7thie8!kE9G;q3|N++v{yFeKKQ-z5! z8<~Zn%zCf0KrK#Yc-FxWl(_y0xJU+j7qbiv`1Zh4#$DR>uNZscjFJTH{KftNbJGM$ z8(N|l(+<0d(@6l2`JriMyQTjqBXv81b-YK6Hb~p+cxNrzpD){^H*lhq@PXajp@TXI zEs}(p8Rz}v=o``&r*?|cETgorz)v{LfUN8L;S;o78a>Ugz)ea{uwZo|0% zJ+vH`WC0~Gl4Jn62mFB{MS*YxM}hkj6a=2r*Cm$uc78w+3tKM1zdB=@7|wHv6$Uxf zSH8OZmui>rRL30r!8hZ2y~(!sSCHw7&f}H=IOVo5kG(1BAJG z`34pwNU(~41b*1KaFHPare*>$fyy#x7blIQ&OEYZE>jc&14c49VBjQ>KU(t4ivlBD ziyKIK1Sw*ZX3iz_^4J;~z{bQ_8?b!&5(*unyC}|dLTD5CYresE>W?UQ^TPZR+Y80$TVEHQP>laMmGHCX=F?<+MVZ?|( zt#Ul;F*9MGUbQ-PtJgMb)3|l^yczUo(WA+ED|Xr#FwW}GScM6>a#EzQ z^WX@~K=l6NK?uMG6HKwiL=?BgiU2VI~cSy!5gP8>);#rH2Z% z?hB#OjynNo zNoKuLDpQd~7=`8LoET-K5t(OhT$b5pfvE@x2v*<<$qP)9w#fz0+7q}alhE&haK#;$ zHrl2-m)s0AtP?6Mhy2MsH%~ACgfs?hflhYU^f0lg(qjjU^GkEXy?bgMKP#^m8} zBet0Fl$LQLqsX^$#$%mDCb?s)T9|?gHL&E4mn9m~j>}(fvFvmd6X_fnWk*2owLEr`v=&Pe$LmDB>hVG2DJ7W0`g}@UY z3SrhIBT)+iC+QkWxs43~yS8SY^`z#s`|cthuX(RmO0KvIliD>2a^3nG9429$(F4jAAMx{_U}Iua^5 z&|y|13K$Cb@E?JggL0*_1k$KMLKE`NG%8di)qba%170mFSO1$~S-_|ku!TWg)FT@b ze;7nh261}RQ{obt*hD9y?>-X}T%xLo39V?&jg3Jd2_{g>iQN*HC7`7)Nx;in@{$E5 zAVVj{NTCaTa)>xUlo1Qq3=96NY$ej8iG0ul@2Lb4y6E3FgV8d*5Ymu`OeC@z$w+cW z4J*adnvr6tymoqRH*#Zx*~aC=WJVL260#oWY8Ol?@beIn@)SPxSxQq941L&B-zxb7 zAVhQ}7@qJS6L6&oOU(eI7^MTM67jkzfJ$_C0BIKZ3DSQ?h&~;HRW&zMwgoivnf%Jw zhekBbDLjU9f6S>M!8t;oiW89~OyN1pDYXDfD{Jg@=l?q|CBr7kQR6vRsJiA4dm4CwNY(17ihKYPxzMIpt4hXfh7>>qkBbcVh^*}LNxZVF_>0FXEC!!xMH%E4QlTenOXNT zQVmE7V35cdZrzmJyWaN4Kx8C&Wu|Q3IBjmmXsp}Kj;PqkUQJ*J+Z74QZBnT zT7kFCCRCpKK^;WEN$lWn3q?uh76ySIqR7lnu|R}Cny^<5{jUkmgn@Q7R^!B~k+F_l zXEwWO-kt)rgzIhZd)=F=Do85|wqTU(m$!x%}VM)0-jRUI1D5*o28 z>N-WvSo*|FrAbZhdKbLT&;?g`T;6Y{SG__OsV2t*b|6k!^7#M0*GfFaH2*lgz|5v%Vm_5j7iyZ)1(ibvN39TBVUc zikb&D#G-@;FoQgg!Y6N;4N~56>XU&L^tdA=Q<^?HLQh}`dmfv){#p9GnEq*|H4Qx0 zgzifDUf{m>y`_r$yM=k+#rxV+G4F_pz77EfYm9>&g4EX`vU^A+dIFL@2pfnj`vaH_ zFaQ;J4in@l^RN+`=7R=8*+_7}|I{#`9z-HqTL!~smk0WGtv-R6_z2FvuUTkc@iitQJ zIW8D`a0K2O&xtM9)Nz4p=f(gG#Q&HIGZ=Zwn^J5jo}BMQ;0kqgYZb!dAi=1<7PJK& zNeHlUNRdzSp55_X1RuJknBVTw)!j!;Kt1Z0{&c;mj!3KDqv2KX`qx+AdiqiFEGAjI z-7yg}v?nKVV0QsDAo_D$lMh}sj64_oVmBBxnB>MZ`B+h|><4f`4Zat^7(i|V#aYDU=8&B^%ef8fUk`gr zb_+`CMH>tP?GQ}+lmtxF%3+XCq3puuF6_c=4$qEB3Fv^J%nX%`?bswN<%~$xGQf(` zKu;7xNlp#&PEepcfIm9R_y1&v8DfLo?g0JvW!%VZ^UzJJ*v;DvDd&&ikNJ3E<@$gGJ!?T)F%DtT z`@m03Y`|X55f@jW4|mZQr-=!uz%hiY-L~odrtTO?&j*VNA&JZxT?hz{hZ+YgzpfD* zQH$8Nks_0?*hmrZluZm6W(KI@$gC#X?#}umkZZONDvHR+R#EcaA`i_$2Ki7Q{m{k$ zk(x%}$^vqyim@Q64m23C5tR|MTqzQ3DGfZ~$!y4rB2ps3B?|&b#!@l^s=~^sC{^A9=e}|ZcyT6^ZXc(K>99$g zAj_Kui6DDY8~@k~C=;?rDxw*)X$6q%Qs}N4nX)OLQX4Ij@2n~VCG-3`WX&do__lBc z`O059rPW@sZqSIJqGS#B%py!hH92HG)ZiP^h%W2$F7-%@^imo6r(*`F2N5wafp9R3 zP$8WW5t0XGSzbeu)r_#8pQg41HGihR1c1>W;kO8wZIlVB>)aDEbjXGpf zHbbl?ZL+&U;6QFlSF}keDB~62PrW>isFE=#>(fRgLK+DmTSUt-uQ8LRaub^qBSZ1F z*h~Wq$0}}u63*m64G)tH3OoC&Bd4(E`Xe?kbUe$GyS^u*5H2L3Krf8u$EFq>+lNs%^F#nB3?aQfB%^DSg)jR-=&h$Iaz)Mz2 zNo|x9CX+@9G)LR!1ae53Jkb^dpaT42NN4K-inQXi=?%82O=a^!XVWGl;Vq#wj_@o* zS0r~h!x(v!cji+^qUKV!v=20uM_TAh!xU6c;seR_5+kAxL9Q9rQ&rz?K;iVvgK_l}f8LQbh*BH#D!GX(jiDkC|U_xssy660`A<@RjJaLVzq7ZbO5GP zKe%NV`&5WT^RV)pEVJ)R2>L{7!dGFd=BBlgkr}e1Su{R zF(vleLM0NeDmbR$9Ihm513D~@1H6$8BNJBN^+1EJ0R;yYGKx=im1YadUxgKC5j9{3 zAr5%fXKyD(&`VMkc42)7^&HmqY=i$e&fox$HU#kY3}R%vE&-|pkk-jLYgBzQQeNAp zm1<0VL7zhj^+vl^Uv?C?@+V9VC`ik8Mi*B?4|Q#K_H897XytZp zdlNXXgvuHMbyGKDS2s#p_aPqgI{>#2TEpRF4RIs(x@c!iP5?HuwmF{@M_E=U2rVY1 z$aqPr4=z`DonoU>P;=v=P;+)zJ@<3p7CuL}ZiTXReSv$URd`;5X~S-`Py!0OjoG9G zeQk~<)wcl-u*04=P&RN38y9}Dz<8HeNU6YQoPop0)_>C$y8733stRq%vvaApdgJy= ziI#2~7)3L+dpQ+?dBhYM(}rHu&8Uhk`GA8V!}GqY5OTrt3PE2R_k4^Ca{sk4rRukQ zx+;WmauzU%!x8}oW_VFf#W7OgSVuqvQs!gO(}QK!uwwIo*|u#{1c~AHdU^AKCs=|p zYv1Nhd~e|>jlGuMneWpWLLg*8Nftw4l9c!ZsS zf3yG~56ttN!SdXg3V>J>J}+fb0A=2nCW{zn5;%!NcY&*wXq(uHpLk29c#3amw6OS! zrzjT?q!7f64;rBT?hb~Pfe*%*lqzlptmB0J=1t*%LczyO^$Y}hH`f+SKai^%&-fGd zcZXvFkVya+ML5SyAQ@^P1@JhJ4`6_OAd@-S3P8Aj2bqwMIDutBbpIJx2(_2U9GMvx zD3ZM>gRv16YXJo?U={8zW^s{+yFegC*?NK$*rp3`O71QibR1r(1vhxox+I9CKpa5% zpZ-ZvskEa&3Yvas!_-RV&_j#Sa|Ap%hK0+G@t_8XIijUtn6Ds{!#EuFI0R6pk7|Ii zL`6Ke>zNH1ZWXvhDV1~?xsm%#f-7NGmC_n1838i!n^^!~@Bk+PS-2WNRC2(fHE~~i zI2nfS9u1@ux}h%qjw!N0P~RXW-arL_gU3Qfc{yMejwg&4#W7Ifj=y0p&kzL8E~gZi zc?}IcPbx1*pnw0#f*|1uyyuT$IHsh4g9Tv(09m6$+JHwokN*xCiMuzEM^~kx7@K<} z;eNn5p0KpKIjZ)n0kl{KBp?M0z@r94m{=! zq%o3o4nV{s3`h(+CIy^X>uq=#J!#vjLwSs6U@RX>7xKY*mlJ@~mM8`qCq^@dIn$Z> znxsv7Xi@qz2(vy5dxd^vM}FZ!8M`Udg|XJehjXxx!&{=n*uh*gv;i`rzCpMi$CT$_ zErVg6ReK1&(gN6H#cY8CZ33qs5CTooF_x>Ec1SAp&JN2YUkey*)j8>!nTVOYx&OLZ zp*y+K@jZME@%KB(#p-2fYMT%m4#c$5j??{+tCm>nj@T=9oVNR zoVp2nfewYk&jzlwimSfd2y&1s?E1FL`~p$y5Xfu9LpY<64WSt%!9V+w=^#AZc(j)!MF;oY`LDG1}XYdHhK^Ma9 zw$rEl(?MN$_U&?~;xz5N1h?lsh?gZ4kge2Eu_tA>?x2c2TY_!T_F%oTpa+_QN8>CSgnOC!OZ%`kGtB98+y%L;oQx%`$h(;lSF+xZkuCovJK3>uU5&3 zuh&Ja44w|XGiDcfAP$79{RZA%>!hww3l!;&842; zJrR3e+f{#^15^#@O#``+ZY-|(GV!exHuv7F{vTGXlP?tI%dey%r2!i7w* zT2$ThS75%=035`_H37oLnGQz^Vwq#G;1fDjYB_tTsT7VW-Gb@Ll?#TLBsda!Jb1%n z$dNiqYB1S>N(2A_SPpo}0Hy#cb^kg(xXGbPl`{fvA^>s1#RLd;BB0!Xw1-lRPc~5b zhDU+{LE@%3fD%p*3=vwKp^~5m>VXlflC?mQRmupo%2H57LWBMFj!_7^FE-z($lb%$w(V9=r(<1H>7uP!arh z6y-0toGgjkqlCQ|l`#v!=Ue$zHj#YvL8nZdH9T^E5QCqFkNrtc-i#_hO98;70Cg=9 zwgpEx+0n-mJ}t1Fa3@p&!v9ey`M{I|=@nvy6Pk#_hXhz@5kd$ofEWQqxmc0H2rl~7 zf(Ym-Lq!hC6w;!MA8JyVFoC6&6<{e8M&w{GoCU!ZK->if6lX&O4j{0>2YrGUUJ(Y7 zMWbDGY`~T$bur;*LUUxWh8FoWS3#v5yr6;_Y3L%I5`TvDUjqTUbY}ttpg9E|8o=Rc z2F&yUE1wSpkOd-$D09&aB=x78LrvkKo<$$lAtH(&sHi7hcR>LmR58w2qgPP4NCb~k zK(Ww#dnFc8d__9MNB>zL3e(t8+WlVS%QEAUxF@7Dchg z7K$K*P!ChABw%$6zeI(qH%RgY527$)ha(Y-*ygRe@(SpOzoJO3j3d@}Vu;h0*i|wq zIxB$>bw!IKwbnl21`}ON5u~9D73Nnih9P!Hxs)`yZk6x8U8THNYT0Ea_1bG2zW#Nv zXWvUi@zNjvVlsiLhMPL@0g7LMu)=^3kcK%jtOBvbA?YosQWYPPg-vm2F-tBez9)w> zioUS|a589W!2fT`%wr1}?y(%wB_yzfd+q`6xfl!p6BfybCxVch1l~boiL+FJ4!}Jse1}yb z{0d*}zz3+L7Nd*63wuQe#q_Gi6dW<1{(Tj9JX^)rbqaXVRz%2=|ZUZDB<`kHsC>#J0 zu(QD7K%kRKWTp~25lR9kpaNUy0$>CP%LXf_FGj>-39%4@g}Sz@5CTGWMZ_7THpnKL zxHiB_6No$7}IfgtmO?DU~IV2VE%UeR$Rf*A1tC5ltLBeV$v}5$j5x@^N&;t zq#y@b$N~~@5@GsXZhVrH?Ra7yc^Fu6dS;tQRx(coQz9lh*%m7zvI&@Y4)Oq{LS`C5 zBsy@VBQF0iA&O>Tr3k6nHo^p;05o zpNk=t2uX$F#grf)GorI=Zg@mI2Z7a;Wpxi*wd4Nw$z_jK~rhV%` zA}wGGhB6EUX{{syD+t%YVf2%GJ19pxYB@0YPNadlS<6g`*hV!mv5QS!V+AEgVd0c$ zbKry|C}9vL4fUuWQuFg+SGD&6RcfZWa8B}vHp>)W)+}X4>-_* zX6~SSdMhI@YARsNQYb=+XcdrA3F7+891YwQU;DZz>|iIbM3gBKM*1EsJa)UAb?jna zz{daOt*0X)4C-MqYf$cPm#B(uZ&6QJTGPr`we)SRJ-=!i+3J_JwY@E@{JYi!YCu+Q zt%4Tv+Ry>m&2YpOiaw6Z3kpHNqL2&5a~1d!0dRo2o5gM`5Bodo5>^p_60wPI5Vi8C zIGBPO?3liAi}b43v+M<}dl4Jo%FegGKkf0_?EBjM`ftc{t`&waixx}+iG z{Q^>M{S_#Zj}71^4|q?EmTS*kaLH&(+87IVsa`X^fDZa+gMYw;r<>_rq2@5Nqb_rt zQ+aAKrza{ka}h>@S?ZDmqSm*rca3ih5;5)Cr~KZtzkOW|V1rfAL=JRWi*0DN9)k^u zR4qJi~`r!XmuR6L2 zd1wItBBlg>IN@wi2~$P-l9hNcv=x`-3TvF>mb*i5`s2(^=HYP!;;1uhr~;(Z^&%o0 z_gpl|rbC4=jHmP2FfXA8T_pZI2l>!9-|d8OFbe2$W~OdnfX&X`8!1ExOdXhr@6h8p z=}PCb(*Ygz*&5uRSkL-E38>pm|Kx9(WaycaD{*KSmqH9~!?i8Eap;OE^miDC;%$JZ zI|0D%^8iO0h!zEK{A;10aKLUK2ocyt3@O+zk3HIJ1ALh7R!dx2=#s{2U`-zA^KLi? z4zr#gSk40|uwVB!k8jd1o!3l1J#7WPdX#Z>RyTkM=X$Rv3S75r57++;j^qNo*I;CK zc49|i*w%6L2Y2=}OF$$8B%lDx*9XDSe0d--gkXGJR7(93Ib38O)@KsoR|{;SN!gY; zO8`51a0Q!01eSFPP!TTDv^*O@dF{t?iKk+w(*^ED2Y1nC8bxCtrGNX!e?Jxq07rmi z1%LurfLC}=t!Hcpn1BtpFANBQ4Hkii^Fp^W9MPu&SwMsJh5=Q;cXna{Lty~{utBc_ z9G^8ZtFt=Qr-GNT9Ly05Z}P^V`$W7 zZB{i%Wq&HhDW7;}ibYy9S9(+Ee_NP(0oaAF$c0`Ah7q-JVwnGJ6_6>|;U-nUU}lJh zbkZ9jFd)8R02hdVFZegi;R2&FD#v9j%E4F<5HxLo2%pea()cENu!s)VZ6ebRjCd07 zCo*cF4m40ZG|~Wp;cjM;iR>q1@YjhJkpg}dijw7oM<-Y+vtwgGRa4k>LDq`GRvD~F zkg_<7wh@Mj)_KjLFm=dn)A2~Zcq+jm5CftBcSs$W!c*R{UN;vpZFMQT1ddK-10`8e zcW{9zrHjOHjd4gY=W!IgNP*_4117k0SOgZ|Fa{goQ|3krAk-gaVUKrF1MRT_;l+Z% zKslm?SaV@b_c40&#SrYY7--Os{I`l#H*nLim9XfQvWWjTU|5TZMi2QEql9$w8(sltFHd;MM zc-=4pUC|VpcoaFwA~jG0twtk8xgs{A6(o>@IHFip6E5Xb3w!`g`Iuu~(}Q;LJfGk- z{`d=$L689nR$qym31?_Al@o3W1-6r-il$+UnE*aOVEpnE zVNjh2005OD0Bj`!0|0G7T2vxO7Rj)Q!p*llO9FcC}0CUYe5s( zBbtrU6{gvQJ!M*SAe2lfEbfGr0J)%97=Tk(kPkYc2$+x-x>L#+6feM`#L1@)B@+@C zqiWzd0zq_5go_LH1`Pz4EyWyf&`3n3qc>We0x%&0LjYL7|KzV8>3?K)S zic&;chzC`qk~We+0cinXcr}R*e_)BO;|V+2lS<%fOgJngApt_^YBgYlvWh5n!9`2D zYNF+8Mwo-QnyYcZtAcPX_G3C}(^dV4dd4b{$BL{6h+6^HM2K{YOJF8_syJ>*2ql!Q zMF(C=i8>A>m;vw{o5WB@A&nGi1J99?!$BNT01pp<5;GuSl;#cZ+91+J3zA5$=D|4_ zqpyx4MgndD>kOAE2&>AO`04)^EfSxx`c&luLzqbdUK$^&sVuQ7E){Deb&#?D zmX!=zaL5XwU>TOLSD_t*bS`jYw6KjwKttp80VXJ_Qwy^kQkqaPnxS*HT$+;xTc0qb60yoX z{OPyVf+Ml21$c0~pJ$cuAue8vHamhORu!zSu(4bjY(!?acbm7fsDKKzQQexTz3V0( z(yeTVxI>4C;|P}lQem1D975LtY~nc8n7NCF34?13)PR>7u(Qg04(A{b8ITZczzzDK z567SnjE4lYpdOhZ2ji&=e!2gzm6$Yd@louF0erBe4~&T$LlKw>!O+733)@VJ0-9F9 zAp~0pz$?6lLPGDuPH3YxrpKnm+Mr|Qu>|;qF?@gwCx%A1H(F#X&j^|*(2yRCgL&7z zj}?+3Nnkx8b1#PyA2W#c8o!>33B$uDHAoASP`~g14=tu$`pduiV81j;4CwGO1geAW zcoi2R1rRa|Cu}n%fvTY0v9~sC!zPo%1v4b@86u_0A@vc&nXG6?Rcj@=90n!|MCEpfm?eN%gLc3M z^vexGKs#*Uzgw)62l4+5_i)1Dr3_|3#uUA$7j+>FG(iW*yQTwp$Xhv(aT-8)TS(K}S9>`^$gBru%Wi7Xu!wM}np||- zSz;>ca!Y_I8!>n;;7kH>uBx;IY`_K(g^I`U%Kf_yXMhmEa1U(M3<;epnaHDYV5?S( zJiqL(!+fw4y*$Zm$6^YlZc9c|U>H^UtC@G82s)dnLATk=%^quLA=}8*D}3WDrcyuw zFEavvo6gY0x3}g@#}NTp00Pn}BhNOYE6N3!T#6%EL^M1*=?2hYQc$O659OfHQ!P|1 zgdw*2uU*T67j6Hr7yvHu$W361uxRQ8ez76qDGY_cKDjW>YC6*OC9K4H(s3iFak|K+ zfY-L6)Hv(N4M|5`bs-zw$gnoG60?YnhXOWY89}paFGTR z)f=!bf)$KWat>^u2u^(r_G`sl3DB0sGG`!? zpFsbWi%!6HjDE46o+VQ2wvM|{^s1`+8`;$5Ey6t+YCnO25g1O$sdz_n-on|O4bQ96 z6I$YXE+(YTtW1S-!u}$4>;Moh7cGJ<+Ai#0%b)8jzmxmj0^!LBvnJ4B>3gHZ3dNFs z1u(t=Fj34mRX`3~{J*4b>h}BN`rXu(Me6n}lB4U#Td`p=!fM6xMdm{-o~Npz=(}#w zyXOZMW8KVzA$?AR5p15MZ6w@&UK`-=Q8^0l`k!+yupscea$tRtCK`1%2%>WIy&{3?AqZx1x{I35G z{@&E^5by%O>c_AfVPNp|Ap*>R5ed_*KvDJ@0!A_N2fq8)>$qxe0i#_JpwaGnES=L+j*zstWX=hlV~BYQARQP0 z`WiXztoFb|%~;NQ?X%imdm}iVE%i)~0G&Pc?Q8BCpbzkn_3)q%VPNW9{Pns&@cbgMc3xBQsI8z?>t%)f*CwCRhs?qvC^X_=qq5iN6da zuWzcA_;g<9FAwv{D)aK@2YG$@HgEoX{V7l8(VwjU(h(g1aVG*ECJs7iIJp1ggN}s> z5juqU5FsXdX(;yb&=8>jTFufmGiT?4#02Ny!PA!#%dt_%j_Es>PiDSz`dqri<${e) zHVJ0P5s@re3JxG_Ez#*>MurNuBvARr2Lp;^n0P7$l5a`_p(s=+!$-x=Sf%E~DJYbS z3e|kfnv6vXQj3;c!NTS0B&ALqI=?C{!v$C{vTMMq_3~p03mijx95?Q@ts3EMl`UVk zhFNpwXPpTT16@pX=+VYZ8|IZ-H7nMw|Gst|+teh)M{G-Eaod0<~@QJ~L#DzF=xVaX;a@H zonew@=#*jB(1Fc%CVgRnOQ1+094#84BA-6801$(3ni-)Msiu%=K|2_Hfe@|ek|9BG zFzMsMDqQ%ZEj4PXs}M-`>PxUx0)vq(NFrrJ zvX3NZ^W!$*dV?~`NfHYKJ1aRbPMJ)MTcJxXqnPI?2B?@WAvu21?m8|!GS8sl zu~=h)L{`WnmtAd1C6|10w%VL5f{iJquC)MgfR!DAadZO*HC#)|?#lu^WE#Ygz+Wi}XF!&OF6j%m_YLv$1E z*dU1=`bQ&=X7*NRp*5vRYNgCJN-EdfXrMow0fK>X2N+;n1=4MT0jbVAw>x>M#%?=$ zhy`g0Z?nn+5#>Y@&INn!3HY4F*fUsAgbT$bfg+gF4Zr^%B9?#wW=dv?)26a?Vgw12 z^QrL{#M#nS3@~7ZW5-u8i;udb;Jhsxu(Cs@SIOkdj5TVO#a5hi?ksjHZv+-Jp?Bwf z8Xl1bKIdkky;(M=gUb!-sD0yh-f!nJX9Weu1pt5w#)bQIH8>C;UmLnJ?`$;{k_eiS zS@8rPy4h?d1~pD(rQe6xP!ArH!e9v}d1lH7rJKU21f?>tDk!)sV-c|0=O*}E%roYBS-Y_QR<2BRz* z4vT0-lTi+J_?e_3NomGIf|IOu4CMvTH?(QqnhgI!g%h1?YX$IH_8>rzd_4dwB}3bs z<`tqUbTJs~%LpMd)&{x7uWsB(NlM631WDwtP&0dHqV0P@Lcht<1xfqMTA;n2B$=;DWhD|JHYj@$1((Q&wG3HmiWfk zhcraNIP(Cb2J&F1V9Wy)^0VbpzUPq)*eHKBQR6vW!U-GYW{!3wVIEol!7B>#k;&M~ z50)^`3@-AKl54^r)WiC1OV1JHuU${VUO zm9)EP3yBAcCDl?0$-5;keX~m?GSQd73?>Ms2e&j-Kq}p{A~Lg>Mg27Mef$!R8d#P- zVo+yrtZJqR;nB@Gl(Q!1sL4HGXrnm zh#-Qk>SZo?aKb92un>8Q6$bL0LR^`62pC9!2yK0WSPAsjv2HS01-e5(Ftt6%m0loaCxm2)HV$!FI0P?9auarUKY)^|z2x=5+;RtMY zzy%{x%3p}uD>e{OcH&%2ud3wKAP)cTpluC7LZaG`yAmj`1p!Z1*~(Yo5?8rlJ??Rz zyH)B|m$oVdEDJ|7SW-Tf3x1d(4;RbW@$T@kj&1BLzaYfl0By2Ua-IV)t2H1La+n|* z6AfUBI?-maDAOYsSsgN!23(H?WvZLwbTZqf=Ej>&>lJTzIbkKZVYuYeA`W`M12lkP zx*EnXTQAH5jX3wiBsQ^t$-33=#v;4gRgEKP)H5Pn7skL#Rz}Hd-t(dtur;HWeI%-A#H^|?C~Vl`y`%OE2PWo4M{Eg6b{kXOPqy96!L2ww@&#=sJJ5_W&H*- zy;IgSI0zB!MM97CVsJ?CT{p@hY-Ysi*VR%Ot|1cVAc!@ejNkxvY?Fi~%3+XQNG6Dj zbBQ-yVyk%02o1X9sd8+2&`3z~WhXbouI|qjT$NT=^_dZBp?Iv>|r-! z85yump@TYdl*5?@pIHBS+wg=CCeRbL)8xaSbA3pQj3XaEcr`P}t)OoSy1FVqG-+gP z&YeKD#)S>9cyZi2dmr}EJ-)OL*-Wi&GCg^@tSpjC9bm7GDNOVsQ?_S{wnb@z0=3vD ztSk=6sU-5-Tqt3E3+N0+BxG&{Bsncmew9LU(;-T*IXykK6ybD1pNStwEJ7>pl|K_7 zf?h~>s2UfoI2^h}xTW#DPNhZn6@^hR?ETh6a@bMN~&A)J)&>;12YJnk|sfabBq<;su8SgoSKhEW+n2(X2E@ToNj!ta}emPv!3^S3)N z9rL>YYg$4R;ixXMfMig@Ab1K<0fn)sBaq0CtU{!=IKTlUz+@}E;X*ewLz;Tav1Ca) zW|6vl%Q|S;y73u23IvYeBZ;cgGD~_NoASdGsUr~-!M3vkL8t+}s5m`a!MUIy6cIF6 zSp|V&lrR6Hj=5o!<){THC<=UFg$qF<`V#?~%L217B2WaKPq2BL(t|h0p{_I>5@e*hs#~`Pl*4FYMj*32X|uqXyuj5tL8&l+DcHv8nX3b+ z0ZsqNGCBysaU_XZIF(gcr&sASSVNHFBLS$07b}v!KZpP+03@Nj!J|+sdPIr{co)Jm zBzyomTfzir0uWXaLp|ewQj8Rx@S>xzIDjFU5$Q$G$|^4$5xT6%sx+XL(G+5IEAJ93 zrvprrER19Hk;GIbIF&i!?mMEw0Qy!iX1A~9Om0ApajaN)VR5IuNw7f)m8Xr?dhqa61+tq1l|w$&5J6Gy*_!0_5`qBj6BvjLI&^I|}$C z{`3IkmmJzx^ zf;JT6r$b;k?Y! z2017^Gm6LCt4N>&qay;K%AhQu13M+L;u=)f`Nfe`RCq(7MiZ8k9M2y0QS|?$)bms* zAyrQytwU;|vrbLYHv?7Q0z51O$1EMwF*UbORn<@Jgi`IY6#Rfjh_zV-(+yw(E!D(i zh=YXSLoeG-;!@N*164sCOz#p_I19;QJ=SC6f=prp1H?0_YMtNu&S1S$AJDBZsIY9c zvu-s7z(}*HQ%u#X)N(afN}U8Y<5p&4*CSn3I+Iq-Tt0rH*L`B5E(k79eb;y83wkXe z80geE5U6{VD>tiyc{M9L6IJ74*nADuhh5erus?le)*x^*a2vl2#n@)MvyR2qkM*vQ zEG&>rv_vCA*g2(e6;F%OhH;(La}83MRk};f)FBlGM_`^ATRNR(1TO!ywzx{XgDs#l z>8^D(T5Uzzi-pESJt5xe0y~W^W{s{N&?Juy*{L0|q)l20J6e!E+NU z!5!SfRa|XxFTP!#^un&m#Z;MJznlb-tP6@?-k$iZC>Ur-|GMUT-J+T>7Cxiyx!1- z-}n7p^9A4fO~-J^HsJ`K;0Fd^3ufUJW?`fXVd%x(mG$8D z{ouqR;Y!osz+zw#F5wfFU=#-8<&E7Dc45iLU&xbKv4GM&dFC;|1K{ z8s_6CwJGgjjjHU&6d<2qL4GY;fD-r*t<<1zm(U>x>gN3P@$PGc3uUwwcQ z0Dj}SW#d1lVmij-IDBM1jx;BpEA<+;6OL&jx1R>nL2-(RNY_1$Atre0C@-DB2fUY^-lrrT!T zWzoIgMuz1CMCS4eV_^p7VJ7BoPUmw5=Vfl=xgBTzy=QygW@wJ*^z~&2l!jE6-N=XP#en2u@A z!0DMjX^EC+rxxmJ2I`<*XR0o0ir%4u#%2R12bVP#lwJ+12J4$n<^_gon_l8pLTjSF zYNK}Nx0Y$9=IE;x;-s`nKYE%Yj zmxk-e7HqpF?6F4dxvuQ@RqVxP>~|jH$KLAD7VXR?ZK=lW!RGAE#%jK<>cnnqxBhFu z?(5mUVbnhD(x&UYPHok0?bYt>@pNq1mh27|?xB%eh0g8W#%;=0?%(!p-gfOs4({TX z?ZA#{yuNJRu59R*?&kLH?w;=Ij%}~b25|po?wsas%U)Yc2JiHKZRdXJ=N|3(zHZyL zZ|&Z0_ik_AhV1aBZoj@a`o7)q((3zOZUe_}_O>YhX6^m{@AvL+2p?|Rj@}*)YXryc zyWVXFmv9Id?$4I*@uu+U&hXnl@C1);4hQY;&gKVy?FlDO87Xk_I&KU~@+^;P!k+CdSMnfdb2tBSFAsAm*XV_YXk$L}JWt{#@A40i@D5K& zIREoOk8=>Wb3I3KE`Rey=Wjuu^EHc?a}hW1^FHz`SM(HjbhduNA*<4^#BiXA$N5k|K<{&^;#EpTkmsK z=X6isbYl+^SSNC`Cha*Kb~I1(Tkq=z2lQ2M^<7u?Uaxam5B6x6c3M~PHg9P^&vtCz zc5dI1fB*m?`2+DH`T!-^dXR%q3;PtjUcE0b&4uWsYQ zolBRj*Sd4p+D)ssZ@Rv6@e1Bc_^n~Kg%c}AOxUkuy@Mk!o_v_HWyy>)TYhY{^5xH% z1B)ID81z+*rc-a_TspLB*Pa{SeSMlXY}u`C-`*X&wr1Vzegmh?dbsi9qe%-#&b;|@ z-_E1E4*p!a_0+{tUx&VZF!grdt9SpOjXk@0-s8n5*B)Lyd+y-h$CvN^`1$+m>FZBl zzw3PX?eW)NbodnrAX?}N7~p~jHn^RE5Jo8BehfNj;f488m?4G(K3Ly`9dan*h9i=w zp*tr27vhO4g6Lv#+`On@i7cXMmWnvGSmR{o{RN|pKHe1LkU;9V+l~pTlVgQ53K!&) zMMhaujC>hME2^~K28k?5DU=3$2FFGP|sscar06w%|@HZnfkZo9wperW>xP-?r=SyX#(i>Z&}d zYwo%D9;xQ%OVxuK%N@4n^OD=;<;V|=m31ZV7x z!5n`Ka>5>uT(ZNLenu|CVpi<(%P@D-jFKG7EHg7W=dAP2v@mlG&&%Na^U$dDY;@4H zARYA5NjJ^((@;D8^w8H({Y=$W`)tkD7<1h^;sr*JppdGuv~IeJ+BBPq_4akPrDG=L>?q`t09P zhx_nj{(k9GBr?DKn%6G{{rB@F{O5rk+Yg&IR~ia}62zZM+57>NqiOB`<+v5W^FO*~cvK(U`?7rU_?wiDq^WhK58W z_6m_VO^kzkjLd{K8+l1BNHUW#FoF}pNltKzl9cE)XFAu(&Q(s*hVLW+74T3#dfM}z z_{^t0UFrWy(*YBgdOSlw1sc#frp<-*$T0VC=uzIuv-h2X#@BB_}R zWi(QniF^a$xY@}AP@tTc6sI~@+DVj#^OG)R=Q%5RQki3Sd%sYBU&#J#1nX3)aRu zwxfmkA5sVE2di3ksg}5gQ!U0es7keTR1IBdSw{ypAoCE+q^L%_n$baK(-vOU10j0K zhoAqVUNKm*7i+Xe)n1aTFo6@*LN;=0thDm6iJk;~k2D%ZKsjV^Sh`&`0?v#n~a zBw~lFhsPq4dpw0EG!+Y8NOTdYtP_QLS!deIQub$Q+Mv8*vCw2Dq7R=Kt$rOb3DWlQ z4~Dzv6x1+Y?O|`4YDmK+y9?CWI{2rP9Dr_lONIjwfSla*LWbFTg$`coSg+f>XvN-^&Gx#7`J*w9fM_zAb zm+am$!nb$`CD$^d>QMadx4)IhGF9`7%)-S_t6I=z4D&(eGMCvFdFbX(&FWKcq9FeO z6K;bQtZU*DbNJ4~E$)W(EMhyKxVb3KL7=It(!sj;w=f>6S938>JRo+SYknoOBU z%?b*ZJblb~cJj;P98KRjq`S?S>)SZSeMb+duyua&134 zMn9e=7CKo2B=Z<4|Q)>pNulqQ$>uomw#$+{~CzFo7SOaAg3208pQR zi92p=h%YZa4xQZw~|*gq@|TZ2K)8un!|hU zIp?e_7J6TwXSrv++!82^sa1hv`sfDdb+4HYK1#BH>dUn{pt(-@jCbAZm^7(fYZ~V8 zZq&P=Uo>N>5X?#l3LC>l>VWDLMhZB39v<=eNIKIK zzRg0gV7JG&Z~>G%@g_#s*jm?m)+IlA%ft1XCFZ0lhf7Zz?(h5HpI`s@#oy@m<&&7j zIK4G|K|9*LJKlN6{l9u4z@+`X)f?*+GltH4HWU2lnGgj_S9}D>O$ML^c~%0u)jh>F zaSkYP(-(nQXK~{+an-kC&DB+iAYGDFMMf1-AlPILHG&M~2w;;js^@Agm zQcyehCo%k{4P?Uz0B3mrS8&2tfB{EamqrHUbO4@!bb>a06(@NT7=09ofq|B1&*eSX z^I=|>MI`uv$7FL@I9X9;P&PO=VCW;M*M3a)2(D*GDI~MujT1g(3gwh+G(nc_a$^CWpc^ zbL>VstyhCrLN8fDZ=Lc9Wkhst*lBS1HK>R`bx3Qog@h{z~{l4xXa=rzGJHg!jTnh1X_s4m_h3T&uqv4DRw zgNm*8jc-U_p|grj7l_Rei+f0XeaHtP28d@ih%&~D(Fcr`M{(gqjLF4CMV5@q*pD}N zf^PGS>xVm9c!sW$Eoumg*vNBz_jm3Bju1%iAl3z+vu4X3hYQ z9)<|=Xow&-j}iBam{(`fg^!JBPshlQGTD#YScwtojM@J%hOYCE{I)Wl=z{2i4GZ~S zg=3T6D3R+142(05SgKlt#Ih1L+E|&|V-EH1b9%2gNDfxGh3Sdw9SM4(V!flaxwHkpbqED~AcNH8ag% z3utBzXf})LIBu;KVkLkGAgKnG^kGP7k6zi8P6$qf#)z9|3t<_BGE@gxWtL}&mP})l zY}uBJKu~Vv3plqR>ZdULa+e$9U!D zkOvs4k;At_2qs)(Hcb#PnP}w+cGY3KHD^YE2o3*`ZR6<)O@s&v$en9t1mD@7+DQTE z37_d{0&7)ag(jYpd7kqrpC@$%O=Ms~=ylIznr8Vysd-Daqizb?nyj#Muz3o-Fqg8@ zlR#Oar-*|D8iUMWdqSg@NLh-i)=CwLfIjA&Cvc)DY6DheMa__rj|rJ!Re@oKYmhV# z)^<%Xz&$xS4lGcFK{gMAh?V2e1~CwPC9p_G8j_Eu2R!NpKmL3W=HfcMdR!|aZn+J)T`DcToprNxs z1$xSx9$J%b7(YKqmJP;3E83hl1Og|DoQ3~6sAASyHVR_b`K2x+LT5vZ&5JQ>T zq*|&4dMKnyDu`J62A?Xbl`4pj#;K)xs^;mX;9vxvlvp+zlVzHwo1hABDT%n6OOutL z)o?Wn+LlV?26+UbyF{m*=r49!gBuEN2xf=CIXFcFW`&t{y!S{_;6vP+oZY&6$?&3l zb3JMhbr%Mb=8z9Spa7Pt0yYW(cLRukr~;Xg4;BEb$Z4edS_|;13Nhi2g{fTkgoC?uxe1S5ZeX?Ae!lF4$fc!24G=Cm{^%$tHKmfrLe2Z6HI8@tGJ4$ zyaSlQs$Rt^r*m4HKM9l++MCas32dIQ40WR&R+BCH*pbWZYw*Lxs;NV-$ zumWN$wa}Thx}~*gO8{$owI`qe6`;2o>u?sEu7p6QM`b!{x>R`t3%RtbXat+4V?o8> zX>rQ9aXFhZcQ&?pGN?BT-++#*fH?Q_2WMwJtc6YEs)3Hgk=xV>bWpThg|3JIYzqfx z($J&xqPv^B6MVfFCIE;w zyww}M)GNUlpu5S-02Vw2zMBTbn_bas02^GrQGlWsPz7mr0t9=rEqsLoMZV^n!|AKT zIt+6#`@_IeFt`Z~|JDt;Mo;9o3SkAno(I4}mQmuRn6MRqXx3nt)08(`4oyHs9!NzD z(7-ILyyJo>2%?m zaG1loc+dpMnaV8?0j$gc3-HRX{997%oB}|vSDOhSHvkHN%M8HFnXt6KOaQ?=3vL^} z0&s9ts|Crt%(Q#~xO@V`oUyh5%#uXR6Wau^ECH=70KXz%z0=;z1U>z+J}u4xbvgb-OzeBkNS)M54G8KShP_gk zcnKxF*{qNpP%v=S5Os_&>Ip$RY5WYJ0-d{Gythq&l_#_~TQ!*Lqd2GRXb?TqF%8!d zV6ar=(ICC9ISr%(5PS=uYbR|BCEW}I09!6y*nDu&hb_Wz{lW8)0%Z*ZK#iJ%J5aUf zKKh8E47Jpnt=VY_zuf}WK5yfV7=df%DpWG0}TYa=B}530>M4fCos~R+OdiG1DdLs^eW2YU;{!+-MyU$;UIw5 zt;KvSl}-O3*{InK0v&)Cuv^+h&xX6%>g}?gz0P!rxl~O*3LRV2Wd%u@(2MMm2X%Nru)47@py4)S%&zaqPpyB(w zr{W9VJZA$p1HF4|n12w?=)B_R?1k;U$j#Eu?)=W_Ti<%TKefd@QfOdbwOG(ubhXX8 z0$xv#g9=C-+gnu0`I9+>*W|P0a&&-I0VF?8E=yT}Ia!6FT0Y|P9pWNx26BK753ZtL zY~cI1*;&Zq>D$>a?%sE5GF1IFU4Dg^O`~e2;_c#*7$YX2wkmTMoVC{ z<(B`0I!vDEhyFaGxCoCP>0UJHMwRAf@OC`|7a%zo^ULePc5LmzZKU%0({d@uTRQvy)u5=(7{Dt>fs-Bk7F3bHtA9 z%T7X;9_o*HrdssV%UF1fIt5uEQ5jn3k-ftv*yd*Q=1+p^HmI0nL+%brOsEyEEH}#5 z#N)7DvGRV7o-9^(8l*_arf+KAAiNe>E0uQ-gpIZS}KBKw*RB=rzK z`OXeKgFx)W?nd=@-jyzEqpWH7hWZ7yP%^{Sd28ONfZkkC<1^6uI^TebMwyEg_uXDl zb4ThgFYYlv{5#WX>F(1~K(RO9RTsGec|iD-Oyu-#vDaR&kZKDe8t{#??1=w9jfx9L zxw=M-4*FOR@vUH{7oYkGT&t*d2L@+Uwf5kd9S7-O4!hRC9Dn8@57Z(L5V`~+0+Q#A zlQ)PABAPePRvJ1_#F&{#(PAJ++pKNm*3l!mbRR{2ga@+Z$dD;bu2k2OB}-`9WX=TH zj3&sK%lusY@q|jge9d$@RMv2uQKU(ME+v=;DW)_Ah1ycqY|0%xsG?Av8U+K^Otn0X zt!hJ1Bdmw|sQr{R?b}j|7(t;^M~>aNaoqrtyXRG?R)OI#c~t6C;a`Xs6|S2&PGZQg zUgZtE2~8Z#n8h_Lw<5E|E`NfKyEMaM9jjF3bg4Lo@m|)4Ma-sgmj?e6+__!<=+;b+ zL9alBU$9WnR>RVumA8JZkt0gz(WSe5sq$vcnAxvyho%#=?jL_LH8Eb)oYJJ`mm*(5 zXnh|D3o3BnV1w@4sA~G;X)>cpA5WqfB0pQ8fD6A%X8FUdP8f+K5(TM%OR~aXd&I9z zDr+nf#rpb8FA4*jOvJ>%SS+(Cm~qCL0$Fp>r>e%eBL#~LoGK#bB-_ixIKY^Q2q2XJ zZw%eqB5w&KBOqXa2oz92g*SXt0fQE(G_5BrU0enjE}xsOx-Y2{6HM;Pw27u?yd&~9 z(CC8_9H5Rm4?UpDX+|kc3PK8^7NEdV3P95`XssC~R71EngggH-PzTvMB$GJd#OJJK zf}*OS2SZ$hFda4Qi!#g{b5YX~Np(Z7K~%{wmSA)t#J)8tYC^04{mkvFsIpk@Gc zVB}nHU37+0*P(OIvkH$si7gkb{q}Q#gHQZ4R6u|4!VACA(w*nNg5h*f)4E>!3oywJ z`xMnsD=w8(SO!aCV+rISYlR?l`wtNoX@mht2Ef8mw_lk!YvmDqvQ?sx!INtSB?kzg zfMyTi`B|WYUI7K7r7gOTYqRB6>26^{lO|ifm0HbUu{{4b(s$R5RGoK~aSu*qfT*M1 zKN;0e-@1rUAgkX1wyTMO6G;o4h18%&kh;8H$f}lOyU;~X8P>5Pj4$@dDg147NUe@tlF6PAXjbTFhX%R?+G$7H zOQo4!8eHCIB1R|MY(?_EtFy-Ec&&rFGbugf40~pBp(qa%OtPpigbSKzWkIa+dV8-7Rq2%1-F_|HF#IjQymkKjrxIlwM2#7%(G46j770sf zf*E*7QBbr{2=?jE2f3MG9dafC)%~nz&Vt?SWGDXv3s~zJx+CGXc6X-UX$N=<%+C@o zakk}f$!7C?x9x(*iXhrDXPcQ4%w85U+HG2z2-jE~V%iSUm6{vSMZR zDkQ&DAr5hCau_n58HcbGNIn>p44&l({5*m^Qar_Vj zn8-&aC>TyP5{sIKB!KIz>8yJ)5{`|;-R@5E$VfVJlA|19CBu^j0li>W{F#6j!1k_^ zAa5z4lm{}R;yiXb176U>P4t|V&?*(=fvqH4Em@O-7b3zFYqOeunz9=(j8qL=U;^oi z)2l67##5JZ*(`b^jAJ^}Wx1N>H1Cj_eqK#?;iEw1}ma+K9b4n zn~*zM^u%KW890TiGXRcDnPH`>l}rC2`iVfXP~eGH0%3w)Ou-3|MWiXY4mUZq&@!J8 zj?sAMnM}ZegrwkIuU*^{nHH7P~+| zfi(w@bI2z6b^)cq5KtSu1|(KC^io0p4px1}l72m{B1F{w5aFFUiNyKLr{V-Ao5U4xmw0;-3_vGA+PLr!QlJtd zg~+TDn;5RjLa~Y+J;~@sTAUd%s;W^7$+ayL==rXzGJcq$ovmna*YOzu4UHIIZ*dBSPkdOwfj>L)FOW0;c5!HCa}dMS3D{Qx7;x$|aHYrM!>W0Ab*1S2pbmYY zC9r&OhljKhA5Sz+#D#qVRz|@`Xp}W!0F&%7tKtcZNJb%~aPDz9JI(Rig{<&NO>D$G|S@yt|^76C&DXz^ZjG7z==VWq9K3a=^(L>CpN=TYf>Vt?No-}|0)*^vPZ zQJg^zEKFnPFzK$Gf7;XSvctmnbxk9yl;WHo?5%*cLytH4R&G~m^M>2DK?&PXgrG<< zm2r`2GdmPFZ|9J6p7T{uV$G!pTRErz;Rr{m89EDq-br70{$^l~vcs^CdQ)L@vb*vX zNcj^&j6GKtBwAqA;MbS`>u9rP zGN^Qdz&Kju))tKay+9%zIS(Guw!aGd5O+aEAl$I)_A8+>0XC7c(R}%5<3ID8mukAD zM*#520{{Sk3J?o57{CjticXM#K~lX!d$@2CH*rISX7DU?3LB`f0R>qUlvQ0v)HYmKQ=&`^OA|}HU zO#q5o01gm3np;YSb_%@EX{RMR3SB7$tZORQpf(-|ficUlm)nA6bA$WqH2i}Eo(lzy z`2ZovLP21L1uzu<6%YUdJQ6rmfg_2428=BXFuDlLFK9xjqG-GEa|jH$y~{Z+la?4e2t!gX-%CM996NPf$K4^mf{H<1 z6en_NfWK;NX(NR*w8kxn09tUiS(p!I$VQcfltwriB~T;rIJO21 zw>;x0E(0_)q=EK26huLU`1qRLYN~-Yh0kR0D)wPrH7n>ZuqUH%fi}QlVdzaR8SBXsUWQ=mMpBGQq%*j<2n+o zr$jqPfoeq29L=LFJGD~E3L-aIQU-(U!5+K-&#V?qixy9efdkN!t`W#}TMN%LB#AVS zJlGijFxUc{tj7F0g}RJKk-SJO^hNqJ!@=N#GCQya`Ldp~&XE*OA?XANSdsdmg&1hU zg_?(05P>{PAMxmsczn!qAV@wdFUwTV+=DV|>AKz=O67W^5DHDCWXDHD6SR}W4P%JQ zgF=LIDTA1np$SI~EDv%dB850OUBJGiW2jK14?#IFYcx&{O(rrzPUdV*=Zwqfd^SCB zIUdQ#WYf;a1cd>)wNH?Ns0jfnO_Q zKoZd5RLeQg$WaK<UfD2S2&jJvL&_Xl?J$c*IY%|AtGffqQ(%zX;N`pshyqZIaf;Z~5-*7%n%t1`B zfT4TKzdD$zd7HSwiwz00LG4vvMMg#VMa`Q_WNTDtiZ(Q(nL%Je)asPTBhFt1OcwPb zXys98l>|yq1T45#^O=$;c(2GLIQRHGOx>y^K~f~GH42!wL}XH+dD2krPj@uCDjg*D zn@teNvWM#?YUx*Rbt7g$0D$CIxXG>Fh#A~K3)J$+&v{dg6jW*b)ee=*_}P*FU5o}M z5C$_7gT6%AM=;T3P%!9>SdwK_4Z}-lEm>$aKunpSl>1RNg3M1mG&(GV%IQox^HdWA z)#j28;Tws5n%7AS(DmG{JG{)rNu$VF+96VuB3l%vEuH*Y$(XUa(D**LXn|@RS(2p# zwxO2_dK;#Mf`?TFVd#WzI9nmZ)3asD-Vld}tb?{)Lo-{2vlRx&I9stb7pCOCH%(cS zWwvbvT#`MghV3Xzr42L;gW1pS@KMsCF)1us!cBsl8eG|pUSw=jK!eO9ITWiL!8mKzoTV_!B~8)H7R)U^ zp*7XklU5%9JqOr;_kG58!vOl4f*Rn1GqYKC8wf2lBNXT~J%e33KsW%aUYEs0jcQ(m zvnv|`+29r4i=&IKwE3kiNUGBhK&rGf?CSd|S{t~Fr5Q6$R~01jX>v#SDjQ)JY- zvee%i00$QNVd^9&NRWd2JA?`T9F6VWXlpvo8wa4;V&x59(aE|0KJi`ktX|2ROY0?E zIxxa>nUPK)NU2TXEQB)_#@R%3S4QJm8U9a>6E8L}H6XjN9A0EbK4Ku&Ukc{n`%M8F zU4;}t$=P7a|8+=}e~~*kIzkWYXZ^fhH%>akh+!gSQ=MozhtU@;tvV{^D>@=t#&y6>$b) zj^ksd-Z@a_l>G!QXO_5i-~e zJWxZj);T8t-r&jwRUl3(933G}Ck^XpyzSjx*5IbZWLMVRVwPQqjpOPa;NSq)%DEG; z6}QH%vMW{9&b&{X7GM4(&D`tdF`CB*$^MblnXlHN&okME>42Iy{^+J?{1h~$;FWzM|%WBrX zzg=c#z6NPHC25izPj{f1{h%Clc3FhdJvy|{PJL{A8eg6M=FE*%%$|?THagYg?0}{f zlzeaahHvvs+2+(|i`|2fCQjK#M(M=Ag!b28ZwqF!tSy z-i&2%10L|vacE<|j&A9mZmzz`L2KVe&etT>na1|xZ2NAZA=UAIj)GF5vXiy*-Zq+^ z56}bgoutGd6UAz_i@t{K??IwAepvlR0+JO0K`7!@v1EnDK~@xIw63|c?G0s8a9%Vc zpHhXyRI}b-%H?g_0n#ZH7HQ!`PBGu`Wu8p`UA!$01-jKCaU^wfd;0F0cApf0&-*1ZqR(FNnNp+XY(3OFPY0R}WTLA9W(% z++8noVAsw&2;LI*>~ylaV_dyKbFfB(e|YWW zY*wxTqu*TRKox>V+rI2~f;aesr}e4_?1UF`t>5}b#b*Psf#3dyC0H9kO7^Dw4QnER z0sQ!`>t^L@0FfX0=Awu9+ExWaLMFpT>Kz-}H1T&wdZ~8V@m*34v=}diF+CUYA{R8}OXli(S}) zG2Ufe7yYR({nodPIRywT0+S3RcnS^{L$E@9Dd_NsLpQe4T(Fgo*^fKTfT%E+?~e~5 zV9<;VK?F*R1r7!nFc9D+fdwrYG{7*kW`hGbC-A&*p-L5C+PD$*X7p%La!Qw`W17_I z(5OR=0jo;vS+=Lvv_dO-HBXu<5ESSvYqkK|v@QV%U@HK@gSaa-c=(V(2#740Xkh}8 z*M^GF{DOt z<`hGA6DINol@i9&j6mKzdIdnDte4?F%^O=2_d?~0v_5{R`l)5LzyJSN6y2oOSuq(1 z(_3=EwO|JgIzSf(f%(;k9byFr7+_K4fyo^od?y`Acs7~{ zE6Rvch^2*qS}vf4e}wFCPj$wlRn;u!UC!KfV zd99y({-@tw9Ie+(8-%KL-~b5D)hMHiz8D(2JWTqo2Pv$=0tqOjKta7!h+w7`PQ2${ zA>I5StZAK~s;|M$)Ny1utI+B&ke3ELq;t9IDy1S;!V98v1Um*u870y1#1cQizyW&a zp2-1x8@f#Y$X>HBBgZGz@aLbk*y1TCwRr-#7ey;*Xim5}nKjl-HtFQWUmWD? zyN&JbF&vM8>8u`vaO6|Hh-+v#jr%%2a+h2=4ZI^0kdJZ3FH}{2ALf{IK0p1PcWZ&r z6LD1k`|UHG&a~57CqQ^vM0^u*6A=10sH71H2!zlM(f+U@d~~Q_&q5X4kOZpTFl=sJ z5Js(jbPqb%&0@Kt5|oIA1lLUud(CmfBD9k!{pG=d;6sFb;DiSnolJe{V_z|*@jm#; z4`-UIAN@FlKL<|40{l~8x&G&<0J3g?fCJGR@_{L5eZ&Ar6;rcjvGf#f9o}?Tl&vac{R4&dsR4^Y5bob794uf(-G-45@a+Rer zHJ?rGL(pu{1`gHkpQe#5DP94wQ5XDfTFcPDVZFaK@S5qgBi^tZ4Twb2oKHD zpkTm(c*;}JCE{{GI*I6T*Rz3c%5+zQ#HPmZY9&Gj3!yRnYa*{CSnU?lIZ)t(8^gOM z;E}hy!8@-L62T%-7?rY>rRQdWQi&+c_qFzA=zU*{q5Yorb}xC$U~G_De4Mes(HbpO zzxOgW6>b64o7zUN`a}Z+%AzqrTwIt7SCIZS6wQN@^gx`_5qtQ0cZFX6b~$EpKH6`O1|E^MvXs>&dsuVObAOSx9F_44oV-N^g$flJr1peFOA*a?N(K7Ix`cmYe zGz$^2U_u}od8O4h;SN8L(JmKQApBY2!WXSW6Cx{zF7*vMhJ0yp{?^^;RoAbD?dyq6 zJhknr_!SG0TWY4d8r5h*l9<9Fp@S!0O&p>VPKB>!=|o^H7P-U*WhhHo$koB^m0=Uh8fD@lhbEuxe91tTP*@Mpf*B!b(+6U9Q z&e42GAmyCmnbt@ZxiLc=#;dOspo9v1kk{|Zwx$}ahTGiEQN7mxWEoV<^Qe)|aTVB* zZlI`JzU|Jh4ZO8q2?=+VW%=*YhTH`^&;lFf`1hpWWCODjyaGUT^=?CaB{gV~ogUFb z|I}kqT%Ply8ZD!N+d0hU0(+$&ZUwRT7R^7dkT`r@!XEBU1BiK3KaJ zHQ!+kfHwu5zpYZ?4lh2>eZ;2*om#|rLxo+IhWb`w1+o?wzFnCp)MelWu5IXr@Y^z5 z*g*n%T{TFXARnQT$tPkY?h5c{7I~NX02G&C8Ya@t3PR+d)x102^PcRz2QtJ>Dt4Vo zoUxH_#^ijhUL9fjU{bhz4=^9n4#j`%h3=KgmHW{| zw@MpqfG@*9wG#D>0!_qhx>kyZ>Zya&kTZY)NTDn+B@0d9xeB?IhyX6eb>jXe%FDEn z`CJiV#G30a37by8`j50eb+Zq66jLAYAvdw&m(5}90;}=fNd$> z`5JU$hv4n}=SvT1|G3xmr=KJZ*%$2H8`K#JTNnVN*k0;{2LnEhh85GCb=Z_l({Fje zr>8$GqhGSL_!bnRdM)D-3?1-Afib<2Coql1@@Yk_!*tu6cbF0=FJPB5gLg!;h=%n zEbxI96wkPA0~Cth=;@yoRv|K!5;|c*dsv7Chz>=`p3;2U7?NRw=l~jiRI~LR_E}#X z2u`uo8<|biF=9w9PTT|dz%a6zXtdx%c);R-L@e0WVf4wD>_wG^Z~^o z!Y$DbG9FniDwz;0Ab7x%FNTP8sa!E>1l)<;A1EJjRajyjBTF3QAIL~YLE@6pl-x13m24H?0^pI$>(uWO(6ZExekznNg}}8H-(CJc5O3W8OaoJE3E9%zx87{)jZU~;vGLZVVg>cWA|gC~&B4Sa_4h=vBVpUv$h(0orb z%!5p(kT9r|7aoW$ss$g^6)--}^PB=?N+y_0lij_BW>BVkzGr*ppgf7@Xig<+R;Bv{ zh*-QL7+qk*A=y{L1a0zS>LkQT=H^)@5|>?6dIXeDRG2Yl;7w3m$%F+a3?d{FQZb<< zrceQO_NAa120F-r2S_7IPQiDksO>PCV+fv$R?~SZ(`3TudVIo3?2oh-aMQ&9OzvjLu_x?y0V-A)orGH2i9y0;HgBj?M&AIT)*a zj9o?jXRM(XG}ytH@mrkmg&>YrHqwK2SZgNCkd>k7ltK-r)|+uX9UWd&1AZ!Xp`{Rn z&I|}ZY=J1!*+M$Rod|6IMtG>#tA?4F6~dQjD5(BxM5xfM=IX8%Y!3FRuLdhBmYzCb zE4>Xyrd6H9Ml8ivY*#Aky)i0HxPV-aT7mUNfL%x#T_{2(+{S297x<@5gb{4E1k2(g zG%}YAVSyGpam@$$Z9INYH1~P zS(jlbXJFrM%7sQMSWF5`1>(iL*doymS5Isz($1>lQiAjJtJ6NMp8iqPw%F9xXDWss zKBz<2Zf>P{<$!4atk(8yz=cFkI&$kQdGE(~+$VnGq&5r{Jla z7@Qyn;B2Y-r8%T+P~ooFpunaoR;R&c;x;D0f?u*dF62t!L$N7iMX=Q2eP^w=UbrR--K-*N@5WgbLHKf&*~c zVaY|0%4Nz%4#2zmW!(KO41ic48ZW1EW!kl7;esb&EiSKe-#9^U^iD7JR&NwuFQJC; ze(Dy5WH0z?RIJTjTNucI(#6y@V-~bFA1W9iO%tfoS;Q)wGA#-AU386v21D(x zMgx`jf(KVEeHObey*lWJNRVTh~I&fZ$&+mkLJ06FRd=5Ux0)PtbR4+F6w7l4;} zs=64c0RJq9im3ox0=!O8NZjQOw5kI)5SC7GFCAAWH^A~TqQ5#9V4RYWa&ko>t-7M+DAcMW#o0`L2d-`~2lpzDQLW{^ zvIxKbF)ozP1bELI5bYOb@g2)%p(u?m4*)MaDnyc{K_lpuG_z#2!MiuN`KHWsrL7vIF2 zuC^F&Ukmw7i^275^XV(Fvi0J2PUH1HP1S3XHnT43fqckeBR6s9uzxo-e_v!2-0xdN zK&~~zSXOq4IfQO8!w1j|dsKlmFJ~-lz+Dz^0YZRq9;XQBp!FFt~H;(FH zQPDLU!*_1;#Vpe|ncX+W<}yVEfN(otqpr&?2KawN=Z-TokWV&$C$I$^U^^lI_=`{% zQx{XTa!D~_5|r~!i9S*T6RrfiU275m2_)}$>!u8rMr@NeJ9Dsm+jfhyxK%(L1vmj~ zD&%+1_&*EuXOy9HpP*1#V$2SZhs+3{ugD5MWMU_4k-tO^B{&#c zphp%!aXDRwi6_BT5Dg7{ICR zzXg*7(ZNt|QzeQQ=m9T{Z)9S)0i1??qk4HBF`zj!IS^hZ2l#qYd1XpkL$k&vz2PMjz{JF2EW{x8ybVB3R_IvYRsCUJrm4l|qJm=m#&R<#rmo4BE zLjt&~9k79f*uYSzt|)vlI)8%dDnOx}-PVD_0)(QsGPY+Vt6o0;)P*9sKm=p>o#cZU z5LY8YJ3!+hWIUU0yf2k~0E4(9ocCafyV^&tjV1yrF=GRP5ip;o8nX_ikpv`!0?FM#i*<#|<{*%n?tlZs_mGXTm(63?Fs97_9oLjc! z%b7QKPEH`fX9uZIw|<>ECY_8uMS(zZV}tLJMNT$(k|P%#ejl`V|Gp&xukt&{x9_*5 zkjr8y^;VezxsH{V2vcm$}5z+@^TO{_8zDy55lez z@vjF0kYb~efOyKh7AGUAkVn9PW3D;=$j6y+?%+-TL?`B%C&wsep+bvHmf*^khAzQ| z89FjFWRobN9K;P&x>=>_!^r^-ExmL-y=okV(+w@H z53nA)4AG7f7eJBE7gPNxMjAV%dFzdHl#ynOP zRaOay89}6wLj{d8s!6kCgBxxd<+MGMoNm>0j?Hl6Oo-fa3ySHY71}^VhCC-oF|t1S zM2~^D)`PFmLlfN>E=9-U_pbpL5O@YgJiy`q;L;BE7gGiu+_clA7{m*#zCtBzR9^28 z06!=oh!F7i= z(A2o35KVGJgqA;kzSc}`zx6gvap5#~+;dkf!3bMcQ~+%8_U!Y~vIh{b-j?#cPhWof zeVbvpKQNW5VM3z`Q3iH!ZdC?MOJKoPn&P&dnrd*Z&Km1+ zWtx^hXwN|9I`U&8S(6GAMv`z{|@;vH{1BNHLn3KRzDu`PNj%pSwKFa;<8Aq6o+!4!y8hY;+K z2?AW9!+6MvzWuFxTln4w@?e$)>H!hxz!f%paEL2;q7X6EoX1{R2clW=SygnQ7P;66 z(@c(Hmcv{QEu_PLeF!(T;0yuTXu#EpU}$G);5g`bt5`rnf>GPu9xvFf47N#w9Q00| zJ_tfXj?gh^8cYIB>skW;#b;58VYpaNn;o=(YG&PJ_G zU$IoDeinMS2KkRgWdo4^3T~4Jcvgy@QSIbCBgQw6@}~)(Tgkw}NzAW|(QeT@nIc@7 zP=n%a8LN0GQjX+_d{h*oIvmpL=x|JAeiWny1OptKsRxp-AO^Z%AO!QM(w6p)IWLXr zVrAOcn7qq_K>1pDcq&eioChkbM4<*i;0K@xMJ7nXNgq(;RYf^q4M6hFpCHqOm}x~p zoTO)`sQO7z?vs`lLnp%M;6ezZOfF#BOe5@-zt)L?2oiunZE91@Ai(h_k?3e#_45Oj zk(P{j)oWf8H^~OF>k21*L`n^NSeBj_JK9Vyoc_a9TNGxpmK~u_5nxVyxr7b8+Ra4U zQjXo|f-rJ$Bx;-gkp^X0fVGYh$3%g|fY{2GZ?*NIJn?C%(%p~|rxU}$asq*7W1L?=QdcXFtm2@I>agc@Sv6BjKmb!9x=GSFzddEF~BlQzfMd4gz#z2-{MTjazl z78>i&hD?;iGnq8@AefNylzB0#0jX+n$xBHTXu^`0B-tGJ$cLE6JZ7i7M6wb&fs`lu z1_5L52~4rY1rogF5H`WY!1zzHEL8Vih6=mU$m$OW?6C=M_{Fdxwz z|6WMiX}LIBDo9dGnmYzF(s+P(&9RR4Is%nB6Pj}*?Cq*KWFo&BJLw>EGwx#_TdM@u zx6bvh!x=LF^UzB^AgF>Z$XsS)j~NqcsV}Z=z~%xkVUz2Lv*;r5D9F5kBeVeM0~h2r zsa}ZBeWq}Q7bS;vIfBDyiA8=SQk7o6TOyR^m4Ps#k+Tr79BO#rbZh)+QR5ik++~Cl zMw)78=wj7na-|6Q2mhbuDp=OU9>EQOU52ee5!WeC#W!0Gg#- z5taM@i>yiD_2wclg%}HQ1DYA?x9JjYK-crz0V}5fTr3!r2tepNUFy(9z&{Zj{n8qV z3HWLOI~qj2-&DVG)pKa-zD85)2T!;jyM96SBH1Chki9QrU;BAy`#)#TGG=)lU1QJBrw`8mE zYA*U1D)FA+Q9dgQmW{IDZ9G<>^7=vmt}Fb|!tRX30RZ4OfQ*;YZ|c^M{RDvz*2Uoe zRPQ&UBmRu+TkP-kX3qxCrMC$D`nt=02&;(C#^hnQspkM`)s0HCK24!%iXfTl85Xz=( zeb!}cmWPBy?s+yVsD=;;mCy)}u!jt<0j2NT8sTU_P6D#3A~;|z2C31witi2p3(u+X zaEjhAB1yim8=y|=!f*@&&Y=vb2c!TMG+}m5v39sl{&=Sb<1mA6;#`=9PHYAU%*Gba zM&wc=0HNm(v8O@C%C_|E7Z(trmJo2RDyl9|CO&`$a^h$_Am>n_4;Jjr96%ZWg#;Z? z3>v?$X_%~^nnpfc%$e*e3_lGNLou&hXv)=>WFQCt@59^XrBu&k#Z(Ld>LIt8?oN^KmcakTY(ievjDD?~>>yn3DK#pwS2Jne#yhbknAxtCh$SVuR z#k!#j<}xcw(j>VO5N1PkmMU6EMZ=(Gz1n2{%1dfC$PLjF%1RFZ%;J#RQY!orAbT<} z=@Ktj=q^{2Kh{9qW}q*Dt0+d~mF5XI=Q09Ri@%}?9hzV<%kf{f(lH$qX?EtH%Hc~Q zNJd!A{>t*mI1{E83pB}P%0yGr+-1Jr5-tZtj9v~rGcq_;GbkerVqEhz14JckDbPMZ zJzF3(g>eD!0l;nr5fEVO2*aKMVi>=%Wb#LB+llGdIWc6HwtW zn}S!$;fwgw6H(AVt>|T>u_>+yI3z(8HS@3(RQ9w1A(AxK*h|Xb#1^1I4dhGmE{nb% z4?ZTsC+t9&VC*aFE-27YD56boM0AH*lQvg~0MJ98{6bB|;zb2B`7RPGada^o(?4Nj z5n^;@_OugL4LF8W9(N}}jlto@OwH8HCwxz`7L_Jw!W#=iC3KH1-OVs4q~z$K?Tif* z-i_IKrOlx2P510frOHfA)F>W63VZ#|jxIreTedHNiDw8zvY3{#FxM2P#uD8b`nbvgC$vR7bD& zW8YP4gB3|5w=)~`Yk8*j7E~e1)(#&9J$|r6w(&=u$^1k&X_UrvWz!&m65z&aTVHfT zMzwEUH5uAxm{6cMOzrA2$0oYK)Mi0fWHb@YFEr>N8Ti14)+jNxl4BXSaUB;}i4<~? zbaJuga;w)|YEN@FmqGzh06%~Pbb|yMWd&we3kHh?!l4FafPsjPBvV%qi%PZ}2}58v zcE2JQdh85yp?s}iW;CIh02h_;HyHN!euwvd!-3#-B^ju8c^is(v({LiH+rjPdcD?y ztrvTjZ6BA1dx6S(mBs)_u>n?r1gPL|32F-eM!>IT_hMy>4La}d2(lQ@Aaj_ux4@G_ zq@WuGo4UA{7f-jhivDXF6#ymWi7j1?QMwomt zfRhCw5k0w%<;~eOx1aCmpXBA zsQYe6h51=|UD==y+KX9lf)~1>IrE{Zw{pSO5-`aCJ@`UPAc@oW42h(40e5{vnuSyn zu2U}=zCjhNs+z|Eio7EzJgg%Bj0hhe+!Hxpj7x1vZI!gaaG4) zIkTT+vpL%{$JT4B^Pwl9A*m+_H;b5?sglxLy^Vn2iaLvTTBn0LvXLT~T>HEDz!_%2 z_>K`YCc^Ppbzll4%~*A`tCJqXLHIKJ&Y&cOc!wxpdJC ziWI}aXuG(?XW5!+d7($PvsWAx$oR$iLqggXZYeN)Mv0OtNnT3CXlBCFP&XjsptfiC zWV$t_|MqVeVPS4=eph!?p8y5;@xW2xnAj8nGAgX|(%`)jT>4)c)MO#cMCD#rj}0YG_eMSO4oWp_4wcd)ZWbCm778)MS(;U~grT#ASvnjreCxH!sof&x> zNC3+M9ODH4&W!TFV>0)et7frq%*-qZTrBYES)S={51t2@{)9n2EWD>~W&4{o0Fxjc z7we44DgAo}plGiT!iA3wRFq#5VV8WMfC>Afw_4qm;%9 zP(-!aTE7jQ3IcxY3qkCep522iNPFSdvs$a${JVQSCLW^V#~VN3sBoSQr}|{?t0Ls@ z9Q>FH-eBkr{Hqe%w?Trb+CAgo);|Tl*$^)Oa%9Bv&R|YGq=mda;zI7F0jNX(qKch9 z3m!Ola3YEg6$#~d*r6jL#EF8&srdloTP{9UtT+odA_B8yAw|>*24zGcahhWMD0woa z1Srn`3X#whP6mvSf;P@t^%K$)pm=75!&B;sly0kv{!E6Gk5Ndl7LmI8Ya6m>)T}|v z1~#lWa$~ibEsKpCTD5GaZG8(jE-+qn>$0+YH?Q7GdsCU3WT}YY4umrpu7JV70K@|Z z*s!H0PbSHE(nP*&IS&vEX&4arY@soR#2*km#4+)RBqT=$x5hywTp>LgdGIKdD|anC zsC&FNbi+j!WxAsDxex&ahtbC0Ae!+)gTx!u#b^nD9e`-XiUAy0H9|FEJIty)qhZjX zWKY4G0sAn0`hr9(H$2hR{-Y-FB0+|@txOjZM9>L)-|+EYZdrsxQ%g8eL4*$y(1!>A z6iFa{8fWmf=`+%*7Rbw{l6i6ft&N7w~@ScZrv zvA~zwMDu}TMS``YP?Cfa7}WxNIB_b)ee;aKq$h5N5yd=7)-c9)Jhd< zSR#joP!VENub>#pExl;fVp}f4_!YA=K5HY3xZLPtwK(<|7?43eKwt%qDT8GH6bB?Q zD7k_vaDkL*G?53B=0rAyknu$XoR@;pL7Z%n_|O^=jYy?nnyX=8lpcgM2SH>ItYC|Q zO?)vKDV#w7#*fN{ned&f+8H%L5h2eiZ!%Rsh#Vw!K2eo)6ag$VHh9|ty|63U;F zBx-udMg}gpg|OekU0HxGU46d_0o0t*bO%M^73BMMoDScJee(!kk#F zir0Mab=TsL?{&0ipI>d-@@ZgD1_Ubr#5nUPDL383q{BAfUyeVpoaVmv5BvgaJWeHaxtcR3P*c7(_3K|Dl72+@Q25 zmcoR+x*Dw>VL~igrF&TeU-^~=#xROeEn>3~`gUZtyi7uUYjc4A`;McAc?3vq0;!`w zDj^UXI4OSrgOcB%P{6@qqB1s!;9%-yL)R!xb!rOE67Z(K!eo!6z+E)QSa7KQiA(su*h{uKSrEcCtQZxK$ zIY4e;Z6X*XEifRR_^{G}U81MGXec;*Ixv%%{F2m&r-v-i5N03a1SvLYP%zYRWejbo zJ_p)LUG{RKdkAJlFN)D(YIGUM3L7$&xlCq~jcjR5-!ZiRMHsfdFACfo*JC0{w>#p3 zoP1P9lLYv;K)O*OO<zQ5PCZ%VKq)TIH%|4bjzN9kWJa z9V=N$de&;Gb+D#!<4WPmiYg?4uC)DTCG?R7y6F`H=op|n+laB&&a1G8J)}_)s|bTm zthoCe13({3g$PmtsE3^AWv^S+&BCm+p9O7by=zSWu!gm?VmvKs(V9LsvXfv`YAGvd zJKNdrj~hDLUriCi*PONy0ajuyy^cFs2?AJ=O$~5?6}!(wnjuUz7(ok+kPC4{ca_Vn zE_N@RS*voEyWRb+S2-%)@tW4W=k-x)SG(Rzlz^qd*k2{!E8qQ{BQNDdM=)-x1^u#? zTa_t~e|NA#Ep15&up};H4QxmR6LrD4*hy2AhdLxLma*Bb@`h8{Veo3%tGDo;7o|ZX z5s%oTaw$eLQwvVC{RPEqV{Z&P=mK8MRlf3_F*&2O+umYH0QuahWun8v83+)Da%e$> z*wutPObI;%K0^|jK+r=TI)qInLMdtBgDHpq5Yw23$bvIv0Chl$zZ7S1>Odbm1%S(7 z$P`*CpBZL#tFH=FEN9un8}&tqNlV@^lNiltW%D&%ONjQO1Oo4kac%8vE+E+WoC?6H zK-bAyMCCDp>6!;3s?-?8I0|kSfu|xLlV3t`VQ@gEWHTUhhai_56G}}38or$eT|`P3 z)eOQSB&-stO(7$^NkS+p?4>COC(;P4!|A5_gDEJymamQ|hiARzS`XvP)xb4*&1{Ta z^E$oiZL^ApAV)n4#Mtp&BVB8JW8V5UJ+joUL{uO(gK2vL7h?eC=3#9Yq?+6KF^6_D zF*$IZAx-~Vsw1Es5pG9n9={L*9A04&3G5_^|DtAHyP7!;_l|E11An+&xDZ3m+9u$|=32~so84EW=W||sy!LEmppyXD z924>@N2mxp4l_AJ-9E^dfqa@XdB-tJgZZDqXw!>3-NH?L5QI~ze$cB|)>5@wDyi|w zfe<|wXF(Xds+~?`S(}~KXs@`%)xK+tmpJZlKPgJtn$0N4!1M5}?a8yP0RN?7F#=q{ z!5bcELV&~Y4Pc0$Y+m%^L>n?ju-tQk^mX&g;QSFWO=ebL0Yn^~^_huG)77NZQb9or zRnr1$0s(LqwlW2yRQV@lKvRHWf=gtF|9fS(dlDyJ8TWg-27JONffs0jvaoS*S8r+&ZVlTGZlE;=U>6mxCPAsL2|QW-EM zmeCBS1QC9A8AIqsDa9_7aWd=VT$|DYg76ZZfO^!WNCGD`?4gD1fd>*`DI&OqTKE#0 z)p8DKQMAToy*Ghk=5`o3aT%zA8#qpDv_`%bSc$eSh-U#Ra2ZU}cYl~{E?|Az0U8VO z0(ul_m?Q!r)LKH~KKtW?5u_dyU;+K(11-o?E1+zc=K$iuB$&W??DI%7Cp({}65=2Z z7E*<(M|l4fhBXlZij!25=Lw+$|9kr<6N>Y2iBKT3uqY5`2_QyszSk9TF=lP}hQs%E z#P=6#WPEDl0c;&?H2_S|H#UvNc;bG&7+lg-7v!R@gw+1#>Nt0#d*!5LPHEgBvkG z8#&-s?O}_)5du^26akYs+~qG}CmXbah7h=h6j+SVf{e*{l4jM6==FBKW^Qq04&H|@ zaH2<~b1w3=hvcGod;@}rkPq)d949c37s8?~s3n*jj}^O#F=kCW*n(AShy!fh;t z2|`(v9e`=HpklfBA7F`SotA?Sz?9ksZSTmGNBL{*;~gXjBnW{3P0|=PbOyw6m!$_R ztGEL*fR{h9fXdc|j?)xq*^q1L3|*Lzcu)gm@G9Y9Dw^<$_>mF|#7R%^2_9KBx(Ske zNeSH}j3Zf+Aq9u$d6I9J7t>@?i6kAigojPQ5XdnaPhyid>4({dlUk>IH&|5(VQrS9 zpV4PIM_@-2O=~YfgfD?Z_4?K2G>B!28A-= z5xUbGpkxFk0W=_5dvZXX)rma0$4gEFFx_c2-wB?%Fr<(S~V}~oDtTATk@jRS$d#U zky5~gkzz3I!9!7`L{mWpLJ=FelN2GTqsd5;Y*?OPah~8qm~O|C9Jon8CS(s$5TnR9 zPue7~I&;QR{~D`mWkC=}I~XN~$TxHnc@-duG@vBj#|#;OiGMNx=s<)@Sf2@y38KJS zeXtn}kp}Z;r}{zzL$r$9nSdwJ4KN_62uTGCxty^`DUh0w4MsHA$2hG*0yR(r6rd~L z={))-J=zo^H%d%=*{S4taX|{O>UpYvVWgdjes>6h39GO%hysu2f<@(DZRrWl!IOBy zrAy~BAP}sX&>Wb9k6{XmJf*DBR;@dt4)Zq{LmET(7YCNt4S+-w9J(i{5N<~#H4du* zD{vI=Iw7N$l_%N~%!wdyDV_82Dy8>YSkglO@*o*11vQF2ILfbn=_3AGwWW%+r7E!K z^ zudf1NyCfYkhlLOS*sKH?v8!1!}D0~pFL8GW|lDlYYtG34{d#D-WIJflj z2b|$fIGD0kzy|uT75BGE#}K%Ii$CFe50Ehu<l0k46o)e#H@g%i0u@xjx|}+mKH8&V#iI}`#-?h!YDU3etGhjc!DI=*!8-!W z2dU9jj>S8|KZse=0iQO&1`CmUgP}*Hc+Prjp!S8~1waUD0!dHS7 zA5Fl+e~<>P)CV6;3yD|+kKs50CuKKL9y4G?zczK&;}) zqG=zl2+{ED8wgl{Mt~cuPyO>5 za8kk83H+)85Z5kgD+$SEUGZwh60Jv&$&4X(XczqA}a1O`735CrDVX!}3fN6c| ztx&!kk7mveqvXHMJDDnm7$60nO~5eF$@{i{myqcMrQ!puW}!1B0EW%SkfQ zwxABrtT$iLgJ>+nH!Qf40O{APw?Fxlf}8G?esYJ%GO`h!OOmb1v2Un;5|dej`L$ z&<8zO!%l}MZ|p-Uc(tvN=~ADEXoxV6X&|V(5<*L4juDy7ax`FL_oEJlGk^xU>2HR^lT90-h1weuITKZ5AJ@=t}qFezQ=p_ zGc!XVm!SHgJ_Y^Ju#(eLfJ`voAS>gl1^G}OhO9@MCp7krbJA)DbyEdDi2|XP#jNZ^ z6c1v3?>cgS@fok{TQg>3<@XTmOwTm(&?p_mPWa<*?1G-V3(%aNHyP`HFgre`a$La{ zeEw{O51<&>8oM_2zv8w~^Z+qQ-nn!7^c|CQ|4gC2bMo>%%y;nMF>Y*JunBW>{M|fGoLaPnyeJ%N$U4 zof>sBbA2$(pc1-+hM*-s-@bu6GL3u9p!u$Fn*lvrdC)b>@LAwJ=I0P9RM-$=#A600 zowH~$gGC^6W`qOjk|~?*OGJ)XnWBNX|DM@F*jV!PCJUYOF!izHg2oac=ESKniUeDd zB8iQr>?@5AsurQ+s;qor3>UwE8O#zt%t|W|N!SVuFK+4rF)t7I8f?N6DI`p>N>)@c zu}KhmOtLUuWU)rfF1xU^+dvzw7F#U1AV?vBWG#*NVw)|4;>x*0wYmgJsXE+2c~d2S);Ae!hH#!MQ285-g60q^cG0wM>{h> zO+J?Nl(l;I;6CkVA=DeexkZj?2`|IEm9vd144fZ%`tfCaY5BSm{40@Pe{vf16J{EfIM zeSE`8Oup3dOy-{3MIm_ZnJ!I=c)b%BH{k>?f(6pJ_Ra)kS^^WDl2Na}7u*f%k8!j} zbczO=;OPZBT_A*~sQ5@higD&4DJUD5F*uw~q`1lqGdOHDx=-XqkfQ}-3!iCPjO0qNw&K6eEB%})DtpLxrS4)?JNI{5O z;NtkzWt9jB&BNg(ltI)-|4*R!cc`J^+|uwmta`$YkR#NX)x9bs>kLmf6cuI9O9j2; zlv&oa@)lc0Eau5(rrAaug~&Ns6}0GkX9WrvfB~K_u=WD}9uPowI2Tx2=_E0!Vp-&% z+}8->nsLFi40tK1>q(H{M4?@_9((MGMmgrrw3h8>H}AChlR}dOVfgzG)L!84 zw5UAMCH?bHi{cLe@dw}t?A9g!`7ctlzysu%fQCvv$aJL>#fvhhxycRYbEDIX=rFiJ z(q&9BsB28EbTA{fwIN^l_!=j&|ECCs7^ETkP~us_r=6I5q8WW6ToXhfH`s`PG*ZFhYhc5^7TDl^ zx(H6Q3^g;8;YsovOBOk}E=C&R}y;-Kl^n zyW8JIn*f@JMs$?Tx)ZU2G}0#t>0c|5W=D&Fw+g&VBQU)x8;}Ed;JJCKho__3#i9|t`s$xVxTv*FgwaURmdbNXLWn^L` z0a-S<|MQ<`tr5A@8U;@vv;y-3!a@Uw0f+jaj#-eXK+8w9H#qYTFxbr}stX|MYSdOw z1Mf2&t0vhJqb+lIEM#@0UeMkq36s66Y&yHeQEv1HbV%kn_4`yipw@JDnBZ%ndfM{N z)~k0>`;jEQ~Jewboc!#KbAuF26jIZ<_BcKGqaR%81p!@Kr z1{53aeBCGqt^ClAa@6>WFm^B!dJEv_L5Zd#y|1GjkQ>>&7YD~&W|QQ5WG2gTCEZxU zl8c;$JU|T`De>%22ONv3rq;{t+_7s->p%nFv#ScGDp*~#+p%URI})y{KnF@t;u4~* z|L}qA%|60Lz1ejb*OT1NmBPYHW|Xn}BOeqf9tSztTm&SQ&T{HiT2ury9a$gbMO zL>>QlK3_&*Y}MWHymHoPlH2l@8(akpnBB~=El?TsNecp`Kn-?oXpmU`CR#AJ8W}ZH zI(#Ex-MB>0ry@O5cn;l#{o#^4C=ihYt|Iv@I8crTc z=5sW$?5YI6LIQ9QzSC zirSJts0_!;2R;jdDOo`LNS>t_iv0Qpe3-hQIFv?^0B$0Uhsc7O;x)RvHF(M_)55C1 z2(}yakOI>_kK?``3?UH`u?z@5v+%P~IvCW5iqyEj#j7qBD1;=GGWYwu2Ee?VXn~%~ zIn6^7p<{_QN*MQvkpFZ=_);Q#|HxgCQ5-*C0^p}mqrnt~aFHe?U63#R_~gPf`{ zUZ6WW$b+GPf(_sS4(JB^sD$MC2~1F&kXS3HpoJD_s#~)+=&L^J|BEK;<2yv+DwNSd z!22?gTa3j5!tY}*mCG49h$DXa8y2vp1E>L6Q;#PELqT*urm4JZk^uAJIoe1Ua1syw zQ=i*+vDZ+LiF`#Aa6&71LJB}gKKOtr^hYPzN-Agxgxp1=tAVc6 zJRkr@Eig+bB*y<6G_Aa!WCDdwf)^npGEhSYxVbmJMX67JSCNksnU$gFxIV zlmI*YxC8&euW^zCpJ;?U8nDfj!Opxp?UTn~^D@8Sqb@T|r_3!;{DM*ZwAOUZ*MtG7 z%LRi>!f|uLxa`WW6iaMdfk2bR9Rjyi0wSCMMioOejVwJlavUH7inNKr`N~Uy>4+A@ zui4-?f*FrB*i7pjL|Hpbbc=+WYO*JwOa-dUcbv{K|A@i$WY6}5tqkcS>|07sJWbSm z%2>&<2kXn%hb`~iMrh7@xj@MtmP%*gM`1VX^j z3$2AYl&O+nPoz^XF7S@(15qaHC=u1HS}V~X>!}p=0T&(57S*E#n!y>hQ5r2u93{dX z#SjV6xK!agdYn&d7|kNBPy0;G^l~gE)z1S>(ECaloRiXOno>b{MP_0GT6EIfV9m~G z(#HW$ih-sZ=r?Av(rW62C%A#tjDs_!(nP3(nChwYj8hXeGA3=(6-|Q6%2PZ&(5DJ2 zI}NQGJxWT{PD9O{%#nlWi=alO9Qhp5jXBau|MkIKBTeAKR6z2rMCDYHyUb55I4~v0 zDfJG=FxAK6Q9orbcGS_4i<~aFFEjWh$f!}xQYuuvQ5+2dA-Gdjy;C0LOkM?6_e9iA zjj*{xR!5a8F7q!Q|6En&lkAvID;RV{0+RwQNCV}lTN0)~eZ`bau^de~v%VA3Q^D4IG+2GQ)Y7cf z`#cOua#(wP*oZ|yh?UrirPzGU6^zZ+js4f{`%!^4q)h~ogC$u{Trf*T*oDnle6?7N z#S9m5*_d6~nN?VsrCCg^*_Yn&D*&J*saRi#x)rsP28bH+Po#) z!W~?|{aee`T&g`>#EsmZO)$ExTf3c$&y`%$O|#1NTg+wM&2`|{|CQeA72M!e-)lA8!qwjM_1EK7mG2E-@g3Rn)!*(# zU+LA^_HA2z5WJ`**p1jHR7FM zuMG?D?KlR$-=96CyG>vTPGJtNV6qipK6+tS$Y8deVX^(-6;|L9245Y{J{xXc95!DF zj$Uk)DhvK$4+i2KR^r;_;oIfd{qVl|V|xT)+Lhgnvmh^S;4t=vFcvm4#@0mA@=>Ne0bJmf|N)V3tDV zNWNq~KIBbyWljd;Pv$BSmf#Q;W!HTE1mN z7UDjRV_Np*M2_ZR-sEC7W@=rhWX@u3j^b_RVrd5FX{KhqduC5=VsUQdu3hF)=H_({ z=XWmWo+Vf&9_Mq$K}X(NZ&qe^=4W{JXL%M)fJSG5ZeU+F;d$O?XVvF+#$|>!FWdU( zh#u&AzUEkdiq>RUwr6rSX>-PBbuJN<-M)V1=$BUD5guug z?r54m>4~OioDNxO{~h04R_KieXA#!vnzrep_UNHT>YL7KmF{L|9vw2yXreaisa|S= zE@-93>Wc>IY=&u#mg=Oo>5$Iql0IsZ-s!p(W@v3{uU6%)ZtJJ^-kGNAwr+jBR45#r1=VcDpW)_F&5ie(gh;RX?@J$Q|03rDV z1rz}L04x9i002M%Kmh;<{{Zg^v`4VuK!XVDDO{Ma;lYOxBTAe|k)cJ07%OVbc#&hr zkMiJ&3`r8C$CD^csw~N}<;a&SUCM+>vnIWiI5+0p`EDl9n?Q96{Tb6H(W5?#8ePh? zsne(Nh9aFxwW!pqR+DOFinZ!is9(c^b=uWyRx_*)7z^i8F7NjJdLA&X7G51>E_w=+mW9 z2UdOAwB@<3S<8+cdUkEwv~lYO&3kuG->2E#&dr$gaNxvK!$y9b`D^3Oqqh4^J$mo5 z!m~ra9=y5r?a9B3|Cg@ad-d_?Z`V!FoxMBv=HX)mPhUQH`;E)%-;e)1$oc?o7odOl z@z>vh1R@yUfa)>$--7M+HeG=c8t7nx5MEdlHyT34VTTziNFjzHmS|ysAEua&fF6P< z;)uYhb7EyP(pck#Dza!?juV;)Zh-o|GKKIRnmH@p{T0b>aMBAI;yY0&T8wh!sgnmuE_GL?6SNCORTf!J^Lj& z)Jkiuv9(rpJyCi^V6=zLr6z4$_VC%XLh zyRSC@``d4h;i_x!x&kNMuDjbVD)5c@mJ_kW69@cnH|1iC4Ko`D+%d-=gS;^<)No9) z$k3dOvdS#4{7lL(yIiu&+0<;!%{U+YvBw1KtnkkZGaT=(0_VK5#z-E$F~#91^9|EZ zHxqR-Qb#?D$+TRpb=F*?5{uVghYj{8Vuwxk*=VP&_9v0B&B)tr$1OM9b4RjB-HzCe zciwXU|Dty5!ayep(W=D&&bHs2^L_W{pqIVM z=%i;|3)N4XGtTO)x9|=vndndT(uKVt~H{$#6z$ZfY@WBT!hu(D@pS<$O zgYNwEi#+mt9MV@$Jss9VpZzP`!}5zI-gn@PQ8m0t%!c!3nOQf)-4{1y!KI z4swtLAG81nN9aKkp1_1CETIZlXagwBpo1=qp$xMi!{ur3hTF@b{dUO1Q`CSb!;SYOHOdjr-u|KULFo2%Og%-CM3@#QB9Gb{P>?#NW3S_VZ zXq?~>WWWM8w$Y7mOko<&hypXxagKJpBOdoyM>*!vjDVa#AVnxhLTXS7!n0u_70JjO zLQ#kE8=@qQh{T2&DHhfn0~4OWNg6`23!VfeC>!B5*^%yxUi2dCG%*e>u96UixI`=q zApPVj5GJ$V}!jm&r#o7P5~c6a*foNzG~=!kX8V zrZ%_P$mOl!iA{9k87BFQN?P(p8r>?Gz;x=q87F%Cw%c%&AVD)=E#T5{su}Wi1NfQv#B1mb+{~ z04QJuG7Mn?2|Z|3i7C-sK2)K3ysB0q%1jfW(W?)Qs8=Cy09-N^pFmBeTJxF3wr(+u zVNBjhQ94(;Uay>$n$)ohWUEiW^qp|C!|^=l2O73h3aBvUCGe05EC$1M&bVhPVM*DK zvM~SvC;=JX;DiDcprHpff)hq7+R}m*w5MI82~rzc(;{@WsEutvVOs*$|Jt^p6ga?V zMXS^Xuu%_P#3)-ep-KaywYXi3tTON*SL$i9u!Jq_H>aCYBVL1F8p4adcJ)$my08v? zfSVr#OWyL<6qKd(+5)TDPu02C9L*?18hi;?j>7Vd0LUyxjSzs{lD4T{0I+}s?Aiho z_zF!O!huWsU)9Q%wzn-XY6;-q2>{>#Z6rf53L47+TA>CWHiCyi%m>jDRlvcuLIHEZ zhvsr&4S6#~`SK=R>ZWw1<~-+*sP#RV2F|>b2(pmPn}s{57n@qxXFit9WF{*E6Z4gU z7%Q7m7KkvvQr1HZ2*6YZFBriD_Hu*4oM173xy%x7t(w!^THTh`|H7X=>k#(Y#Tq2p z8S?e0inq9DCkW8aD{e7K*N`|FS3+7r=N~rVrg+ z#FB2k)C?mynoR0^d?U*CrED#3SpY1zp$#(NvYF3}W(AXZ%(TXJt=YWhUb{B84ZW}+ zHSA|TGnNm*$mofi`m~RJJ{c zH_0px?Eoy%PxZ)B& zE_0tChwC267_P{+Be-q9o1IA_^7@pxg;(iITkKKieYXNTdO(~rdA>=ecfHk6(Jmw0 zz$X5%(vQw&f-l|R{!V<*hsji|uNApyjl6NgAq6oIdQsO1`y@tAT}xQw>}Ltq*ri9x zw!<77DrZwAT-b?rzd^}%&qovpp$5HEnX(!LdRaYd;<472@PG%t@Q1GSf;%(deP;oJ zM+ZR$BH8)SXD>ZF(Mf42O!I)VMCZNTXq4wiUp)qI|GEziilw;W5S9XLoovRPm=%^p#YRH4iZ8fB%Pc{WgOD z=xeRx3BT3?G1Y?z_(?wqPC!UKM>IL-f^FG0fnL`xA3`sCwOz1qJ)OV{l}2rjk}+6_ zg(64?8CQbH7C^B#4?4GkIwf&ePy!RMI!vZzGB|Jt7GO3ga5Z>?yQBxC_G>|yhjqn( zCsl+-7=lQ+Z97yA6{soi0vg+u4H}p`r#B^8|EM(B5DRcqhI9vmYuF5T$OkJ>U-ZO+ zXV?Ii2!n_qWe(?tw^oXQ*I>3bhr<_qt{4NEaEDT*hqP#8dpLo9Xfx$vgw`f*WhX9A z2op^hjEQIs(Gv%V1q^+V3bF8rI|DP&s64bIcdB!VnYf7(M~0OJhY+BKOU8z&xL@Ju zaI9F4=7@MT7>haQZxU3CxA=~INDMRcjJsHmy(nG37&I;QfwE$ZQW!R@K#$NUj|91e z;>Uv27=~kbd$#vq-PmNKNM?v&iftH&t@w(oNOTM)i=?GSM#hIA36Is~2$>c$x_FYi z7;?Tyf%Il8ElO|%V#m@O^oEY|>pV?#4Isf;sM zdO5{jYCtu%ATjU-4os5=*Y z6$P|en@s=&Q4n8wISU@OQ==Ax8|h?ErC@%iS!OhfeurSbMFzvxoW*%s17!urIi1Ow z0}xPu#0gr*Ia|Yw$wrd+Cmrb0vC`5$?%ifC=SUbhX3V_rp8ndTA@xg4rlmh z0z?5mrCHO88UrobO5PcmCYqv#`Jx#Lk#FV&^Uy{4 z5Sf<8fZ@{$V3yP)7crqo~rQsE7DmZr=RSS4kPjAcvL_G#Vh8}> zpaw2$3o#1-S}+d-|FEtFfU^vMvMk#KJqrM~Pzyk-3^Z#FMayORpj$lar!yO0LwmI0 zN@00`LCWe`pQTT#lLj#~nWGei-Qx!R3Pj;kQZQydV3#>|1FBl-3*RQ6P+B?El5QvX zjrzGfM)rkKgRE@|Svjb8LCcA}38qIyIxtu@vF2Hrb#Eq%S!qB30#LFv%K(u(xwm;? zQE*{@V7Z$6xCTJE7m!&%0J;Ex0coJQPd5Mo0J(W^0F+C+60p6U83Zvz-y#nOS};*lBXaGZ40-Fn7mn7pXIWJ z&ij$ki?{gqs~Q(^M0I}@hrL0Im$r8|^W>ZKdIAh=pdfI-=8L}QJHR~bzCOFZ@GAhH zRlojwzW6IaKAgk`P`XRpU;LYp z!6Qg&wy-?WYfmX=bvy+|@TOeIHkWja51Ei;$0ke5010Hnn{T;Zd62I-%mEA_0THkO zh@85g|I2qYy8?|Yy141SnM=9>Pyu;hvVCvOf99S}V%lTe2s>2REC^ z7?254djW`i!!!WPE)dIAP@8GsSEA&=3FtTGX;&2d%Uo={l_bU+T*gwmyyxj&mq5TZ z#8}gtXH}p%&K}!%rhCI%;fCmHMwB;NCL7TcV3rhnK zm;Y-HNNWH~APx+$t_D00GY|nxD-J&E&*%IH0j}3BJ*_|R0TQqQ zuj~U9O#~Kw(KPU*x~zb0sfWP)(Omq?xd?*&=(f=UDaV{H9{j!F)2;P=yKQ%JVO*jr#(x>u}O)4pxNmSY9w3E3L0KXU2~UkYS+A6e~{F1Nqf(G-(Dj%`Avcf+6;H# z-*eXu0p16V<;*VJg)QWcbiL6O!*}1?!spf8A>G%{4c$*F-5Tt{*8Sko3u4~=IyuPL zejC*KnOG>k-m=NrGd04D5YC9?HhyJ1zC&}#^8+)!fiyL4G|oIfzJW9sPv#kbRIsH*0A? zi_3FJ;Ou-F6^fKz>7poHW`5>!i8cJ=$G*C&Yu-D=LpM>_mZcCndtB;tzUno8>f!U| zh92v{d+J;4LW2I_PBaCH)#fks;yx}s5sc^~QRTyK<%~|<&+y@%rfi=6o0jfm+`D8| z<6ddC-#S(4>rL&d6W*b|*+1?(-jnJ~Wa^8M>Z$t~`O@>go;%?;bj&_oGIx z=SY56t*}WX`^_=M%#-;8%uvm^cR{u(M(#a?BE5vhzTrQUGRUsw(16RDq~m*8xP1HU z7mx9L$y0ayX@xtNi#p%mNT>VW?b(z~Ff3lL?(Wf(?(x$DH^B1l|9ma$jqdO zWv*fP@l#VZy5N&F`|OyS5AJlh`wG2woq1->LDet?_FJ;Boo!x4A%cw)S#w^6P!-!7~P53$}S*yqlm3!T$(u zFmp9e_^p2Pg1-ryL_JgR4Wf9Js37f!K;*fD@cgRezhuA({~E}UKj@1eUs?2iF3j{% z|M*lNuwHrfo^LLqU)?Yx@fxj;&T6|B@A{4D>|e8i_FB z!q)AW7x;7`!juQdi8Mx{+)a}wSkXUFwu&L#HBeVk+ooCL*4Xi5c#b1amR#AgI&zpb zXHM67vo_A4p^X+@x|!+Hy_gQ3lFX`_On%2joNd%L|30JTZav5~Az|m!j1<}fqxxpaB|IZCH7;Te6NLdrCxCjauuEV=5m_XA_ zDM*1+PCsSH23|<3k+M@`Om$TpO>GsKU;aGthk7px zk}odB;>fAl{^|y#G6kg#mX91X56<-VnC~Z+?4z^WZNpWiBsjK6!dwxm^2C;;BJ4|# zbxjlj11$es)KQYU^1;bMCzNn14Flj10s}1dw9|u8P}tKI5{{+S%g#YnRT?X%(N&By zCheIW?KKgIOoluXw<4cnR>_ckom5hhq?8f_vF25yIV~(2iIL<4snVQ0=vr5fIG72i zIa`{kQ?Q&4(<)C&uGMoT|Gd2uPjA8S|KaMZqiD9nv(RecN~2b(z^xE2csW9rx)K@> zketLSg_R>+MT&w42w=kw2TnNQ5*qIN@4HHE(M5?VE}UW+FNXM4WQ^Hy>5qf871xsU zNl4_mY%3&FV6D?`%C5_%S&JExHN9xm^S~yNF0w9FF+uw{v zzGX_OgD@a5J(yswxnAL}v9Y$nmbxH!3l=6A3A+}8mRQAdKKK}Dc(VP?NrJj9)f)oz z5#HNx_C3^CRm549KS#z%V`Pm+Xt{`@S&a27SCUn(DCo$Htxjg8ANWXtANvME6w-tQ z(n?bevrDl;VnM;6P9oh{ODrC;|3N!pPdh;oh+58rJE%G8Ucu|#d@S_65X?&i1#3VJ z)u+DXiJ)xHi=My0pe-oiCJG9`#5c5KxrU)mEB?aTzPeB;0zff}>9gSt_qI1yU~FRX zSo%AddQ3%Sr{$Q=!F-?<&DGw8bVTl%^rxdYaR}9HV z!y0n#ik(`E3@b1yCTi;rz!{O&gjHZW%0866ztV#d*56IxQ<5$N)vMsY=w5L^WIkn=-Q;7(h6*ktukv5HPS?G#&yjXW1%N>r*~eG;W;4bc*U zu;`~5;Zs7MvhczR=yC?#NP}_2)XQEH=9Iv!VliDxfj;0UnacEHGv7xHV8Dt;P~co& zN(xhJrl~V+#Em@2QHj3LMvlIU;5fT*rA+xnqE9lGCs47@MEQqaj67rLh5$V$5JClz zgb?T0Lb~B}G@zMOPYS3L@s%)T}-=j|j@g5TSsOHhA-@>`YVwWAoJvJy8P8v1f4dsRWA>XNMSI z09)9qN%c@=fNYc6?ILT&;M~;#5v}Mdxdzx65|*QII1HouI9x>?bP0|nEPB-oQB)eh zvX?#BF~LaFn%3yE|HWw}+QK9fP18AVz?`#)(HMI~I34F8MkQQuSc%kTlDN&SGD>J$ z-|9k3MMSK}?5Kb;*3$ygm4I(N(VG;=q*2tBt`9DtfVARgtpx>3Cz_?4Zp_EKW0;Z( z#EZA&wN%ND{}rU7&_+<=x+QM*m6;$YRVfk-k(m7DuYUpjWf?J3v|}c)BZk7$7ZhhK zV$D*6|AGlu#MyCnsDcYzP=zY=if0;zpi|u@kRkY`q)%8EZF^{Pdpk~dZyK{{VI;1dOye7Gdt3EGAr}PI%$pjVQ7+Tw4 z5VHQ+@xFcZa0NXDuL)4LrCes%@iIF&Fmn-^U?g+1=?4r0Q{)bJxgV1lU;ze$BaWUp zh=uK(0B|S3&Ur?}en9l*yZG5Zdqd|h0;q0@zbuB)uN6IY-FSkO}k=RE1ED!r8v-2h<~jt~001h`wieXy8>pCt2NNd&4VxIgJUU*4eJ&avP&!{4yBj1EwqoiVVv8xz zr3W?HQ`K+n^n028nTB(T#n21`&q|9~WDJh8f)L%{$?gN-MkG8|I?s}aR8h($%4 zrhM3uwcstv~ zt9nJN3lpn=2ELdMzIa67a1VC@M83iX9}$MUaG;H%!5oCLNlZKLYakxXg?bDJNxU6P z3`JQKMP>j!K*GP^w!wOVS$H6`7|4Y#I?Ts=DA0pYbhvQ4Sb#3tb42 zD8K~YvH~MWB3&8Bmf%Nbgdk+Wk3K*xP)GwsfC+xozDnH3?bAVe+{9JzmQM_*Iq1hx z%p;!RfF;157|Z}X>OUQdtfnxehFnH@gGk^LjQ{HsmZU%}v$*5w$cC8!FB?gaJVr4C zI=8GvG*LL7X)dk;x7?d5P8l~niVg*c0ojulShJ%uiNWw1n1YEq+Gv6=Fv|71Keh8p zLjc9ml*)Ld%I#ByOmqgr}w(B*_)Y7+l~^C*qT&Yq>PiLmuU-9^Jru;*i+_5z7%N20_Xh$beSNI{#EVMnSv( zlt=4f7Y6tPGl7a5JCiA(ue{T{j0;pToX+*3AME@~z?9KpXfIkszh+1ye9Mpma3`tD z(FS{i6_B~bd_F_7K^0t${{T5Dr4U+lOVb=b|0JtZ4Vrn7J6dbS0UaQL!a>?xP_K-| zK;Y1-)YVbw)hX>zY}lncML9qmR?iH9kQ2fLdPN$_ltCp_kDSp&JyembR?!>1HN@7? zVpLgNvov5g+!Hs3h*ZQY*Gr2|yID+#LnN|b2Jj(FdUOr2oP|+SA=(Vh_1Xkq9luhY z%~g0S@#_Rw<$+RAIP}=cxw_R_jYa+Z)zP$7UFA>;-(EA9}ONbyYJA2m&;ygK!X= zt2?MwC^=0yJF@A@q8!!lDGRF!K9JK_NFY<;nI7R0hDe~barmU5fCSH>%c%tlujLn7 zsfZ_vSki0-{Zw0sh1u^i$VI@KCeS4!;8?Zf8S$(I5zzUk(o(%ufx-f`Wn3q^Pvsl| z`%+6l^Vk$6S!mS{YDmD&8r;WV48skrY^_nneXNO`xD9ZC2Pof%;=5zHyp}xLmlF{1 z$s;QGu~F30PXIqcKs&d^T=1jqS0mEhO%#Og%Y{O4%>a?t011j^c&qL~ z-TfWjww*ieGv2n%;8DyDDmYNjDN(!inpa`~0gPVGE7=xRL&3$~?QLOZ<5uKdTp8xV z`g$i8Fpf0(6xA(+5paT`k%0+x)7R6mD0LSi1}F_CSl7&jK=e`{p1>$cx*CvINT7n+ zHHZbS1)$a4tlXF06$eNt1$$h$ZlJZ2`(G*Gz5k=2V2BOSmqpVgri1GNg<>fRP3Wrd zt+;d*)Q$98js#o+gHaaFUKbuPXmA-nU^11P;qYbTtT|fJF?Og((46^W$O)+(GWPfg#qKu?x*vv!!!+kYuKU?}Oq54u=@y zfr^ESDHdR?NVvHI%Q5Z*MTiC_cwkX`g8yM21fiLS>6KBF6 zCYoJjKb>JHXwAf{0e?Q|8sLMs6X?@*T7*K@S9pR&3X^X#lQH2{Nl4=Qbq1w=VyWX{ zwsTF_9)ZaYkYApwf{o=dMu{&*3IBSW1zW~q3EgN(zxcCX2}ld2@L4V1zz_a?tV>P3&Ny0 zHQpM8**cVMGT6l6O@SqD&6Dos`}o&!SOtpKT1dEP`|z>Y{omfUnzqXYwn$>S9%&1| z@U~40=y9?i$mj3{Kqx9<6}9G>_FHa7RK+G_a2^dUBc^N)m`Wkvi9_!Qbm#9Fz8gog zE{<=tB-JCh>X1(2&9>hjHq0QNSP2f=+HE*7AaJ>h0|c+#Ww;MvupHJ^a@SaivX<+8 zHcAXnZYiZi`yAFaLc=zw(YIuj(dH->cUiDS)%F?{|eZ_ZQ_4lU?crQO*SC`m!x0=-s zE0g1%S@+6PECeg~l>hQkKTQ?9?B3>&484(L_C4=-SUOK?@AGlCtC1(~B%}Cn6E~b< zBLUhjmyr?;a^EOa3FD1~!w!VW|{I;|7Ftce1QjerB80Q-M-JPfBV0G2*qm# z5GlR}2n*)&;Zee)3_^ohv_K)?fIx%-0VW!#SP=t81rQo&Jiu`R$dE2h0MllYoHuSM zRkB3sG9ApAFK3>7W|CDUoH~i|!sWB4v7kJM4oy?VsD+LmmmVl!(P>2i0Hy|bfWYe2 z4jG8B>Dq^uI&_#4fEY7ym0Hg7-0!7Rmm4%4J!Yn^CIVeEJry)WOYeUS+1VxfCo#qgJzS&B^uM*n>|I zCT;gN?#E1;5NejMTNbXiG7UuKc&?i;dml1i;PAQg2o)}>SW(@&^(bl7-1%Cw<;w6h zWsd(2%{+RV=h5>_qhJ8k`10jXtdBLT2Mzk~kL-`BN)`Q$bQD{jz_k~G3F@VSUvCK( z$0A}lb{u0wz%Zc?87in?H_muc0~8nD1xYuvG>BOz;h3nRA+N0X863N~Hlu2xD3Ti- zt=MRrY>?>&Mh0@=(+L*~GV)1qi0zRakjruKWD8MB`NNa&0ij(Qh-j(HK)(S4)_UNT zH~%J?>}3-snrWJc2_{HYx72)3Jy+jE_wARTR(NKBr&W6*;OABqlr=+<8ycE~2$9%= z#|W!5sElR6=^26umplq7A50u{Xk;C_70fD{E^_H-DbDlPsdaej8H_U0NMmldXlA32 zwaPYHZj8~jq=J-2_p1cI<}s`%0~vd2Gqfmc4YOjBSEjRPnklWC)1KER5Efh+Oj2>q z3E#K)*?DK4Xc4ayg-{W!2!U#%2N`Wxp@$~UgxchxtBx~I zaI>81s9$*aRmh_Z*J9Z`%((h0thco?2gkXxiyUp7_Q0faB~vLKzQ9HYLT?4O_5X5P zqJUykv(Z9(bImzh%T`Up4XdpYY{>v;Qyj_Z6rBT%ORl--s{8cE75b3TM+_)MwV)w= zL_!@jL~Wuc{~kgqGvgEzf-q+DF&QK^G#v0`xro#1YjtB>qsATg7!z~ zL(TV(X3+429F<3THSW3RF5;ma#El0Phja}##GX0aa5(wQBFMtY%!1k7cYiVc#va4& zx5&H#DI~IpT=~YZ^Af9e*RxM3p{Qs4B8|?QyPvtW?PJcK&;J3PEaFNLZU6LB6glJo z>&{h#*Ab9iVgSeCT6Vhq5$$M2uSEc@2W+Ne1AH!FFU&poQIGa!l}{`P6rfYEbKaMtm0h#H6#& z5hD))($;YDr$7ER&^`uePU~D(Ko;sH7!@;zE^LvHd~~adW_$_*_%M$#cJE+I=mB0B zw!u(QVKQ(#jfxsHng&&D4;PD`jb@a>FZf^;ps}8tyCu62Q86_* z5qzmD;~CKy1u-5=mjp2kUZenpWhzrsBkLN3wxEjV?W>PpfDj}iM2Q;JO;(}srmIx= z$Jg8i8;3;Xj}(cZIG~|#eFH*o*c6gCWJ?8?0e>2gG*hWtb+&{LX{O{R;% zqBb%TSfY-K{)-3zC-s3ng&`1v!zD)d;yc{o#<3F$5^vMv;$zhLBP8&OFm%{X_@Zc?ljgxUyIdv^sq{mB zesX?F94J=PL^Glg#fcz+=YN>8z_BuPbpi;W5E!YU^JR#gYyW*KMkxxYv96$@7CGos zwAImw%5{q{T!o2*u?)hn=BHmEr(st^y-iSWr;7C{P(k)WaLd^oJ{QV1}}S%ckK9 zcNWVl6btu|mzBN8BdSzES@kMXJT!y{F!ig)2C+@Xjx({3{plt+(bzfAYK4%UEL~#y zw?&GEG%W~TA^61y`TC2#^#UG#^E(AP@YkQG&1!0ALfql9qD?sILnYF{3I%_K!3x$+ zg7c6Rvf7m+9<>2Vwc}Jangbo>D5lFy5;Nk&HZX(BsQ-1D3l)t3(;^B`!xTTef$K(A zHlc`y-ALisZF;xI@_WHLI;*P%whLTXK^kVCL%B;2Mv9UP`M4!lN1*x24eUE1K`LD zq+_8Anh(3U=_X=#tmEl%^F}{@bPRD6GIXlffUmeTAhbX#xN-WDZn-3=KV=zv#R8yJ zwz9L>sO7VK8O*F!99n8iVWm(71d6KyY}Kr5daRnuYtcpoEAtBo5oTRnm|Jgn$w56k zu-SKgLj;@AgbKQ21}#l5r3Z5{y#?}{iVoSL!~ZR#>K!_T#bzOqZ43)c$AL?^phA?C z2|N-6Ob<2Cv?g+31$xzh47u2M&_<1lQz!AiYQhIEBqyMSGhADJwzaKe9cJ26r6PYS ziyc4_7JTS>j37{&uao7Qrk=qtYF(GRgYEHy@lae&yqInE#BSnqb5 zuszQyr0q7FzlHY|*$(P@)5-1`&b#TW==Zy&;5I(EO?>pGI@CYCEP``AwaC~iSHCXy zX^=f<*n;AJj8X{zO#qGFf(lj6nw4=`$OUV_HE??iaS#BAK$R)^PL*3LBD739j-#u? zAO76zG*>%@bJ-fLjdVDajPjPhe9(cuUH_kUd%WP@`C+{sdP(G*y-DB8Cf1E<7%&XJ z9E2Er<0XY!059tH+AP?#Zfb)U96YB5JMhEap6mE5!NzZKFt1qax!2N8G#FH%vbOMP zBNq$j_R$P@D=6m(V2UmWf}5T4N{QRR;?jA+R|4Zw+7#W~J%7JtcO>pcXTI}6*Y={1 z9=Jy0RP}&J4(oY{>rK$R=@)(#;L~*iy`-FR@gAxL9|T6AQ1F5)ZHp7F9d3c2EWI7l zEMEYb3)6IfRgeo@Nf!H^phW>7KLL>UDc*Is9YjDt4~?MfV4h(aR>`=O4(=efmDlDe zn&)xM=b@MWB|(i$T4k|YI}yPleE-nB@j-}K!8hca)WM4~!5$tYAa!LS#WdjVb)DBO zkyeeJ@J-;^ft69X35GSr+s)D}F%1o7(MuVc&u!ih4h9pnpaD$Sl`J0k{n?8}!Hm(E z3*BIBn1M1}0y8v$yIcY)IN~Gz!I2QjiH#m(Nz?jeqV?UyAmAS*lwJ{9&mbsaLeNg6 zFhmMu)7sdc06NUS)eF>7-6|&HzTm_4H=#Vj0BspAw;xcop1mghM>)9 z(Hve|-oRgYSswP}A+wF&FPhF8Lcog6;HEIf>CJ@)C;=kyQWFqCiSSH0gat8(k%={y zD)5&(LI^e?SslD16rhF}*#9H#Jb^ya&boEu(R~x9mEu$#j|RcZ+E7M8c44F_BxOXx zGEkW<-lFg2qAmuX{Xo&s{Nm~8)x>Qed{l)C@CgY*#xj;9Gg=?%8KO(oPIv4Uh}Jrj(+hJWBA8ijy@NGh|a$m7PRRq}h?7S)OHAJrOHOQAfs-FoK*CAY%>+kV&3o z4;lgwK87=Vk(NXr?l>F+^dW`)S(dC(GpNCEM9>@U3MQu4jD@2G2|yMs3NO_c>6nAp z;N=k*!U)aVy@3b|kpDnviY5w_=4hTKX{P4dEW$RTmkTjg3K<#ZrB8A!1Yi*fB;2NM z?j~=dLN+~PSc>IXQe;I+pjWL@B|#U9F_c>}#Y1s{TvDD6)TLeGMR{3|7eSI=R!vL- z(?$Xl>CD_3G2*TW-2iC-v^4_;NG4^155Qbz7kmI9WFj++05LuX2hPVV_)YscV>q24 zxhX{DF$sYd=n@(zf-Z@INb-a z4dli3twP)7C#0eg1Qdd3*x}u{U+k3PYcA-NcB-e!pOsQ1mToDja_N@<1yJhher><*%QbMSfssmCB z1eOUpw*S^}*(-517$zVeb8cW@GFV7roE`MtZoy249qc{C*2ob;CLD;cw#%^MiLts^ zEX5ii28f=t)E(~N7=$Ua4uHOfPa3!uEoiHLNI)k%sy|!fOIVdQ&;v9OtWC1r&N5#?7uN}+N0)bS>o!nsp(3Y4oo}_U-j*Ds_HXf_L z%Kt}B_Cfa9KufxmdE(rRp=^7u<%n&{jQzkYsBGN62cGWP1f0#_j8jjwMx^EKrt0ng z1@GRPo8J!9sQ&FbaGV_+ZdNewmc42$EgxFg05J&e)2`gbF)le=W;llA(G?4*tz2_~ zD^!fEVZuy4EG8p;7wRs6*?MVZj;yrR=X$k(qt5ON7J%$}ZXwXEA>J*cnVatd@9?%O zsQSV-KqT@481SJ)S3K|o+Z~22Tuo*RnMPlm^(zA{?!~b|6~w~EahwRVQ3Vw0^w|Qg zNsujEZ7x*|WP;Iztt`t{0Ag~O1dWmUo-PiIK{;f?A56#lhL69rZvCQyLy;6U#{V!0 zI9Fa}N(osi6hrH4^dq_+=$4vX<2EyX5Z81DmQD{tQOe(|}BP+)$=VkU&iKhjirnG z)YkDIhS2G}gZbjp%0+fKBW{S3Y zG`G=cA-C&CGigW*?#Tbtp20wb&ouweqE@J>#=Z=krCiZwu_I zGVN-12=+YYi%HICVI8!V+~`whiEsQWU8F7_+^G^@Npo@YFX=Dtn*Z(}i5qyO-mHMM zS(md&yC3nIau*ZlO1E|~xb|CnDbP@+5-*Mhn~~Dk$A+$HttR79_qBQ*^3BcI(iFE< zP&JLESH_&@ph_k>$mHX|mP$F($y)V_=_zz&^+e~@kXjdzDfTC7Ti%rQS)cYOsCB%e zv|GdVsk%w`Lhsd3K+;x`UfVL8{mv=Zv%H^y(xZ!!finm27Co7OefEExLtUUBqF%x9{&YWAxv7i)B`jM9}i=M z!(a?@^L_%(Gg=}*-W(%QF^GqF0i*XVru2yyg(dTgNXXiq(l-D^#Q>=3NcNSDuM2+j zc8xRRoaaTN(sTC}hem_<>3JzF$N?f_+_X)t=xvzzz#<=Q!pvs4L|AgY&SVoC%62n0UZLZ8E*oVHGH zGr5yP>PDg>O~P%_II)~!qL3X*&5E*@v+Ib1Qi+!(r=z%Qv$7;f*w)&%(!xni$2hr+ zOU3~gHi(?J1DA`cDkJnVNhZpC5w*H&lC~0FE!TM*W&hSyL;!}}!Jk60O16|EZid2PTGOtdJ z2RB&8eFZrD2EGnQIwln@vM|I-zz8%HNW45jONOAQx|8sCt!rK9ziKYX*RL~Aq4hb0^ZE1bf`@k@I>1)uSgca#7S z!1n>8He4?o-Q?bp#r&4o6HoSJUV;|HDqdp*BnHQ4d@X-3A=KbI&+l`+(NNWTLdkAG z(MM_}OBerocR?rh^&ES{!y`RD0+rqZnD-v4rvHQpkA?Go{y%U0Q;->aFn|jrYMP}! z4tH=@ltoGShAyZ9X}|^z(1P+rfkcGF!*2QYrasB13iq9=g&Ly=`!fE zTriQ@^x~b`uRa3E@$^bJFZsa^0K^=Dia3or^GjDITLc0aaCm?K0|G$uDXbVFS(Xn$ zko4gAF-b=s%@8$X0HC2VbANPnJR*tUs9U21esOtI!v+UW6qZzG%M(yr%RttFSm0q& zq)U}DZHi%59FI4##5`gX4kS=`u(4Ccab(DoZB(&Q0dr*t7cyh^3=wqbrYIXG zNB{!0oj%OSb!&!=_3L}hE~C!i0=MHRc?hL>BPZw8mve5wBB_Rm`J1(K@PdFLR18KMJ~GR!YePoLKG3O5eFmeFvV0{ zk;K1NBxH#RRCvP{6=Yn&0@QNcZ~s5i1~`p|oE{jGIs_ab^0o|=RMLedoz&3>5317u z6xui|?Z@O^z)THvE|G2n>!>Osj5{g|@`2PCArq^CcA<&9DUWoIPWRS8vWW&9no-aE z{QMIssJ8NMi!Eje2Pi1Q`6E61<{9TWW~jL2jwv*x^bS!zS?v=u41w^|P&=6gk~+S5 zEhvSaD`%NVI1J=N6HlC#RuqxB71vyWAtuFMcV#6H%PbfI5g>Myh(6_dj3@xo81SkC zA*G#`S|X_husb%%l#+vP7ibpT(zcACg#$4N=gu9p9dq68g1ZT(nBXKd&S>XUqrH4F zu>sGcm<3eXKLw?PrZ-W^XaAgWmYBtsHk^=wjD!Y~ITKtO|?6AEnSf+X5E#n4h>%`a2etCTR?SZWV1r?Q= zK&3s1A592@Nlk7JlSww_w3LuTnQP=hMvdf&AF^1FZMNm;5N26rp7~~7dD)zEoO!bm zVnlY_ELn#by83jY!3EN3Yh9lj_NLucS8UpCKN5GU7a;IIY;CWdU8>?OTaq{jN}I^F z>l~7Y2%vx)V1Q>S^#6E@<>=$_7*p1Tq$L*6q9?5>Zsal5KxrfnP}NV+ap#-?2Yz$j zx8;|@E4TdeS~KSrXPZ0kLk1SGPmPYHoYkRDb)^Zz(ik|q-#svF=6Rj8APBtR8P82- z6Hf7vN1gOQE?Bz(P8Kpmfw(E_0DTILo(@2QLU^TY?CPDAbmssqpejfk5rGgu;-Cn< zB?KFQ0Tgyqi(niPJGJsewj%TI?KH zL9h|QTR{v#w`c$Z9+uI9^a|eaVu-;G5`i~C(IXN+;e%B)hi53Tm<*awu}Lus4}9pE z78r>KLnd-kjsIK)6*@wY1%d^WU5U>FZb%aLVbX^`1Y#G!R|q7&Phv`hg(dC>#r%1r zHBw}e6#EdY%{c5vTP$D~O?NdAOl^W>grgaurY;T~q7a%ugdW(F%w$I5X)s{M*W!4= zHf7L`t@GnDKj@yIfI(HBm_qJw_YyY%2n^*E!wkP>9&8OLBzWkCLg>J|H-5weHcZA( zx`4(8l|ht0904-jhf0I0l9jzG(JNciP;$tUiV>w^EjP2XlyT-~q|@R?4QMn3L?Dd9 ztY*}Zv_<7qz?~yN7j;H@K{{%1IMw-JzS7gN5Xh5zP++M$v6(h>?!kA|6OSBkk_m*( zqY*^VB>z5zxEn%92%cjJ<@&$?gGM;alp44~%eJf*|OC zParG`MZ=n3fk23htkjz6bqU$PgV-&+KC zdH*+qN=a&mr4_=2{=gckgyEl46(~V{qKmCY%o#)RDq_IeTZevlF~1FNS=+;z6<}gt znlb5$m0L6)xo(1|QBB7{`xE(n{_$&1|0zdNxE#oAb|N-b;+_tc?+-}ui6OV}++aBVUV zddgC`s+H8}u!k38t6#xpmWkmoixz__6w{bjC_X{-Ml%2q2rHj528Ezw6IydvCJnrp zX=P}P2#n28T?k-ALb8(|BLBn_=S}jGUFu|4KN-qWmYO;m919^_?*dwr(kGDn+5a8- z^BZ~ClbYo)Nb44IkhZ92G1<(+H%m33TmT}4PdPtuc%rs>-g9n4jH}iltPdYG&637F z?#?t8-56ywQ#{&OJ22y@@s{@qj#lmjRH0+g*|CzVWSJnR&U7Y@0zLwZ-j;6g)lMD{ zlZjfL2+-{}9iTylKK+5}ekP+CcOJ1DW#S8*;EK%v4k{$2YVJ82fRZ8UHJAx2{yg zGYPf6tGwmW7Q|~~p5d80F|3%=20Q414t!L>=aGlH7>WM$ACXuXy$FjDj*kP^N^q%Kx0uZ&uT+nWe|NaFjxaUW6267u2 z{{s7Z00BtfN0hf2@PUu^%L@;w!`qypnXAP1x!?Wreg84K@%PWtb0z3uP4h^{H!YaK zezn)~{k55Hj9f70Gu1Whh1%hooumY_$lihyPT?Ak>=q8bjD$3FubX@?jaysQNCda2%Y zPrl~rxynhdKrR1JPna~IDu(Ldu7dX9i>RWE0Gozt5^(p5YLKo(irLr%CJeB3 zy5my-aP|bS_J}aCE@c_Tdt_(2;4bLzQsiX5fxcMq7E%(d@X@uF?F(J78}SGcTrn%@fQzn2=DO=j1Uk(;C!;{0mv*^ z(gYoR!}B-*cR25?GVl>a?R+rJxT+vMpuu~-Cl5R(!8DQCw2>1dgg^|8M!qo|uP^a% z?upLP9Mupga)bijzyc8HfZj1CgXy{Eksfhz0QE5deNiVv;RtER2k2l45JXCkO| z)A7&Ch%`@;f*o2*bt?|FB*(r3#2*AOGbj5+o%SR&7dbl7g@W2Ytt0 zIAXrsi6gG3W1wj(E3+%t2UPU$acHa=kO&`=;bXMzEOQGbOS99Sg2jd`4(F0C?J{cI zL>GfF9|2GlmT8&-aRcNa$85kg4-@~)NGPw&K-vfsosG1#a1%Sy3qQ_!oQe_Zz=GD(HYLY34}C^1MF3osghh2CO!IQ2XrLS@#n4KE9t6}fb5ut$(=CW-?-1!i>_;x7Q=x{mBw^`D z3u7($lt{aibMjzU2nb47%xV%a>&TNyG7!bYQYa#|)Ao%Qxs=sT)PmBaS(rx=aANs9 zYAR&(Oog+|?h85J)UtL|P8A1lRsmKS23A+ea743D`{Ga4As^JC3c!#1n$%DaP1A%` zSQ}<~(uZ>5)mCp+FR~LJjtUbbFX;NUUx|jVF7JyJ5#K)V^B|At zWX$mzCWpASTVK;E9u@%MGe4C|gO(u^4iSvfl~e_hT?^De;Z zL>{JfU;Pzk|Me9GHvJf&^O_E4!z&j3QBu?340Lcy`S4rs(F0Ut2$taqE(<0W!2&t9 zV?CBXxAFr+mSokZY7b{F>eXKHwa*#@HR_kZm1-B4aU0ih1r>6=K|BA{7m{~!Dc4EMYWpq_P+(F@%QJW4?qV7# zA|HuuuCcI4bam@dbtP6d$0l#nl@4;jg>qp8en7iIU<@qI1Budov$86K=n1AkiPmUT zgl~=dmz|Tv(ZM;f0&S89HGEB;XTp zxE7>f7E<62c-ViT?{|TBfU#CjBh-memIaYCfftyHo0R(=cyry+xpHu>(1T|tf&@~a z5Lgd~!?7c>}jj}wWLSc!voc%RsSj~5MPp^8I!`*b#oWs!Pw zP+D&4i-%Z@$ygVjX9Nx<7}PijGvH9h0Tbz{dT0S0dV^uR#Rj^EH!K4ek4TTh!czo9 z6$rO`)x}(rGImX+hyOz$hs9VNY&3?gzzb@ik|)^;On{P&7;i|RhI0T>P=LTL8Iv`6 ziJO>sA$L$mOq4}A0bsHnX z7APQv2EM?OyyJTY8L9=^xT6}XEjWni7&5~oTVsnk3VU$ZmD$vYr419PC0nSsn~96s zs1pyV5jd2|3LPhb2N}pwYbd}3+6g+Ek>8t<{koZF_moD^ki(%4U{^eH?b?bZa%w~@ zeLKGvE(Gd;1jH*`65LYy7NFkm4jOZ9P@1*Ple#}Pv9obi1$d~5nq>{x&%_(V$-9Bg z`zmPh2LF>TawG}0$si2%ND5K_3SwNw-*=?&caLIQhgTs4^!o_n);BT(~Qg3%Df1`;#@B zbF7&A&Ks4drKG|b&f~lU2#L;FfC5CjA$!+$CtQ{Hs6ukQ8{D)2qvU~7Sw!#3yH+5{ zlQH*BEKIN4Rh927Ep&1XFWnp3Sn?ng*BU>dL?Lf`~GO9Od^m|6P>LV%YKsuYrdAfzd_ zQOOLR26Z>cgzF_Irkzm(*3pH;TycX26EVn*wxCt zBa&voe#wW3b-r$PmqlfS#nIr(_U8`kG} zOrf$pRzd|bv})pR>yR(VnZw<`j1Y(8%A9iTAx;hn%=2iWg}NtoH)cqJk7Vy57>aYG2f}o z%O!m#;49F&R)9+J1sNj(N1(gLqCnh3n0(hu35NZn0eaQ4DyMh1D>9z%IsRQNj@OmY z@}{wma>C@lw(x;OXFA(yUQZmPV0SW1v|~UVAZwrY8H7zm4*&u~id^5IjF_d2Pl_7J z!Yyp*1<#Zee3EHeg%Q$-I2NU>O4dn=poVxp3M(0Im7tG;V7UwxsMj`W*8iy0you8r zH*MK^^7N_mr_g7}iW)s?j2AGaOmR7V+LV>lsZ^;>Iu_AatY9WvjyN+OuJsl?_`Z3)jJt2yf-VLrXYk1e5uIG_JAZ8TCV~FtO2Y*MFhv_1a>f0d4T&t6B`#GfB@p6bLh_! zST2y`jLbqR&Q^U*$w9&o#vGt9t}!Az4C|V@WUuH1MMoIlgGpYsh~q3g1R>RV0=`qq ziX>+-bN@;e%w;o)=G+NjP(uL~V1P&wlT?DmLaX7k%R#9gTT184{XhARw>+lS};3gjr9-c4JpcrWSx&tZnI469?$e@D}R&uH-6XMvJ15G+0zycL`s7yJ{!zU>N^1u3$KmRbz3 zriBP(3TI8HB$Tzt}TL@@F9$`Zok(spuU}_l9m;WbAJUd6ZmwiCT1z38h zS^z4bvcVBw#zG6dLUiD&nRzs$OEY$O^%-YHwE3oydPsrY4`R@1XF}wOFaisFfRjsh z4j0N^c(o*houM4Z!2}p+gbeA&@FgRRM~m^%D9VxK2h%MwT@&h2pmr+YrvTQ>^HQZg zDB-Dnq=Krdn}y~otiPpof->_YbU=qvZ%B32zM7?m4YUGL$P=y(r^C6*h9I9N(#8^k zY*QfA7#OlyhM5fznPrO>GNeJs0eE{785CbAVlHKkkRSkRhG(f(mL$MSnIxDgAw;3V z(E?2rwb`=92dq|zikn_3yCQ*pl-`nC)3JMY?ZtUCB z0RZ5w0`lu=Ah09nppXTHWkFGlY>5Z{7C3-WFig;JnwmO@u8`c3T2xpV**;+`A+XGA z6lqV#N=P4xkmnEzc~}=JmBTH-0eDjq#tzdXhEnJ{8p zYp3~2g9@l=i($D%n?=hes%#LLFYb$9<$Qw=z6Q-Kph*PoG{o4hg`+S$js_5;kq2-? zBMxwtqWIBI3bBs@Gq|Sf@i=CdDjbiRF4&HYhB$cP;-$~r6wh; zVK)Ix{v;?T!<;E&Ynn_`F%zegn1mnLU>RtBjcXGi#M7eoMPg)=sM$0I89|^eb6}#W za^$I0?FqlIS@o(FysAU_K(HH38$O9|`-lV&vOZetxwQ~$_D@ut=b+NvtV(=t8DSM$ikuw0-Gt$p#ERnUhG zoI=Lbu6A}ZU}Ji_(xd@IC${fo)oinRG}@X~wh7kl=FHG9AVt_bEijDZ=GoxfE_cJu z{i}zgJJRV^_qrt3E)s?LUGP5Ero$kWizPcnW*!5x=uNK(-o@UaGC;KR1t)!_q7NCk zptD=t?;2YhFIS-^l*p2;Kc5*-1v?n7P>!;0BeB*jQ#hWty#*F9so^k7vcq--v6)3Y z;u4!!v7<`yF^?J9IFom>MXNGT^Yn)tZ-^CN>@j_jAtrAsV5vst+ByHrq$SrVes-R# zvd-#YF5h;p7tU%$7qurUJpW|NwMDL00%1Wgb9l_7=Ie(Ai{><|IlF5{v76tF6YGEr?jeHui#Q8+h8K7PE6nZEA_0In}CObzIq8jaSPQ)+~Opsia8fTWdDg3fXl~ zE!XF7_ITKn@UgaTRRAEgPYp~??;6Ac3J$D^1`l84xID^;-N->nTv)V|Bj<54L-eDO z16+YaejGQct`N5)*MV^)?hD2i-FHs6&EdIjcDq~6Cw8^G>0R&WH7LeJpPrqgiM7w( zw?p6jwXl;~Y}Fq7HUGxicgA5GF!ugxH5=;IAIPGMaaAb{PXbXB?vU3oV4yF){xbed z#Bt=}tq?5CgIhnLa%nkZ22$``U@aYrP@M+q0I%LqH!K*c<3?;k|&W=2l`r7FsS=EgRf6(3X)?J6-49- zKHXg8j4$kA+wD?c_*}fxki7dZZ*TSIJM;SfZwPd)^wT!|i{30f1N1v}PK8cQ)>}me z1baaN4bUZI5&s!7!WoP(PQw=(bP$1C^j8Q#0}<#K-KPOuFiv{FH=q^jt6g%$9|Ouf3rq^MKfbHW@8`HExGqz zqPKtG(*}Og7B;YOX+wY-Haq`wfO9by3%D&Fr%n(j7SXl_C?XbN*eVY|1m`mswjf77 zuucGkIbZgC{bDyQBLz3Gf`I3CF31cpxK+d8HUy));~y0dGSD(%6jTF@qDuWF^OViE;>y05XoJh?2;SHPwyah?A2zjyp+- zl~`GoRfI;EeQ>BExxykPk}KfjjtHoVRl_xJ7;fcuG?;jE0Je(%2ay>NP6uEnHby?f zQvU^Y1z;`^0A+LrodJO-xESKF5PsK@0C_-$zvISRilGTH@O>n}5Y8ZySvM_4KoBNy zm))_F_s0i?@H-L*SRs}>i9jhGF*4_ujfmABKe%EmCRsTti9Bg*nfa5KSaw6mKP zgNx;eKX;jyxt{IWljL|6-uDLs)LwjHlyotgX1t$4FBam~AurYqv4ex8f-JK4Y4XGob&U z?i4>{NA`vZX#Rs@lo|NDSMjEQ#2veg< zs`FQC*K-U7>J`VbG+q>spv5H$00DSYin&O01UP&HgiZRnJ|!|v+}QyFpr8nAJ_ji^ z7U~YsmH}0OI0?WE0%;{qd6wuR4j1zSmryGrYE>eTsW9jqzjz%qdY9gc1KZxMlxUZQ+S{v?;+m zAt|-yH;-AHXF4_XfK=K)uVm0Ca#0Yqq>a189;W0_GN? z__jx~rsyT9s92?9Xr;W+3~^~a=Kv39IX^BiZ2E8vt}qSP*9&?#3kQ{biMzPnAw%TB z5RRLti=#0Q@h0DDo$-4S4WPL@`w>6OLMeEHIrM$CP*Ah*1bw$2l=A;WFJ>7eB}VNAixl}w*jLY z3#Yf~(4b?O2Ks=#^+02xhQs=R!}cJIMKHdX;6mExoPN5vs{;dqz{Jq;xY%I@_94Fx z$g`=_zII7Fd&RlhnG3Q*m^A=0BYlJ8=~;sRz>l6tQ-Ef~Y9c9TyzP1PHQ;he_vHPlbE}M)hI7 z_LJv8A?9)nmW;y{EX4R=E9q=}P4IDy+l3lyt0F#M>y)tZ9)u0zr!iF(}Q`JR=HYk7!6$ zt> zER0|t(s+X~94X&Pjn6ENr_=)uV4Z$AmmW+7?7=50LlOYJ%R{=r2@R@0R}-H^h)M(Gnn6@ZMlJt>f7YUxZSW1oU z2`sdI-{;xiMnoq#+6OHPrhVEE48dEy+N?dhU%mgfAP5TMR*6RugGqNnZOxcnP*Lo&kZQ3^jt5QvwC!QvV4M-H zgCF~?#{NB-S1sTYOyHVV%>Px)G_Vj(p3G~W&r6&nA;C)Z1JH>8&=K{gweZ^wIa=Xj z4*x>OcQE1-$gp&0n)J$_-+5WY*V_c5Y8k zuvfq+aDv&%0tkaJPgNY_s_M-?1aHl;9=d;m!J@gpfKd1fbLrK%&pN_U7X zM8FpqUhQlU7JVB9aII|5yEhYLw<>fiBT~H+N6tPx(}JDuAs*uTzzNwsWGsEsZ#yU& z72DO(0t--sLc4b3I!dNvtpgwA6Ept~AW9JPeeAqgO;aE(`5wRo4KnE=3Nzyhz2FEU zs8v%9@mQ_v?n&{oTZtDRRc!E+dv6C(D#+FRHQpu=`bn#^TKHmh2T(9`9#8VAO2tW* z8&#lMtym6Apjmp`iVAyYJ#5K8Ki$_Y^xiw^WXA^8u8$$x4Hp9!!3|Eku>hnz@N<#) z8S=KrL5IYy1t^SEevtzBn`x)4aTQ)C?~3F?gh-3v2y8DCYu_Ks;7o6CSh;R967NKH zU-4Zn$7Rlue9!%sRpd+Vpk5&bQ*Zd_II9_w;9xbp!2Vy1Z6eU%g}Sj0JaHtN70K{$ z4nBPDML*5}aSmTTg7}UdJUIV}jf;hAGQ5xzfmt{z7$QXJ!%!mwW+8xt(?lW#6=pV; zxR~Jr$hrRcxL(VQ!v;nNCp5Y=+U!$!7VeOv&XesU}EuoGua-YZ)$I zvZfkCR_vN7OKj$ZF|;SFBtw(7UBjA9?OL|h+P-y5?b+P9!?a?xdl&CsUVW*u0vuTI zV8W9W7CE9r!{Wt$3_pf!*s&k49TYRh?7%?;C>U*h#zaVToEotPpCp6_;oU2kPv5cWez-LZgK|N}P82!J zDAz|p(^rGhNPwN zXetht>Xqv%hfh8aCEy_G@L%_9Z z?vV!`<;X$^Ip_+SZn|@LX^4#s>`35_9~ucK1YA zWF(zvxx)^P4g`=X6vV-gQ3HPi=Z%*N=|m8x*65{0S4@#`MPNL146FMKuhu#2SN9S5k$&Q6V>-)Jy^?v`{7!2yn6>T4~vff&tYwc;E+S z{^*FuCV9LAA1ZU}vNuSJ!i2al2fF8+7|VPvy6LDJ6}vYX;I2;2+Cxc${rGI*&jv6M zR6G6*rGmZ`?4xN=l-SS$K}{=42EU9n>Oa^~)sh(*bFA25Z8Z7YWcf@`fe9`+zyN7~tX59iUNEg&pJ&Ug4|;+_#|Gj& zX$EU_(jYEgg3@7b6okAUXx?Ku+%1d*T)<_!7XHvzqWyM@bI(8QTg9SeIszeswNJgpckBwWna|yp+P$_FQUXtP>bHW1w zB}_Hd#T(R94*dKpW0?ZUr=H*jP>rfom^)QnBxAExk!nKFXjukHCn8&Eu7fkn+^%Be zE7s-gbv!#3)NF@42I!7=Dgcgn(l7%7U}*-#tKIRAcZd8CX*HK&1oZCLjP%h+IY-nU z5|00|3+LD)ItUqsSK8!)q0E3;wWtLL&L{CxGXhNhVCfCtKuU zQvkZtJV=v<#bL3Hl@miA9~2=z#;jx~ql5)LsKIFva)X0Jk5FdPXpHxw>ayY(_#hm~fJi>6s;&g-QS3 z6%>X=Qkp1FLdsH_@-mL zM@<^El$w3GL^Nmd0|~VBJuOWHA7W}oLDV!0P`GJlbUM^Tg#ZS}d{1Y9YR+>y27)KL zCuO9_1+b*7o`XE(RTG)d)2)b9tLtYZ|B0j3WyVMeN`OH-dC(c+!3H%H-Y9kGP}nfS zN1lm-Y)+~MI+nv0mq82{sxZSU@WK|V0*mX`RE;E(Gz>C&td7L@Q=l&O1DD0@WzVU= zCuEifpJnL}D%(uW`hc27g+XdZYgEt@!L=@ZZBs4SN4-#mLXPPiSX{fIQfdEHRxeXo zaD8h}uj121V9gmM#~M&4`KU;H`qx@x2;Ja$F^4Mj84(N`0c>4{2r5`>Mf9q^{#8b| zSnx{c*pS<=V8OkPlx!Ky00z@awg;ZoksD$gg!}gQHLZOuenZgT5de6=2S#vxV=F2U z)>agg!H#Z21y$nGGpkx9ZdV)mE5uMntgr(yS&?;Cg8E2W*0LXU6)FlM%(bDzlfhAf zG=(f=Rt_?bF-B+rMB))I6+Kg;a!2(GuS6EUj-;=D>DmJ{fHt$;z0#AR>`(*?IKlNj z!#f#Sj23y=D=x@RRn<9Mc>)*14R#S%`S{+&h)=y^f91X*OUKkRIvwa>JaSC zRY7m`#4JQ$89zz$jq+0rN_8==asA2_*&|0cB25i8hdHDFeJr*@>rgoOSF|;5=m(Ap zQ#wds${WA|Yv#1mE?DWxJ^+g!PDR_^R;Ztpv2+-mI_A=qxsYqwDuvaY=3G_!ku|#Y zSf94SI;&tmC`Ra!pln(fFL{WD-WYc^DpRZ>S_BgTbT`t47+-)y(u{^IBw<|+j^62BL^C5t_JAL8Hv`vP9DGp(_R1Qsh8bnK13=}{}FVm zS3nU8XyMQ|xPe6Kvn-@sqLQ)bMiY?I4Q2RMNB+AMl}vetRIafoT$r*5K0#RerGX`` z;elwQKs>MG9wNX{fC#r_beEn;;sJY|9_ISxhCFf%! zd*nEhs*9E7^J<~wSuKx3Zfs#K8uq!`u+dZle4>RKIKgQjVF5BnQiwXWxqLJI2}mAV zC!qd0wl&hv*OQkTq1<>0Yj-uFxutY$dYuUl@b2!{Z3C~*V%nf!$?!2%V7CjQrQ~^V zWZHCVidP?<42Eakv6|J3|Gl^x9(*MkKu9Mr5<^2h{|-#7 z2At9S=asRrT2zGbK<9uz`3Yr7hGqdkq-y|FxCC*!gzvBeV(_s+NgTl94|=FQN9hk~ zQ?*ahm2e^$rx72WumfR{zX7W) zJB{nUU>i0K8zGbVpkUi&PhTD(~h^c~b_=f*l31aAj8ng*DU^~mm0tDd&IJAkt>6Wk)muyr- zjzW-#NQpNPkgt=(HJGCPQn0s!5DPQ2Sdo>7GaYpTKVBKaP*V~k6F(Qb18GtZPI0+n zMr=qlOf+tyF%0m+L6eg4z(Ym= zzEVUO_sGWm@qszil1EX2#F0o;djfQ5n5h4gBQbQw0_w3vsRIXe$9Zt2n|i&sBM*9P zN(s?E4$8;h@*rMwFn;VrPyECrq(4(syj4I*j%4@3sQVEq zATX~BiZv=bR1nG#Y#5c;l0P|04$!y#;Y_G}O4N+X9efqO%R~*rN`LI5t^CJ;Y|G0l zlt2R%E$ql@y22||OGN0u+x!++T)erI0d5M;{QF1^2tYj$2?E@=lElk_*%ZUv7KZRc zrSYy=$OlK!Omdn-7x1h7KnWgV1W^AYH1B+aA^D6?=u98LOd%N!IRV7sLCg#fD$BGiMlDQE&I>dEnaBpkJUb$TXwtT# zLq*T?#h_b5ozhLf44{;fCeW&)@*0#ABtm%0&hs=0Ij}G10~mbkg6G4pjQNl#2W zNVU7M&cZUv%7XoL%BEye20DXF1W?y}&6Oe0Dm~CEMNm^+Eu!Kn zGIfFwUC|g-(=}z&4gJkG8Z^2DG_27}65W_LBFPQ-ggXVa_>#vbb%=N4FA9_cM>zu? zWrF01Q91ajL(MgUgE#nWRM-D1H6;s!&XC7FQ_@XkswF+IPqjNV&@U+cwbm3+(y>z6 zOx4-sqh27$Fm2OVrNDn%76xHLJh)XdT?IZ!R49x9aLb|)^6q2{0gvbg;f4>t=1CPck|SAT~a``!FxPa zs{}&9gQ``f*RwgB$&foroW3x#R_6;dB$a|4ZPXNaf=?sZfQ12A^;KYP(@jk+Y707n z9auBKEF9R+AK26wby$~*4Tb&HZ&lc81yyx*%7|ms8FYlGP}!BWI0t$vv~X9nkWzfq zM|q-Gz`I#hEzp15*+u{IS=8COpe41SP0xJwvgk8f8*JC2McJce1e7h*KfSe+rP`$x zg-D26r8QZt#n+UDT9ifFs|{N?1HXKlwVBBo8NJ2QR&}_%9o)MOT*F1&!%bY+Dcr{OM73R8*tDl9wb|J0N4S;Szzag2 zU0ky5T$cS@&kbG09o;PD+tbBd%S~G{qg;4>8Cv7S)XiMF-Q3ze-O}aT*STHXCEeZi z-QPuB)%8BeW!=^Vt`HI4*iByAwcUB8SKft;-i6-j1>U;dnd%MRvsm8CCEm96URv>n znr#c&CExPh-mL$e-3OB1ouyvJZPm`j-uRVY^%cCuI^Pe%-^exI?{!`CRo>?H-<+M_ z_FdcoUSIYtl>wH`HM!rnEnfbGU~LfJ2@c@@WnSSGVEQ#+1cu)YwqBRb-ubOy?(JU5 zec=72V76#r5>DS0zF-gDUB!ao>($^H2I0A7O$UZq7S7)aJK+nXTpkwU7Y^VIwqf>_ z;TR5K;AP+&cH$>?;T|4d{r%x8o>Cni;t?)lBd%fJO=4I0;<%Mz*9l`HHe+9CVl9?p z61HL!rdikJAT?g&Gv4AOhT;O%hsUsEI&N9oP2(K)-#K36Q*Gl2MqWAQxGd)5MgHJD zM&LZYV?O`>V>-s-sT5=>He}5i(9z-K@kM06`(*RwVjG6!QAy)ZEaOOy;y|wCRxadM zhGTeLWi+#3MP_8$HRbp{9`{zGhgCU!LY~zGc&W<}{9GKbGb@uH|9&=Hkufne|~~=HKw$M`{M( zR;Fj?4d+D$=5$_Lyz8xM-Z*WB;5TO5W4>o_KIj(4VQ)TXH2!CRR%eKQ<#z77$%SW4 zPGMO_+lc1pcxC8>{%9O z>7D-;XdrgyOpX4e7Sp)TR0CTW=AX`Tk^i{50HPGqE}>Y`rilIG~2&T6e@ z>Z8uMuLf(1M(LE!s*E-y^sVY4#%iL5U9`UHv^HsC=IWmIYN)nrH@0da_UOD$>#m0D zwRUQ}4(z&~>$}e6dY)>c{%gJ-WWa80#~y5AXyeHCX~Rb8$6f5VHtNNOY|6&zO)Tt* z4(rOcY))oux6W+ErfLlO>~_{{%_i*8rt4xpY+?gt)h21&?rg!1>&Ra1*#>Uf#%N!o zWZlkf*xqX1PHxfm?9g`Z=KfxSW*OrS>C{f_zgF(u)^6x#Zs~sN$v*Ay2Is4+>#b#^$IFZqf$p{+(UxrtbM}YWvP_ zx%Tb#_V2;oW&!tU`nIjTHt-5x@bkv-?QU@RHf#nrYo^ZN>y~f}|LE8@?+j0H6Ax|= zFYFBmY!~O{5JzkhuWr!3Zv{tj_GWPI*6|BcZrdL5AfIs>xA7j&aUBP69$)cM-e_AX zZX#dq)>iWU=5HgPax0eSd1CS*ukp<$@GC#^D!1Zb26O!m@DC5?)fMsrXLICU^701n z^(J#ApJ(+Z>!_A-)L!#8ukksLaw}Ky6K3&j_VYl0aOQ60J(p=d_i{HE^FdI5?;S66 z7%y{4FY79A=_gloM(6b34s$a<@JfetPT?3PW-nsS%#ZhbT*uT&c39%akW!%8W^~ z=1hw=bG{VG66DUFJc0fU`q8IRp+l1%Map!i&ZkhLI<>j9sa2v^vsT@z^rqLZT(yE7 zOIE7cvQn{z?YdU2Td{B3vPDaGF5IqX>*lR1*Y3r*fCK;a%eF7ay?pNyMx2;&PHr>} zFIGI5vS7(mp<2E?xv^)xkac3-jFB_x(VZQaHC_30YS)<=pJx3!HR{^9bHkn;Tea@r zz=I1PUV1lK-o%qD7tF5s^4`v$w{C76x%BARr(f5;oi=yszPnfdw{ASVc<<-mnorMu z`+D-(+oOjrzyA66@Z;N`4<7&i@c_Q(-+$i`SYSH=CP<%x0ncqmRNqlP4&6l01i1{ESs2=*vqi$MD5 z;*2BSNTZ2H8mV87I}RzNI#33=WRp`$2^x||;;5ySTvq9&lU-8zWSC@@x#gKG>X;^z zV46ARmr=SYCz@IQ2pbeHJD_$d>Zh2N>gl8(nyM#xA-YU{AV7Q1V&#v$As33$3&jl0&Vs*t+U0v(0jQ?X=)pOQE>rmiz6w z+FqHix7`x!Zn&hH3ooDY)@yH)=;oU(bePK8?!VxMijKhL5Ipd~2+u2{Huuu&O~VgI zj19yRS8Q>)kY%hf!xT%}vA*kycH6en+PiJSC>NYEGb+b9@yjg7471ELKf{d8*3g2p z&NuhWGc`W{EcDJq7cGs^Im4Wd(n)XZ^wSqZ9rec{i|p^3sbc)}HP~>?bT<`)-LuNd zg!7HrXqTOg+Pbiv$t-ZMBKO>=)NS|Oc!SbOET8oM?f2e)2QGLcgEL|{;fF_(_~ME; z?)c-7I}*7lzZ_0^-<9{hOfP(QzBwT$2v8HQb;sHNI3=$`YQNS1$P?v(s+- z?WHKbH{EnI1NYizi!;tV;0#~9@yHvWOx(=#p7-;+M=w3#)EADpAlGM~2;5xMu^ z(;+_YBl;YuySYt}d9^Ux z1vSyZwt3KP$|JGS%M)Hl94P}WF#l4NJv_8hL@z|C4aa{Mso6# zKMW-(MM=sUB9e!vOo8|!7`|4z@`_b_AQ#7|#WH@e8uIehtls6U1jfM)On8DAgxQ5T z7W0_JoWkx(7`Nal!xI~XW;8YEM`w(~ntCAQ+Zt(sL@v^i-!uX^!x;l|N-~q^Os6{6 zc}{j(a+L6t!X5BHPwdrmp7*q;KJEGc%2vv84PuO=0%_^BeF5w{&`=*S7dk~bIJBV) zHJtMdtk3 zvYj@CKs?nzQjz-fr$FTcPwP2Uq8e2YOcbb6mpV|U0&|N6HRu5$MGb`x(}@jD+(X$g zMJGmKq7~&MMy)y4;%Op}&KPM&LmJkRzK{Y1D1al$5Yv|Cz^*i{sa{!{*G^I(1~+A- z3H~Zr!a@>}Ep5O_NsxnTI#!w;{bMvgc-G2Nwz5ctqZa|nS*I#Bs!}~@f2?8Eh#syA z>2s(Zw%Sl;hIOrEjqO<5#?j0F3bLmt7=i)-AlGb`AOL?&>0HI>1mPMNugD#z3Cg+L ziQ`OX4Rd(7c<%9uSA19Hs+hUPz3z*F%UE6HILK2F zaxRLyXeyl5q8PLSA8Qcl8)LJ`j=%-yOu=a^TUp6nWU?=w4CP#ACmq))t@reHWmdR) zXRNS8e*YWbE%TIwVb1lJXYo}>-xjE6RzO_Y>;=W%RnH!-^PZK>Y-aP>&(DVTZ~01q z3MbOgAe5%Bp*dkZko3%uE-41Led(lTIu)W zTFo+I#rmNXZQg5jooic^Sqq3jGewor;s6l)00+3UoR_^~XP-FPbjGl>|4iL!H@Dhx zy+SY&&f^cCdC^1vzU&7rO=Qca6%tV3!nv2WuxWsf(|%|-9ETBac^i!0^$t3efT7S; zvs%`yfX1hY@z6uKR*)eixR?(PZeUxK3IqW8olk0lW>cKvAO?HJ;~ed>FPH0!jO%r! z$zzv)8@xZ3bT-op?r)s@<;9EzHAo~2S9luKQ0=+T-IMAD>YH+vNP4Ye!9)NHJlj&A zIw27L>p^}S1&PG2pzU1FIdlHl6-V*d^KACgLmRqiKV)#jJ6ZC&{`Kaa=8%s`b5r>O z-8z4-;0G@qRJSkRM^C)c7heZjKmKhOjeIkjkcnGs^Z64{FvKBVg0OQ^?4%ca*s*VV z&>J_DAXdHqPBT1td7kk6M@7F;75?p6m>Xh4U$-U1*TqqX7U)8^)iTBx!Nh|C7!E@u z&0sB!0DQ-nN0Iks%a;cXR(TF~30Q{$bftPG09#kETo~td ze90$u3+Q~$7gmupW8%~RS0F<3;0ePudY*@D9Qc7+*nQq-T!=u07uZ}BKm%Q%f)JI4 z_eDkj)P}#b2)Hyg<#KcP13yi+dtPE{ZPYUMW_YylgRT}S95aZ6_zC!RfJsPjOK4jU z_<$P!7IC?@V;;z1lbD4W$bF-iea8iYcSSs6CIcAA0%|ylYsiMC=!UWcfL{|e>edVN z7kD_BgRycg``3eZ6NoOeib_WehA2-0cWX&l4w>MI3P^PhI5&=XVNw{0$X1CPIEk33 zaeLNzUq@WTm=9J+Trxn4FL;Wm=!T|4h!@j}NmhS&I8}EN5(?vcwg`yCP;$B8ma0XdijfV%tmyt{Mjp>yNMmo?1xq|)&@T51Y!n- z2Z?bGIfhxtebT6u-~?x2h+x>rQpHtNQ7M(46`61NhN3W%NOL+b!^3(HMeWSeW1BiGs;tWR(d82$|9;nRO`Mt*&XQbu55|EY{!*8n^ALx|v? zQV3WV7;#oWpjUuh06GB|7@-{Ipb&bX6dIrpTA&5`lZ)9>!9Y!S(2Y~MS(0g@Yp_4p zi8OWCR9^X&)$*BQ8J;u>p5n4^{*`aOa|f!}lGh1{r-e5?xnSkD1VnVCrgQ-(U<*Q} zn*n)X&2UY~7KtUGV;#i;+!lt_HxD`B1zZ{fBUGg`HbM{pO%^r|TgqMkISv=drW8P? zSLz99I)ytn4vsVrUJ#LEifzHLrC~-_AUZ--_?RRbN5Ak2t$?CS^+vY;q@AO)qC|sk zfMY_yfTwC;tTq{@)-4tG z@UbH#MTJ_f-|AGhLkZxDs9{8z)oGpndJX?JnNI}>X9S~fGOshbju^u?vmiXhXpb=o zu(#!Qt+zp8YL7b^kYE+1LaALKhon)U0sue=$}qQe%dr(U55ScV27m#48vtAi0LnlJ zf{O`ud$>8fw~A%51Yoj-`?#2Jx8?u zR4cn412Xv6Fu1EXyPIoUR(!6h2aZKj-4$uslyD&gcTW0?xg$Z%ONG|8s{3mI1F!@Y zz*}+)0i4^n+groyOS#_L1n$cKo?27^4`@`egssK;{E1pxIUcRbw)`W7&!^h`O9%r% z0Nl*IE}REGtOnzo2XZULbBn#7e7@IPVR_)r18@P~d;g6*pQs?fBuYQ7+{&=r(5swjvK)(5Qpk#O<2juU-PQ~Z(Gk5WRx6shNHGj-*4!#Zhfva}x=|x1r@|aa7BvG(ddM}+q?U&Y zrpnUp+0uNw1dkorQ=GXyebr6SxO@P(nO%iLz0}?C4Gz$?+^e^EAlY{y*&a*Snd`Se zz}kMx)u_zYZwNRmgjKBN*0|kNx$VV<0N1NC%UTvRAOabBrRjM)q2-- zX?H+FFpL50$vbt#*wxbqov_jL1O(y#P($MILf?Qw$L!kUaL*t6+Bt0t74X?Ju*dD4 z4DtN}x+~A&%?Bo%vMEs7D!{R2cFaux;9DI9$^c+dAls6OOn3CYQ7lLs&Cn3u)_xF< zz+K_CEZlazRC!IF``xe^RfVwk+|P|;$qYR*xy)^a-3C5S$lT(y*`A|3$|j`8UzH35 zUJkzIXg2=hk^7n%D`389QFSAXx*Kpteh0PS%%;qZ2p$JB(6xK40*VEyt@(lxzT2Yq z+ZKK2c{$uY%^*2ttvjb~{hF1#PzU1R%nDc(0=b>dq_JxwqJwcVQY)O5X==Ts&K zdQJyU6h3`^U)r-hz*p#u!{j;tI+d;b*|>?*4AA1njpbLqx=t3?TTa(eL(%xE;Tvv# z`C7baZdTfs*lbR)Y#y+V7w30=2mMqV+OfyOo{G|P0r_1 zfJ6e#d`}eVuzcH1p5xM(F>$89E4(RO@lHfz{R+K>WqtLwG>-;XAw8P3xWQM+e>?@uK z7Afegev$85>6WhL!A*$J&XR@4%1-xu<<4NjOP7jyV9pz>dpw&&-aBGulN8nQ<9+U` zUOnGqKe3)bh_eTXdOPs{{s}R^O7|WIAxS^9-tRm=o#OhbQ-FpK)zClC2x@a{Erdd# z3dai1Kn&094G%H1>FioF@xrvI9EAL_D|fQlKHwP5mJmGytkr6`~B=}tR`(?F%L zK)LSj%ODJHpaEzQ@33Ttxt{a+&c!z0^Msn<#`K3w+S(1yYkn@~E#Sr;HMm5|U9g*~?UMp5BV#KteuIDbuuKEX&)jNo z(l!3!WA^u_`}9!%AN5+J`I%pf#!a`N&+%ISp`vS01%6WSz7ezQNdRS=^%{rYtbWvuu3VEC=pB?j9Hk?8~5!_mxURdeB;PW ziWrb)q@cRfGK(3QV;Ua((XCk;H7_bKsPc!_M;bDja1lzYV>3iI^QdXV^o^~g^KRC8 zf>LNqKv}n0^W{UdLd{{QPzg5!H!n&{mYL8g!%GN56Q7(Xe_Mk2kZGTtPz!1_kr5O* zRA0O1TRW@&DoVKzMx017q(h1oRRK>33rC;#Mf>TEOHi$}2>02M`)@j+$zCF9YzgqqPbnC6W3M9uaZ{)&e z!E5Bm%Dfgjxd}d7I#O)03m3D+F^&XVFMBOYK9-8&g3>Zk<1(hoSs{_t zRKp{))gtM`B_4&-&Z2`_d1Q~7czX!A-)e(pI3|5WZXsp{Yi5P!oV&xEV4Rs{hQ89! z%#+uyGNZRKE*g?NBG6Q`JWvpEuRRC|7{CApAOHdr!|oVyJU*+$FTef#gG*2W3xx|o z0~1aEv_bo-^6rEvD9aE-3?+r^PD;}_iO&*GOz~42pzvwErW~_@#wl{-1E0zyJuMPe zGpTT#Pa*-*HZ+QpC^p)Tw9VErg%j4e;7*|qC3PKkkS%gnf`>TLvyNFN?$pDRDbeH!Cpm%=Q`?YI@?of^)ATl@I^`t) zpaHC}rhtU4yXG2&uS4hHeP_w0WA?d)H+*$#MMf+L-j(n>8Y>?y;{fs<3%D*hCc zC1&`9i3q503PzB5WKCpo)N2DBJPHD5DNKgujoDk75VF|b$b{)un?rZw=`=x{RfR`W zUqKha_ADL7Yt`iIyl`d-nc5#_sf9k+990_4S~)1d>H^}-+H2&`6?=u{9cvqH=;`9u zY<~d`INa-cFU?@Z?qQW~457*U?uYfJ4_kO!tUQk;GN3~Q1*Ujo{ufWZ{r-^gKq||p zE`gG0(Zm8+_EBYp?}wINX^ zG9NRH9^OEO5>d-rpBNXN3?QciT(E-4YhDej@Vr1pFKuQ+RP~;BJz88sWGw=X8d#SY z?}4vw;~NK_I#ePKpbrW!a8>}@7XlPw@J{K|-a`5&8Z6;sQ~L`bMjkPXi`gO)T@WI7 zl%OjH=89YQrww${$?l!y@KV8kB5u(NU6a|x|Og>)(~3zJn6phhF8)(*-}giaNr)MMxl zANnt1=mi*W>jLxq`iJ^OGco2!OGdx9QDfl5QXe&66i3ReMy$vG0%`c!W#IFaXz`L- zQ*>Ta?)iieK;V~4c^b~rr-ID_A%;pbsyB(mgjGa`n?y}Q5!0}MsJf7<>3r9@uv$nt z1W~PK8j^`XIuy#Kb+}&rYB}2)!?;=%c?z{_UFmvP>n-C9K~dKWY-^`i-jbH}@LqZC z(UnT{Bq_-npBC!i*vD4ok?Zs4eDO99E@{?xJ&i(qLPUhnYNEf|EG-(!H53BOpj}cU zMQi2wqrfzBwzOSA5TlS=5T%Q^y`?~%e&AIQe^s8^s^%20;>Zn>PQ?+FKtT^sJh)0X z#x|I%b*p#X8W#u|d)@AbXNX40)&epJ%WGlJdk$O>!v*U9l@w!bs^6EC^@PAIn*IuQ zI=DSyM3%EBd^OVsb-qzMEj$29$jsmdZ{kNUdo4+{REKU1q^M}9t&8&DTnEg_j|Y|E zAW0WBd4Bb^GU*+JOuSqeByI#%n=06#8(m0mEyty_F^=(?S4=}xq9u*j3=Rn?Pl%Vj zm2d^ASuqJ4xIooC0cHsiHja==5nPiSuq~)nO*>H*ANlnbxJ>xMCgiuIQ3yZ*)bKvj zrV6z_sHUvC;Db=4SN6WKO{2>eN(W94@OMFRzcMX za6tknzUuYjGH>pM?@;)bY6XXiSv^T1c0A53Wxhv@PtyZ?G=+#YW~Ev#dRh?)0a~M0 zo7#L>#k`A|?Nv;B+8NXdhW-8Gqa(f8F1&Z;%n;BpM12v9QF-J#;UPW%#RnnSV`Z8m z$(C9`>Hh}!j0Ij@wAc9Hsf+~|f(uxnm_%Nf${PTar7VkE90c);kKM5p4CZtk%JBsn z7zqEZZmd^6Q(!seFsnI3oC;|{tDG*79c|20dk}1tN(%IMl-flj^KnqhZOiF^2ne9U zh&TWM0w4qbx;_TK=K2Rd0I8F%8=xDxFh2ADz0dg39K=XD9B8t($+F9sx|Oyxb_=f8 zUJ&IICZUQ1KA}xKG$G^c4F&+AA@PYbV96EQ{==TE-;brAZws*%w!TG+VaMPeR8Ybg zUIZgBL0b`Ux3&G@-~XJ`JVD5(LZ}_kn*}x71)|~up|QOv!Yxez2W2P#*Za8u0009Z z3~;Nx1yC?LS*zwkmM+T^XJe!{04=Wyws)EnJsZ0}NLs!6BK(STZ1#ysgNrIM5A+|q^f<-y(=0sDkjkNe7a4+jixF8wzR5yyG~8 zG-N{!q=QWmJK1XG{W>yn|vA zkHR6so%jP&9DzUlfLhA}ZWJ`B(Yhb3J`l4#r%JhY;kP((s~4QHWNRSjsoIyi21gTaqLuNDtmDCw|k}`9c0S;h>HyDf$LricG4@<%#75T7~B%CbB zzUw%-Lj%My5WRm;kCu}Y$8gLGl6 zR#Bm#$%QB^OQ{-w6M%~Uqv*yt&`C8>v1${soq>bnFhfD>Hu6A80X)XWNvhuP%QOp2 zXN(03U>(CW34cmFa2!myi>K|>5Y~8~Lhulrj6zU4 zO3jKJB@-zR z=)g{cI}H`Y3}w#$Gt>>-08yQ5#@>*Sa6o|#jRQzXDmRq2$*j30c%PaR9B)JgKurP; zrL*PiCTa7YCJ;Z8G{g*wlzIyRGAW$GFWrM0%!!&L0m(H@(?Ao^5fw%hg|KdT0%5QWnuLScA_+TUMrIVu zlneqDRZ%rV0-Z4qXdMM>Wz&0vYNQMR#hBjqWhn>uvX|Rx`R*+Ruq!6r2AOa|;!W9V9+Os6kaFGB` zK(W7)!kPg?og_uqNL#y4MGLN=h8zUdDD_x*a=@Ad#4ck1M!P4!dXHgy5ens^6d;{t zFecwwS)RMvRe%CC!2&EWgpd4K6QPek1Bugkf@`IOiC6_;SR=3+h_zjXY^#cb5Cl=- z%_=N~POzg*5E4kB1B&Sz)zH0W6TDyoo9keW!ci)Z7y$E-~!pv?~j>XeQFeAIb>+u9M<2IWomsv6mUkBWPJbM7#LIG%UMO>U*fAq?V1%px` zjh|Bn-9$VIRLcVE(-K9PTtI>0%{GuBNeEuULZ~9vtWM^gMKz2-kAMT=1z_nFgJ#vv zX2jm?eN0BUTi+?zO|e~bWEabIL7a^){A^bmuHp2(VI1}s(S_fk4OgSp1Q$u4tQ&!- z?Kuoc22CIVJ$PM&j9v@zoeX`YXKPO799j*w0wo@d8UQ*eP`uB}+SonbD)L>{I7aA& z1gNoF|2i1fMp6V=I9Q*H$QY$XjLlw@EaAhQm7(FCRv|C|8(KWWRDTQ2@^zO^wP8fg z;k&W}j7c#^j>Bh+WFBtfBo4~mVPF)30tuMI6jowM4Ia!4yLIf-P>00$%&&}=kMx?FZVu)#vkKY?ca~(Sae`c^1$e1b6gY%@9)Uv8XPS(_RnSLc z{0vaQiD|Tvq^e998B$eF$<#HRA$*otCgzS1uquM+Z@Abk_=V-=%^hT8W+;Uq%%kHt z#sX}dkCvunu2!xcY?KbnHE2yoHJ=U1L&o^Q=K8WJg9@e)Uqa65g!)|cb=P;53mX_} zqE6bqLuwudJ0VW!tTXE*wrZLry%wlmI*^s4I6&3lL9jAtIP~QaxnKwOJPFp||EOhS zx1NX%wOeC+R3b@3j=g21Qfnqg1&}V`Kmbg_F6@+MOfDQ1!MLKxRi>Nn*PFduaK3DD z?&+WQ3UO8tRg1I=dLatZX7?WLyaprJg}7j?;z1)kIwFs0wQV7Q!VGrkz3k+J&EQ)m zDhGyygI!w)KJEdBV{MaY=cW}(o$kPlHj<`p?A3@1H9R=W1P~Si+iNb6b1ngRu19<{ zoPH~%ByV!Y;d#u#vixir>gEWDUr842e1uvZKdqIJ8o}g2koX{z?t$vnnS#b+I^Au) zc3`ga-3;yqI+f`ERm?50Yd8ysZorKxH{LjNmTk~-Du-^NrEuy7=yMC{|3F^4Og%p1 zfvyz)G0n~FQworEpeH-G^NN%4KZ@BP`UHO3k-!PZPGJPaL6yZx&#hHXuMGo-Nu#N7 z%;NmovA%H=USTMBWs8LY?Ytc#*4l}XGey|jA&J(L91b&7Kv4fnMTmwk=r;atb?JU} z+olC1_FJh@-1f|7)FU%BC!Q0(su&Y*IiCx9D0X8H$jwN0WbdK1(W^Yy^P-%S1Kknv zSt3iRX2S|2$*>S(me+|b2w_gEKmGIw`^KirZ~YO7m}{?XbHwf^gW|JNHHiGTB84=jgbwjb(|zqs&IXOjo*0Z4g^!_Vr^we3f81&`li=>r5|{%%}bIl+{`m~ zUNv`YAdL{AjAmN`1+alypKwb;s#f)GK2$n{3hy}C>C9d^|B&x}-;aBDDL3JFRq+k` z7eauG>xs3{drWBid#a08}E-O%7n&-eV7FZDj^0CG(a!_0fPn(G{B&MVFQQ| zCs2f-;KGSY+BSOYNRFGwkRL~qEXh$Mv5l-&@@o0)rOTAiX40(5D-BKy79}!NSm0+u zg8&2!MPT#+QluY*(BR_24l`TjY$UMZ0z`!bRnxdggHD$@S`x4%DT~Cc60~YL%yQ;z zmI$(N_TcCdB2|ht|BVyoche&`ts*B@RjSk`PNOt$ z=G^(S+|YALQ95-C&RNl_(ayM(DPKN0M40@sRV!b&EKI|oDU+5*hz$;gF9>S9_``%M zRxoeQ94F3~*`|X$iBhD=>e;E+wBFtONtu{fsqhK(z)$q&69y>S{;1Lp^5bW)inULl zDp&2>sFDT|B4Yif))s&LhKgIOM8Jd@k;Ea!6HLS<#8MOjH(OJ)S;*H~pSa~j4G|U? zM^wp#^G$AbbdwBY;V2f^iOkIP8I3ir77A&m9Z}$F1oG%ZGl%)`n;iLRn3fy74Y_0p z95{)ULs7O++yEFvF~om_H51G{U4EzCcG!hE|E6?bnhD-`Z2)0Ge>$zV9zyK77vFqG zd5}S!c-pCFg-^C1p^t$MNZTbXNMWKff_8IQ9&E@Mitl*FkA)+8vZI2#r;t+T~WY|zKR8~g~;w( z<~bmjU~{Zy;0A({I^z#uAhyAL!!aZ>ST0IIn+!q%Vy98w9VsyfJ&N!Hk)Q<=T4h3w z(ZzX$T;U)H(LDee@m)@UA%Lo<2P1(`hdV6a;?@VmAV#in?6ZmGE+PXZu7HV6{F$#x zmw}8`k(EC3iaDSYg**U3{~~-l>TS(Sj4(v zDIs9aW2vDI--{4Z8Fa~EWHO(ee5yV{DW1xusS*I4Wg039yV&*YX9Bq2&}Kv=dbVz! zIHlqTjN&)WWwZbl|5;N|qy_|$5pAJ{JXJS@u?$ssW@4P6DQ9NNN45#pkaE*$3a1&k z6FeuYQFW-)R?&!w$^ffV?lheC{(yyYFQ6~wDvJn(_N?xn&rAMwYgf@!lqY!+!>n+8_w zcfVEmFM$11i6{Vgh6H{Jfd{zS*bQJP40r(u`ZG&(h%s5mf{n`brd!_zBS2#j?uG(M zve&8SBAhEA{}5_P6}K{0I8w~tcJmnBmTGglHpXU+1^H4z9+qw~o(ze;F+oKUxyVM= zMIrDI2}WerlR~_&Q}H22tZ1dmR=%=UsElPRvzE2(aj;tPBb8)TSd6xC!IoWnvUX<3 zVn1M=6e1*rrxoykJZaNUj!4-f)zq&u-ljA|0qAVrQKmTFv0r)oO|x*982VW(azt>9BlL}j*bSm&(}N+O{tkpm_nOk)1IatbxcCkHQGmz zhO`^>|9~PVq;bwQ;6nsOvk#&a%TkY#5wX`t$F? z?MFZS(I^4w=N?~0m<^b~28E4P=06ZbtySi*hj%o+Z(h-QVF{ofh%HbyUG%s>AY+9S zI?W$@`p!r#+c9X87NkHmW>Va z?HvopdR&4SDorsryRbsE)!!=^`S}3!+`s%+_C%2?Dr#` zQra#rKRHd{6}q`0n{omzJNR?!v`por* zM?f@TD}O>9`G$}^M8Hno>aD`m<(uopo@Pl-<%owG z*Z}6lPZZ>ye@p<`@m{MTga@46?`0tH)y3&x0ry3QX(Y)G?ND%#7ytt=Y1?cDpp-)z|1{eg<1um~Q6|H!7M1l|qn#Rdow)xm3 z4%!WP-;Z#h5c)`t{Yd8BjZ)PglDSLJ$y+%1-(b*R?x13O0g%0Q+5lpqc3_VV;Kq!1uwS)>6LMWM2Y+_vmt^j#wX#1OLhk1ebm;+e$;%+j!YRkj#`_6&?!@$U*;t+dpV-L`oTXawMPHUs$F-9dk6SSuHb)R~A<{Y-C z2)-wLK4W~&XERPy4_T#b?ptln9)Ip9Eio4&)aC4O6;BYr&s;_6c;s{4oPx&65@1Un zj^y&O;6eaSDYam0nc)e>kSzU2SWsvBS!XTi4N@3pPZ?%+%AvrBXR)=Yi(=>`@rWgM zB05gSQihK{!l!GBAPDZLkD?GFDMpU?LX*X2e(I-I_9rg>|K|!MQACENgsz%hzDHA_ zz9^q6S#RJ`dfq6Ke_i1q~1*sP@GYz!|9q4gYR+_#h*6)&;S$MhRPZ89LlCF!2w)Gp{qo+TA4Onu0`CeRak)CEVe{p_Q;*wtSOjEz|I&4hVm(u)m2!< zp$!TPY&pOSAgoUfi=F01!y-i@)DjmZYh}J@8!SQy1lEjxttDcfZ*(l#e(bd(YPNQ( zEY2czP?$E{T2;WUsm-mbC0tJ|oK`>u%E?;FB>+);72vu6KBP=y;;Vx$3Uu(I-tjB3 zMJ!_>(P8wC(sJp=K97shfz!4@Pu+t?KtX{3{~szGE1yyXi%~#BXf2xpYRx&Ldva{q zn(cZ{(<>0^eX8x+^4s8+8r^PL#nmluVyLI$+L+a>nfYy&U7R}9>89~){xrc`zQz_Y zZp2#DUSgQX@ukd_5S|?DuPSU5FvF(M)F)W&1P0YQDaXVb#MbVlo6Z!Vz3;<-8^^fr z>%#7HD7Q#7Ai+)Q%EEh&Zpnx$$8lYw6?#K;52L$x$)1GKOOhQXu;*Jz6 z`Iax~nph3Li|O_ZAIK5L(kT4GZ=- zwF#9at8sKHd$7T8f@%gr5%ZdkaPo?)qH!9d;VfIlxE!CWWbYBk$4nxC0DLcWXs#OA z11jgK0CWPx&T!wMuNu1v#Uk<|6PgXa4I{Vi*w$|(6S2rzGRZ1&7e=Z}=wg$i6CRFb zc`ls6@=2&B=tgERf#AvlWWiTb|ERHrC`F`9Gdybu}ltLb|!3{vO*VZCb5R@ zdX&ICqlZA%Fy;g@dNkbzG-%{$o3#OPeP*+@PV%B2G22!LCVz8G;GQ@;DMie44U=an zn{tDd-m0UUBFpMSw4`kO$S@x|W z-!+3ZXJczee|9&AHl>PoE=o-j<>v23#6yGdfy$k0^LAYeN}j%RGSW2@Omw3(Tdc%k zKg%%~Cnh6c%Yapafb&=7466nlb26ugHBMuOD%b=#wjo2a4hME=Y?}u{Mr6>hc5gRF zd-g|PGI-bSCX=*9nDm)SX$!u`0T9Jns!~k9H;lWtjLUW4EYMd)08Z2z9AE5y+r`uR z0OmG9-Uai)c)))9|1q*UczHr}0ThB9O>=c$Hii>Og=;uAZ#V6F_;-hQh$mntmpD32 zsB?&5GkC5* zk~??HHg1zcqeqegN`(_⩰c`jun(Sd%qqr!}NXYIwiT-;whYJ(yc-Y8QExaI-gE z*A<`L%0k(N4v`gFaY)_(ft?!*mQr}3RgFQlH&jcBP+?h^?4wC30V_G8;GPk{KG@Mqh7iucSP>~ zCdibr0t+A3Ly5*O1k6fkgt;*B7H_F*Ss&b8YCS>t?XnQ$)(Lo(MFDe1b=rr?G+uB) z`x&iV`1V~tKor2GrHR{={5uMe@2F}cRRo`?0KrBS3yUYX zfWY7|WS)~L69qsqL1rJ0+4P~+_cJqf%@lLyOt?y#JW-s0%774d%`g%3SO`FX!GNE6 zFcDHV4@1EXQ;U?;(NZEsix|0V>^LVRNZGM*oYa64NSY|wS8Gobiw8mnSxD$q1B9qh z_3P8WiR6K}SfoBagctFIP8az^HuK=$UkUlX8gPuL>gWW)1Xo$Glt)@QC6HT&QHw6P zD!lM6yo}ip8PGbMi<)Pk0r5jjN{hmiG#FzH1l!oFY$)X*qd|x39=H+595WDM|Hljv z=utBuL7=ZV<4C(ef!hesZAsmb1Og_4m^lcxhUP1RJQ`g1ASEs3XbuaGJc`aE>WZ}S zffxSpj!iZCKmf;^CX+#h^z6K|sQ0$m^NT6)sU?^@W+CmoD4Y@K8`ZSSWQkQg>mxAF zmdTH+N-eE}nN_-wi6!KIqvexVAhaZ{5G~9Q)l@YskuMKj1%?+^Af;e}707r)g~ud= z5l<#%%m7Rst@CkMG>x^%4E+pZrg4XdXCKLxW&#* z-pcK6fjB4VmEHDW>1daD*T83-c_6LDO&MZ1G#H5z%T7(0CSk*mzqFYp|B?sIT6p1A zy6HxcLk55TTPNU3rz(&R=Of z7EWP{RdxrSd-j%DE`4P(-5D4mR)hoL1DabiNrLX9=)}dstc2oR^W1ZDmVmr=y*BS% zaOprLk9<%OG+!g?b!HZeWfHWdN+Pud8#y>;*bqYw>_opVSj#$J2P2$NE(}8!d1O{g zMw#)HS%KMz6xW?r<|d!L+;W?7)_Lcpnf`p~lu$}nbkBdjvgy=~gF0O5%0yFz?qn|) z&V`=X>1&$tRAUhwreh+%D1vvPVhemDq={+=)uo9g)VyP`P%X_H|Bk52*bjyuRQruz zekZjAmQqbU^>D-&SKRT&PwqoS3ZQg?#a)BKTz8<9Y_8bNJ3pQO{(&BybJy`7=v4PK zkx0#T8iC#40QV*ze1Z?SQb1*PGQkSU?1CA*AR^+>15S+M5vdy;0MnF=h-86B&6J-4j;5$(8YH6Tc&#l2<(WRVxMfnqNBcXTt(Y1YQ)n z1#}`?2i>BWp!2~pIYmr9<4&8-Q#Tl(k!@5O7VDr^9pB|mTh&u(G^1&QeAJ5>KVWG? zHV}b~A@4tYOr#Zwu1#cEY+1jeM3Fcw475w!}G%iLyb0ek?k{L4fw2u%D_(fi>U>uR|Kuhh{_~ zrvch2W(^cUphi*;F&KgjkD9pgk@mEvZR&tU|CCxELN#Te6wJ9|)FFMG=;^@Qtn$*w0l>9D zE#?tIFx2#=EStp{;Nu4WK+m^X$^w!#_SSDA^pBUNul3N`1{t=Kh6xj`QVA^JF4x7t zn^f(|Mg$GIb*eXLo@(xxL^0>}$TQgBFgiFakjJ*j4%$J@1=1Xh)ub?Jw21;{lq8ft zNaVOu5@~c@Y*84;Sb-wMFgD7&3?lZ<1B(bE@)lq%S|9=g26zE~ZlHwmG!NAPyEJc7 z7~dykK|9Y7tGd~J8(S|LVU>h6R@@v;+5&(kCfS?K) ze(AKS{Zd;X05ijOfoG2ECMOq%2cWEx^{IB`VkHa^lg|Ro!@K_kl>* zvad>D8D)}fZc>|i$^CrN1un+V5Xb>yEa8!486NBT_OuSIskN;miEA$F8gbc>2epgc zabbHL4#VKh?jGx`h zWF9b`edO^i4cn_9A9nq`#_S3N|NGqNsP^755u_mHxi@+?DARsSI%zQ!2;JP1y%S-& zUk=vML;sdt$K|Y+3!t~yj+S06$oZ!Wl1r>1_+pC3vhUoeQO%@D*I`}GuP>6;V_)a& zExv2CBbc?aa)sP@-}~NcXSS-b*o)Xhvz&k2x4?Z=^i0!y;uU-igYTAILlD^NF9o_l zcX#BAF5`JS0O!%dyx?2jdC%_^$2U57AA)}#3siwS;6FKJS2U|B2Ai( zaw_2SP-6Vr=K`X@L=q#t|FEqIK8qH1>D6Y?;m}}uGV9(VNdIUrwEWBe(hkf5Pzc9N zpQy?c_6-45i2<9)0bl7%C@%taAPk`J@wSiqx{nG=;%u~|BIJNKv;hR7#Rf!h1UC>q z1i%XguD9641z}JIX|VNN&jxRh3$!BkD6ZlvY0J<3twnH2jB^I7-dt4F9oG2E-4zqB81JZB@ zv&=#I?+5#kEq3w122gTpMGY)R82^#biqV=HF&X1Z=%TUR8jTqrFdEOmd7_WEen1Xv zEQ-FQbflSpfil78fRd;t_b1M>kVNs4%x9RzJe<7!#D7-_DJi${tqAXu@}=! z1}dlSl;i;dKp+XSAh%_(5b}wfaUstD8dR_fGO9|N!CYSPe~z&UZ=?*Mk_H$7B7QC% zPx9er(T}QQI=o&b>u_aaK_hk~~1AA8dAeo~9Bki|X+@(wZ@(<~{W z5X~O3E~6{c|Avc4hzk@V3w69HE430Oxl+lPE4`p+WA<>gU=ovju)t`N7t3l8I}e}& zD#A8T0UJ;*N7FAqXdwycE}t>`5`qRgWiP33d+uPxgwhF5X94I;3myqjS^^*bATfXQ zj~0_FE9DM=OB`WnlITEG06{V@(>Y_ZGIMVTF;jgcWitT^C))BeF#|5Iq({8d5$%#L zweQ_%lL*f0FiI`bOtINeXcA%RVOC-snkf@7A_`0rIOjt+hjR)rY!>`eBD72ret{k* z6FPT@a56Ix8z;CL)FwLv@7an$EITg6CUr8lIC zGb0p6Zi^=`PiKHdGZ#fezcN475n!rh%vP=PUHuEzpYjhT;$GhSn zR(G^c%hDb#ghRYxQuE4OvxAS>c4(R3pQ#6jh^j#!|CdyX6eZr3qMvqPBw;lJY$xc3j!iTwfJkFE$h~ zmJ-<2R=upWEORf|zy(CM`Fg}s{jTjoZdi$R7!R=7P%c^VfgUiy`7q|>#8LV{R9Y$O z#dg+A74`-?B52Db`T9U&^%F5CmRuFHV$)S)8HQu;^vgbWIwxfJZU}3yHftI5YO6rx zR(5Cxsy!PNdPa8C7>e`e6>ZlRT8C0g4VE@n6LqS`M8k7MgP;($sSu{gx&jHx|Hea? z{HhbKZ(#MaX`S|Ip<-Qqv}321*Q$0wwN`TPN%sbgX#A0Wj!z&Zpc@+qaApY#7K=Pv ztcyBTX-l(ikG4pXp^cnD3PvF7{0cz+0}7bn1D31>B$7s$pphVpYidCXt_VN~H_F=7 zViEUgqqayJ_i-V2awC^<0#R;Qj%Afgb4$kK>G|Loj z?^Zlh_jHYf4{9J$Hj4^amzL}+7w&fo?zfikce9cK2K?%N!=aJzAO(hucnx<74>xgB zGGj-DESc92o!4rimx3d=5~#O&!&dx)tpVvx6B+<^VV7d`H-z^~fcthR|9L|c!b7@9 zNCmi0DS9L zKbZFQ6gY8}moht6f~B*9DOhe?;Bw!RNxIWI3b+;;KnnQaTBHDeX-EPdN|I0jg$Za8 zBuNCE!V13t-ZcMYf|`YApq5HxBBsD~8uhL((uwf(Ia``rucR*Z8x9 zn3lrVh{4wiB)NzIm;{^{1$MZ5lh*5~c!3)jB&-;M-!%a9RExE^61KP!Y#C$13y|Dfcv${`KJDu4A? zl>c)1pP4q=z=>x`)=PGNUYx~V?oHT8R}@Nl_U$oMgQnFM;-3JACmb{T)enSbij&$1Z?&sZFi zK^P1HqdX!5FyX|~$pAEgL#J88?1gHc!z2Vnt*9t_M1ZDqcYYh$TQooo1fq{mENA`J zIudiARmeW7ucpZ+cZ;`_``MpWS!!8%CJVZt{oq8g*E$#)cDT2YEg1#es1S~rmL5q8 z1gjvqARgJtgqFc|jzEu3S`8GyO(5W9s5wUO^QCKI1G)f>{~1aY*A_Rbh5o{XB*5`V z46c1eSFvF+VyH-potk=3xkn#ZpmCKtFPo|Z(Gq;7gV97tdZeoM_p?S@H?{e*Q~;yT z&!(3WJ26{#wmE$ww%2(CtUIy3`)a%qd2 zY4^lUk@ug^!WAgHvIjbPGuwhU`>Gk50kFmizS{`u2)w`hR`z$4YZ|=|N{`2gzX+++ z5-5r|v^}vZz2GkbVL%ITE+*^Tuyt^$Qjfjs4wB8*{NP;0Iu4W=IkR20+^kiz+t54 zco8?1sT)W$$-s6|!>_x;x0^kCbzV1*fwCr@vL=kx589HZ zvXEz8Y^6@rjD}GF5|AQL;fhRx!dNg2uLJYk>^p+mj6#`hUI@7to6-w1E;j;zRa#Cx zx)bW)${Cn}Zvm>K8pD^j%fB3DV%aaWd%IIm&w|J>7AlpRMApk~yi#zMIAvw5V#Vmd z7Kcn|4*|bkqMV;XZs}Gir7rw(uD4VA2W({X|1fpJLX|PR_J8W2oQv$Im|UFSR}0wc z4TafJZl*r^b4`fE3qSb|xJH2cMk}275b^^OxH-a^;Sun|-p$|BbBgyBo?U(hqKYg;?IEyISS46IP4U_qM;BBUV@73N%$`l^{* zjIc-Z0eG9)1Diw_plz`0Qe92}Nbvcd6=*m&q%zHfs(B(MAmm%S`kYwGyWpNfq3Zp8%cB}{37&GRx=Kv(-1^u)kx(~)kLoNYJG_J6 zRN%yW+Fqh0(oz?Wc0L^VrUwc_o1vf%|JF-cd=^}%0MP4a(4Sq=(J(P&{!)ntVh0Ec zvK@4P*MEJ!-IpmCQ0RUmIybv%CDbU!n|PzA03oBL-sV zRMZRf!iChjyT0IMnbn7d1@DVuTvkPJBjVFu92X-ET-GtI3>`Ss23QK&p-bI{JrPD= zoA~RnsBa2cDYcO$zXN^cTYmHn{RA_B!WMnDhzD9@sho)bf5AE&T0rtKp!!5P0(MiM zqX=~WcbPQdlY=1#00PDc4pMR%OcpL#sVA$F;c1Ac&Xa<(@Fc4^3n3qfrCzBDCF$3; zYtN`zJDCloHg7Cfy5yGfCCr$}|4P=piPM-)UOao^^2rKR(4l>XRx)bT5|ScJJAB~5 zK>`LFAPiK1Fn|C9OnIVY;tDoTSg|WCq*0L7zzM4mUNAtA!9!dfbdQ9=VmI&76dIK! zV-kYT!CH9a;?Qf?g{@{w{!mRI^+lQ}YB0^1aU*ln2Sl`F4Ilut)wc!!7+8h5j8nx+ zfYEpW8$lYbwh+;pTX#oYDB=o*@L|xIa9!PINNAYiX(4A&q-GRO<+r9>%_U?pSYx>s zf!1>sk{BmELx%A;Eg|k&?)qHlfjMITL^W{q^b7{|OXQ0uFf8QA#c4 zRDupH7#e9?nbV9dDzwFb{{j|LXdw`pM6tqIT3xt;9B?I=8)8izR-Px{@X}a<$?#F2y+K06X${fkwuJVmy9rVK;o#PjCur;q&lgjX~>w0%qgbx@LgoThZN;kLY_?d1jvl)5+N-%WwkYurhb|U zVa~GzOy<=A)HY+zJlFclgDyZUNp{&;HBnq45@RANORp?xySrOu6fdUZ@O=HoY&_R3<@;1H5pe>FGM)FMS zl_McA0s#O7gs3q#2)ma91lS!+l!cmI$U*$YbTl%>j5n}dSDP&^VMmr+$IX0QiPWLQ z@^upCC*?|eAO+O?COzIsp9lt6ivo(zJO}K|A(Hp7I9Mz^ka*nv#M2S*WH3IF>cylu z=ef^?5OksY&*(_^snelOITnyX*RuB{+0CvBN4XvDYycs10O<`fUJjRg+gX&`h1LXz^Ldp$Q8pI$usJRb{aFL7*SqXD; zLKF&U5~Mns%NC@%D7;WB18_mgY*<4ZKn91w&|w0a!NW!jz-0>hk`1tk#3R-3Mkb;q z7vh4;4-hQ_G~lHVt_VyMnTT&N(nY=8xD-Wj;ub{0f+Uz3Ov4=Sd2p;IGOfwQE&j2P zd+4S&H}TDIdhC&nBw?i<35-ZavUCNS#4$|8N(IEsh4T<4WQb9ToXIesLAjmQxQ31i z8~~NTTVV#U_R6BE@k9%yWi-7f%!k%bU@EF8aZ+<4PGpNIihw3IeI?D1GVPkyyk<7F z|H(~ps+1DoOe8tWX-;&iQ<5sHBs^2chaPP*0r8uCPErEEs;2zrMfx~xG% zgy<|mYKVxA38Io%s6?BRH;a}sIyG{rUc3s@X;M?HBrU5DPMT70rgg1`U};NTdQK3+ z)R8isDNT{$&YNNqa4VpvDC6hT1pE}AL3tu613J)DDwRP^YHHME>8plH^`XiHD?{a0 zzrhT~8CrNOWY_0WvWj%Hq9rX!(aP4;wso~{4aDZ+It{sIijj4->s=|SLe$9<2tW{y zC|U7DeF|2e$S9Cu0eV!$veGFJ5Df@|=UF387PFMy=w;VtS-DkoJ1J;`U(~R)|LlU+ zsiiHid9Aoc)T&mst@YqAKK9bNqNKJnWhd!!`?9T+pttp80C0mV*x|b196!nDKha@I zg5D{3MDtM}91Bqhe|Eamy{=TND&D3r!I)AQffSk}vBOmL!sHEWiA-x&^qy6{>s_f^ zTk2l;nnboUjwyXT1*#m|x26WNq;7NS*Qo(`xWmOoQ0;kGfhu^h2aT3vdr6BvkTHzB z99CnR@Wc{5tc_xA<3fA6;=6o7!*qli7nI=}MWp}+E6PkLCcLc?!}GjFtMf;z$>J8X zcF%5&aW&#A-}&D7wmZ&ijhgy0$MClb{spk3Jz0s{jS{&ASTci|D~8E{|6&PGCMEWq zRDm94VjQmX)eQO&qb8`~3?Y4i4RTclF*ijrwYi?GnF55_2u%-WL=u`|CLB~m0L>zX z^LV@Xz5K0i5JnNayF!sfJ0bS#4i*C?_PD(rVipf97V9^Ya^iKiI3rSBJ z!M%gEQi%X>F4JXewsFt^Ofwr&O9+-iP($m8V~$(qCJpm5<{CnZ2?)h#9-`=#zjp+; z56s~dD~b#hT5vZO*~r*yyCMxN5+g!np{xN6Iaf>CWob{F+AQ@mwqq;XV#s*rGOsy~ zyPYzd(#4=t8~0WfaPFTbz=p6YK$0un6(wD|yecS1;c_Q~31r1f|D@KTgcZJUR_BIm zx>&e~PPCY=Cp@WOpNDPqFpg{RF%)Q~g4^Ft>>j*1qH<;)B7!xT(K14-Fv1MVPup=V zuH41oeC^9=Y;&3?ew{kzan23BnRx!8i50EN(1SkYv}h z5PQ>RL9hwTXKEP{Bby~`OhZv0NP^M#cV?z?enfcG)_sbnZSfT#opNpCM}F-Dd58uV zlEGU$*dYMuAokXNMu$6{v=$%oY*1roIMPeub1IC*OH{Fhxv?PAQh!>PfEF52!?5I2-EnYiia48Mu7XHB~c!5M5VH2QdO9Vg=M!OjB?`)kFi_AxC+Z z1(DDRf#@(R*j^vxg5P%%GpLA*cyp8Fc;)8@pVa}Owjl`=17*<`vvGew7=$|G0?ws1 zD==z4(>+P3We#YHsOSN(Qy~D*1apx(orD!pK!C9r|0@UQfa|v;eqe@#KseyQ2ORJ; za>s#Vf&mCI1SH@DYjy?4L?(h^dz_^(gA*@5uo2|}Cq;x>i_j1W^fW5CeIT(6)@F!^ z_&JP-j^tQ_cy)t2hXHp0fLqm95KwwQQX%&^E0^~~q9IuImk(YOAt&%lCDwX4*legM z03a|K35hzfVQz-8RH0^124w?%00IJFX1}&3A0`6cVLWr!ML$qJZDk^S!9yLEFb%LL zSfmHJPy+(($oGcH2T0*~3$9g;hxmd;nTYC`j!1b|_a%uZ z;3Lq{SEkoHK;URv^F;V)Pic{f8uE_>#~4U>|BweH2U4acxipsWvjp&lkgQUK&lQHK z(uyz8kP#3i`zSB5XqR*>XeqXW)l;=SYL6*@&f+nmPAQ$R-yesfki~0!su% zT7!>Rd7Eh=i#D)%RMZs-bz-?Oo8!P)o45cKQhOXGmuq1Fa3q4WSqnC631Ki7SOZ;Glgr{=E-4irR+;3v0M|i+a2O2Rkw1cm z26dnwn^}DeL5P;ng3@4)f`*z0`YEf)|C&u1Z#5YKaZzrZ7=#nrex68W=GAmM-LcR&QsX#)ufg}CRA0w!2aQGsPcaNx6=c z)Q+mCE_*efmIrDFV4G?hMH;f6C>CLw6ms+^l@(Gg5OM+qc^17RWU&DoEBYB<>42}u zNomjv4cHVGc@^c>dlmRy;-eAYi5GikpK&-P<*5tH_#KE-Fig{udhn8X0H4&+nDdDp zEO4YHaCdz0q*{a?jUou$H5dR&|A+%>l(+hVV2YrH_K4<3rgNz-QGuMiIgct}ti)=p zP_%TZ=zm@@r?iz_1d>;#ZAbXNx;(x(;kY02>!Ph&m6GdUSt<@?a zl+jzY=oGZaA?SdMJ4YGh2oHyPB9(9s`fv>TU~Nxv2HoIv6|s@Ww1%-DC*spx^GYW5 zstG2kl0&K-Rm%rCP$vZo|FAt^s^ey)TFaOYqXZ@Q5DGCKX-gQ7LP#q(j#|2E7rw0^JD8)+!UBD($`i4+|G5A_RjshNrM0i>H zrE^=Fxr(5ud9inuv3*OvM6jwp7y(+~1TyI_h;VjOu&gE~zh!DrK9HiL#RX-7xii}< zmZ55kF>*gERHfSs9JLfftN*(Cz`84jr-E~%nGj|U;Iv|so|YVAS$B^lbDESa>ryk}b!!ultfmTjQ zslzkK!!(6kVy4K8Tyk!V0;$@*AJ|#Unu-`OdR1Wu3vepDIdqT8Z!I7cWt5h8d1W+& zqU=0FM(@4EKN!0Cf(|WF)ZoW{hisxmA%6 z#aI;_cPIvVr2iY;fyP^V)f;fU4IPaM!zlOCjZ>OD80}(yK*Jo3!*@%ud5h0=4TdBA zWHf-P=W7L(8FHWM31oL^PNQ*!S(taBNka0QRZ!EqhFrZu1oNQNk{#B-n1wvuD`Q+r z`B1?03=IG1bosE&gEfFM6} z2*Yz?iQNV=`@~O~b`=2GJDqh9X4!8b+1c6wVK7>*mg+@{&w75j)h?87%@vSuch7Y7B8Y~yKizX3!7ALiq37nMK$5Jlr7NjKAt zeLY~}mSM;R0Jy&am)QixHK5xQL>sg=atzO;!1jO-NPP~eTvcok22V8Lf$7he%ZHn+ zquoi+V(BlJ;e2q|uQ$FCy4}ziSIq(hjVT}u!`&#w4F!H|34uW9VSr&N{?>3US91N) z;w#O4>jGKq2c+T$S!}-z3Kev8K$vtf7D8o<`kuBT}U z*Z7vjJ9)#|Zc<3sn;wv70UfRrfTT|@# z=>C_*Vv6rr%*mw1>4!s=%6=J)%eY}Az02y!xfJPH*6J;jv@4MY4XGHIpbcW-0?)|? zM4P}_j>@Rq>%K0u8(##HKGOtY9L#VLa4tJnsgMVa=2M|+V@Pmnq9T#mA(NNt`A`ci z@8HOC>b&`nGf=_TebJ6^?&vNHA5`gvzV3zB?sUuI%-rH2jaSk=ME9QWxJBvmz5lFJ z=-W%cgI<5$PXx={jMuJLD=Pti^ zK4);1$8L$EUMuIe7!n{bg)0usa66K#kohnLCW)%6M?-SplsG?)2|?oFLG-b3^q-%2 zmA>@Y+w{tO6G{p7wnbC*UiFiH^;e(tBR%S0QfoLRJkO zK7<(2)TCZvj$G8BLj#**6e~^i7}BF3HX3haM5*C{g9S5yyaA^sij0{xUz)@8q5_@` zEHnDZ5Y%K!qD5EwK-UbKEq5+D3|*Nt&AMhnQL=$o51zhr$HE)~%N1TevibBK(|0x< z8@46cfOx@(MmRhXA~d_x7OsuEE%+o$aTjkE7lm!u$ajfFttk|2CGKO#u#Io?v}m%D zVpd{Iecmov(u{L1O`RaoLG7rQ!imdpy*>yG;Wj&#{LC?wJGY;0v8_S-osFCDXViip zN1nWRLW0blON1W1BF2+p*ogdyedKMZL|e+xq)dec^5w}t?xQnJ&;OO3ZT%_QUB1c; z_?jt2=k%${sQ1k^)9f3oYJkU@MFaB#UdVy6%d?u%N1_0IW)-P7QPru_YK=0_*?*SM0$jQGC1_<-i2X%IBO0k3puSpb~JUFgc)5 z=E5{U;owrk23-X(31WORD7d1q%!*v7Xkd!O4%?7bUEUxK&i@5795tU1JCy>pA7hav zOJ9t=h8ToCB1w`xbQ7hwX{MyMS}2LcHe1;k>O)IzBO3E6FvHA_qA$ZW>N`N$ydVJp z255HzOmKP=z6S(Ypb8-F0Kraw=|cyaJcv;SAFEcvDT+20CKQ1L^~tIp0}mXqtVu~q zFDSM^NbiymLj_^b3QFV)MeyjdtCmdq-3t@O3b{4qeA1W(tRMN1w?yE>-P3|#!I3c`M! z!mbxa#LJwBSi)e}=W1$GHFG9d&|(=Uz+7Vy9aMs98uFpvNZpfCd(jKEBb#N;ZZ5CYGQ=m$^0-6x6gLLkZT zIY=W38|uiLEA?=P$#Z2aV*^C!Z7E!4Jl7GiqeSnhtBKwVodupK#q_YPil6Bh7UQ!9 z56(^q9xzD;^02-k1nLZOJB+@nU<>Zd0Sj~RNV10W%{d~$oacPz3kgFQXnw$*?;PdN zg<$!Nbz8W)m;!yh*Em9SKnEPH6IAW}3H zToMv27Ak6Pg@eaqdfn-*qOsLrNi42@us@T*7t7wE}2`J3g%Gnl< zctxk^G-FVO;07?*vj^|oK|7BM1~Kr!k~~mqROtznsUlUWQ>`jg0qWF%hSQWYOeNAJ zbi;!>r)m$CXe=#SQE*YG2yCgIE<2h%h5hm{<|%10sjz^RW^)NFB}yW-Fj$!kHdGps z17Q_QQ=9f-c6NZCPQ6MjQFwERLXGB7k80Ir7L~Jh`Yd~-s?X5^bZB2~j%wOc$AuEl zq18w$^QM+owyqT-+_?%z;3`*i_5ZR3RM5l9W_s6|npCtjAkj$x`;-+$&~&D{fdoAE zT-OmIsam`b0`D^>Kcqt)@#w~P!;4$%s4{LEQ6f;*Srj{-L8KnT-Wkfl!8_zQfjv%AA1CI9$@qs>#K!5p*&2ST6m3W0s;3#Z-&H%nh#2pL$C1SDW|H~3(H z(u|wM?WVMf8m)_Ys{jhjC^sGy=mQt1L*I}sOg--sLpR+ViFSgPvU?kEBt`#7(`9J~N zX#h)#m88oA6p(^)tAW>xu1<)$Nt%K{kr3>HmDEdtMXIDbAhnRY4tePYRU5uWC_Wf* zEU`1e4%sx{Q$ZC-jLn)0e26~D(X~x_CXGY8>GPHPdbB9;1TmroH0wADL%gv{ypmfy zr$IT0dJ^QJk^qB0D%_I!qYn1MtSzjzCwLte7=gaizB&#QYckS#r2CNDTKvqvBKY( zMJ>@Yg(EmFEWIy$v=KOfGTcByBD&@xq%~wilsLc|V1df9pcEjYZV1M@z{3SVg*}|4 z+5tp5NSz3PphNl-T46p(V=f|48%^oF`*4I7%z_g+jL+a4##%VUzz3SqCL){yTiY_| z6RAE>z7~*xIV;z%vbTiG`GCtl2`)`=z|HAMNb5)|CO7cCKcMr5#a(Y^RZD}1NHN#^!Z6Hf;&lS zgz~}1Q9?U;+7zvPuht~U*i=HNppYPXqH$+B7g5`cDJ3%_lsoHiM{I zOs$CA&9j_Hpd!vytI*lO->JWrwbKP%&LPa&`=Xigo&F#`pTrgx(GV( z&_jrfXJXG1?I3wNP5mS`BD*ZiVy)U^&$zT99u3m=tPTLxs?EAS3vI<64Nw9-(9ir# z0voUdZP4}`4zlFUE#*=NJr=KtD+0s3@+8wUAkQpdO?gUzE?6g<%%>5R&Q(Lw{HzX9 zvQwbI2p3XMBz4a@b)h(wP73A1{vy;NEz|!T(<;rS(<-G;h@n|LHnf65S^w8SbbGntSohA6w*mH#?+?0|B6Q*nzECo6QEEP1>9-Seecrfm*`3|>72Df2%k))W`=wsf#a^hrmTJ*i@D`jFN{Lto zR^R}xU)rkP0*2qeqac8R}=2v54K?M&0qq?U>EjVz|~$#RpA7# z;OHG;mBm>SPT?B<;J$?|7}ntU{aZbe;Vbpw^&Mak7Fr$_VH95C$YWui4Pqf4;uyx_ zBSvDu<>Dz`;V8c05dPmD=3f4F;-!*PyHqN0qCSndoU_kC;hnnLszT-u%V~N^ZjsLY`C(L5`C1lne+)Pg7N7iI$ z^0>yrsPyEWI|TkPUhvY<>X#= z`E@xkUXJQ`UX?AC8p5%0{W_)hBcK+dbcHw%KXYBoFaK2u2 z)@Nki-bVIkckX9VF6iF{=zun7gQjI|hGQ*0;HXvTg=XZ4mS}Q*=!wqfkEUo_KIMZR z>5FzhW^O2U-e#CpD}okikgjNwPU(=Y>6^x9l>aWvjaFn=?&yY|=k}ZFT9)aO-f6#8 zY4|nPpoZsWc4{~C=%S`-k+$k{u4HxIVh?`luI^`?KI*DAX{%o6vCiqN&S#8no~r%n zsP<}U2J5RfYq^f=bT#WYUh1>H%|rHSes=4>red2GY@J4Gy~Y+?7VEA4)`@j#CVp#I zHf*{sY{|aox~}T*-RS#$?5DQu#17}lrd+Z{Xjs&2UM_9GX6Udk=8~pttj=q~R_e%A z>t{G^zea7&9&4b6YkO{O*q&?O-r&d%?WVTj;t-D|!k?!MM- z+-_{^-rd1|?y_cX;dXAl#_QmQZtC9W>S8u--4_e?-p;n=4tXSZ}hfr z_3mWt)^A9r>Fu^+19rmMUXJO$Z>To#0+(w1j_j%SZv|iP(QeiSeeGslOZqnN`>t(| zPHzTx@CA>JqdnSV3-Jb*-}nv)03rDV1rz}M04x9i002M%Kmh;<{{Za?tVgflzWENsA{Bw6*k3PLxv1-w*W6Or!d3J5vuW`o>UAnh#-M3rQ z{@qx3aCXRx1D|Pp_;KRUmp4zHJh$}d*NyYe?tFUp>fqZc|ChaeJa_Zw(|h+@UOjrT zuF`S_pLH{E{z{rvw0NS}BF>IdL{{w3I-f(SM!AcGHv7vF&kRtTX&5;|yB zhV~VB6^8~==pl%nSqR~YB#Nlvi0-r)VTdrUh~I}asufs`CQ5Z9iYmV7V~;5Y86%KU zLG~9pyy18xPCE)o{IbNJgs+p#pZp!&3oOFtLC7Xcui6@$R+UVz=dftGB+2B(A*bYG{+N z(u%0Ctomwfudzy+Y_ZHXORS4vMl0>Kxq^snwPnV+>$ca{sjRolKI<&D&{AtIx!Izt zF1Kg0>u$I3f*UWp{EfTrz2%~#@4n~WtIfLrV-xVD-V9u@!3PJN&5cbW?C`uNg}WZM zz4B{uIT&A@F~!|%+p#*#2rTl*A(L$K$>fx5jmjvqymHG6Gc0q=G|z1F!#5{v^S5^f zt0>MR)BEvv@%MUJd4w`tP(XUQ&&y3)v}4O)$nCi2 zF8A8J_pWv%ivV9o@N^Jw{PD#c`aJi4o{KX2^RIsdMM;O8ma`1!~OkoyOxWX1L?|Cqc z9`pi1L-f_~hNQru4sB?OR`k$^w{QjG|9WPfOp)(74;Y*P1Sn9!GM1o>F~Hy%*T}{)lJSjdG-Dgv2**0sagBD&qaOEo z0S%7P0)6Zw6y)&4LK^arO^o3ZceqGKieiX9!`GBbafbJa;gVfQ9wFP8$rau~eo|DO z|C~{aEqX#1ro>+<$HB$V=^=GrT)_}X&;T;VV3sntr4e?C%UtTxm%yAO8(EM`VFt69 z$4urhjmbY(^O|+QW+D(-NK1Nihn&2jBOqxk%J@o&V_?D)*4fG8 zZPJB5SZ5tz8A?8k5)r60WiVQK|4Pp}!x@AqgwzC@31Y6Gmbp|zFP$)ihNe-O%Y>*z zCmPXaTJ)jL{NowLs6jq>^rLD3sYpeNhmxA~q$nNUH^a%&N`})C>_$RGm>{L-OUrD#PG1`Cw%=Q2QNNox5xliI=cCMa9u5yn%*Q81T zu#WIUbp7Dcz=qSXhh0}<70cMiM)$kkZQ&ncH{S7Tp_EM3ta|?!(liu87f#(P3epGw z(wY_k4ak8Nrpn*_lEJ@TxPpK6%i03(*P#N|rGX19TdX#?!K39R1n`?sT>yBCRM?^w zFpN6>cKE$>1u-f8YDq_kB6}y6u618}4uD-KA(G6Ohn}(BPUemc-kn5_$tGU&o)m)i zM1(o=LF6JA*%p6Dg8~r&+JJ`kjP)JDe@j4G1Sc54Smy68ue@3=FSyGC{_=wzOlAk8 zcD55fz!~wG1EtEf{{|TbXbs*A=MHmtzoe}5Wx@Eq6Px1CfY$MJS6oBvCde+giE(mh zOhX&@0TkdBG%IjX3`$Gdld1FZG=q#kBYPUj3hir?L)+UZPn7~uZD^KP{bg2%nacrw zb*#l)=34Kzp%C!xE+-J>3A}Q{A;v=?fPG%eTDGou)`J#2y63;LFK?-sw50<*=x@SQ zUPTPhx~@@UwMn`X#Q0aDh|$U5%v;Mc2!%XyiH$LfOCa~m} zZtr3u)UM0TJm!;&;dRDBX9TM>wW)hULyN0JZJ!n`5}QrA-)_>{f`+*%>u^u(-N-;d zaxI)3cshS|jAxcG1Z?Z&$t%6ziiiC1A8bqEvo)arT=q#vO8wRS{o;8=FCRK7IE0nm z>^Qx4|8}4CcD&Pq?w|;J>bKR16<$q%BECJ5*{z1G^M2-prz#nv+D}giz48CmZvmKi z1c-0SXMC+?Ra5s+Rz*Uc)CD>thY6E z_YBe^b_4?om_~abCV%r6YQ1-V7^G~lMMj^aO3C+s1}I=XxP!>2dC|gXYk8 z@HJ{pwreraX+*|QK4^6O_Jeq6fI>KgrNws&CV@=Ygik1leP%eIHw@n=3X2eaS*V4m z|2KlobABhNFybaMpFl{5aE7a7bNIk_YOn(KM^t-tY9%0t^UwxThHtZ0hpY&MkLO?k zR%>*~hkR&zxQAdY#9f5Ai-Q#hvQRjOcraAx3UrfL%F~F)LMB3!H0JVha7Q)K1`OQd zGA470Qvgac$83C8ghj@N@#Tj8g@Z<>P^?&vdH9O@mX7zPj>*@DG(D2!D|H&>W;BNu`)wtl&S3y~;^(zq?qkdO+wkiGQ>5OtvQc4+8nNWZJGm%5)jZ*1Rsss);cm-8;WMH+GT4w}G4E=_co^VD2MgjX}Raao0()o0hiJb*kTG&aQ#8jPd|7duK_nq4* zo<=ZyU^`2=XGId%}3 zy!nN{*#urleTGSYHn2b^AVL_qoRu|D{`8#VNS)&_T+Sd>9~y_1X$2(!P`(y#Sm&X` z7osO>R?c8nr<9@~%AzDH19Fz5h(G}|Ds_k#qec*;JZf_e7MXuApVgI`)l;7>MLkR! z2PT%E`bl;-iFV5{n^vKhsG_97<(MD)j znv<$Wmk&4xu2Dd!pc-#)cCPaP1Z^O$?D_^W(5RAXdx~lS)A~%8H4l9-1G~to zIRvXs%CA7=O;P$i`3Z6^6$pNVtBiuBJjpRrC~S+l3U{D7ua|zrYOLs`eQxHi5K29OO2DjQR4BN0Z!mDzNkXe%qrl@4&;ew#m;^-9mWb+m_; z*!V^HpbW%ia3ni#17HC7l?RbqUwLqFm0PqIfVn6e1(SOLX)w7tYq^k{1{0=an+pIc z@VOA62B3SnuFC+Y8wEgsx~f|N20*d|@Ky@Y0Kp)isd={2qY8sypRM``W;<9W1~vxU zQf>R6qcV*TQ@84-X%D20%yqpR+q+Fgrvx=X=LVUU|Czf=dvFQxxEK&+5jI-#hO&^m z0ttY!6V^u*HopwOxlx6_>61fcd%=V@Sa}nBAgn4Pm@0EiBLn&`kj4z^HVe~Kz46yyr6joa zmu`4PrvtT3P?U|HM!HLotQercHjuugH30b=1qNKiCosSATfhD*#711i4WPdogu3!3 zK}`(60x$q+umBaHK*`FK3Pq8@pql8!!Rd6yX1tmR`!Y&`Gg=t83B#*&n@QrZZta&s zcz{`QwQPzzr+yn(c6wJpt29yYjSuRqv1Gdu|M0_$yvSLs2BUkxnoGJUTg9`Ry0qK5 zm1_Wh5CEJkx;I+^lbpl~bh?-^H(jLw^A{o*hutcYeX1^3XAjNqVnY)_}d#~^lbnQE$jYfn6fIiLoSpP#q!kWYJhV;l4TJ{{3arlz@C`pK4ncf# z>CDcZyn95OIypPEc^p;cEY1$_&>!21t7XQ|+@CPys$`#NbNQoh-9Jo6gC=)cDLf22fNNUD3Z= zff>zXNcv6~-O(P+(SZQch8V&M%Pewi(gMQ>??VJiIFA{m34xn^*C*Hjg;~$c!f)Eh zae4`9nwyZ|)0*~Z(j-(VV39qm1rcy_{cH;cuyqT-G@)G%5l~bMpa%F*zv(K~Z`!g= z;M(b`0%FX+ucwL6;MhRU<C|w9em3cs5yxV1aDf)#q-v1{oH9x z*L5w@yrS26J%%Wq)_DBKcEwNH|Jz(1Rjg=AR6R}JlkGJ>_6NZ|bIFOEP0+1{yoP@u z4(b`^uD8+T;7rcJi~zJ?~ynAF^yCl1~-2(n+a&1!DsC}rdj6z9Tofpo4&nj}1JbIfyofqM?6 zeNIWnW91L-!Lu;SN!UlJ|Kryfe&K?iq*$)ykRIv99OfkcKx1ybcFLHlwBYe&-u;c- zEnehr{^H7uNO`W&eCs_WqKK7=J6_$|S=}z)v{TtS&Njhw ziLMSk%ODJ=06mW|>+Wix0h)uwVTRPcPq z_W28Nc{s&x?0^Z}`55u}Sbpp@1@EO&C;XrBLfZ|F8wE-tLrCjOKRpc;51U-tw-xKKLG4bnfJ4uvrweYFv+;nT424u9+t-}V0j@m?=5+ogmSAF67Y(^|AN*?jikaEc*M z@>4J0OxefRss_13>XiLxE*}YFK(@wM^BRByFu(H@Jnw^V_%>hjVqiSJ{*u8d2=u8$ zjG%o=E8M5KLL80u{S@_W;F15WN@X~RkIwL0-^S4EvkOxuj-xm*O`SU-MMflvXXK$$ zix`1qsS;MoID1~Yc{NDY&6{u@eS#GS=1)37fB8%~bf!^NLzz5^nWYgyOEQ8k*^Ai<&-RJ!96qu6Q#I zg%U4B{&IhyFY3)3%3sXa8N%GW9mg3Yf5CyG898gu`?E1EUh$KUX<}P?+nu~ z1q+%$M7P>*TZqSeGC|0p87gRtxsjry+!Gf;+R5p)U&jV+q*muKVWIAk)irTJMmGL_9EYGV>Sq znz9x?aLGLIy!xEWi@?_Y$g37L#3F(%N{219z;THf57EL_Ww5mhFdQI53kk@O0)h)B z!Qg`v2BD8o8+KS!7g04W##2+Z?+1Rlg0|L*WTmyCH*CRAxEp`fPS_|o_#_Js?u<^! zDPqnJyKu}|*XE!&Opf9@W(xG)`Zg)kO`;LZYP@ihu9t@pe9&SbPef?JK?i|T|AQk! z7JSzQh(NJn-az9*g_%R72x^2UR#D4Wt@pj|!T|yZ0N?{KL^$CR@?Jsjy-&!)#K;1F zIAVxFrMP0$PFu#2_AoA4N7`cb5g(Cj&1+=G|1&v{CFT7=0w4hMOHr9u+D;H$xdbdF z)$^j|+E=Cu>)OW2%l5!=wZdKLHkoQW$KNc3)JUt_u;9oDTyrigLl3MOFDeL|Vjw+A zZ`uV#ToxZq(n_hi;o>6kGx|aGt=#IinNeo z>c?$J8MDH~44&`-9I1dp2;r5lOlGzvG-F!j$V7o+w3}|IMiqzX27nfq|AQGWrx@!6i~LP3o_QH*Rzx1ws6-k*c1^5J0f75rjTAnK$Ehvw zo3-%Ien5mH&LLt5E_lIM6yThfNd*v)6pjq&P)`f7%nCN=3oRS-Uw)>005iiOm`uppYap;&3iM>Cr0LO0HG|D=aSC@xXjOU$w~N1vih zXFUtknbuUa>;pz=*}$*A^5vsFg4H-U(=nkorV7rXl_x%SPgdDUjuOg4p1S!B*@8EP|Gpqa`pemT_7djB=E!)sQn; zi?zkvL&Ele$2qc51sfDK7=b~mt(2-wZf->x&uN2iF@}%6)GuWr;%189u+#&nGYVV% z&~vpWJO}Ah$1-VacAb$6kYtdpTUglW5J28e_E(A_|IJt1f+i?m5h^Sg!!Hb}!pTS- zjC=lN=`VvR;LsA-i(r%rrtta|SU#A;tS|-^&oK!MyI{7xrHBMC;$aYn*ey20qcY?G z11Zec0?vsnh_*R3cMj};#kD{IfbeHhpuvFMTM+#6dtFrwYM)MvE39hPvwUs=3IsLB z5*)DKB-1d}og^hKlvLO4PLO8gr54O#m*(g zbxF$kfue^oIjj&?ArL1h@lK5`#M)r6$q+1oF-J*3uhc*FGI1y{u_Zg7*)+~f>Qe$b}d27#EVV^EC`485ur+BTN7RHxQ8URRYt zWDRq15+gXUR#I5Qnb{Nf4aSlrk$|L@IfZa&Y`l$ZymK`i$4LRy?&zCmBQ zIiT@vaWlXMB@PCidz6O{uN#7m3$gE5|AWf8;5A-D7-AD)winKwof5_R%2GO*mx0$^ zCuM-7Rkphdw|82u3>Vti`XG+a5qKIph5B|+QNo-qB`bKl?LG_>0`S(0KuTE4Y<0l zSt6SSC^zyZs-5^=yJNG$yNflL^&{|ci<{!;tZ|rMfDE7`F84Lly87j+nPY zlb}1al|iY6uHc_|A{ScnJv3;#f z-wBWsqoZ@{Kxl$PXq1plX_-8ui|Nsm<%t16Isq%;2-B&$Lu(F7pP&?_%L?>LNZ$Pd*DkmAb!xNyz4``XLaS(fKLf7~@Ys{o7YndXN z5DQ5(T~abGW3~h2jA1N8|HWy*Gib&4%8~}U6gs>^lz|A696mit#M(<440#~tX$^|N zKt9ZYHrOg5va@lRq?#C%CVZd15j#UjCMzkddek&Rp$mc|$~c%rJ%KQ^tGi9uf+n=b z5u1((kb`rxuPneks=P<%7?y&JO16n8Dg#JOtUOKV3+Q03Ed-ced?+xy$ciLJj5J1$ z1T;Q)o+OfukU2?*Td%%69!p7`bs2)M%FAv%tGNg?Y#cVb0ZgO{1_hx6MF2}DY_CXD zJc7iirzFZ_T#CBdg-2LBS{pUi3kU9^M|zCPOr(Rph^$7~$6*OC;W9{OV5iJ1$XLL} zTtfx8`m@Ok0kN>W{}OW@&0|YT=|zf!%ecfyYS75e1PR^Pu$eR$J;KGnj57;ZwsrA@ zG&lf+IJX}$yRrDgy#yyI@`{V_gg;YFCgRA$5*phf56Ps2oR~hH)XAfS#~(|Cp1~;8 z32>${SRy=P^o1xYDtztJM5TE;v3OSOesP0^m#wit0Ep zIhuWH(#ER4|F3wZD7+OK_|i-yuK=vLxiQXyi8;+G(J<>Q6cxQ{I}zt}&-jAT-om}# zkd&KoMm0MKUR(hiCBz?^oO<#mR+RRW4o9cYMcK zzziO4eRP(>={J9t4LIuMBx$ud4-IpB1 zOIYEqR)tE2W7lewzo&~*TZPY*aIbL5KC`ipRfsD-xL5elQY5{)C2b;|)md1@f=#?y ztR>jeyC%<37uOoEVEt0yn$Z-af|rB2-iR)mp-6n^VFjqKy8{Q?5>NIVSIBI{ zV=TxNu)d%xUmo-{L0|%Ac%GW+P4|USJ=xxUEzO$gzM2U+?duQuBiJ;Qgj1*qtDW6{ z-Mvelvbr_1IxSXS{9RM}sKB>lMY7W_vkO@Eq=LBegcin{(fzSPXhy|M-8tdaAHznQ?M8P zRf~!^Iz?MvQZ*1Q7`AO&G(O|eGEZ=OS%dmb4&LB5t_2dn10V~z5isE;G6^fl<1Ieq z9a+!s<=)5=Ep7h*)e>s!s9>AvxAt4$)oiWr?=n|1>6^Ra>$g zDrQS10xHObTsWA4p5qZ{W;|%&ktPFaK2;Ddjw5xD#`zT@P@g=YGc8iE7!(o3ANE#6}3) z{F6iJmEN-qmuwV;)9i^wouE76&Ilo9F3haS(=do`aq?YSYBu41xsCzSr02 zWORONQ6}J1sN$;jR~sN3Tyf0-*;OlySl+^m5hH7BZR;{dYhaw{{{(alHD;f`)NL-* zXk-52T)4dLGE_Zi?34ya>Vsxzwl|VEt0@Kps+QWjTuhwaVdt*g>+@*%jjK`!)ue_| z(I#*9{RYeJG_`r>9&m$D&hD#C?bSx?EZ}2$kOGUcO3L}1oB&NgvcEMxtR_MaM0T zZy~2$fcib)w71!oWntZ~1zzA{)9rnGhT^rzWhM3Vc?VQ?hj^fe2@|STcXe22bw+`3 z;+egju?}s3rts<vQYCmX2 zc*XcX78>PAGSA|AJXPWl2ZuQ&u@!;kB3+jK$_)5Y~d08e{V# zfF%cY8%l6f|91svk>r2|VPSMRjSI1;EIO5)-KlcI||_}G(P~CxID#WUXuDS1o+UXh$_ec ztvCH~OMTT({jFDhw2$~zv$)NNn`^D6$sq`IU=*vDv;SU~APEW!fx_qH(trmzHB4b1 zo|oNf5Ae|?b}k*Kxx@Kc_%yp8e8azf#2?^x-e>Obe(%3(1JLjC1;r?Y;{Ua%yA!y7 zsT-!}eAvgV5guOQW`ioQ+fJ#3`mY52-+%rGh)M$ORIyU$7vlc_hgeoH~W@>>0^ZFlHlc z1_IMVNzbH8kuoLH$mq_fU7S&^IgS3;J_jQ$dVsoU=T6kgv<~qs8|8y zbCr+TxREaHO&jXer^&HS*V?sd)v}?*u08uUZD^RHSa#^UKxD}jFJ>%`@i>S(1(Ww# z?!w@6OfZ{OkZadE(@#}r9Ch>PDEOrx!i#SjHH4YzHKwFk!~{$!VW^UY`^u!$D*4@B zc^Aek95aKcHQ-wXUY8hJx73AQUQtC+SQbplkx4VL(9lyEXk@rpPYZDHVFi_mI6?&y zICer5Df0COAwZ2&4oleJw%Tho+IZuPqv>cHZp7r4M--ST1d&6)1s7ZZ06=zJ2TeMu zQIi**1OgidZA1f=FFaR;9WpV<;1ilSga671bh%#CSpYbO|4sp!o+Bh?$4o zbcoC%1!GWjBMzWmK>|rQ^HjCppD%th%UT3FYT$th-g4lVQvGu0QaHJx7#AEeCW48f zMlkA!jv1hw6M9(X*NcAuBjb*+;W#U;Ip!AYt+%nH#zAT1!H|(h3X9~DOcn>F2FWVR zeiYXCKNN~rZnpmr+Gg5d6>IW!Pn+v#9@d@otVobLk53?9rpP=jp`cMJXHmB+F1bEC2Abm~sFG1)DW+GtMD`m>CJ0amJUYm}(j>P?|87 zi#V%{KwmO=)HkSCPJ899Fk9Jcpj-UL1@KHUW7kT+0vELy4UODErpnecf%m@E^&DX1)$g;q!%VPIK!~J^ zwQ=(mq!xoIgNSlcM*BSUqU5bN-_=|1_cMS8E_fcjHp8Okn=8I}WX3rjx#W~vmxW+w zFX1XeIsZl^{RV8dg2#5d&Ho4^L7#r8xLOB5LcI%6eiBD63K0&hq{Vi%+sYRxp|9Qf z>r1|?lz=uxE!ZI<26{SP6!b)xjy8iZia#0;WwT4o~kumoz50gHf8v?HeB3C{cD8355j>ctR- zGt^iOxtEZT<#2~QbV$m|hpb5s@ptL70qb)3hKDJnWScP(18(+2%50KV9e!G{!dqW={AnbS9_wgvp4$(97V#RGv!JAB!vjfQw*QvjhMBph>JTbRMw zh-bJ1;4cj_(BsGq^@kDyQc4<}-U|&`Nb1ordp^3AkM6h`U&Swcoou48Py$3s@?;Im ztfbJ+qZ26f!4@g9L2o+O8OkUiA)_q7Izh7{Rko>u`rN0xzCkZ%>ZC3FI%8HMsGvEp z1$V%6=+_E%rww^$EYS2-9&~t3Enuc_+LVF`p(ln5y+@GSFef=>D9%GBGM%iD!E^A4 z33qZ1o{VeUJcTF;0aY+)M0{wuM)d|b+gjpZ)DwT0S)|kLM$;s zg=4TOGH{yHaq^UgEKz7M2It9l7WFq5n8Ae*AWxK#7AINYU)@mazSP!X3T(t{8FsJ% z2V{=3i`1$CRIt_Htdm=9QYvisFo;r^5w35QD_rJMSL2=)s(TGvNIBXH_oQG7$)f^x zvy0u8T9>fig+l4Z8izsdRHx-t$2!uhURlYe8L4qEeBb*$qH@+Z%(?8@&_NFvG2$w|b(m@JRkdzvBsAi3-VlPzW@jVjCn?1p{o>$@K?m`u}dsJL9Y6#>im#-fV z%O*rTViM!z71Q;iyjpy&J@a|boj6;?!g=07k3^M3KJ=ka63Rp~`pZX#FhvyYf;89y zy-s#=Gpb+`$RuFW+5BK%74xMTbVE*;umOd5!{vh#lmH+Lp)#8HJ~BP^S+|4pSvX9W zP`BBxq?jOHmI^^RTZcb%wzHi{4ZHQ=fdzcPu9!5DJaAdT3r@&^j8!G=LKF=T6btl? z!~d-sdWggYh)&6r7#(D*4%x__B|xOnz2r+{+T4h)0zN)16zNjbIyUeN5^KR^oB_wP z3tlvDRVBrSmAj#M<+pLwGC^Su6V6#-ixaX>QJ`4O|R_6+ZttFMuBON_^(ktmvbZ4f4p786e5V*~k*? zFtHesyCjURf;Aharwj~?YuI_gpIAhQJS?x15ou>zaUEgwPU-vOcnIV&RfSiUf zfz)UL$}yhP$XMwxUgSA}%&89KwGPox9=OS0ZPZ^75@G&v9%nR?lqDV05&s~P2mk?m zfB_yLSyu(1+7%oGU;-07 z4(Rm6u$_krbi?t`0Dr|-e-Xxk`CapY!7>5QYc*p{We{{R#$zed!>N%Aa*7-LfQnJu z`8h)^%!1@O%(cW0wNV8y0%0ZIoF(=YCbnZb4NfpZ$Ni|9eGwc)ME}x+m7)#^5i0iM za|N5$n1LA30YT0b1yWPm35n7jgjQ9+3S7xpF(3#YMFfCjEy#fgutYiN7cqVV2(THN z0LbB4q`)NuNxF_nGR;Ub%`(_h3Ie1cKI01tP*p&ip55f3)Dn56<2ugYCAuR}B7{&K z&X#qS{6HE$PLi^e*cJMtQ}QA1mcWg9eNBWgjm@jj9a5NJctQpUfDJ$cUWy%CnuAOj!VU0YEg%`Ifg;mE zKsP|a#YvxZ_T4HjA7K_I!3g9)K7to@r+0p*cXa28g~BgDres#$W&UJ(rjs3XoM>&-SNl?EW#dZy5{f2=8K6VL*-%J;lOM$!`AtxCJ84d zkXvz5R3V7ca@r!RQNSuZ0p6(@h)P^cDx*whXGx7GiaMZ~t*C&osPdU-WcDOauBUol zWsBbG|99KHqh$(449$p6+U%K3{oO zgER=Luny~a5^Igt(3EZB>tW!d^^ote!BTQvX?2T#6^^wQqhW!b-1*@oDcGucrY-i- z2HaPN; zvEtl}4(bu^N{o1nExe=06y+oJ##Z_tC~yfa=$DRdB*PI{r3Kjcc#JuG5KTGPPPECv z&=!c~UaKwVXFcjoOy7du03nVYsQkf~vIu|uL0l*c9?a_}9u#Nvi~!oA(oCe$0%Qtd zV=MaWzXGh&rs%*HY_ZB{Y2X7r0Mxm4YSyakWF%@sJn3r=!0Dk{Ifw+kv5LKEEeK4J z9;_259RLTtpwTT0B^*}mPW zCh4ZRMo55B^NDYalCNdj-!QbW3kTf`7b@&2;Sw%~>ONAEZRKcj z1VQxhe;sa+_1F@wrBDp1S(S}H(f^I@H39W@s^f+$tPyd75-&tOn~AXNTGr*^?uD7s zM5|f>hBha`HJA)^QVFmfLj*_%_gaWKALfoQ343q&qA(>6A&r6?p!mBkZdH125>Y1@Q5d8kEEW zvJ*lE7X%RkDznQHsca?Kw$P6x4@;tEg3BT!ATlThI4C&$Kqr3>^6CatCV>MmF8vTy zENUe@u_-KHRP|91#&NF*%m1-0`>G|-G3c7c9p^C&2b~QS78$2AyD{f62Y?W`z-Xdg zMmQ>YIP=?*hn#E|Xq<7}>>qcjN)#+EdHn5M)r1B}fT;=q78h=qUaev%OBXmVC~>7H zA(YBum6KA_V12F9baAC_*qg-_#_8cKcP~BHGb!XU)$SBO_p(vrA5#Ca3?ENtIa2AN zOek)|KI$VwldD5Ni<@X;%H<_p(&e>u0YBdLZ!$`{?z(@_J12Kf9ThErO#`WXIqFiHeij;9~T~ioKt}E~1 z9daKZ4(4DJwqZLTJ}0(fFLq-$HgZz{Q+rNxLun61^+|JgSzk9^k!x5dvr{|)MruT2 z-N9d$>sXU2+C)hHNSV}y5I!8^Qo=PL$23H!KwPUcLgA}jAeS8~A54NU_x^TuLf|cv zXMvNiKJT-SB>(tvFE|c0*tdf1yFnoUSVku!^i)^3g*P;}IwgK;wJoHKODMxKj4KM> zT1_lLF*-uGDDfu)#WC7NO2c>c#*c=+xQ+8?_Kh{*1tNe4xPaU9kQ;cBBRFv{c4odO z$lP$VLel$+t+ZBoSXcFSLwa_D2Zf8WX3u~{dileM0&b3ZCfm7~U8-+!3#eVu9n%%mOlW=eod7ukAk{@;d2^JpN_mB~#qDuL2bn=8# zq3Xql>e*YCMya$$xH=SK)E$4a(2kfpN?ED#bQV%uaf(AItEkh#ulk4#PCNyI~B=EwW z$F|7F&iipkXPrQ*KuRZ$ixgpiub`lC_Tlp6VpF^ut#phVx2Xy zK@_}-)fSr8Lx>OyF~gFOV{M3*!r=?tV?6QgZGL>;PC9s_Pm|pETRIBLyZr`ibK3^5 z*eQz$$UH&6=BnTPEQmlq+WSlL`ESE<3HLjomqx&!Mxoh&>_$H7a$48Uxo5I&r%A{B z(*N6QE6H<^mJ85><~M8>+*-O6Mif*&@O_UtkX1N@{1ID7hjV+>Edr#5Hx+v}%ztBF zW-*MwJl`9$<TZA5%BK^t z?sH_*>#CBnR|#qLLD_~ZDD1b0Qb;Cf!DK{OE%?e`Y!U1mDLQa6gqFi60E7oD10~HF zb@1R3Bm)=j(9r?n11Po-RrHkSjYUvfI5MGfz(4^4B1x7EX%b~g2(!B2NYbOBK~lHA z)ExE5k`kRedGz}FYW4KiuWi+=W#e`gYgMgTxppO2%>Nj$ zu+OxqQ-`lg2DJ&+vUTeg1OvHj=hjW20AyahdrJZsFxN$uKG4kJ%Xb(tVs&xXG6rFz zPcuYmFwszGQ_-9+cbHKO@OR`~0%`KuDtxC!I2bn?775aN;fr#aYz+_)pu*`63~6#g znE5j7h%^4Ep)8p;B3esox|E_9h~AXbQLcRA5;jBDtb4x3&I2ZR@nFi3FM3|+lP>Cw zwl@Rl)J&<;sBN`>->cVQV2OrxS#?EEU1gGlmNI8 zE)5WXhz1wV@UX+c`mh1S64?mMK_Gx&#IoQnW9FL&Lp$lD6>LOg9@JFZ&i^~tTFZi& zDAGvaHW2c#<&)ho^X(SjTx?Cl2O{Vr4I!(%@(&xL3_->MWKaVF>cr%$n-E@7D1=Cg zYvhMcmVn{~;<`x&l{=s~pY2p$Uw7_GDni$}bfpB>zZQ^d)6JNF{xk zQcEwD_`m=9V8zB2oIpf|3HOp~+mT2$AXZl+dgx?`P)>Q)VQ1YC*IRGJ016H^@F2}H zd2BgYW*aJq$%J471PfHBly+KbXRvmPyhIL}#_E`HV;4oLo?{tJPAks1FVnrKg?MQb zo0uw26bwm)Z;9rXw)uTz6@2S>LWPV~yc-He8Gh8^ha=@L;);10obbZM!laLmDHsB5 zkR%lC>BvEoxzr3?SQ*wWSzftWo<08nXP7~2RRTIhP}v7DO7^_x?7jf{wtG37l3_ z?-J7v9bd9Y^liAlzR6jieTV}W+;I2_XK?Y!JpNQV%5#g_*n+9{;5dbP`ZY*k~Yu z0WhK#x@gzwLWDy01d0hi5l;%(r3>-}!x_+9l%uXOjr6I{Z{ASi`{Wlt_|Z>ei0Bml zLN%saaX~F6+fV=rctDqZj&uObpcfg~z{~U|c0*Ih1mATLLJuxJ;6Mi?r1DE}Ofd4PgL@Z9N0J_KSA4|NBH z5z&Z8M56lu$HXQIj!K$}fi0qdApAY(Z@LU2QCV3aOun)5XdkW)9f=@T3hC5QLS$p0oS0z@IcLKFcVC_&4zL?v3apiVp} zCLUD3u0lvIu#9E?ZbN_!P_&{0>>>j#*P)2+uYcE6T^ERI(~$u3 zQMg-ictD3j#BgaYq1`dB_8si)EPH@jCr(k}!+iXKJd^bf2jq|-)Wj3C^sJ{+^{`1J zEY7Ki7@}0Cn#8KU_9_$8>TC};1PC3h3Mz^VSQ{`_vQ`eQtBchoxqwB`EfQycC`}YR z2qDVM<4q#HYj#SS*JR2nhi$P|heBs2?;aOCni<}BkPs9cC@@H1_|8ihkr8}g7N|k^ zL6A&S)YOGxo=2^uJ%9PeAfDC;sdY+J>Hj0ptF{lpWsxmyaiP^zHoyXu$?aFg`p{3| zs#KZFoE?&fxMQCw{Ti}pIJOnl>lxRPZQO{1d`*tM7ue4uZDqef<>tW~;G;2hIw z3s~TE1Eqi-abj`YIN$)D+?+uO#5uPtbXKPwTElg8s@a!>^T~6iWPbIV)c!KHssCIA z`V4&F1S?p!t;G*npwVVGUr7ypg(%RBhg01 zlXGNgPV=kPOjpJeQPECee96~A2Wa_UIC$^U)++Ij>QWtm}%W;P4RFlL)D8V3(B^QJ zASX);B>-=asRTLfn%O+sL8a5ku4q)Tn~U7SYra!(w(liBrP=A(Zq~g zm`sp*5C_3RISH`t?UJu4mxO}k2PLcE3F>AyAdn-N===mc<9lnP!a=l1_3|$@>FfUj zbl7TkErO4oj&9Eb1OpJ=%2m#Cmrw34)~3$LS{|=*pv&9ePI=2=R)GR>o7ynG!^C%< z=xoVF4$HBDyhlp!di%_QP*)~Kn2~TaF!K%=C3uY}1c5B55M@_-+-(_t0)BTU3ec-~ zttZJ7jPF6$8u!STJuYCVvj2AE#cm(6PoDBT^ZYG=7yRI5KJyIU#RZr!P0n==c$FtU z59siOZeh$FFf2fSCz-+wj-_d)6SLlurm1#LYUx(KAOZr&z?HNlB~NvI>|a-zM2d5P zvNz~K%V`yEPe`mqUi|I5?seTi{s>{`ouC79v)M}2{I*`sF_sfNSPMTL3{{}zk6HY; z=}{Pb*rNF$%B}6bRA`=QhK69rCJk7GahI6kyG@_5lbS>6`uEHM7*@U5uRrkD?-s&j zPwO7cv+l<1mS**8VID4F?R>B8woLfwj`%$3?h1_9NU5OCAp;|Ze$t>B)ZpN%kADQ< zBvgeiYN#>NF9lZ+m;Yu3rrNc23~V!pfmM1p#Si z38>BX5K2L!U^(`$w4$H^9T4~;kPHrs_<{}ZM6S%zY{3|e7&u0uzzStZ+O`Q#R!d#&G7!@bA(vtx$&sUC<5Js`DVt z29*T|T?Y&7uxH|cBANjaSMNAd#YXDP;J~g`e$e)YF-34u^&}Arqp%Sl5$>8%?ot4s zJOS7|PPL3L3;(t75-~9oy-=&b5Vh6~gh0p?Pmv8h2brV}(%SG9V^J1o5gi4hN^~y^ z9%Mv>3Kwkv(R%LoGHqorZQO3J7?-T&Q0(Y%+sKw*>DPgR!5|Qt3wvih< zac~sx^aAXyGEZ_a1ZL1s4au<_&uvCatKHZm+TUvo3%kQ3MuIztocG=K?-b0Gp@^vK4zR1?w!0v#+!!e-$c zJdGz>b2k05Hg6L?1f^E~5x4Y)%zl$G8FM)A^C~IA2%y6k3xMX%@GI94EJ@NVy|XOo zusW^tN^GsKBqFy6p*sz9q&8p;mg{Q>5*O?8Ez=V)nX${V1}hbhCVMkJK#4w;ZwvRM zl>e%-A}@wGB{K#bM_cNQ=DZSsEXUQdY|3&}$vmV5WzjRIGe{Rm11_MfT%$oTuW}^R zJ7r}7BTrpu!yJ?W{%Dg!JJdtjGd*;3O{j!KOY|Cn(;HSaEUc16=f^6iXK;QJBR{6M z#%g2$v^fV-S|sllN0XI60o}@INcnXB#A__21Xt9uArRnJ){99kO4;gQ{KO_gH}pz7 zB1^T@3_bxB@Cj2f)qEJ`_(Dl4T}wq*v>n{+*eHfnPZd?&Objioie#qftmSMbq1Ag)d8QNe z(5O!tNI?m!2rA7X+66LN5m^fsQkxY^Cly+YLT|v7DzY{2D1{tO4qXTDAGchG{er~c-EO$jzp4HjXMGX9)ZVHfs#8kS!=6=Jh7 z8v#d&x(^@b^+^fkv8T&NS3;iE-(RJKlU()_AMwQ@+hJn6LnwlY7SEt9d}bR zD1kRN;|Reo4hwc!Wp-vK>1G3^J#*Gyb~a4)u4l<~O{s@o?10(!mT#GlME}BXT-jA! zL*g2V}SudB$)I&ql zvu$w}ZsYb^8&e{&>M^$!b}LeFxiV%1FGMI#1&Qzv6<~L1h8S^Ia*qK})e%^Im30h( zP}R0aq#$5&VT9O@)>so~d>}7HU>2VDQsa^~PggLtRCVQJZhLl8x-fQKAx+O0eXX(* zW_JYjhdG0bw7v{s>PTxep?+8Mc!!s3w^n8~H)e~8c@IDf!l4G@=8jOIO`s8zfN3HG z=6FT`VFJdQ@PJP600so1o9duuZI*kz_ig1CTgBAaUY8TiH+@eSegD}v97o1RYsq;& zVgy8p4}d2ZashZ|xDcG-4Wjq1+>&_t_c?#$xS|e0|2J4vpn3bN29jZb$v}s5VSsD+ zkfuqRfGD4qm=CBI4+^1)nfQ>tM+zUYgH!i|QCNLh!3E$-(Twg; zO;G{u2wXQ1Crs8G0O7v z_gEo~d8MFcn0N%7fdq0`l(D!F3K)kA;R>w4eg)Wq^%4&Zz3X$#87I1; zRd+sld7w-cGB^5!l?^RZXhQ}uq+^ho?I;3b_@4B-l>fJwhO-!db$}4QAsF1Ku%(zh zV`vdtb`EI8r=&>20;vF)sWft}#ymh1^jL+|F9cMmw|=SePzN7$I0oJzADH9TcsQ`p zGkTefq&GM)&HAi$xl}P)8`s!y+<2oM0ZIZCB%NoWe8~$W@S632sS7BIJ5IS{Xc;Ce zGb~4wG9kiBTLy5E5TpsHk7c|pxOXsb0la%;`GA|J%<3Rwk3ld&+U0C@f|^%bds~}y z-}?b28g(oBqSe}$-^Z=hG<|>JWkS-0U9I7GC0)4--u*vJm#lsGs%3T~=dqEV+k*}>iW z#f2cb#|M~R8Yg61duZEJ5q*4d8<=x@qyLG#$jv)9dpANp0|EleN(3vgx+IKnYfWqb z4aTU8a(EhLy?g3TXNWb`k6?0Pk@Ro|#6#SDky^2I2SwymBFMFid;p-QoVfNBcc~08 zvr8_?>ASQVo+N;R;Uy^~t{VoWPXHxaD;_?+)S}VYww)c?JATpYg|~BxSEPl)u6>w0 zs9hz1kfu0Fo53itv^3p-eycc#c7X@34vx|ph|n$7Pu+l2h1EUMY0})@8z}+)QBK|5 zadkD9v}TyPz}$5~g9QSyA6f*W*@rYi_IQ8_VhD~}-~(WQnUtXuURbN&&e;7qodrhF zBOdDAJ`vE6ncsF3)F+`SP-DLieRT#IWon8V)U{0){>#>;$68wHw zAf5k694eWP>`0VXI_q(`s2?5-92%G!ev1j4o&PCrf=-vMz%N{LSmfB}rO8>*tx>r@ixeOEH1HRS(P_z z1&`tNM;gtfp@I_HDQD7eFzS`SjKx{Bg5+uN0-r6EA4iVOS5Xc8`S;TgZZNTFfwKt1AAt)3g_BJ(trFoT6oz=Ah917E zRjaIuIO2&W+WL!%xor^L0|CT^7z53iV-6t`1gn^E3>>g*T|E|%n3pT8;-YfaB9t0# znN%c`ng8|KP5%f=V!4DTx*&2FA$0{-KwZI@qeUQCC_?WX1*J*cn#k@&SCaNxBF7r9 zS<6U{?%I$fcM9vF(gHJpG|_qJ@!$al5Mdxiq7&5;4hkc@mx2WUX|bOtw*8k#7o4I2 zMoqI=n$t)<#Rn>copq8bDM@jK%~P>{`18+T$r@sc>wn_EW z%Yxvn8fhSw2fWnodhNVQUWP}Z1eVrXgW{kf?q%dE;|Z4*5K^7i1Yi=iE~(jTnFf5^ z(Ockx2Yyp&^?HD~!t{8P%E4zYJWe7XNPv?QDG>S{6TvLHUIS%QOTt3^X{J$lrC$nL zreMIqy8p|Ne2$<_3qsh*dV=JzH_0(094@7>A(aY+PVTgyeZn7t%Y!b%?E*#_g%+=(qxs87!VFuIS5qdP!+10Q0{c+ z!(8?dDnTR$@a#lH;bFmeNmODJR4^e`#?e&DlUjR6gezQp(_8#I$6eYqqXelQDjjeU5a$1SLvlOX#aJh zU=q&`M#K#gjfu>BC=)R*2n#dQB9N6{k$zhf6Zf!ryy!%M8Fs-RWsnq3%ZzhuHxML3 zjSAI=GBv3SecUJV85;iN@e8QAg+b_LiK)WVpieEURLx4!IH~ob7~Ml#ISSXKNMeUS z#3e~{Y0@F~VJxW_o?j<12|r{js4%UCOoK&?>s;}9&NPN%^+c?mcF}u;b%?aKs#dBl zwVdLks%ORe*`zYbT%_GhAl+t9gA5e3V~r|V6*f_C#x|{MO(k31ird^8Vz<1ND_!jh z++98;uXKg!j7+d3=uX#_5+SI1esh-78RzV{g z$;?;2oux!)@%viW#&*a0-EV*W8{hyBco+nhMqPv3Lj)(-xWp_b;_jdZ2ye{}(5>)v zr&xvRV$prdknB&tc(o?TmOCoWu{dvxVj62!zMplm3|xYNvgyqOTtzc_$!X?1>p8y( zrLB*33+N#G))gT&vZ0ZTXpt6q(MvY2DSFXlu&2k)0nnuKC3# zs0$M;(&Hz=XQngA>6_$5<7ZpFX8W>tZ5}BJ2W4zyvDVjtINmd#{rqDf2RgvNEpVa_ zjNrcdRl&#&>R+PV+)7t@x)!d2rlCt=4tx5`-nAB8a&)bIUczhA6L*+yR2Xk6^$c^_ zpQ;ghN!mo_wD)r7jk}P}lR46wBv6KF@gs%}usED-=CzJ?BOz6 zhs#fEAaPZgrKJ^dEy;9Ehs2Q#*knFZyJ{3wNv^%-3StJgXA8q}eG)0KmH!3f@m@c`V-fguy$5_^w_6XV zcEyJw$j5;hn0(CFe1X+$-xYmxGY@`pHP%;QR1+=g*DU660ucB;W5smlf(dd20TYme z0bm2v1O!qAMmka#FNG%Dv?D`!5D$PgIYI$I5H1M=ds^TGZD0h<@C242XS;C{py6}{ zxMC~PfbKx#3`GO% z#CbXp9qDw9E6@i;k|&UHJ)LL+GeRS!R!mC(9Cy<|5cDwM5Dv;^dJa%=84MJ2j5sboEY`uhRzMzaJ$&AbBj2)m{ znGhIT!Wh=$BRgn~(j*YtxR3rbEixAckGECb7gUHS9Xwc%2CxS3IFvCcBu7bD%%P2d zS4=YU9KX^~{#Yt7@DT(Ebt|GqxX}ptasN3X(*S~!I2IF;=28O_U;;5Ak?}-6xpr&9g+_c4 zmuP_rn1Z>CB?3&gvXY+2CuAaiLSh(LWId26P3oDG*;q$1)ipq%nXM%PwV@L2b{^4} zBeNiHN$4x3DI=l@V(n>^qxm9OB>wYd+tD=KxE zGILBysguZ}pjKm```C>!D1^&}n(*X5&818(*B38o3*E$;{u5oJ>6E95pb2o3>6k?9 zw3Wv9i$6dc`xrFhB6pWwW9|Z(;qcJmZULSB$@(bpqvpkp=TJE7S%E{vvn{= zL3sIA0QL&pc?~%lhaSkIJ(^NJT9}8)1Jx1&j-sg`07cGojY(>gF0dEtbf1~pb*?sl zJ4b~9u#Lb`ikhbeXCjX_`TwOw`3w(`ge?F9BDO3UI&}KTsRCdJl?0(7^8<1g3|Zh% zNCjiLxHjQ-2oD(!RN!V4C8Fn|n^a&ZwSb~^qzP9bfKzC5nqpCA@PDg7t%7hl<3U4- zDw2V43)pFlI;v=q+MUa$QY*=z^cVr`sVtw$m_4YhLFsij@B+!ya)t320uUJHM+jk~ z8_Ts#NI4fTR}Y5z_dX9GL(i03$KEN2pRFruvI1JkL198(AgC8r}m0%Po@rhVn|UK|^5OiIVvGqm&A1mobAg zC^8Y?jxo@rM`#3EBmX3a*lUS+2{xiG1K^k^Ku39lrAbJN>w<}*mLuGQ4eA3Id({Vs zRC(i5Cuo5UE`WvTupZ)&K2|W00@VRw03e0Bbr2z2wZj4aiyp@`l$43ul6dn`N{@V3n*3li~>}NmbfmTXo=jEhGA>4 zP$oa5=^K1!7tR4&8z7)q4!^@NRuT4;f(_wL_tX zt2yRTs#tkJW$3uZ0dT=tvvWFyny?av(yiQTr-gtA^Bbtg3NrpE4D{BuD1#G~KCaFFuJjd|5eRD0c zi@7U`E)roJp5p|blaPb}mApGvy&{3y@g)ZH2}T8X>wv)~~3^cvtR6XJR*KYUeF0w7xDP*I|)Hn^?+vBvDWC+;g39?1uUia9VL3|gSY z0Q_IaK>v(9WNdepeA(HkcDQ^{D@?Gf#s(1*q(ufR@u4M(5}^E~KY3r>hKYGRf9|G; z^*MB+)Ev*FUu=NQ*gUoM6b6Z`565r}=kRWmU=M*?e_6$*;p>3z11G?o$?Kd8HE>W4 znTtESoS*@ycIrVDnK6*y5oYthA&DBlFuc1q;=zTitaIKnL)w2cBfD(K?HIP=z`1#4+K|HS)eMkXH9Xh4F0^g$qH6qQd)ff5DhdIRt&fzX6xVG+$dG#a0 z+i|7}X~J(&1t-`i_Yl)sFxF5gr;NQ&5G>I( zP)goKyM~g~iHok9u@ei?-caz_z!%k0%{$15&|}QYkH+6vExYk%1rlA!+v*92K>s9r zgy353-D426+l?IL^pEHgiM#!=>*CvK+yQR@-TC0F2GNr;00d1Wf1MZHM5zh@fk1x0e<$7+bG5?h;Mef*=QOs`ArJ{TF}5q($Fz(69dT7&)fRWgzwH0F zhwcNqih8w9>$FZJPoNREj!poS0{a8OPk^zvn^PbAh=d{M(ip-Ix_{Zzk;q;~fR&<7}w z3DA+6D;&e({LLdW@ZbvI?A(Fn0@#eT`;<%oL}iix0$^%#b@}Ui>KuD4dMu%I zf8b79!d(pW29JBz?v&i~kOJ%uAzCa4Rxb;*pUaMb^#B&-wTx^M*Y)}R^|qqGWAE_A z|L`uH<=$1IZ7YQBMoDR~@d%gvtjR@bP~CigJPeBt+VC!w0FUYzz4j2pbLsertoSrv z{=9(sk>9U#QOU0@W^(x6KD>l3`%q#`Xf=@BHHehTnEgV__2o)-{P=z$CYa%cv2opgL z=05Nsu^Sw8t1-=*(FPq%QnU;+yEyXxnv^UhmD zl#|#~=_#I!apV|3tY7f5ihR#l4^Kwj)5_0*E!pW{zZ3p1aKQl-@L~T)|D2SKvnb9~ri?XK2BA>_0HA_V zwYzq08U#4Gi93#rbupG{6>wh0zG!P+%W*r`IA;-}`+i?`s;2HWH(@d)45it(@rIW+jFH2K=S@?Z}Eec&iGy4zhCd{-~vTq1d5 zdm2KPHv@K2D{KzQVHz^zIWYZDdA;$P2G#%Yhc?`hd(6WKUT#L7LI?s1T`1rA`sFY# zN`-xA0aSeILntzq@o)JQbItB00-DO4CY1+kz;`6^4LHH0Kpu#m>}n1 z_bkz&NLH((%coBFx-;=57}gn~4FG8a8Q6jzDy)el*dRL;q9&8gIl~fYu!c*N!--Jn z%@2wPoZL9kY8<#ClwdFgyPQ%4)QSR>ipY`jAPr3E!X7ZQ$OI<%VT=7i=2`qi6F`ko zjb^-`!JcWKHCB_2hV$PS5EKSB9D{Ll%p1mLC4u5BaF2OppdSI*fRYK)Y_>5Z%fOO> zOGIXg%FtPf8hOX?$ptJfAchdDU`hW>#z#o!K7e> zEPu&|i4G?O7`12?q!3Yx3gV-%q(MlbFjBcup%bY5C>B+zN|_SD2gEdH#ngAqNP&?H z%|xa(^H)D=PE(D9~57cG%ZM*@TLeH5@4wJBj0 zd)QaSRIrOZ=}je@Sjn>CPX6kmQykGg_c_#=f`evgMSIk1Ce;=H)4>eW0#4SRrDakz z=LHVJHLPML4Hbx2V|W-Iu*(1QEhh~b29SoB7A((OD_CA$eWx|cJ%I*DfG!p?z)d1K z?yV11ZV!^+hQF=7{B$cwJjrs_t?W*@D3j{i_ottIa+96BKPXGtQQ9nPH>>MZ8E*MOa-qwDEg-qtLIYJl zxN(KQdcjqAf(#tdf)zf(XgTP@31CC>7=Eigdy;1vumVLnPyx3%GXv9#i$Wz73{Lwz?_=ICVh{ z+9q>DfASqQfc=%tDo2U3KozI+u9`W61#4Rp zN7m8-A82o+De(VcETGM#l`^B+>oU_?HKXpbRz#P1AjkuH-iLEj#~q7iGrjGgbcM)a z-TDNi(!pm7AivjQkm7Ww0}j-n5}c@{K6t51i{N0*(8>-^OT;BEw+oTV?W_AsDxY4+ zT_eWh(uo%zC=eYsM-Bz?LSA2qz&#PSfR=QAoed2i%U5#2=M`Cw2>%&{>zpMDE>H;1 zS^fjPEM40u`x3i*m2FTE{RGv)cVOpLxGD8Y(I;I(sw%r#+PmrO2MV}gEvvqQ`>pF? z&+OHs3^4&ah`P6TIJ92dnP9iQs?Qf_1JyS7y7TzsR**siU{3P4Fd`TuAXl)JC*ko) zVF7gsMq2;xinD}_Pa{af1U8lsN8HXs`M&d>=Ra?Q(04^kQMWv@$)NEdkc03bvXX!& z2p?!5x}}@C79bJunFl$zB_h}t$a53hv%SgUz(5!{{;HqfBdz~3EwWRL+eJh8Bl zg#e%|aJ?o>5^%tVr^6g=%Rulk0|ewhb+`mAtcdJ!F9K{rpzDSX0gly~2Yi^RQCc-3 z$vq*^n#AIQ-CG}N071YJ!N5_tM=?RgC<1_U9~Dx;mI=QU02{sQ4v-KoFkyj2q&^z- zjF11LfDsshjuAp=(}fx^JeU(UAanpOFgBROMDAEO_Ct>Cqrh(=2Ig?WDd|7_lNs#L zmL^b!dys$)khd%#hz4A%B>*}KERh$ukp^@DgP4~tC`NSP5bVgMUYP~JJAp?l8_K$` zY|KXc!bXAfk7$|(`bm^NWSBpkvf`_|L##UyK!mh8p1@GV-k^eGJAj7J14MATZllCX zgcnEDFgR;~Dfqh_6h#`ii;mg|U1)?<)QD+pfqT4zUEG!6@VQz1wlASbQ!yFeDTi`^ z#bi8z8Cf)gyq;ieL*lt8dpVV0w1bnBJ#qUZR_ex|>c$R413c_6{V+KE;J4rFNuB>9 zGHeROc)GvplR|flml2=9u21M0bRcMyQ2jd&+xsuaD3Ift((OoT$GGn^nNK zE&wJokpp5Zuhap_&-omT+B{mw9F%!Z-m=RuXaaPBLCE5y+H#G zzIrapyi5zc$q(dAQPVV_1WnP@&;4XLuZS@d0Z`uxi$si$LYNK~;3yo-0eFdl8xlo* z6i@)-$K*t_9i-4748jXdKZd*^Bf=<)fWtKyC0#4edg7sdNhuK}E>eOL5#_6uQ35+^ z0$HL1M$o)KdEfr33yV3wXBoF&X8mohk(9nXs&@^)dySxJ%EKuddt8M%-CD1Om z*e*E`Qak;wzj9HV;?pGPQ#&0jF>p^oWi;rzLqk1O_d3)^h19yTiAnzrgY+nv`*8`; z;=}voR3?2=pp?`wu&d}IRTZ342<3s9S}B%d)mCNI5I|KamDN-A3NwhMTYZ8kM97vh zDX^$f@>wZC=qp3}QJK1;R{^wAoz*}^)L6~8zw)h11Jgmhw#?K<5zRXEh@;v|4SBKiugH5t)MLy1ASO0m=lIT_^JtlFjrp(a8Bh5*K@jXvXrgHVzPc=vVEWP-$ zijg&0QNS|cGuJ2m1Oh_YmsMGpjo6cg*_yRk1H)O?YKfRt5R?D#*#Y9ICAFrB{g;e2 z+9WNCjwMH*WLnK?7K**BsKq9pP1&lo+N;G{s?FK0b<&juS&m)WQ9If_{5q(8G7?1F zr!`x#`&zEe+P8JvxNY0H)mgf=+qDHduj{=uLRy3QN#JYRyv5tImE5~Q z(y{H^%T3zO)!eilT&GnZ)Fs{BWh!+tfuU)~($HA^T&30C z?bTc31z-Ew$#6a2a@^nRmEQja;3kFN{$1VSE#Hd0U;D*h*|i4prC$P;U-S*&m2KbD zwO&)GVCuc#xjhFCu3iW}VD3%e5oX}oHQ*2iVgHq2#%*5{8sH38;m1wk6As~Uv|$D2 z+ZzsF{l#4xMq$v5;13q!4Sv!NzF-%2To@LT{Wansp5Yo^G7{e5B<}=1-Sw=XqJ>_R^1n3vl3w7GE@ptf>7Dgm4@j`4r!3C>6#|TVWwP?F6yNg zW}J3vqWI~k25MV&=b8qCMvOa3GZfcW$YD}(T zgQIG)?&+x}=(*17b7ty;E9bLT>b|aPwFYdqo?;yCWO}A+vu0_zrr*G3Y`?y1tj=q} z_UcQH>%aDCw6*J?&TPo$?8eq=q;_o3mTb``?b05{%06txwrrxtYtMG=&o=GX9_`rP z>4Dbl+g9zXZtc!qYuWbgRu1j9rtRJ{Y@goh)XwdUo@wCzZQgF~uFh$Tj&7qUZsbmG zXJ+l)cJA$t?9yg#6P|5>25y}`=jQ+2?CVBvzSeH;X7BFi>ZKO$;U;hYVrJaN?sv}a z=Wg%j_HGfLZ;SnJ`R;A-Ms4f9?`4+i_1M)3T8YzBAm42NwE zpY05X>ObV@`nK=1hVTWyaQ7~2>E7@MCvXu5V&fih3qSD{XYu#u@T3;-q7HBu=kXZF zZ501*(eA_HzVQYx^8MQJ4+rhjuI>ppWgq`>q3&=RxAEZ4@$h|b^o4OKr*IQb@fJUF zBM0*At#O`&@3Nic5P$M6=jO!DW(*f|D-ZK4H}fEW>p({60)KG>S8Ot8Z98{!I2ZDT zR&P0<^T}@UJwI?Zr|)i-@jxaQ^f5PdNvHEl&*WxS^B%`+H(vD6j`2u8bUz$@--5Xd>pgn>I6DmZwu%W|;3=2YxXmFy% zixewn#J8~{Mu{Lp$`eU)WXX>xPnuk*lBLU+FyYCJNwcO+k2rIt+^Mr?OP)S|3cX1b zsL`TGhbApbZl=?qN25xGs&uNxs#vpXwYqhw*QYkqfu-8Dtl6_yxu*R%w(VE7Xya;i zOOqR2vU20)g{zb!-iz(t?p-^$u;IIZ4=d)ISg+&7kOfEHdlxa|$Cn9D*4$X~X3sD) zN3CpG^k>tjLBCVIIxy?hd`rXT?7H>r+plrw*3BDrZ`z$@|K2@(Ztdd8d$S{+T)A=K z&4K%lZksxD=y^$7ufAP6X71O+i~sMA-8=d7%g3unUw%FKis|FCfB)Tlefjv0yU(9} zzxMwCwm08^{uLPDfp#INAXo`5NT7WN^7o)*VmWBxf(<&Ap@b3MHz9{0diWuOB>uNy zQx}qk;)y7Z2qA10hDhIx`pHP6jVZ>MF1Mr(kZB&cS7l7p@as?C!%}ix9FLT{wXP#kurLyp_W#vsFjaG>ZztgO-d=K znPOV%pmA>6DXOcIndPUgrvEAGsHCcDs;#)P8tbmE@(Qc3vj%&ruEQ3a>amF;t1Pj= zGP~%jW@fpfH!L=5?X|y#1*ojcc6)2G-g*n|xVn;?q`BwDIPJRZCWtP!*j5GawzWDN z?!D-MoA18-;%hH9bOIc3y8_SIuE7LTt8l`0HM}sx5GOpZu=E=H(!0L)`%O6QxCF+YRy$YY12%-I{8tv1?evu(>Iv+OK) z&UE|SN!@net@qxa@c-?1-+&(ycq4-oZusGhBp%7)j5of>HQ?0t{SAU!9a-Tl}x$Lyp9tY)pucG_zyRV!K+vCg={P4nW zJdQJG7a#ZX%%k#$^M6M#INx_vZ~YXFP$5Y6++**3BHz~${^Zw7LH-irr+@zXbg)l; z=|GfZR07VzGUYvtPVG>0k@HfBPMNV(NYu+q=H@sZb z;&`>tMFux!!Lw;X6BW#a2$?qp6p-MAB`~21SI9yXy6}V)us{rFNJALb@P;pxp$>7l z!xZrF0yo5f5C4ZagA_oah(}Cf7Um~8`8jcR3_QgGr)UT&Qi2qr*hChu$VES#=qwc* z6C-G6hBBJbfwNm;{2(a7cjafbU9;y@1re zIEdj1u#6=X8mLBD-YtU7d*dAAD97Y|@Q%UA;~s}8LLdaP0Tc+N5eR9>LMrl^kc=iY zB`Hm6TGN`BQ~~z*fQN2+^PAudCnv`lzB8V|l~Y9LIn`;-1tRcIUFuaUWogS!)-smR zlOWtKSpP?Gz>${#9VYVN2sb_qlLg0YfDD|8&_im0p^1#>G$$%iL?)A>*UTs*6&AhHkjzH%R>Pa{|OwJGSi)j~r zP=|u@(Wqt{M;t2{%zn<1poAD{LkSQ78IVB_GTcPLz=tU8Gkpn#f3!Rgfkq zL0Z?kQL+}G02gg)FaJqbIM%g;c;)3Ps1Veb3iYpmHRCC(rX|$ZL)vz?y!~x(g$scIB-4*0d$$?N(xS z%2-gWSIAJVM3g6&1^J3o4I9jdmbc91OzgJ_CXK35H4(!D6u1p~pg;i@YT*jI8UMBt zZZn)MEN41%_|ABC=9;@o)fmcvxq7gx7P@J{)~eK`Hnw05kf*&u3{=PH?8WDPOz9~T z`L9M!rI$Q%*!L{il}%Q!c2fZiu_-KQkL|(~c(+o6+5*2^=5jNlkOwda*wF+I@c<00 z<`^*e&2hGKorP`aU|;yy8_qLa`3!&qtQpW9L_`kUcxVeEdc}^uu_jmy$Lp?vbU|YU zkfVI*DQ-j4p$cP?m6cNmR!_=7CUwD(g$*e$5fzjLFsx&(<#N#a&$m{F3`Uz=l@>R* z!3MT&Z=3Mdrncb_&$GlcyzE0#m8!f{@V5C-=xy6Ex>oolH^cm2JVJUBKL48qX|nyldqSB!he>O42Pxdo+9iw_h| z-Yzb7&Mcqn)44(CH<)QHsLbWYyf=H$yoqh*92p-}~yhj_b z=gn65xK59J(-q$I$V;A~$UpuJRf~ru7;%=^H=p*i&;3qna(dnz>i@Sj-F7H<7Z8C^ zJ?m%QdVRs^fj-vx+UEp{S>jutZ`8e(37-ow-+J*Frch0#RL)mt%y(4>IBaf4Y>s4XMQoYKRfI?V=4fuG%Wm#J$ZMmgH6u5yH7=??q zSRS}K(_(sWH-hQ+Fecb3D42rQFh4B_3S20IW_X4G<7DE4d(9AdyQc;~kPJMygN2u7 zG7xLamxn}{e0<1&#|B$5fDgMDTP5U#Qh11nD1}vshK#5%EdLif-L(vJcO&FND`04I zV~8R#xHV1FbCYFMW43R7fDW;ygL6oStCdW6XkZDrhcsqqQAc&MxNx)ReEVh#Nw-Xf z2!%|Rh*MZbi`a;ahzpPSIOaBqGg67qp@~GYi7zOIc-IG|c7;1vjn+sEiL+|WBVSpz zg!_gLUnYk?$Ahh#w3}|0Vn2UwDeHF-yznG7wWNN3AItxRL zka&y}bAs*HDa2eW=KHrP57vfFu6`)bB+DTGz~Khv;Ws_=ywK^h>X_4ejpQ+O|cE= zl#n+gjax$u)yR-XDF@pWk%XoO#-o2}P-_3*(==fhv@RBhJlRcI+Hp!1Q8G@tKlLJ{P&gd+(P?U+FGA@{gOj&An^N>$jP*(PP z{`WSRkO!|;l`Bwk7&%P$1%VKTUl6#GT;-9L_izd3Tb)UZqIrbF6`F^{cwI#SX_;TX z6$7hj4z_Ry_vn^yDOf&3m~=UpY}a<@w}n3GD8K1`_7a01Lk`QZV?>2-eeelNd5ySP zn2*I$@#RfkreFPKeKxQH*=Yk38HbaJRN#PL6#o{45O4(omWn2z3&|7=9~BHKG1FMt7# z>7A}dQo^Hz40en7IS<^XQb0-uLz)jXW~71Wp22j7;bE?L8j z#rdHX(wB#@3UYXuNco(&IbUy6bUE4tD*psT+DW61ial4!YQ)o|z9p5iW&xJf2Ai6e zg9fB(V5E7#r7C$gE5K-$WmWS)Yx(f0Q~Ii>dZnuBnpzsFuQ~)?>I^Fos^{2ICl*ZI zV0~xWf&Ftu=Y?I0l|^c5PLhVAa=M$xx|5{Gp*TX2!uf(Q$fxwzH@}k);vkVJ17En= zn9E6>8F`f;G-%qhuIkEP?&@E3NKEi&=|Hn=_k_$r_;(s&_tCKxwp2f;Fwd>9aoxqS|`8A9^#%z;6uN$J6d_*yw(S~>MKL@*{j0}jG60KGW(|T`?*CH zx~F3`9eS-whPwV+Z*s7@w^%+2>1YoqW6`FlVApX|o1?S9I4sHsv8DnHpiMS_0gv0Y zHbA^@5U!YbRs za$LuRe8>Fty$evIjQ?tVgGX!4Kwaj*2sGf9nPaD8AjEEf$(oG6M%*f+i>(qvjqfFj zo`eSVb3|xd0SmCp3vgPt(6!0%xW7yQwh+U` zoUh2cxPhx*`KkuZEV$52xC3wkd0PS1Jj}(Mu)mwlEjs{hI}0ovRSCe$9}oc|kOAk+ z0h5eGyU2kM>cc>M$(t_2oYS%F4+PUhyfG30vxcf5zwjyu&M#z(IDMoXp67EgSRBBuinhCcrXto z%dsmku@#^K8UKyZ))d^_bjqED(0D{&nH2@Wy2c=2&Luj_jWE&EnV6lRJ!^de0Bh47y#x%f zsuzm~AAP>~Fbf2r$|yYmkF2rbK+NK>0JGrI5Q|#-It7Wn*fPr4FM!i2u&Nd?51lXr z?c|p3tby?LJW5nOOO)A{UC&5e&%c1DQghn#GPFaRhDr@fyJ*VC(^*QDz#sRkv8_~p z;643SJr)d=XN}fsy*IjDOls!Amv=H<&FM8k={g$5Iddp2t5}EnZD_Y)#dO^OJW|+ZGJi8r_L6=1oijX z?oBw{r`h5&>TxdTvi`|YEsQ;WhB-^sulu?w^-B~PbW*!Z1HMn<&^G@CH)VIP;l$+N zH`Iy)>6Ogsm3})hqd!cP>B4|Joxbg86z!9aMxY+*DW0t2=d%2r*|$_aMdW?CT?arM z>-0V6`ui`o&WJjW=X$P9TX2 zN#F)z9Pt+K=(5A*{o}&`^uyu)@uF^l$!a~AP(l`MJ*s}x`ufl$#(Qwce^R^F@BhxT zvtIM`e(Uva?~M2h8F&c=|L?FF^lSro{vPOg;NS?q#nWw2JL=Udy70?6=|u(R+y3da zQwq!A1`nVF-k#~-P6yt8@r@ok8}IRJq&^r}<(8{hT_8eduAL9-+%V1rnaxibN5ySm zRk?@T?mp_(VB_;{^L75Ga>CET*u?}hH0xB=38@VK{>b(f`I0|tafpNG=-@$bVvQE; zn-vTsuju6}Ux`rZ4SzeT5WhQ_xi{eP(q8sV6!x66?WeB_Oki^(W8yl?f+K_CX#dV- zwBk!=++eQM_bNv*W@u+_U%G$_Q!w*@FYkdr_&A^Q!UzmJpYK1fe~xeYK>r$k=$ErAk`8>5&K4;^mw46GctN$n3V`g+4NwK!> z)o!SIwn$^RM&vYXDI?S-3Xl^kRGffdLFCz$P<~Y9PB7+7Y1Vp32$VN&&bA_^j9UB`VLTh8m)vR+e3nI;r~sDc(|kODBlMuKYzz+}jyH^%z<%ffEj zx+IgfIAf+QzgD?pnbRbjsuRW!{FLs*>%q&s@7st){W(W9b_IFNElUd@;c z4UAfYvO6fX`!WJ7v8?U@X1lD80%)U^*4Ze;ODat?*JN|soZgEwPCD!4jtcDb<5SQ4 z04(>xLWx|kFHA0sLf48`aPT=v8JzSEuRPLa98?t36r^U%@ia8iHWPK!!&oHs)Knu5 zjt3zq#sIrmjq9-rEm8tU3XN%yfMZOm5ck&~1AfsWJV5nv*u}oI^2!EkE}&)-rY%9b zoqOgv4LQBRc4%$6-4>c}*Lwz-aEVPpK>_!Zn!i5%{Qn9+so@e+!9y>c(Bw~k2%+PW z9<*>@csWFbE=+mLYaxWSQ|zE%85R}cE>@*c)li+hfb54b$TG_<5eb=t2-M10qtl3p zM8{;Ph`Ey;sdyCRVK;)pfCUN|;B%Xs)tP6~eFoZEqO&d9=%iam+L(b5TcXfAARWnE zurxh4pQ8A~+AFPPB?yXlKl-Q-4ziHo2YX{+_t3>^mwu~`bXkU~ZVDc7mdQS0Naa%g zfTEL%R57H+giLkmGEz&7<}rZ^6PpOcn-zW83G%l&OA$oKf{^1CCnKk60$KRbj1a*_ zLWLk7L0~2V&P6bS_|u=xq*c0_q)v6K<6xZJ*b8UM^n1vj9SGqxLc(alb{t|O49#V=-PMB_Z+Kx0VerWr z67OpyDG=23X1=E(t6JrR9 zX*20O%YxN;(i$Z)FFuxm38z_(9Z#SVE&qJP1{lZ#@4mE09~d(x5d5PA2T4dnUJw~U zyB0jou}C$=vys<&iX)d+jMm^wI?YLgb)q&kblgLCeJYdob`+#}@S&8T>`pQjAQBl; zNi95DnU4xJAzetPCCPKn8`P3SjCfRhK0rkt^7n!yu&Ro?q#|&V5sq+hFPJN3#DkDk z&N7y+XVYq?9rC2Sj3{B6t@vMAy7|Q^FatW`Fb5R+2hKg>)N}V+W;&r&Eq2Zmp7PXc zJ(*!oVgLhnTnN+8CR9n>p+g%26$?J@krR@2WF-cj3qu=9uIFsY9EIdaH{C6>ga&L)vg8?tV#5?eKG{j z8o^{qp;BI&PdG4j#wt=Ej2UgVv++tp1U?|ub509q&1CRJq6r3R0s>gX6J87S$l%)& zq<&M)*%+1C+i4+Ekt1%?aEXfx(Yye;cd-i)t|GNPrGppi42mx*LEX6a%aRE#WkU%< z1%q7&9V-|nbN^alJ8_dh%>Qhx5P{Hu0x>oYj%C?sA-viAURgLW$X;e)I;#6_LIkn! z7X$t%01u}&zH44E#pDneD1;%X7~v%vrU1U$w5+yjY$jeUhuo1Y>i{6`?QgRRToRl3 zxy4PSQ^v}d(N0N*<$^J%pkW6#n2ZfBa6tlGKr&9*kW&h^YpFk!SO41bgxAqUMV*#I z&*h;JEE{yJ@9^Fyltg55uvJ=wie<)1d3=y#YB&HKW-*I$UJ`HsPSwmTXq!`T#ypz< zV-n%Z2II3a%eMIT98L~gc5RZR0^nvwRYI%t&|779qLHTPr8PQeD5k?;W>>posUR!; z3U!cJ!D&w?K;YK7&;NCIe8N&Y?XgaqwYA0d$>SOCdLqyeNJ0WqCUgvWEE|amuKYgC z1bYxr$X*Tbz+XBhBpRUzh2tE0V9_>UX8H>N&DLvRW72>w^TV|?YiNr@GDivpJ0V_- zm2(W>L)&f0iZ-K70CcDOs_S<5Jmr0Ltjin4Tu78ta|F?S@8=^2wY13s-f86q1A%hg z5swq@W2aH&HxJLqXW60#>+xU&pGbp*j8L*Vhc8fW^d8>vkWyevIElGfiOEs^jc&}6 z<)O#|Z`#9bmTL|DuVEMz)Yyv@ZWBMK%i4f z2mo@oQjgZoCjVe);kH|7_I%``lXlNQ$eHikd6LOy2SBxtdMnW{8GTu8q<#ZzHgCTr zKk>btJ9`EXZIKm}5ZEYyJ*h25V2i;554JS`;t_UK%obG6axJ@|6tUrd_KywnJQJr& zSbz)AkO2UI0>BGGs5u5uo2NRk`T-*dn?MK~JOlxY1&O^1P=kabJq!yisCzoz>bH7v@kiT09g_TPwim{m3 zBDg3x!2b%6G&kf6a1(;g$vvD?Js3ax4@)=S z!zx5$!Qq3%77QWdf`y08HqbOI#A zzQmXqjX1x)LxhR*uQo6qE5tuwVJXI0Ib<7z`~pM9%dAlW#yz0G)T1^9L>zn3g-k$} z>rjI^BtYrdz9+#$IZL}c-~nF2yV$}oy3TYq_OJss!dnvUcEPpgd(&Dth0kv8w3yrG_ zmXNJbE68&+!gEZ>a)1Jn#KO$sx%XP5f>4B&!br5lFJ#0xj~oYQU_8y5%T}>T+`_1N z@H!402j#Jq?10H8GC~{ROKFP8^UE{x6T}e6Nu}$ku~UI)6H2Qpw0Ap7q)f_rlgHwq z9CV2#l9-@viYRKq$2K@eG6=Oz$p>sCxs=cv`x!JUv`dqR^iD77IUDF6#i2txT9ClP@t zXVJY79Li%3KA6=@`2z^j3^V6Oi zgt=tBZ=j5Bh%;bW19M{i;(B|vXEC9GL3Y@_i9O|GJW~###^AgJ7m-3RW1F}nE z6A@($&gFbCfEA!KmAp*Uzbb+R{+hcd5Vzw@ynu~W-ptctomdzoGyjfoPMvEG7}e5n z`Vf;aLR6ao9_?0dZB!r)R}DHeay34zV+FL-Mr&oT=V(_i@y{X90w>5%ywDdZNU22F zRUhgpcX1h8x>qc)zn9g<478QQbDUO%$|tbMM-YY&$&gO42&$FI41pX~O9l<$udf}$ zEb`i6!waajh<`QKP?&>cJz9!ILkg4#SCUzVc)(@M5R59b%Ou$Z^HFdW*P%I+lU-R# zoKJJLQ^Yla zPjM(+I293Jz0YEUTyfZj;?$8EAhyr_fw#@V)0hLeO# zossQZM%9_Y6$(fVTyljEmu*TsgvslLIS<%?LL{sy2vAaEf%VJ5%FTtB$fL?tLd8@L zeQg3UP1msJHD8ia-|Gd@{2GPInBd<{`~8MP zaLZ1};w^>+*6`wFu;qNsWjst{{Bv7~4rVo80{;$fK-hGP(|k!r9yff+#d(Rg1RFth zi{3x3=Bta*5Pr5qPFx+fxI+?W^n&R;YJda64IfB>T;+hx5T(b{5x{VnARc3iV~%Id z-=<53WUka5NnboL<jv8?|!NgmAC|Iw}Oj zb_OdjY%pemsPIN!Qq`zzYceUsQv=_l?{n@TSJIw+^%i)Q%N((N&f)aSVOhWx-z3$lORRCA9QNoyPoFq5ZviS znvZ5)zXqKa`y2+8Yeftl313tZ zTW;o7%6NP3Cktsd%e@OARjROVL*9-hEH(M6FJ9$M!!QiUDDPuMZ=tRjnS>RHyG#1D zl_zks)eRA0=!C4@1(D2JRrrF1?eRsRT46ZLQpgYhTLl1na<^pDM4+<)&us$-(Y#4< z-(K)KUVvs3BL@dzbwqCI=HvK`#Q)7?2W@$WdT4Wccyob@bGMpvI-m0blkR7Wj(Aau zzkn_i0=24zO0TtSPZbEvMLgvIpnZ%uH$ z<%CW*pH7!xQYZBz5NjA0Yl*GsW%i=2(BV_mVFy{$Q0fAG4tsz4S#MJk6i@fK*n zgyP!sJujh3Fq{6Y0%bo>4p&FdL&Slr7VQDgN;Vc4onKXx5(!555i36#tY?&zW*4+)$M{i|_JP7Av~Y++ z-ljwD15q2%UfHED5CT|=4vYr@kr4OPD|K_krr_!s6ADwu9C4tMs7 zPq-}@e84XU2m%8F5EQuJ0EP+%7(i&~Z~}#hD^%F%!wZ@=ZW_ID+?bIZ$dDq_c^ujC zBgSmjR8C`A5+%%(#4x3SP~ZU0n>iQWG$;^2K>-LJ5bZFuXwe@QT~w&j#|G1=E~+33 zW|fGLElHGY?dtWbkT>I;>0-j8hnqn?aIDSh_HElBAK-%ZIz^nbD|o0RN(5(5DPf&v zeE>#4xxWjy2Mi>^tAzgz+M{aKs&1t?QKCHRGUm*RLe0UV)hnr1p-FZ2Z2ELvB`VP6 zqh*#zmrTvdH94y_Z5XUV<}L^k;dccNz=Qt|fT2m0A<5OW$)jsFCSoyPnmk$E`gEBt zvun>NeI`w)3U>A+@AGFtq#BH}_u&2mi1-<%vXL6KV)EQvQ}6$ZqFY`e)PV|Qo&3Q^ z9U<%{*IZNVb>JU~z(7}OQs9LMA9%c0N;mU_Q)<4`r0LzUD8wqm{DM24h3Z@Yi4?Cy7?!C(aLEkTvJd4 zkwYXngyab_pdb+mGH4Qsg4A~P-=NN{BH)L7eOReD-AJKHVY9yL=!%bZp(QG*n5u=|a>c+wjVB$iS! zX)@g~rm4P~@f#YzGl&=b z9e*5h$hoo-14Ks9I;PLtww&h6G0*&Xj&#s+2jxp%Vgt}Wo5vI93@mg(MW8_O#1C08 zvId+ffXHVds;65Rij&SGh`og*Gs`P~Wv%ax1D7^5&0vcSq`^P9qbSO4*aA0`h(xI& z-AqJ}_Zeg8y>Z`uYaclGBNvVdB2idCrsn03w@`WpH10U$?b{V)+Rfyn3q#v3yZPsY z;Hv+&ha!08dZL^$yE+JpEUSnEhbGmDCwvG}%g`OxcE=gY+yV|Uq5~L!H!W3!&0Jp_ z4HQyTt2R)e4Pkl%5US#d=Z)@WVxV62G=@EnZSQZ~>yGz|rM8rv&wPtZ&!5=mJ`eIo z4#=yDhcpKfgY>X-owEg$@)1Q{K*@hz8B8IvIEzBOP$_(<0W%;02UrLLUf&SL7WNRC zEk?r-Evg`mzy`-re4&DKm>_44&Db3=^GWN_cBiFg#pvnHM=0kK*S`%pJTBF;&96gxrvdbolnj?e#z zom0WYGLeU4(Zh-iic24XDWa;u4n{rNOle*eM^aUWQ@2r?*FGq=4yMt9xj~K|cIh^f zA%qaN2!<3dp)l!55{8xJ8x1+O$#s6mOXEu+B5J8JmL(2N16bcy0%3?>t}GZ zBTIT(fO8X3!TAQjvRBPuNkfuLGPSb8cBV4Dqr^x3*flu~l86Iq z%;cKKLoL9wPg#J$^de~nOUTfY>Aa*3rxP7w!4nfa%p6cjDG=ghU?@dB>RhrC2stpd zsZITaQ%{G%ex}Mj1CYQ#-ziU=*nj|Gx+(!6(uY^^2AB}_8rfjML1%95cX$8%6h}Sk zR%DJec+{NJe&(2|6hJ`+eDN#N01H?tux)gPzC1S23AQ zWub#v=*%*4qok#=EK&uhl%up%nj9#YF~7}zleJI;2dz{UQ5-a^hYh)w6Cn@<^Np1) zW<6`bOf#e1Wt6UMjVnn1bsFQ&77Kv%VD2{23OYUQbp>qFb85vp?dmcT7^^9D(D~TQ zjtC#(vyW8j2Z98om#C)nTr52sRG zLrNd=1auBoXhV8K)sbFixEN)tTtiAK4Wov+5q&PTPVov61HxX`J@Nl_Q!FS{2tpLk z*qeCO8Q#e1w+i;Xv3;2W-yJg~0X!BZkb6j9hJnCXPU0_rX;Vt}3fLUAVyZq#+Kfv? zsF8%=Aro!6lL&_um)<07Un5km;DYL{m}#qY?bxqd3=*#Utjur$X6Cx`2gG1eg<4;s zltD4C26*_Q9<1U7E9@zoWPqC$yijP{Jtsq_ZLDJ zkTH#4B=5HzCopP`@zGlFRnLvXTbc@wyx?1MnLHMF?P~SZKatY8%oT>w)@l@IW}ei| z$dz-2lZ)2FnClTE9`Ov4K^8ENjtZ+_ffmw+v^@D4(A%a0U%CGq3v5U_(&*Fl*yt=b0;!yh} zY_w?zPr{?jYbF>A%Q`t9m=222oV+t-5QgOvx@F`=DOaDvT_)b|u2*K>XkW+Pu*uQf z>rvS`pD>?fww=M!r~Nq>Jx9KbAG2fq)8>B5PD+SI^X(k z7Cuo8Gd8_fK3~zDV^9@)H^ejqLZKx>Dh?*8U-g}2EoY3{oXn6Hcdo!5`KJJHLFIE) zxWn6En(zE<*`onbCY)94g1#3pW(er&vUaz`DEzg!p>c!cL}g_Qy@0|snB%)p*<`OzXo2qO#$6f{*F z*vw3sh4Kv>oGqW5{hzBbTXVghT7U|PMc0M^$l8ofxH%rp6#}@e5%~RC3~HCFg1q3eB3ilpB&O& zB7le%(av@m5To&&;}s&Kg~KX*0(h01(fyzgre6@MUmg&P1*8GWWMKhb;m%N9LC^pL zI-&kSAt{1}s~KMx(ZL#YkpOPuZ9&A_T%7NT0K(YJL)}r=IKUHl0wO5j4^Rm@OqOY- z4GvI~BuLI05r(8_Q8HFdGj0(yGGi&U!t+fNAC?Ulpn({;fu#u0_f;2)p_rnS!Z|r2 z=0PGlrXw#@;`_zZLj4RqF&O83Vi2(huTB4$8$RF|b^&bd&^IO0K>b(S$&>xO-)!6g z5Lk@bjG3}^g#*X|FE&GO&;m`ofko;PHkbn@6r(jMf*>g3(DB?wVjmwcqe*I!p&6n$ ztb#S&W2a;go{<;;Vk3eL2r)szyxfb>=_D~#4bMS>XwXz6nje>_qft)cl*n5<{uDvt zj}N5>2q-}l8Wo+Dq7?3vOVYu{#n1f2SK93sJboJa;2LZl89rFioSB9$@**Y(#2r-R zNJ3F2=s-5!fCbTx64IZR6+lVWA^%MwC2-L6$)u0aWF1~0VkV|4Dkfv56|pTA4(6aa zYTi+D9#T%p5rRqhY?Y2VgggpWeMSG}v;0d3;UggEqaS@h90*z-P|Whs5-yqDLb#3R zV4SVY2LGI8-7(i%=E*?xff+8+E_sMu{+|?df?n?BUjA8HB2)4?XN6tkOd96wRi|}c zr(+UZXh3FUPNt+?W@UCBFPs-9b`~dk=4Yl48;oWV=tmS2ViGdg z#UY^1R6xWeWGn~?Y2xBO4oZE#C3D23Hbg;jmd0`BrE)GOA2{G&KBpu^XGs-iA08$i zcBqGbsCEvQP*wvE_MnNLXo-fWc&?*TUZOh!jgQIKXXS*|3Fv6L=PeG2Y#r0!El628 z!0nx9`6$afae$ia-$upA24w$%+AN9yAm}bxWFOFjjkQvQCg+ecCj>#~rr4RPY$$eu zsF;T6jBsa)Mk0!yDVnC^Cb>;7{+B?;scn5Gjdodml+{rY36lmKGc>86Ql!q^0ao^? zkNye1{g{?1KyU4UfemO{Dk;KT90!1pYf33P?2;Mm2LnO@9$=}b@+BT{Y1Z`L{(b3R zQs0=SDixZkntJD&svk+LX#0W3$k3{-VnS2S=;kDcX#JZy;Day%NI7g}YiWaKfe&lq zsGQ)bdk6>brDu6eKo(GCqt029BEbSI125hI0#v{jRjNhO!`Dp}DgoV>AceGy^1KSb|+Cv8f)AGQ)t?H39Qm~nviNvsLf`^F)Wlg zEW!>&?}3DqVMvbV$XIxjbBqJx)aJSNghGAYh_E@f0T7t{t4Oy~PK#dLmRN*nF^o{QaB2;c9$91rmClYbxtKs-`x;vd zMK9t?Z}sZk^$ub@1g1Bj-3s7b%oZ=M!aXHEshriOg{ zhEa(vv!wrt5)&*}F;v{0fyW&iq`v7u%uB^2C_VJTTeR#0a4P-jtp>y622TLzc<=`o zUr0G0b-IxWpYUj;@Z+wqin=gnvZxMWnSb(_rxHY{g@F#R=lVS1doHV?76(`zG9o8( z0VL!pnHBTmNNihU+9C03^R`r*7E=bLQR_02uc~hKS*E`AfT^+~Jn6&AKs} zE)yKbajVWT`>iASW}<^3s_7NPP28|hKym&xA+y5g2r@_uREl85>9wdGU}=GD&;bJi z6LOhy5zy8yX46o7t8ii>IxI@5pb`}%QnY69=JX6j7{JW-ExWFT!igb`u=CB5u)MM| z^{)TwP-3qfvoJ8wu|9_{JC@L90+nnQz?qcNK+ne?19Kw_GNUf^Gz)Vy>7t{dWJ!)B zMVlerM4`1_s{{A|Tiz3ufomEVYcn|HdOT=(j4>I1POgG7@nRw1vIQIG>>z!Vy{0nH z%JV#5-$^l}GF+{AvZ;O`G0k;4LnXVSJFC%oJCbx1AIDa&@ zfy+UZ(w8>ZhVyAQ(UhTcU`HEjh;&0{HP!Ek?0>6M3;^4gnWsPLF#tJ!sJ zSMPjdZ+-7JP`Bng{d0bXrKinkVgmq9^zuOq5&K-Wfmd~0bbyZ!Ide-TlTQDcf{%lF z0k3m?fv}m%-4rUTP%cBB1qFYxXO8jOZ8?bFE>#QOMt!*n!X$jh_h{g2EVFov_cmY; z;Xfm4VI%LE==d)q%u_J}`byOkzYye{h@<^kfuKUB?gpKVqFgwF4A&#KkAReh1 zkVB!+^ayEFfjDFN0jQk@pZB`HG*{4>9le((1*{Pdbst2JujAx#=d!EbN703U3*(v4Nl85|- z&hdP%CEgoTP=z*QsvJ7xb0gG1n90W?%m-LVWe@U>|2SJk`a_1~pGNp8(OYuXl9rpE zykYbs2h6C)bE<;;Eu4GyA{}qLaO6snzXmM8w!GE8e2mVzPTa7MDXix*EY)7sQ7M#0 z%FptAI~;X|YSjV|6nt*2hVz75fIIdYKo#x|MME`$Ds{DWKj38dX55DB)1&pLc6h#} zGce%>bdn7#*)+&=XemHOG^)KYz#N+MbYE+P=dQfVFD>TE{CGU=R79=5_wvzRbJ1dI zzqrVehmS5Jarq{F2qn8?GeX+f0v`i`InV}^+tIJ(qL6QDF9!eG=Btn;?8lV{uwKFq z<5TEnz(Uz9T$%T5yv)Yzb0)F@Atpr{jvbXjx9kY z@I9%S+7A~AX@$=lO!sm9f^6ggL5S_Tu2!X~&cZ_>L}JL0jmQf)x|v-ocsHR_BxsXj zNFE%u4P6jG$pv+e`8IKQF*#^1l z$X1jn2)^3*!%WT%EMLBa>O1PzsJ(4<+z`6q0Sz^2r(0|OFI2fYNraFq`L#T+2b4dZzgdLLHe;uTr*{_ zm7BxY=@%?j1`$4-czq#uh!#h7-1y!5Gw|U1l$@}ljUEdxAuv$zUMA=G@(t(&kbuF| zpbz-(@9+P=0I6EbEyCnjYJda-STI2YY;Z6NP-s9Q93grrf-u4|*ug@#24qVxzyO0GU#tI%@n|iKAYyN(MCK3QH1{zP6T^po^kRa zw6i;v&T|5hDAsciz8i6D&_4Y11Mo2a5-n^|4==<}L#h%Wl)(gBP$7f?r|3p4NFVhS z)KEdZ<*&fD;0r_!QEZV_6<_3U1}O%ewKMs2<`|_pLZ%=y$zaA&MvovHHgdCtmgriUmf9_aVP=*(vY3o3Gre765rU?3^UXQU-R@32xm%Z`@~WVKf+0N7 z^SwXCq7mPGy<)I}LlF>A(MDArm|#%{bYTC1e;Y(N1SmSZkcm$X#wtUK`3g})5-FC1 zZl`Qf-+%sDMWDpLW4=WGG_&5$00h69Z; z=y<{hCQ?Afmgc}&?zKjw`-R+c&($v7@#{Iqm2pL8+)@#&8Czm5R^caZAJwy z*wKk4{qF;WgK``0i7QSS3yeoZHDyKIc3=h-8*40&OrKzlGkqOzyv6{ukV3D&iHJDq z$ksZFD8idg##jkBC@OO?#t07e(=+EG*7TNR66!yst=j6Vugkh!uD9-kt9TLV9ne5c z1}lKXFl77RyYXH=Zn+1yyT43N%v=BO>aX%!FpGEfk2W{!-*N30}BZyikvY zAlhHXv?qzd$pL(bNJ$#Pr;Kp@4}3omod}kAxkwBIX`@@52ne_>DoPP^B??0fr!c_L zNX&D(m{JwPP{0G8F@Z(nffP`2Bw;OSSi?fa?5aaS+sR2?7}TTgxI?cau_sVu%Z#A- zm8cW0Fo(?pN(5d|0`{SR1djhCi}X6MDa|F4f7mk--`r-G#~AKrOc0~^D5<<1TIED= zz`@-L7#dfGrj;KsLy+=O3odj@ks0t_H}H@}1O5_$XGq|$Tqi613J){Up^$NN z6D1feY6*mWQHlT;zySXd@qqy;ROp@j)8578InV5E%!d?P!{~OZuysPrD6Y&DK?O%p zh86}8Rhga!pumhl1XKu2*kv;!`bE3;>^T_)=0<@t93^;klp&3QCxVeBW|s7%a#E}| zD3Vf@a;;}Zoj?>aA(y3Sv!*p2K%o*OAkW&NJSoA(=gOg+^EpPSs$vGQC{rooHKDSe zI7Q!D#c>L(&Q3b=67 zht--E699%rT$uklFUY`c0klXaBGBoTG}O{L#&V}^2b$ZfnrMGhZ6VM8P{S9dCNfj# zg8{Vg91nD^8D}B!3^=6R3fxAY#%jWhU79qBRoAW$W2hPT3N#=06%6kCE_flRCPyp- zBfKL zlIAW(NHoVgE+YYZyj>rNc*w>q@+*$q%sVD_$tM+0N66vjFF#0OU5)juCsk`WXW2cR z{g(}hkz)Tbiy0j!*omzN0D=HJM=qBH)h8~Efm;XQL5@DIC4)-NQCRpGdcG~6_hy$# zuZ@_txa9(r7($uOAisrv5~N*HQB3$?5d_Y}Ei1tZLo(}S77A$Jm$qp8!Rx+xw z#%g%9K8}9c zyZ!ceGXO5|Q$zEDQASD?w667uyTGOU5M7nk@` zDo+35H&@A`wJq!;nD4gF$5T)@3nv%4kXev`!~lC>K$ZSfArI@hKtA<3VwyJ%99ymT z((m2r?Fw1y52U)0&HG-&I<~zbN!Kv?Xr3qpLr=KYb%0Vx$=avzo441!A|X}QpZljT zo8&vL@~q*Q^KSv&?pAGqmP9Zkos)Pvc#1y=|OITF@7K;b;(+a@-5iYc#hGR(I6&hL!x`1nT(&<*2m z&Pfsn@puk|E-Fz9uXnyh`&7WR)-3$O&+~fYXSmCsLQV8Wue?m{>X>Z3v_@-i0sjBO zW={}dM#S!@aH2o-Phj{Dkw8TNDa?9^ulRb9R18q%u*DW$#FMsy%$7iea0F0VU;=r@ zSeU{jBn=b(rGvO{@0w==%V_+5ERDv*HAL^I_eCA=Xe8yMqNG3t3{|1rGpi(1}5KB4P5+ z2JaAubnx0Zi4Xb4pXmTuz`LQE-&qz|S?5W8Us@FokW#G?QF3upQ( zQ{WEMwoom^Pv1u4QbKQ}7J+I$0u>3aA=GdMA8XPAZ2qhX%Sy(za4{EkF^6967qQC1 z{;n;WB@51=2zGD>9N|FfDpCyUQSLFd_+t&I0=XJx4Ju1HvXB!yF_=CO6e}$gA8JXK zN&V6>_1KXlT@fDbg$c{%rZUbR!zl-Mkr#W>7xyq932-La3>%JQ88m6eq9KfEArxA2gD2I|S^F~V0gChTTDHg_xDKFCA zobnSjk_e753Za7Inm}9v!4FCgE1hW_S+6ABkzL?X4Q2vf#*!>6r6qmKki_XFD+X8C zl7)g%w}eoaDlg=?jd02W6soN+$w>ohVH~_?SR9C0s0jQFGyD=$BT?rQIP)EqgS^

AcR z^RZZgLaWm{r?WyWG&!TdGoR0<>|@x@(jFCKhzjg{k~6+E5D5P{p*=M$HT5wr*-Qc! zWI$xo0cf;7?Gq+Rqvn`P#$uxz3lk%s(hEyr6)xdOlN2^Sp<3F+1hbN{B2+@{!5jAI z9h+>xCT+mB6k0xK{%iy^@9zcjYx%aWXK10cknQc(^ax>3Do!&dTQm>fGChfs18`J6 zuYz#mb02Y2A^%fI!EtDa6KHfJTn^R9){ZFNT-&ld)Lt99%q?(9glsi5`6h(QgM^S}l$R_7E}Gd5jyfCgxRh<4Kd_9qhT zLKy-&vx?j56+@2H3BsX^ zQf_6}qJ~&Ux~L!=qDUHnlQ3OYZTZzpw!~lCw&32+7cg{gE0lC^wh};q%!Za0ZIK}* zAOuz9e;Ozka>0!Xfv{GAUi*}4(Y0e+?sq9HEAXHJY?pFDXB9w=jYwb>YJm@2s7btM z6G@g)_vbMeh+DWA-H&3 zz@LiOdPiVs#_NA=>tlz-3|j15kN69cqO$G)NY?_ks);eaZ)=(42^_#MzQ|Tv7F2@N zQLqID?l&AL0EWK7fj-!HYXE}rz;-M68)|?AY^h;p0gTPKmx2^AO_+0I_H#j3)mr#M zN7r;)5oaxQkSSD#XP6G}(U5K!1xBC{CWClu7lIw&sel-OMI|CzKr4cwir4cOhNi5@ z_4SP8rR32OiA0X}l9EA6f|c}zV55~+)jfd-YMeKV&$AksX-|xWzRz;eJ~K z_JWv&CQ=v7_IQIK0lPpeAmPHw3T9w0Eca;z?&E)YGX$urRxy}<-dQ$6&>85UR^>(x zmgJ3O`BN-%b*4yZa9{)kLV;?obly1x7^n-BnS=$Ene(_`pEQ~u)sO%9noqhCbe0V? z=N7ry4k0l;r~p2^%ZHbSc^^5MI$;4W*;D>&Km9EXtC!m}=D}bO23Z6>*5iEUL1g_; z0~~n-%C7F}u6QHCimw;|4{;uTr=6N%6T7xoS4oStI0Q^XTsy?0KN_URw|rAEGa)OG z;X1Ajc`)p7G#9ag!6vV{Z=C-n*{{`diy0xj2HP2YGaME{&*&Mz=1g|y4D%>0=3w9u z>WdbWiVt!bR&OJ$FwwH27_9LztV7$nY%E7bZv51`nJM9!+nS```i15CX8oY9O%r+~ zg9>INpJeE}`udFTr;ahn3J4pdlA#d30kfxS@;0b38nF#&6{`&ccI%@or~5uG+XHxn zC%1YQfS?B32`r1SK50h0#apz;8mv3Iq8G<>IBKCz*qQe@9ZULrU)!W%c(zM(L{UI~ z(npf`TYlarNl3uW9HwKMVkR4>&W1to0I@A( zCtkq6W}pU-E@Z)bi}{U}=Z!#Aw!!=7v#X7^kO+y`h?mv`wY!9^NBVZ;o4$7(b!B@G zGayteK>Bj3*oLkG8<7$ZKm@$0b-E91jpezQXOkZY7LtHR(73Q4sP_VIsI#0ffKIwg z?x^@6qJUz|n}W=S?Xs!>tkGDGzfmaz=*{Ke?i?nln5)`|phuT{`Yf8Z_~4xfLdL;- zamdRzRD&SsfJy%a;uDr3GDD%y{k*lOS#;-`+hLf{@n&pWh3GJqc!E}F$Ku6S#1`bp zom)VQu>1!$nXgv?IKH6H6&!XtWYmKfs~;e`QGJ^Opu?f(=3HIGK^4+uo&Q$g8<>Hk z@!c7u5ko50f}4bizE*>EcN>xD8?+!(Cb%1PSWHqN(1-?%1wjfRp?atM2@W9~grOSq zybiiS7^Eho&9JR;JkSNbLbGi?KSs7*W<4zBaMpAxpLJ4V41PLjNYcO=&<)Eoj6}c$ zmK7Z4s{({uMpb+-4|%l?PW>xXvW^j>)otKa&a&+kK5yz=nFXP+Q{Z?*tgxLN3L;<@ znuK|`S_1!|ps*$q_AVExGC zNp}*yy_1OOz~>?Mh~C_rkP^+r-1TY5Ij-sH%7nT;WkSQbf|{4g8iF!+J!g9+0r1ip zTd>(M8+0o##;+XM0>S9CH%QJ@NqJL>^4G z2pTMyC(Vi#Q5Xy;U_gb1iCWTtz;MyW4IM{r%n8#mWJo;DaupKCCq$=`xlAq)c|=kY zBsBkQFnhvii3kJ^9MoL0HJF6JkStdTJ26-a_8L&=LrQ@Bo$y3~=JcvUUTwUU)OO)8Y6UfZrcdu=T?Y;N9a+rCX3cQ)Fy zk@<%G+ZeCm#EXN%g`Ac0<$WqKe{RYTNTVH!_LQ)oDTs#`Y&AoK35a-!~Pe~Xa; zOr>HJK&(eV0t~5WU3K%(2a{WC#nnj&I4DtMX|ovBkxWTM5sW8@C{#%|5t{6AhkQh-sPIx3@-#CGSektM0hZZbKk%HYV(CcrNAu_R2Wi^UfZXI=C zMp-xAaJGmlW}+$F!rr>hJpk>&NX%YroBwS*s{^Ufmq5D6o^Ut3@jCwNy-00%_-h6MzyxH>IQ)4YKk zf6M@W_}~Ne?RS6b^FBWmsQuM-t^>sq#^n55V9OEYX5p zh}4O{C24+juvdsmaU(j=0da?DAeSJw2+9=9UPAE(qU#NLki+(j!o)T{F;uRKW3z~IIuURH z1VC}Y8p_tjE^2IRcx~Hvvybvc@%?<6I^0DKQgfQj;iL;l_gLu~FuMWCO6lFj85?8^%sKv5OQd zPxQ(+z>H&eAOy8|vP5o{Wt+!q=Ekt)DQ{|2r2Y91i*)~iQZ!)eBcnK9Hc9$TkalgR zbtEVJZhFpi>eQV)eJ3L!$w5illaiFYBqo!?&s3zaPlKusMz2$m3Bd4H1aQGKs*nH| zWYw6iNTqOg_)u2{aa|kmq)#Rln4^5kn_}!}GGPj;k;3#7&^)QTVA58h$yKH?$Ck_>hzGp1+taYN9$z-X(3 z=vXIORs=K!05otwMw6LWj>MLke640K_o~qv*)>*o-I_<^THE0E6|lx5?o0@4*yJL1 zx!LR>J%=hAhdDN%dW!7mECwjjv9M4(qE&(lT8aOyYSptYU??vd8rF!$(yUL}2^Vj> zR=55Zx3c8~`pkyS*%tRlu`+_*NVP?eaFjrScT1}!N%a$OIfl9gA>zBK8*f&(+9%LzD!r#IVroA*5m016tuQ^x9Nv3l`r=o= zPR#L0-_nK={A~nQD2W90ir)_l_rMBX@P6UvUh3r2h z_w0!m1N~EnPF0Mxx*Sj%D6I-K^pF5FL062Twoynl4qXtu3f#*= z`%u~pj*CPN&_n5os6+BtowOrQ=*BD>&^H|YXo~}8p|X-WgUo265gefOHxbAO9~qay zf=#!&*KOr@hx)>GlD9s)Oz)*~a=ZUx#`lOZ0RaY7V!ZwyttvA$y@Iz-h;gQ|&nZm= z#xyD-GRV3Bz_Pk=$XL+KU=&L8(FC)DBIDoDx8CgzLtFe^<5zfJfffye$NT^>-H7xS z+Bj);f!r@zqiG9fvD-JuvD^(d`W}v6aW46>9gNP!0NsgQx&va(&jZ zBPb0qfsT1Z!Qd&8GfA1yhFvjeAqxQjdU78OmnqM&!e4+Q#WdB76D!@fM2 zo#gE!LqVG7IFlG|ZQs^tfzvmI#OyX)`#s6zS+o@568*TIr*b=eI=M6{9m`qXa&b(L zQ~?-uswPIV6ap$@0l^nGCe#0K{$_hCmJlS92IgQgUU7C^^aLw_eHUOTlfo3o7d9ei z6iPuoxT2mg{8NOteAxXDUiOPg|LW1Q@365NN`EB0$C>zwUl)zlZz-* z1rQ)RP{AE400iF5_ZThVEIW{nHIM^}p@_Ngj`a9(NVrC!xDuuDAE6fW~d(H(2=@#0U-$^XHb-5 z=1ZIBNk9<*8SseB*nyeChAsvb0#|mQBo!BFXEP^cd?Zzk2Z=dHiBUeqh(g2#^ z2}Kz<0F(?QpauDoN1bqlD<@4mWsmANd7tnE``B(;$(6PFm97YuU-@MnCW~dshqFR_ z55hx-k`ZjFRd2~2aygf}XHmkqR~YaI%-N7z#fPty1-JA7+Mxifl>i{-0ES{cCdn}y z(vmMG03YBG92k>0Ko?GMnUsi=(~$*~cv9Uk1s&&zFgGNP2AWgZEkcHr{Uc6@k(wsA zlj#%(X5{~x_@NB=VL|(-RJ2)z{Rfb@iJO43g$S9KP|yStfe|J{OeZsu5?O|0coE5l z0ma0IOojkuX%2sp22QbB1`sWjs73!&RVQGWgo%>k$(Y0h1cJ$3kph?);3=3#0YhR} zFZXTSSfJyyHvV#Xjwnt_0diUplOnHSkwtt|yL{*k6knpr%ifielJHW4oE0F5UPFW9F)f7hf_$iix)m#e zz!#4AZRuxH&{TOFKm;v;0+A_X`r0*rpmB=VE&8DaT$&7e5S06wl!0?jA_NKadYRsxa8aj+4Nv9J>Ju5p9H3$K|$DSfNiO#osHXxC2;6oz#EJrj1 zU{NU#V6G(y4JYSxebLt0_^ARXQ$P3m8Q4!K-7^861n1s3>ZctUqI&-W>5?x5 zYNO~Fu)%-=ED#Q4kOC9Cc?hcsY@--M;F)CG0UODbP4Zt(@&sh6E-7#&g>g7^i+}5O zx5o;@B1B=+JH6Gro7vTc+dG4zu{2~bDYV;}j^TaY+jHnEv!!E4G{9`+HCki|$~CdHbXA~F`*ShOY{H5xa+00;zk>g z0#uorR6x6Hb4P*@lq(DcqbQr%RLCD&n}n>apjN|)tjO8B5sXY}jfR93{21Unght9e zOvcTiL=Y+v0U!BEnG2oiF}d8DsegoE>5R&Ls>NFTzD40nVlV@M)&-_2gm~b1QL~^X zrOrF^U#Z!Yuy979%09GxCpA?&4Gz@ecV`jM zn)pY#^cnb!2YXo^CaiLJG*d%Oc{x+1J7}up2$VWEn)vdtEG+-g%tg_IY|$5ude%D$ z*sHx9eaTQT#Nd$wy*$UV9KJAndEL;puq8mckRa_7J6c>t7|bYCEH!rL40vaDZ_%^C z5Y%RIM1q~zMkxyOAlS9=bLD%xW%YQ`3@?*xrbY~~waeB*XW3=7IEy%r5xZbhKnAJp zUt7zRnGuvJtaA_@(d$NKfgHULBi4J%w>iww9FT!nO`oZW0zN^!aO4@wI|ZUBien%I z&2-Ostq@zBMyzz5l}nc_qI-p{3k*?OgU4Iiw>(v_I5fm;;(2_U4TDuvJ#~BW2Q8S;|7v0i55HHG%reh*(-3**-6sXH1hUzuE2D+eY zI#rMm#MXvGF-($I(DnQzeB`vcjIW1b-!{Y3ccZ#ak}f92h(Az9<)XTqLAz3p+=?^X zgmeZn@l&L@T$H1xV9j)TE7lpEdS=bdYL&fgpypxFzQ07UZ{8Dq(*pWAWFPwGIoAT) zXKf0>*U1JJn9v93%rVShwD~)DB@W&ws1J*-59iKZ9+6i5HpQ!2?~nrCrf#iGy)xsK~u{yVj; z;J4j5Vvf}Aezy^wYZuN~3bw1~I9sz&zJ;PY1;X}%`2a>48d{{&d zc@855@f`{QVlL3=i+&FJaHt6&C+R>8|4TN*WFGZg?K-j|QDW!mTo43I%pM}lbp~Dz=C+CPUODV!{jA5Z z&1kOd&#Lk^v}#K1@*a?Q(8yQQ`i05P|6f2Oi!_@x=dp&D+z^S)N$I@2w&+`qE2eBF$a!p`E zz_tXLmBEprZwi%jG34n5WEqR#2&_?%%b*5XP#OqI^7oa>OPC2*B;H3pTuVM7B=nk*8@Nf@|&h8)v z_5B-Oi6-`B$f1fB*LQFeWT+_IC`uk_1Mf^sc$dp`g9i}L@|k)9jNNJ-hD`aAfvQr)g$*0c=4mO! zhXo;{m{=`gqtsTuOoehKi&}?H97oG!okcs$tTAG3i}`^Y#?ahQl79KRhU}R&zS;c# z?aM7NHE4hb7eky_tzx#o8uMbzO7djNl`WgXeC&pf%?a5Yn-sc|Xws$Ggwep6;%WyY zrlbL<#)66yvuRs^ZL3OzJ1`Vx`IF{T2Gt#mr)J%mH8Xn6w3)Nx;5bErh@2i=9Y&r# zeDmNlno?gLdr8klLI#K*klPEol0WL9Giq9 zsw$;=N`@;$zD~!AH+N-d`Ry2&o z$QFAs#>-~BY?51wI6}G7M#F4J(%wr>q1LE)0~{bAKvDrECx9S|pZGiJw>-{Upg1e1 z!?L1_oTFtTEQ4{*9CNUv$OZ25$>%#SwM^qV^*-RB1x%KJug;N9s*iv>y0p^<2K>uq z3JHLihbbcJT;a)FoZw&zqZWj3iFsnnr4X8`GSmhqmC=OL3^iON6%Q!`@v+QKEAd1X z2}`j>7G0DvR$L&Xl}1~8v?Ry>&e-t7#~yzb4KCt>l#aIuV?q*u0Frca&iTgsVM6hg z%dT213$yCFkWS5Pv*(HdI z0NN_jkV8o1uEg3nTIM9kpsO8VMwc|yXrq}oRkK4MwILd2%?E6NhU$UcHO>bHGBuP6 z`f34CV2KUQ1Bg31oheTL1YbZLKd3wjG=dev8FRvBQes?e!yk>8Q@&T3VkbAoV438S zXU;3MV@8~d!)-{9+0~a@)r;oXWzTt5TWzmw=UqYjdBD9nO#7Yl(Fd(xE$?duQ` z5F<%Yfe12I3I-CGLFeQX$EeiNCL(Z6`{V6pIw0M2nDWHu*Q{aQ+G)k>+ zl_^tfLmLqw=a} zMxAjP2^SD4Po*N&vz_%RQ%^|A45;w3{2YxRiuun0GHIa3E2!RnGa<4n6iYHJV;LJE z9SuzX;fuN5p#~6W9X47AJ2GKhZiRxym>vfSTPSQ6TiO{VC}FYIWUFA|gw5s_7C87h zNMvnF1Qm|#RyyI;X)>$XcE+l+M&+y{=_%T#@@zfng(_ENJ2TS!v8}kR_yVTJVWtsHWGFLbGkL z2Iwc-crrIp;Eiw0T3aq7Q4>bEm@Lm|92v5}w*;I;)P#GK61-+AGu#Yv-&{E=yvu$6 znli^;+e!yQ+!6>yVKKGLH&A7C1ESOMf%&SZjC|1H#M|wI-2TiJnbt|OY~X3m9D!e9 zg<+iRMT?M!%t-e_GPI+)*{N$p+MaRRXF=O9(ExfRPcErqn83t{HlZ+XlEaT=(feyN;jb$G;ZYcxV!96o%N`zN)G zG$^P?sC?`iPUYYhsyXIi9KV!>z;d;EJxv34x?*%iPAWW0tyEPr)v0weZNAa#uYfzl@ZLsjJMbab@fu}cV> zAKAO72LzQu-=EA|{w_1sYbVUZPAd0!=xk>f}8p&T)?4^@2dBL4NNdS!G_dt3Ap77S)oi zRob_oX0;kO7*yWS=iu;x`ed_ySRUQ$DEGbN*j!Yx14X32_Yd)W(8AyU7xw~~lfS?J zvq9x|uM`<3mIzLdTgs-^1k+!8=+2sXnX*cb;jyuiD{pgN=hu+zhS zQw$>eAi+2cUqC(*EVbhs6%@j_XWF}nA{-<<81*AL$UA_V7(5>T!3l1X#pw7E0my| z8mIyP$wugN0<_W-4I3t}kU^m+03%c^Ag~Dvps6cxIhZ4X0^lFn>Vx7D!=)HTSKNU{ zS)TpFwI;AXlNgANv4Klqpghz(qu{VEGKYv9CfYm3E=r(m0w9P)MrPc%4g9@lB)brV zm1#t$LlmSzG`L@|FWXu~g<64Cu#Oyv04uOQ8Ek=cRKH;VGsl^XyD~CGcI*IXL$C({ zvsI{)-U18ATO7%gHmD?nuh0S}T!H~<8m!0yV%iKMsIh@UMzKINHUKZgk%OJc0zFFt z5cmYM5g@KuQYZxbu5342ZL^QB7+r zNVR;Rwe&R5bb(KJnl(s9YEgnLSU5aHvHQ~_2V#P;fGiQyP2;Rhl)5wx5vjqH&L5l3 z&k2OJ_`p6K3_y&=M_Nh6Obf;o&+%+Yn4~1?LeKO6R1L|rk|cN_Jb+J|l+V)4%rYV@ z`@&E5#F`_D&;I0tI)FM1%PmwutQ+`&`HWAM=nO`HPb$EW1|6*G8c7MwPxp+@+gz#j zd@jwxJqm44V{*>w0g3Ss)Sy7XH0(UPciK+wgfH++%oSAp6elwv$lrWY za!OAjb4v6Z?jZOT~oE>R31&sP^C>! zrPN7<)JHw7RGrgRMHyeIEb=5oLRuYCDXLM+&OJTMz4%jV+|{*%kwFF26HK+GBUYe6 ztvZF(S2b2-W!7I&R&{&Usgl-XO;+)ARbWlBo^q$M!&T4fPQ&2U5|mY6{Z(uw*Jy3b zWmQ*domO^r*L8*0cx~5erB^{TuXdtQZROVBv(-QR*5OM`aShjVwb5}U*z=m#gjHB; zeb;(*SaW^Yh!xaAq*#9K*Nc@=Qu|JUEx3y9Rf9d)g%w#sbyjRGS%^JZlvUZ4ouG;R zSdZP;n8jFs^@i>oSeU)pmj&5!^;44nOHQJ92)L9MKo3+@PT@|Ob z*__4Moh?|YJ=&r*T57ddiCs*t?Nx~7*{ZeLT5Z~Y%~i9VT8<6bvF+KRb=#p8+OECY zxjkC68^m)+TH~wOi&Z43H7c}?RU>2Dwbj|ey;rQI+ssH@#0}5IHQd3~+iitWne|(g z%-9!&T#6Oj$8B53HCo>B+{2|?&dpr4Gu^VK+{%TS(p6j1wcF4YUD%b~%>`T6m0HQ& z+gqJn)V17yW!={OUEA$klLg(mE#A2eTj5>Wfjiyi+XY_b9p2cTTYV6X z*|pwY#arbiTj%XS=VXOX5y^iAISRoe6w-~Bb; z=q=xDXj}aaURhaRq}|%tb>G^x-)Te<0S?~>?qA*Y-#(?)0d`>4?Op>`-~--Z3tnLU z6*b}jPGbY~>M&UXCUgIuw-d1+y5Vm7IZscJm<-e`jQU;7&R^wb2 zGG1O@UH0W)mgZR|<~$zX2!3P=PUUS@W@mooZ|-DkZsuV?Vqvc4Yc}U%R%dVK=5A(Y zXpUxZp5}NCV_IHkbq2(DZs(niuYI0pdIo557U*EE=NYxD|c<4l)=zcEgik4@Tj%AKM=z9+1GX9GSj$(hN zX^1}Qlx}H_zUWW?-so1gWy|GgfR$->ChD0E>4*+zjLzw#)@hY?>Y(mus5a%Irs}Fj zXM;XyoZe}lPUxlnX^j>>OLpg)ChM(!YOVI_wD#(+mgq8#pj6UZIGU1U1>>X}u!#?b;&TF;CYsK!?#kgh0 zW?#Z}?7)8J3#RPBp6t=a?6x-T#RluB^=qU4?4f?_w8d)4E^XBg<;q^^ga+IKcI~$Y z?ZB>Ws5WccwrteaZI}+$-ag;hhUT$jWWhG>)E48rmhR$4=IBml&wlOD{%v*+Zn~ao zfv)bg{@>H1=4(i<#_b;K^@infHq7w`Z}`4#^Zwq##%}69Z@y@4<@Rm%_HNpiZ{enI z+_vud#_i3vVdQRXB!g}LX6{o~Yo;D>u9jeOes2SxaPdBHe2(dCe&fxZ z;TZ?s@4oTx=3`;b*nj{4A^8La6af1GEC2ui06+mi0RRa90O<*|H!vQ-gZJtwM3|7_ zLWd9|N^CgsVMU7=FKVR7vE#;!AVZ2A`LU!)c_iVfT#1n-N|!HT#+*4)rpqeYV{U23$c)2B$I7Mx18s#T#srC!~d6)MxNTETk#I`$^FrdY$O z<+`@5Sh8>3mYw^PuF|u3)5_h;SFc}#ap4N?8+hd}!inDS8;9iYvXy+=wKWs8@+NR=6UKW*H~rhd;t7sIm?Mt` zws@qHOb+Q}kxS}WqdG?#8ReB&CYfcEPa`R0jt;<@IYJ@VS^erZT<--qjE+H>7;ogYH6dGK1!*i ziY9vKr|Y;AYNwl$YUZVxQkE*Gtl9?ZsHVpMT4|}M_DSn)#v*U3hc1A z>dIoVye2E^vamMmtFX|XO00-!Mr-Y~*aB*7tDY*$?X%Th3ofeShFk8rcmur|kyj!oc_sXm9zUX43ufP3zqm94>L!0iwO$MB>!sl)q?OM{Bi{&@=mSge7 z7*mXKHvn_oF*5*zEb_=Ck9>{DCocoC%Cwk_^2)TN%nZvi$87V=HrM>j&eZgLvNb+q zBXh_^XM?c8NGDw|!|HBJp0q^+EVIr)`&+We95)k=GFx-)_19g0oyja@uVVJusH7c> z+G}^hHYjd?BKO>J({1 z7JCZ07n@$k;ZW>aBOf?Lk1i0K^Dm1j&>Y^5%9>zK63Dn zfDEJ{{W!=9l2C((T%igTIRQq(@Q?tEq$DRP#PeCgiI~h{6r)JRDv~Z)u<)N4oWFr|V1w2^un%Hb!Hl?S{DL_w>qD0~uGD%KPe)3;f9Azn2$vst)k`cQM z-Y&(6hy<>&77VSR<7D`@uQ`xVP-F!K654WjuE} z3tP4^7(k6BE%m7ke{z(A1t5S7uHc1;D)gy^qyR(-Db+hx)u~a9rU|YZ%&Knnt6$xq zMNN`A&bDSl`sS)*x#A=xHoP~l$`DThZxyiu~ zc1Wxv6nj%s_Q93&RNzn5NZDD=b)OyWXD=ZDfB+Oz4>AZq1CU8jD>%WlrA;ka3rbpo zuGY1$g{^F#O520~#`dWYD1cOr835h(5~zhBLp{q#+~U5~r;@enNEgdoKk)UrM*Qoa zTKdv0-f()1Rcv#Y@XlTS;iI+O-R*4Gj67%|3Q@=`Km8d{&KBUa6cBA{QR@Y!+PA(; zMP`2I%isR`_qMeSFm3~2M=OMuz*5x(XoDM97Ua_i49rCh9(Y>eE?BM#w1Qg`0R>Sm z))b-uaflb|TFOOL?}f&9Z>CtXqN-0019+;~J@;2cVKU7w1g| zgDs2Z44--bpOEM<|0b6TP>7hu%`Gu>rIBJx>1(hZ?t`7ROJiQ}3>t{O1C4ulh_Zb* zvz`#Lq=%edEy#7fMy*8yzPx19PN2!9y)S;JT;)<<`PBc_vZ}E>gc5X_w*c653>17> zP2^zB6|QT9HR^;o( zJyDBEOS&1AzVt@d9Nz$R+6uBBwYsgGYE$<&%I)5A4(6R@S9|r#1Smi~gR2J(n--t` zWHTQuYt08=)){(8Wr$B!CvG6a7a~Ti#Kj2S4~km|~@4X(i~* z-cy0cA;l(0)r+=M76VbzkcHJXc@3>c{`t)EI@sAZ@`g1dSD(nXE6wQn&o{xmq$fWA z6@Up^pcCDJtCjuD*UxlP!+!R)r#Gu*Nq4f6U<5go={jA!|GV~|{{YB8j?gm-b7xmq zM2@!}YXW)D2P(@jd23T+u~2Q$a(zkz3Warc+~;}auvDX0dLRf=mPT)*^<(qk46oOI z?ze&|n0K?4cD8c`uXTU^7l1W5QvT;U{MSiK(}TjIe2S-o2zYU{qH*$4c3)R{5_l*S z*n~pU2^k1SV01tnm|mghb}q+s5a0xo7HaJGg`?JjViNj4C~?&!tyQP@_=yw1&vU7b_NU$7d4lcXi=C^1J`Xr_6O$wpmJ8I zW>`3Go)!aL=!IaIiKsP(WGIH?pmN7#Oh3ei&*g@s2!NAgG^*%>1oLo|(>Q&_Yz*Qq z7WgWD_;G^<3dEo!hlpsq*o%PEO4^19l4c8L2zqivYs09B!WB@LxNe#Ve=`Svo9Ke6 zc8mu#il=Cb+_+pnvx=aTd_bsY756YK#V%DrUD&5OfcQ3Z*o(WEGl}SEXlI6#wsKdf zZHvfnQYKZ;C~BEFjhtw6)_8X^fQ_zIP1b~C+t`rZI8Mbd4TlDf6j^MObBETKhg`Rj z`O;Ds;|;O!KYyTvEmLj2c#_0$Y?ar0!suP~sCM`mewmd^73PnWc1P_0_kQU&kXTk< z@@HGDR*ibp3_*s5r3FW+DaJU-g_OtFd4+(n zUzF-&2r_n1lB6k?gEoEos8K+W1|D<*uZf^AL^f}U zg^#EUk9cLpSOUNWp%69?hKZqE*alXhp>9N>x&Wfo=?o=s1s-}w<3I)n30fcep%%Jt zM`xm@^_@A;qQP)}OLd9i=|?OM;ag ziJwZ+nO3R}7}#j|R87nfK@*6QVHsoAM0$!iHm<3lXNp5p&~_pCK;mT%0y&{v(*|?e zZ5NuMd7uaRz;ZOIMk`>WIhtM`%A%szry*))kv3}>s;3pyhCe1ki0NV2lv=hjV3}&mM7{7=5@U|80xC=cH{tMYvI>RPqf=~E zo4tcTlT}w_x>4;YJa9B*TH{^@U~U3nZVONVvDO5gyJVm%09s&fFb8G~5Vl};0jP@s zp34AH@VNw#2Ls?)q$;s~1EC89wQITnAhrMuLIdo-3?Q`z;Jf}yyQ2GLQGfveOa)L& zwyT@JyZgWxkiqgQN9Zb-=GnIBnVuoMlzN-J+G{EH3Ao+6FxX&+s1Ux~bhzDSzGNe2 zZB@SNTe4{xo5cseF8PSRqro`T0w7QU8Qi{$_@#ZauePrL;F3!7lP1ZhxSQDBVX)dWo7kfi{^%sU2e499XD#}Z4ia@e=4+N#{k zgu;@2W`GGQm!;bCX67pnj(f7b!%w#VP!4Gc4rwqpc#u(fBe~BLtbHKC91sDTyvds@ zTA45a81TsvAj-CI0KeO>xwWoF`?YN6S+C1-0jvhGJj*0N%Fb%b22jesn*yA6O zt^8dQz{#0h%pf4f!J4)W35z65$8+q=)|-lVe8TIp$4;m;c4!7a&o zjQq$PHOcP`&&cCV)^yKCY|;7P&sY7=TK$~{-PH)4tOEennGn&JK-T%-x@S!V`@96& zimo7Z)%Ps6d%DIbt(2EKeN!z{)nutJ4b#^9*K#b))Qrbi!ZdrVm^uxBKSZa=bwInO z2m0i%22{yoJkJX0k^-cUaT9z73T~@;2h}Q@<#hqyFbky33>e^RN^J|N-3&93tShj{ zDWK4y{0(Gm+m2YR7+S``y4&Ay$>Q)lzrC8@WyZ-61$Ld-DBTUnIzd}FvQog(eLdZN ztwj|3jf2g?*s{Xh4J|qU&6q!=V?K3)KRsDF49`z3zUe)D$+M8;E3&NlX#6?f%y4F_ zS>GT7d>nHJ`8{w2$4cQ)xE?Fu<cX)Lq?xu(vZ! z*xPN>3iI9I4TyUk$>RB@w^La`wX*W`)O@qvV%p+jdR;TnPK*ZPU8J9lpo5M@NijY> zH2%%m2X{)bO$&=j z-6FNTME<-`9_3R1&M!BeG>eRAR}Kln`g_8MPifSB#|THsy1oM%Hr-I#8s%$>Y^!As z)xf7buhYhklc}oCutEyws`CjRx9X@)=W-rOu*l4Lp6gETVRXIIG@$1>mNw{RxIYlx zJk3p}^u~vd=!wq9i@vdo?sY#Z2#>A97-o-_PPsv54!l#c;I?gqt4rFBp~&-Sqi(p& z139%mKB)eHXuvu@?uT<8N^yScv;GLn@VqwVPF$qx$_|0TyhGSF;j|E;5N=)yX~|;F zFTmgmCPE5;loI~ZukGda6q{|3 zUgecLXz8QMj@{ZV@rsT#jgIw{xieP|1$=<*k|thN2==Joh~{3_LHvCl`=BX{@+q(K zZXfrIC%vlh?mLg~Lxl4LoA+?P#%4!pygmg*8Qn2e>dHJyesAQ7nFblmy1L2*z257J zuZ*Mr2D8a~+>cM)Q=j4F&ujq``p|;@zR2(?Ft|n4FPWT^1SYdPBF z3lO>xc_L_T$sL3VJN;SMP~oRiix?fOQ!&!Th>zUBS_J5o$B%}<#K|bJ0C@{ zl4Gl1Fki()L(*oqe9q=c>C)5Zot)!p&G4B;*`G|!qhnVp3^`;~uyy-(oH^6^RIgUHnl)O~s%@!=P`j%C zjVX8(mgS8bDIpnZgew*FaQCcBy7{0&SPHrEmmN1sS5&>n30|FclE*F{|{P#YhtY?7#QF{1t0(d5pafT!K()$ zEQ+LSh(SggzB1Er!^0l)ki*MD6j4OZ+^Fj-7%(`23R_r<%{3THgAq2<%6g9{DVk6S zElhgs!#9Q&OoM_Ui!+X)KVk#tG~FV3s}@~Qiv^?Z!n5up?7&N}O7EhZ?z&B)`lLM| zCX}kYVEEwaDGb=MalbL3+E2Rwo~U!HxroBR0NWC@U;+qKq#)2g3njr&6>M1lF`Et> z1+krV9D7vK5h=6G(n>ee&(1s{#KRL8VH7nVY+jUYJ=4&VYgH(6R4xx6tB3$045Sdw z2vv_f1j!nM$)~v|qa261=$2CH%PznBZVj8XOV+zRde|ZfB|hkaL4)x0stH)5T=S{~ zcw7ojGs5F45r}*uLaK^5)wV0{rud;Y1Oq*w06zg8bl(#C_4iN<`nZ%Zf*pnDoJmIn z5mScet8^Jhijokk6rzyPRD4F2?X=Y1XzeXknPPRfK1!})hEL{o6BAsiKv!3bdmO77 zh88&rR*$EgDbMYgxH3yQnRT|ynKpUJL>RuxHI0&)f>Syu zm=R}|c0o&~mNHkpGYjVE4IqF68c48heknjf?tkm1fE9)f*84++Db4h6hG}u6-H}^$ zGgQ?w)>xa5drg%mPgl(isrl~UAd3)cy)_Mar?ALK(3=sqvQAd2EH+ypZmINi(QVbE zFl7qFOQg9hIvQmmibTQ+YI8f^-|a0FLaPE)3QzpPLro7&z|JkFHr{BG`2QF&5MMsC z7hd1E=~nCefH!k={DjK~ykYbqMjPxPkpg14#Wf}svW{g$Yhsb>SkN5N#fpLk32+Vw zGEjjGoVOGl1DZ(%7V@zL4bMat@U$X?i>#eQ1o<;XK0*zmOGD-)5CRse zr2#KY)V`9Y!l+eYTE95|raID~IATtLh5_O4uIEa))=noHuSj2lB5qL~&Aupp~ zMe|9qZ&WIu!Augv zLj!{mFmW;=3T+s|9}%G+I0(s*7_>l{f)I);iASGxS;g2&h&$PV0i2h-q%Qr$Nx3bt zDn#JXPOJh3QnsWSK&U_(VhNzxU;rKYNM|i+ImD*m(uln5UP;5psK!81n8PIIrNn0; zV8oPJPpD@ngLIZ!G}D>sh-N;ZNse>CCN|fchZ^Li&7SP4Op{E;gYLkx;bdz|6ROx) zgcG$VWUi3W``?LPkRbfLfHn$PS3Z$Nr<{sIC@l$S5Z=eK1r$_!;Oiv6R2e80Vx?pr zjp$vVfRu|ig>nv^RYxokFj7YooZC0I}!1;tXiOp4U|DC(GWGTG9dyf_>*s~ z0|!Q$C=RD_1|Q-ejH6kn2P=ST2IA@fgBprZ+|<{+4httG*ia)f{DW7e05;Wr*G?N&M1@ed!(z#Ba5ao}@Qf)_wCk5KG))NOG} zE}dvDbw$UNJ2aJ%OhWqJ|p)L*N4r z#^Iyjb;Bhvt%tC=nt@saB@L3&JS}vS53?yicCl=?LCF`T^^Kx8jJ>cfgZax=#3C}I z00rTl88r>2$H@}lf&^rn0WNrJ|N5sBLc8nFxRsb3(lyT7HjsG`?2jha8CwtTp-p7r z%3I&yXc$wvJm<}{Gs3Zo>O>>b6#c6ITqD4?w&l{GYcpuA-L&2~ouSCF=%E%+Ca=4+9|Lhv2t!#xnVL>IeIYyWMrlL|+ol}&zlPHLIh zDktBtSDs&#gnZ#$r*d{#q-VaTz>p!mJtRMDIq2J zH*6LdG&BAYy(n>`I19a23cxzEGueQZ;Fy~Ml8-j>xS^wq6c9ftsR^MVjeNMj>L@ZJ z6C$q*h2X;j_Bg&06owZQL0bxf6(xaRz zuR6#m{&@lO1Eo+iqLFhi!@IJ@`lb1cKgN?n{DM4*D+4UtKPTg&fe63{;=E}?hxRzY znQUVS&}vHQXwECxG1A8Ey$(! z!ZKl_Li&=wl&iwY;y-w4geFs&Jg6WqBnVwBh^=Z12V%5B(j`3^Ix097Xy7?3C@~uV zMv&2+I!QWB0wUSFrccPiny|)e6t57RG~^P5LG(A~Duq4hxQw#<2Xgu|&3FD@)9{ zFIYr|giOr;wG2b(A%hrLnLp|#nT?SubxI{hoXb#JrvPjU;RHd)l%3ld zg;T)EJ>bS{j5L`P58`_Y9srY{%*?0!gyFaf_(TM35P`q4IISo$STH?QI0F%agS1?R zSlT51K@~mHBJbh~AGm-%iK~TUIFfrWO$s)}BgHSOIU0? z-h90O;IvU0D#7KnfFm-+5j9aeDztFuJpf&Wt%EK|D@PhVR873mC@oAZxX~Mpgoz42bg-37Aio?sLLiGW-(kc- zDN}(&)4f8G{bIma; z4?;vs8*PO`HB=v@$3zuR8%jY)C_zE8CD{C$TF3~q%2rLq(P$ykkNDJd4KGHUq#U{y z(ilxu4FNyOwVL8B1pvtQi`6E=uicakgsD|q{mn4jRdij!tAwM1OTVWnpayKf&7nd6 zi?vuWSkGlhrK0E$)zgg#`zS*=g>*%-U30D>3Y|KTG*y6vPDnaP=+iTS1Ytmq^rVBD zrOb6~h-E;*{RC0~;m`5QRCUFIPL)zm)y$_l%@$y-7FA2tY$Ig|Ie_#vfX%Xlwa~-J zT3kg~-&ELzt)&cbGKXuK=ux*zE>4@Bw@l3h}q=*bkw+kWfM zuS*6@3rEQe+;#jb6*a+cSOrCZg=FZV<71K;J=7O-SEQ|6%@m?qV_2qpHo=oqC4|*8 zHN4!^T7%u%TSZu~4O_7dyA5!F)|G&)<5v>cxEioHJ#YenNISPnnznsg=AyO#9%X{f zl2jE`+77)Y7AQfrr2;4@!K6FhuR*h1AXnmJM8I`0X;s{|RjV3MTX0l0&7_t&2v0?z zT+6*&uhAf&>lz?>$3%P@;5ra8t=b;D+R;78Da>Ef_1cFSOrq6Xa|B>6a)OFsktw5s z5zvhYC=Vhz52r~4EKS>S7&6C{pe31LyG>eiEQHgzg4rFL?)5jGkk>*WL71&sxS-VY zDAzjp#$*u3^_<>ra06k$H0{+ii$lu272#SsUv{MgMFLPCw%p~4C8{&r6t+rD6g%wN zu`vzEs}0x;DP1b&+Q<75Q4*&Z+F~ChOfUZ8b+O$KK3Nh-+#GS5rs81#@Pl9!fD+W0 z$KV~nd7$9CHC}Mc1wI&oBx?z3IyEnyVEgRE=X z+D!&27y&(SU}P>zP@okq^yahqR029qz&k6K6nPXNY=*4s~8t z6KCjp0O0$ndDP5-@;Tg_YIh zqSed>>&NSz>E_}~nO5T(Fx;=vIus;Q8U9wl9%^wwK}Js8K;FswlVoQY?3NCbrgrK? z^hehUV>7Z>&F!KA3bD2L+$#HOu&xX$Ug)x(;xwfUv{vi?er>o??!O;|KCUG0;@jqe zXaQ|~U%o=*kpR_}u9mm1IkNFyq})mGW#kemh3{=)z^>u75`(bF?r?w$npELPNb2)t z>eAliLhA$Yy3ec39j4^%t3|89Gg|B*+y1`@njF^`hvDCk4GpEOO1x| zNV*yxaa8D8Al}bR@ZWe|SJTGU$aR-o7E}4fZyr17ujX&u)>=x56edR{CkL(2f^sSE z?E$afx3Uz3un?J>@Qci#enJkN?K?2%y$9FCyRcdRnK9!?wIxY2@#s#(OFX~wCW8>j z#Gq8~lf=wE{DQ^Y1k7~vRtUsD9|d|Q>`j*D_I`8eE{Cs4gvv{juW`*K&KziyfF~aE zCyrty$6pO82fuNcz_AcaV1+y&^-92+`-$iQKf3|w9d1*wjbe?BlB2fVC}z}&(bU!) zb(1aHDs%W}dIn-ezw?^zacLpaK__%%5B6j?@hSyRXpc!~e{((;iIz6>^;V5sGfnYB zFE*ERJ&=LlIU-i?wLDd8?s?@TSJOzzm`Z|Qcvse)d$Sai_w-X%RVOu82LWx1$BUWT z7{OvYI2lEevk?V>#GJ`_-hx8vCT_z{M_cy);yvHC)+m`a?V>POPLTF*jR(_nHHW%oTSeKuew|Dt6KshiW%54J7Ik*RKL@%X*3fJU1rsz9A z?MNPCI*4h1%Y{7z;=nAC?5Z?|Z-RdokG9yHcxnqvbfFk8hW5 zH?nl~(|J@xcYQg4O;0(*5TAQkFwe_k8(;&?=X}i{c+Ss!Dx*!(Fa2_tU+cOYMz9cc z2(8tC26@*x#_{*gTlhEk0M^C*+;F%56)bd$H~U14%44VaG4bA&y~)d_=eZYlX|H?b zFKt9`bG=V=$}J?myQq6E6Sueo;o3xoniovJ@4saF?wKK!OrBI@G5CDiOE}}FG=8RbeC0mj(Z0JxSL?t3-IXlRK;T||Rj`Zm0 z@#6=O5l21@A*dpvHhlO9GXv*UsZyHewcPRuQXEL7C@H!rY*i~byG9W$%E_prqF9zP zeR+yht6NPUE~E;E#vNt(x@Ga4F>VYk8|2869Y3yY`7)+77A73%T;Q{3xCBO%1`t5N zfVB@gs8-ARwF?j#K-j4r`?ifGRaAnyXq9AHh@VhO$a&~fohfyGganxd3iRl}e>bXV zh01ayHoJxE{*CL&@AJOJq$vK;(ey~y&zU;s>CjMLs{a1X?9)gya%cp{7^VzaUZuqW zfh~YwAO;1#wSWpyIOs`1!OWA-R8oV|mj2_gepn(5Px1(|#;PIU@C{+|-SsqX!PDM#tw4+1SDLEDs-hJbYSi%H3 zUN}`^Sy4BeB*or(Ub^T1rI_^9M-?exm{}hhItrM91PjVJrv!TKH4dH=;;9TU7kbEH zVt{@)Xo!UplNk^e)P;#T5Tr;Nq>;8*BaJimcPXY!o(WM%KFUA>s1%6$X#@%ap+H>l46BG2bF?!0wf<|{aDc|-W75lPw}Y-W_wLZ;@+lcp0(yffwaklS5H7^ zU|Vd_i9s5fX`2bRwlHIhBcf1a&7Xm0Xs)1!l1r$%d1$tR42YQH$z~07W-n(aezxLh zFZyevr8{IgaI$O$Sb<#lDhxq_3@@zk3?IQ7j1?$kR|lTZB@@mn=B3apt4QVoWJ|*m zD{Qe}q&Xk61!IB#o~Hf*80EZ$5R&KxD(H*>s9F$mGZYQ;H#7}gJi&#y>kf7<)9I>9 zC^1hX(x(lgs?p5UqB{KWiTqL;W55K9?J^XML^s17Q*1q#0&Tk;*NOKwa6xiM7wytR z7$0PFEs&s~q<6w#x$!uV%rf4uK%E@sdu2YwvdhXAe9Fvbgyjr`QK;lDs&8GYsWHOk<-m>!Vzp1h;lY>Cz#v=3X-TA3b z(B2HxhR2%rRg10Fzce(=`1Oeun>iI5x4v7NdbYTl! zY~^^#V_*Y0NTDrlB`pqtP%co|jS-BX4z=JLEcS4d7b0mVF7(b$gk_ZTk)kM}U|di> z5Q`N~g@WqSNXwpOld}Xu4VehWUeo}A5;+kF8dWJ5M;sc3e}KXS^0c2^OI&XK<4RE!cz&rXdZplbZ@0<-z5-1%YXNqF+jALQJvn zRnYQJuQY)}MX2nN8Trx=6F|9|5GV;IG#%L4N9e`ieN5Mo2SwQY?v#6+GPDb*im8L@(7<6^`TfSR`@ zikflRS!gD(nFk_lH+RTm5f3*dj%ku7kR!@Dm7-2gCb5~4OBN|h*}Xny3u`z#!Gprm z4D84f222>87q1~uT;_6@Ph-q4dubU&X)}SEIp!N3SS@9SM4nxc<#RH+pNrBVqj_u& zfB<$pt!Xn~%u~ThKiD9OI+IyA0ZI&!lFpgV=Z7`DX-&%%xn`A8Q%>>HE6#S86c7~! zL#4t!`>E8UUG$3tEvP}&wGMi?qpDP$Dps?a7_Alr9bUEPNF!Rvz94D|ptw*UY9j|b z9PWzE|6oj4%i%3s$d4AFB-;_p&7Y!r_h(9h{?QANq)iJLzE-zQJR`^vYMzfbg1u1p?bnaDo@z zb*GVipJL6KGRlpYk~WoWWJS5t%KDU)L3I+~;w#_zY61_Dh(&4v$l4C!Lr*qH0DuE5 zU;)o10kIiyffLL?-a6{H00u!l!$REQHp2x7Ht=%knl(V2a#LAAMcRU?fjj;ShzEPUZDKUg-oMeu^fY5)ftn63AiFmcQAUj$cys&)N=BywCI z3~CS@grE{48c0ade8vV(gx=3I4C5HLv&QL^X^+pR<3&?~hw>TI`v@)RPWE&03|TStfqZVEVCbe>v5UN(qH2LT8fgrz0~|J#{B z(zc|wQ?V!Xl&9y}V$j@1)=>_PX!S69$2JaSj*lYbAhWMonEpW|3?k|uts>NY;{&Iq zAcf4;H`G$V!@xZ&@L212)`P0`lrjFgC@6{`tVly8Kpt{q=MF`tN%p^ZR{=Ztk=V8s zcD6iWS7S~31(&czT@JLiKI;r;eQhJp7o|BT=7SK`b05+Tk@Tf2Jy=dZmX3#2@2T?` z-%!D+jE|T^OuGRFx1v}LG|Q?V+3(<~#sdoyK8v%b+8`5GsKsfmab)qG;F0 zkcXV(CzhyN1b}kh(Q!3!Iy~YNADcV=Scy6U=mMsRJmxVByKclC&sAV8{{pR5P4$lR zjxh;`@{0bvppP@vM{oK?(>&{=3%?nhzA_EFX^&EN@NDH(;0j_y7^cq4tv4rf=zdDpH2NMy~? zY=(LpXif-jfCGDTr2e7oLg%Yjy8MW@(N$l)_#MH$xt`1Pn7`59`2Ew!6oRRB+6dGD z<{;InMO-*s$L;~&!f_VGC7<#s)bo8H2p$SA1d|0AfjxyBvH9HfNlNz35%=|uJH=p) zf#2y>1sYI`XXMP!bw+G$QvtMJ{B4B9pq~BF2Sxk=^~ixT%oR3#|BHn*!!mq;h;@a>9d$sEE(V0|TylIa4koYgIHUh>p@C%%19}K#i3~C3D)!_JL8ngfx`>j}Z-BUF!;(yJG-So$e6_E$5K=cp+ zBRrd%O%(n6~9?%M?#O1Y3GWHg-tq2!vUkCV(AgT}Ks8<$r0}`gy&oqHx;nVC? zQ?U79$61JO*aaLu|kvrH}(~fCx5OBE`Z2^xrZN|Bl#6oIXC;0&WavJ(-d$ z!y~=hQdHXPDO%;!;y>+5MHn0}?qV{KVjwux8&;fZ$srwvU`8HF^s$l8nN)S@8NZyt z9|jE64dUcv-L=^no*hrmjoUKzS-L?_S1FPJsvdo?HrLeK^+;EL-ZS=H9`J(&NvW4H#$gl4kre`?8FE9ZRg~=>F|HT$s(z6_*X)>J~Fi~o*CKCbG z`g~;dn z-Fzk*Y{sToN(M2~rZI>hFCi=4)6_@?&(XPYb%N)jTFok;`<0NZq9 zh3Xl@xF8p(BVs1zwIP52T<1^b!E1?t6f%Val$K_$D2r}p4wW7ag5SP9o6A+8B6Ju* zy2=E-<8C46AOTp4Y=#;X7QQiBHNK<(&;T%DCy8c@2q2M+sA$QFXIuUq9|$CjKHbyJ zD0|WL{Vw=QdOWqW)-=De4>+ilH!JWz1EEU8tBPm~!z#Zn)uFIUz0JPms8l zIe_Y1)eJpa;-t~hf(R0MvS`k{p9dIG=~W{iEkOXx03UIVyBHS{VFC9+fef_hn1bY< zS-~fvr?Ez8d%EYK7Ak!j>Mt^C99D)p;KOTm8JAt_mL=(Kl9>%eCYn*$e*srFX#;ch zM&y{mQi(u{t?KpF%R3&7fE0n~p=c5m|7R9Bfc89Qu4YSDY{82h!mp0!4~_~7=$Rfo z0uLYh$3pczBs?9GHP+E4EVVAe9)ItsE?U>x8Ho zYOE}Xx`1321h$!Kj4i?R=tUoB(66d0u%d{8)kQ(1jpmfY9|%m;iC06s-)byD^x*27 zGDEBIY7*2*c(&`Pz~li?0E-gq{PAh%aV?)htg=RId`fJ^TI};-?A-kYKCr=nEnCQj zY)C2(nPJ$pqHG7D#(?z!J)}YYJQs0(+nxyQ^;wa+iH~7+gwqvKN6`Sj-f9&!?bFJi z5CyFDUF`yi$|ABW87u-194qOu|4dTE(8Gf5vXU*u_Si4@;2S7g6#ep ziO7y@v4P9QZ39h;?(}9t48$i`=@U*5$Kr;A5X4XV?WroQE)1Q`VxE{*F3&a(T}kH4 z+GGvdpJv@m<7&&}?k!gcpyc`ut0Jl8k`zgm7-!-n`S>57*4XHhuGpIH#GcNfUTn1P zXOIHZ9Zp|tpGsXf>Mhh^adB*$f*FM)1T1C&T0p$H$mD2zsEPK}rR zvL^kWF6?QN{hBT7&L$G~|A`XwLZ7tUF{-7*9I4J;CiYFrTS6li1L6S#ʹfo-q` zhw%f|1$PZv_9B6eDD8JRK!`3a=g0#aXu#-6Zi-SabqP`lm|O3vs`1)qTxM|~KBxq9=?*Z6ZALP0HRsht?htnE zNp1ut{tSh>@#g%+dD3w#Pyx@Dz$0>o?!Y9x4D8(K?7_M!4}Zy+=o>Tra3Krv*f#6= z@F;%%sQzkXe=adYJuxV!-y=?P6|;zwzF;O3?KEm~%Y=jz$z!*SGdXWB9l&3V?T~aT z=EgMUM0hWD;sZ9s|0yiTz${5i7dX1X`QMFYJ`FBy5Kf*vo~8Ey>jxqnkY%@RS3jMNsEBp_%flfM`B`P zeL1Gjuo6KSSLf0(3^x!3?4j}qYa5(1M=Y#4$(j3(<`)9-K_jy=TO^~-rb9oqyS!2i z)ba(u%~8hDfg&xrw~EE&Y02OK+mOt2snbf@LjLf6FtjF zi0bvZRUX!6-oTQIXsAK4s&x+w^O%V4=puGvQ^HcmAz>I}bpe11o1M(VWR7oOWhQ9&ooLt#icE11H)vW5 zGACRXdDUe+8FX{M4K2y%!cTc+gl28`+DiJR>Q_H53L-xhh zT^@9tM|QBIYwNaJt2QTIn}KSmUazEb3w~tbI&Op}&UADABtLn?Cm_UJ#vp5V)C9G5 zLB0nSavKPzE82)TYXktDwySd5DbG3qdJo!q5+Fn@cc2Gxf){#YN464c!+*|hA29o} zYa?>mEVTDUM6+u)(ZvVD7AP=AC%P7(QJ-vC+Y&lKEEElNc{&b)aj6`PAP~{1Ycg6x z|EQydv{)O-HZrYV$c6^kI*r%3`3~p+7eEf9GoO#bo(nmR4!e=N!s;S9q7%lMt*r*X z?b`}$+sv(>X{_BggEZmo7;9%(OidRM1)0>JhOIzq3~;HmV}%`w1Z;RSE$(X508j(D ztE+esCgw)G@DgVFokYwH?6x(720q_udDa2J%exP0sfPjj{HA6dAjc|xf-t~>Cy6GZ z<0m{U&0{w-#Ag}BhyBRzu4xolaN$qkrkM>;sFqC=G3b*@c;>L^N08v*inye^&{YeZ z+RMMY5&eNWkoN$jK|%Doo68iv&qz4qyS%h^P7i$y-*+Q8sL}(uawEKrI{kYl|H3J% zqUXcZEFSqXKYVLReBxDnf`OUEk6GD2*luWp-FDEH?Z&*osl1tY8nk|XYuJRKjl0Vr z^778#17YcoGTZJ@?*??!FT>P*CRV4 zL-sJ>g84%!wD^PS|4yABT7hmI zd*+FlHC^aD8ned@+_-b&wta#sS*^UY^oC3JsZ^^|p?+<p19JpEqCT1M&K)h%?CYRd9xibj^6Xl$iEazY^*#W)HuX#A8Vb4Ce z04Nj?qJZGKHEaB?t&{=-C%ilIab_MoL}E_{EykIL!1bcIL<(96v89qeWJ%=`W|(Pf zlMW5}&_ldtkzy}G_;4=|TbfxWFjWF6?6Alrn~}!JjJc7s9Cf5D|Bp)OTY-!>fS4e) zgY?^vIRau6Xgl1711`NNrPN`EDy6cBJ1c>6j!T1-TOg_k4zcdKGR@Qyyzy3|q`a5h zEW@YtZsO>@I>F3siJ*|&lh38zfCa!^1U(}jWeR-Ay#xsiX2cinNR%%M*1ISJ@;rf5Qkyt63m0EB3^B)Kgr~+>cbt{7T5Cn{v^G4E0kzYPOcK~7 zY13}X;A(>MJT#R(6HVU+K;T(tsmtIzwSHsvP4kQsk6ZHGlqi%BahtPFI{UbDfi70d zug_l*(gRR#zgQ#|HO2wUL?MTmhrvr`h|W+0Q)Ic9{Nn1GuN5(ZPOHQ%{J4Fv*%M$e1kM6+Te)^2{kE=g3xNey>=9I z-#zydNcf%i1T~4H2b4}B8d=?5*h?!Oy7S>by62wT|M?cC&_WMgg4w=rWoiMemB3|m zmGHv7Isb6a62Emw3e-TP{m5CCUvhV4_8fEcHV+-kGMNbLdMhxS7O2pOo>ZxB{3Du4 z!s9iUWD5?c+koCYp$pP%&lBnx3#dF%2>ZCrZF1;D9Y|<4t1u#jVX=W#SjZih_#i9I z!^jM=2I)(EJuBsOi8vGA(F8(8)HT@dQlb5EaiPqrAd1lc~62X_5dDINe=;SAZI7B^UuWw!`LCUuGw?{D1m9k2oENz8}KJbbHGEkW| zPq0N@_7Wk#4CCj>xIbfZt^=4bM=gK=GcRG_4=NH<9ES-Wq*c>_*U2N5K%-1$LhB%F z`^{OBr-X9eaRcYv-v;2w&MVD8Li6ax{|~k>PkLUW4}{PpCHLt=e#%Xhqy%U%*htXt zDG{LwJs%Vo%Fu?U!Ev7fO%^TM)|f?WqhYi|6-?$(>1YVA0F(o9YA6Kn<-r2Y+0XUf zmya1xfu$`~nmS?XHE7P1m{Rf?FtebL5dz3fECNa=&>#~)7?Y|qYuj0FXikiDQMW4T zi}~A*3%KBdN=Z*n| zQO)ih+-Rm|P^=?eGKw~V&MFmJU@Y2W5%I0_-V@!!D;QQS_j)X!my1kR$RAL45t!P92TxNeEZ7Gq$9_MjSsXK z9W6&UIsuQ4^rT(v8ev7N3dC^ojycWa*ihOa#3-UUynCjKZRM@VR6ztB)eNtYNy_Vm z>6N$j-k8{MV0MG!8)$LB{}*7OWx*yD87upTRVxSCVm1*Es`I{H}XVtIB21*7h^D4Jv4F19vmv*2a&gTN@W_2j|q%dAgg+E|8J+j z)U5(t)jG8^ack!iuqmrK8$ldUMVO(J^{f$Y68C1gX}MXFY@VhN!FG{;z*f6G)eN); zB^%BjPaP_m+Tduv|2BGh24aPEg>4L~(&aCQc`QE+OT$<31SuNQ^+oV0Tnqa|a^S<{0(m%?nSM6(G`0oH`w~(*U zl5hFUN;O017Xhd)HG*J8&$N{#D{@7z3UO)gc zURxN8B(DA5m0yQV=c80M$P_e+p;KHx;Os^vFi%!5~ZK8t)9-tGn00Jn= z(a7uoR-o-3z`&Bj_27;|Mo@d|u7CFM?)GoOa3DfPEeLg?13+Q4tB|Rw%}b z5FtrS%A_KY3awDR>PQP??fW_~6OG0T=?mh>1{JhvI>fO4@&Obk>rvdp97w{PXn^S+ zD#cdOLVCl@Qe^k|a0l~_|9sFF=K>?}hJyTV2&I9yj`0W;BMBGLp?D+#u?P~Q(9%+^ z606V>|0&Q+8mJQ$s9U~|^sa-dP9g)rBOJ5iqweg0#z;;kuiubmukwa$axNsaz=HNo z7xVA;f-!F*=E9V!5Xa{b5fO2W>u^{>5p$1r?gKx(C;+6d8awI=@A2t4aT~E^6Xit< zxnLup;SSbprobTqhv^z2QdyoPOP-(!K;aY4Q3rRi2YWFtDhU=BBKK;tVt(PE1`r^T zF)|L1AQ3QekPFFP1EcgqqaKnZw{Wp2vLY=I{Jt@060qTb$^%j)6QV#Vu~AFv<*-P| z_PU`77_8*%4j=vS7tb;-&PL;&<})~CRdmuAcTz4H!|(>@Q(KzPBNJ&M`oQy|$uU7^0nh;)oWM{{DV0c0BFl1}t_l=nC&l6oeB1(| z3^5>mk{KDwLU<&+QgD}wGS#M!`ljp0)Fo_4%UW1cFb$J)3bXq<2Lm23F&T*gKB5A$ za%dt`7wiRzItYn~j4WF+GkFn+24QZXU=^WiJy4+(Fl8_TuQbQXGKN7s%?cM7(LB!+ zJuSp4qrkaTuss`6SYR_Y?FvPX%qGk0bym{*bTc=fvLXqhXM*!K;L|ri$2eu;{Z4=u z^(HKzGdl0gEMb!W1R)La1v|CVExl8$!gD;!(;(HMxI$D3MU+HK6dJ-y8$4lM|M(Iq zl+58CPwB2}$RG|yasoe{b3gf0ns~D*(Ifg!P#hC!0hXjO4>U9RYAlgw7x6AaCDaY* zAQ2kj5b|qE0i#8_Q#|KV=92MaI8+;MEWqmU=6*~DXV6TS4ozEY=?((=UMx*%Y#r`l z-gFEEqJ`2bAk#()5$wPoH&90tt{eYUnrv?g1e9tpVGRq_Kp)dcq4PnX$M-f9Ew73? zD-}!Q(%WXFLve&067A+D4bpf{-9WVs+cfBCLpu0nP6wn;B@F~vkx}@xM~y~Md308Z zpiVFnZ0OUY7}ZxF)j^k(NhNhsozyLkwNnU#RWfx;y}?VLp;N)MRYR3p|3|f2q3=`^ zzy;!MRROHr(k&12vV(AxFm<#Gq0$4-;5|C>ff&oO7_(8e@>h!#Sb2z2jnz_Uy)y0@3TU{Uz%0X3gOfqfk)PBuO zY8{iL8Xz3-fJ8b$lvFn4{7)>#L{u(Ch%BrHB*1G=1aE#|w31XhiIr^im2}V6Uq_U< z>~d4x)7`6Lu5eeP>ON=mIA(~cbx$bf@%xL@g+Q#aI=n_KCz{! zHWx^#2L7TJvWIW^;P0Aut07YqPmp7)eA_zJT3e#61G3LzLqpafPSbG`Ne|LBn(@79~ZH+;pnW;X>c zwbTHYPkq^!Z5wXgb~twB*N5k~7AU|v8-Nc)1)P)#61vBKTWSrurv=>P5wsDJG5{}t zw~s<&3GnnitT?jXKpeQLaw*hGr}TP=ClgaLU+}3V!7`{)U~zMSg1-S`oFN5Pfp4Gp zgYQ;@D+p{Gn12~}Z)*T)hxLWa_AC`9Tuk>6QCD@>2ZwXG(Ri4LxH`x0a8Khc7(D%1}Z~^i&N7pyS>E0sz1QxHS(ZqD|_!gZ35=cA1y| zPY*k%2~;(nmWhG=%d<%6%dkRSd_W2yK~!{zgeg=Q{{>-XnJo=<$Uss6HwhVr4cTnV zx5EUFk)Jg)T-Ti0w+bl51}vvH4C#j}*#j;)D8QC;7jOh-=XXIlY5U@WOzBg;0f+YB z%2I+l=3!h5K*JJ*09N3X0xWrL2$Q7_OhjO!bJvZb0BE%DH>4^#?ShR05+W;@p5$WPh6?)jlq?f(|GW2}FdC$bK$Q17!!*G!7MJvBY+Q-r zeUv)9HsboI;77-fKxe~*nZ^!K(H5+>XAlhJK!w}Ey}Ad|0n|mqX4Pk<)sH@tbg7JGNn4wi)SwEjAv@%^;X;jht{DBcDk2|+|+V1KoY0kD; z&mFA5LO`_q_?eyAYk6EPwv#U6=Ahz$IikP4j5yjD{?$XwOqAjfggJ4Qi>kUHD0#nw99ky@Fu@uS3d;{>V8H>S_WHcnQD zN^>XaZY=mRWug<5fgWSk7LA*v|4Q0#B@Zts^J^Q|(ev)TtuvFr@(F~Y8!8-*8m1IL zT?{Tz<0i+$q~fKMK0rrGAD*WQBYA0$^ioGbrba| zQmQ=M+oX!Q7mjIN)eRS}C~K`QE5UX`un2kw{*DNu&D`Ng4fDAj$Uy(Byt!W=57_wOR?se^{}G8!8zURW?b zKtXDLveXJk!r}5;i@f6_gb9K+J}d0H|Jz>qIEw&+L{=fFw$- zt~{eL3z{MT!iBh!=dKAzwZejsGR_sw;zuCm{CYTmv=7_zzJeTi0Ji#n!)Vy4Pf(8z*l+)WD=E?{^8_m7%jOKcpuAr-F=JOuy*@LSY?-~=sK zrMOX6!;(ujY-LgUV89(j&4LBP7QyKb9Gk~hEfj)CF3Y*DqIx1>)wjbTG%1rJK|&qZ z;xvsA;Tp>A&ov6YhT%hYGCWY8FmYpi)^OK?AjN?+oQm@0UXuxH{)kklK)tScr)CWz zwD1edn`bCqV)|kgGlBwHspCfY9sP_fc`|8am4hoQ!vp~X0=l%)fCBcU6A&;lpu!43 zpt3-In&mas|5&5>_6;9;b>mEF;&k#sUXmQafgPmD@L*3p9YE7KwnPz?A~Ehj#1wt~ z)zV-(8iv=1OvqtF6>C6-0Zuj1l0k)?eb7aRJbd6<3b%z8LTcf#=GzY`2r{J}&hX_M zY{Kb50&}R18OL$Rxxt)o%{{?~b&%kZj8${iiHdWoYx z#mDHPrW|vh1`vI85+bW*MSx0&qOO1l1PDStp|oU(&G2=3u>n8VyMwh@ueX>fxB`5fpJHM&KKWMuhq zK#3e=tf#rI5_Vu=6<+p28_;fQmKhZ$JfRbDNhk`LFj>!*rKBwoPg_zs3kbe;#FQYw zWoxMd5r3vWWzDFFw6Tnoj$=eKgh&#Dz`|3wc)YUpY>eZYUl7a~qUxbhjc{8N{{jcL z2RKS_5OJ)d9V?hQ&85e4eEeX0K!}SF!pA=NK^^LN1q5JKtz|B-hzlnnq>#-JQ?DVw z4Rgr5lgtGZO^{TDh8V=e5Yb0rWD2=VIXx(g?g6j8x~B5qNJRi=`R zs4Qa|-A5&03NwLk9Nrz>cuYgk5s#F}pdloe zxWENf(1#5eMLQRy-jbsjG(J5j?>DQz!wW&{qYDCL0RXI`>mshQZ zG%^a!t#TERUiFx183j^pMrNc@pnyqJ`a*FMXr&%WrzbL%Qn{+ugo0Uus-n2iz20YJ zZ9`~Z87o}EI-?L*)R!7K!&t}OMPNaltaU3p%*<-nqTIzJMnRj^|L}^IBIGS^S3Mds zu!fbla~=THOlMBxv*Ok2=FIEC^OknSjKbz_SzFdo$!@;# zZKOF<>XH++6Nh}!W>Yc{1)StZOSt3Y42(F7o0KY?pX@?iv=oMuIP@$5qyZ>N0;DGB zCa)x+vJ8*hng7ho8EP4Un0v5^3O{zCywosu9qQo^cX!Sq{_e&~%s~{77QJ3AT}ZWw zNY^eBtt?F|N5W}exLWc>cM$22D|Ls*bmlECLR(#7!ozXQ|HCn>dO^25(UT!fEq|We_XUTH}_|PNnONE02*P;N%5(Ne_0>$_g-n z60k%vn8?&?sF#U(bVwKZ5cQAqO`5800BiXOc>4Zba73i#8a%kAn0IrVx8WeodcAOn zYE{^|wP1`jA*v?$=g>E2H_pl4Y-h3hZ17H7+SRr;$grK-7vDh(EAUT%f&^~b32EG& zBGzm3TQgK;a+N1g+sYbH0vS-?00B53nQw-?$|yW<|LHE()bG75r>h|b-crhsQ`QEB zJ7XD2aJVkkg$F#W>&z72=@aFU@l^+BTO3f$s$m&Rk;9qfC118xF}U)SM?1wUA4bbv zo?i97nA@8Mf`C{G`PRAahL{jc%Gl!Qg_7E^1%klR>8&!mkSpk0*8>|I{f0E4C z0A~iS6DK6H4WMvz>{CzsSeAjPB=}dX(X8fOm(lJXf2AOFi1Ff^OYeK5o9WI|9}Z7DK;P~3`kF-p)PvU6>l&WZ6{g6 zHx{j=d|;6oUl(^CM?M>d5=n^rEwD%d`x6@} zI91OfG|X@sE*O25aB_hsc<51sk!XY6cZ1$1e!yf%70?7VqX|{B0x^>!>BoMf=rZxf z0DsYQ4YE2Z0|W$CUB9JJu7_zS001YD{{ar@04>oVkLPF+AOH+9Zh%&b0YE^dks7t8 z31Q)ek~J=Spd@n00u*pXT<0v?;v1}&3nmb6UqJG89MX69Mum>Su~M0&lZ7566-gDiCE> z=#U%Mi|f`DlJbi$^ovU&e>rjiE^&exnE*0`ZYS{oI0XSXc7n}f8?kX4wst(mGZoT- zc26K#5+-}>0%2_^D_+7cn$Q~)P={RdOPf@DL!*LmQVVg%3~BhCj|aufZpODiR6V0ViYnn{~+VmehKmh3AstF2#Zd^HQl0ROF)u!M+p1}W>zQw z2(Xa=U{{WY0KW(VEzyyXaZ`k0O8Y^ICwP=0*a zXqd)mFH+DOxln>kl8hhNhf#$FWzYq7QytaeD|1v)jL3Z`_mrU(9#N@%ROu+Eg@e*{ z76}qhUN{tz$0@bbbDctjwzGLV@nudJ1>g`2TL%;R=V?xt2?RBf@&^Gf;Fq|li=(oU zy(j>r=6~LKgx$#$U$G@eDK{Oq6uvQnyEAEC!7qK20$72Wcw<;na-N>)jd1sYrXmK_ zgFt4W8bXtHNXbAT)|BP-|Beefcqb;6v}v1&XO&M_CAXLoUgIgGsG%UKL|ik4d8bQ7 z*PK2P0C#D5H8Bq5K%Lczg%Mpca5+FdD?6j^;?9&CwNdbD$=dps^`~ zP8pT@>Nq(%sSla_lrkwCyW0cnId)2smV1c>w2QqpCN1wKs5jC?#~Vm~&7!MWv+M0*#v~0X2{zIF*y0*?aub z0Nl8rXBVIRVwBi}|5;*Sl$?nta+p|pf}pMdn`TNe@%W$+8lh{drjw9RBr|lq#S_iS zgE)nY4=Jsh@oI;4GU*pjKxc2FrtQNv2C#cHgkz_C_2e=>JiVW%Qa0D#s?Q*730jS*vLxpdH3tV4IdB`D*{Ppe11cGD_S%7izA4@VYIdw?KTv|W{ zK6$rv%cn*|g>502k)exnDv_eGfPMnX07g0o*21(nbum|7AxQ-GURo}O?UoN8Dt zxip{PBqP8aN68vXTMI}x94X+msJfobd71oD8X)(nJgJkNxj#&xuyyhpUBM+%;3n8{ zwrE?X70b58OT5`vn=Z$u$eK-ln~7KV2EFyP%=KU`%Z#r$t(ljC4;fEE_8|V&6ls8` zG{c3irW2KLK9n0GL4p&_Kn(19A>s%Yg!rk!PzY9lo}&Q;LslK^NwQ}LrBD)gYxi-l zI+Wa4{~U3mKezfB3JeKS5WIdjyucbwk?2{-tFeOQvChjC&>O4%0|S>ibsdSad`iN1 zRZ5+9WC6sjeVRowlxpL&bNNUkEVTqWY9UEFzws(J#aFsAPDh z1yJz7y;~=aqdkhaY`-wU$4kN7K${nwyw-%9sFhdFn~6aK7Rji(Vp6ILdu!RtW`O8J za$uoAmH}89K**CMSX71hL4|sXT`Q%>&&Wntwr@j8VH=PIJ$!brw4|G1C8Ei?8$bm* z?7H6AV9IyD>#}xF3mpWSjK34HOIgKNj4>E1$}`BtgA{&W48|vs1^9b)6EH7nx06AO z|H2bF1~O^F%u;W`ash6cB;=%tWX5TcS!A|gJ(XJyo2(4rOCz-4zWGK65;6nbU=z*o z%p(B(huNr|*mBF2_n3xT7Oh%LIyjnE=cnSyAlp6Sj%Z6`{(OqC6N zm)(x=NWn+F+1}U1aQoQ;76xq4kOSs5b)DGFB{mD-yIu1H_?h2FR>qfr|B2~(6SxP_ zLUMbQ=K_GnXgOgHj!@Q8M>7+SiXo`~$-Caxv3 zyQPU;7lG*CLAPvDd@~557iENKdlSjR4*j7+~TNHlHV`7b_kYHrzt;L<6^|P1B4i zZ!VI{Dcp-|mmSXGU83ORpgfyX`QD_p|GbWJ4ED_`R-WbI zck6yY>&nVIFb)?-^fkf$18a%qs&dCm;f*JZI@-|8H%l@z@#hboiRXaDdoEUn#O&(; z4d@U&Ceao#WW$S`=s?29&LUwrzJBtu3Vi+Xu~VY z9*c1LdU37|FGCO9iaG<3?fWk6sN(|gKktLx;6%tCmmt6eD|J>x1{x7? zK9P4eRwApKGCSjRc}{Dv1qVYhek>ReyLe!hXA`JSpW?4#|3Co$2@T&KNgfDLf$&Ya zRSX~HD)v>g-n>Pi>l{n*ZHna$Nw*7tKL_jE!rfn@f_@F)p0~WG4p?L-zgxI7=eYRt z@Bk0iUQ_FU4ke!sZzCUXXmN6c}80y1&4TV7@IUV8nW56sY3fqDYC zMaG?I6H5TmE+7sHc_G9h>ZDFO0}m0s;0LqLMRt_%8C294+7YDXStRBI5Wp$1K`+^fLI)UzhADW4vpUof?q-bK={GX@(+mk_%IO1aP5Q(570mp z?%;nk@Pl;$`YSawmhUZ~AIEOFF93ljSqcWw+RUcH$W>sFn(SbYUsDePFXsV4Peb40BM9Z39~g=;GA+_Qet zwq=|5!Cn|q^nhD2!GhqygY(S0r~uE0y$%RrvSmvz-pMyAH*SV5GP)`bJm^heF0*J5 zjWr-(0}o$5d-m8Q&^v$~u{7Gyz5c*3lZzj`6Y#$92SzeJ5h7qp13W_o8x@*`9BAND zx#l#MZ{*`dB8I~keK*V(?oBQ{!7Wy|aVNYc|Kd{Wyo5P(na!^C>k1tjNRrHIRjEq- z4?w7t;;KLcz2c=T1r_WijJ4Wg!zZ~Wq_D!e?z)UGEeKJjFa`o3-~tFB=UO*gGCQBM(E5EM|$!Vp5b z#6tB{pM-ERwh`PM@d5(?2%v%y(?G1b|KbGL3y3QEcmc*2^J+Cla**RCvml54Kp8}a z=+P70@~R{qeXfN^79sPhjg>UIjndW?!SwTmJGjdwml=%vql$&9gUC%az^skV_Sl;R zg9`BI)5Io}X-1E9J(*%(lp3nyYU(UV}Dke~%Mlu;uJ zzp`^XZxSMy0^m#@`vYGppy)!3|1p?YWI~7wl+Y5C@r#sGN+9Ut`d0FTCcq z-5g-9rlGgyd%GiC_1y9X2)I7;nY-m756Xas3SY!3tSu)C zQ)tQi@bHA(^pI6QkOLJC{~^SJ710Mhgo1mN_`Vi|z=|*+0@jcy1r|uL4p5wd{GR8$ zClbM7i(}k{(!xLf0I)s=EMOc12(kn+s8m8o*#jX+AqmFjWnaQzxiYvxxpi=7%ko(w zTZI?&jLTY|~COJS0b8R_@LPzE8CVw|56KG1_z^3aK7 zbmbLanZ^0Zl9sm=wN%Qy|F+4U3| z&r-sHmH+f)DS3bf|5XlDpg=pPK{v3_RyH)D5B&^8MQO%1^3s?3vrAf}{Ya-d%>ye( z=u@I9O&)MzB z6X0_j6}UCM$8KqrBg9i>D=UU5X%@0jU?>feXp-FF1**CEVcmx=i`?Ljbgy?50?LB9M?a`^f(cbT(!);&~(7IK% z;t#HDbt`jCOvmPi3N4sNw^* z7%ZDpP<3Hkgc*0*4%02wN|jg`QfmYM=rZ=QT>fGl(s4{rnxBj@GaC@FFJ7@n^k8G%RwpUolLuPm(Ovpun$bc=! zk%-yft7|W0=gE)tL9mUi2kmf)Z)rkygdkv}aH;1a>W-neQ!bjvCuBtr3GIz&<8(aB z$#+%+6eeQf^Cl=gZ06E=n$XRKHF!G2?Vjjyt8T!iAakvi-nxh7TI0HU{p&Jrv~@w4 zw2xT8BZC_pnegcZK9Il)n)qA7$zD!|gJgXNfcTOlAUz7C0Lj&#YzZ6=oxpD(=UP}F zLH{B`F9tg>J?8l@a~|^aarWUbk@z+`xV1%81ruJubi{#WJolnmtUZy-K+~+=Ce1+r1jgw=yd?>k>W?>VnFe2q4lRDbNA~ zaKYw_fagnzNm{kbpop{M8*p3CN6~y8*0G z78fXqDzJq#2_^NQnv!S}8^o1^ph90!hWSG{CGOW38H*c3^lMyx}l540!{bgtf8 zr${U(2s1=q8?_a5KIY34M=ZZ?jKm_dg5--VOw56t%7N_TJ1symopQw^{5yidCYm!5 zHjx8;s#c3q!I=$0yQ-KWm>$7!+3o0Z=f9H?Y4u`zqEFtjI>Kh$whI=mJH0nj7gc~Mz6C; z(>j^OG{L(P11Zoud|}F++Nmm1%*Mn_n}h*Wkh`-xz6blYq$ElYC`6?y!W}w3&wRts z%FBYfObOgd*L+P>l1-IDf`I}{yR6OIJOkQX#)4YL*le`lQR7t_S zMr>7@(Jiait{=A1X^^w1Ao5Sx%rz7j!}!KmQ^vG^p*V&|ivDvw#cv;6oNW z$>owaY-mmpT}dz9x-T73t{W-&*c>xGQ)526! zn6%SgAkkf_5U#^gD~&f|=!(Bgu?_VG4uy)W^Cd71Q%fUL7yr}KO5IaS#nem9)J^5o zKlRj4ozyI?H)%vMtINYKy;4V2RY`qRR;{B>1=Tf;)mf!gS*=xD#nmx|)KM+eQdO>M z+GcJO`$w+Ooygr*+zD{8+Qi zSF6QZudR@mh1(I8+kwqmwg$4-2vuZ z%)wl3Rbc5Y-;Vj<34Ywn5n)M-)YdEn};;Txvd6?Wbh zPGA$eU{%#poPA;Z{ov`v;St{95?10Q?%^LE;wOe-8O~p|wP7YkVk=%^ATHG_W-Cey z-yp7E4=&=Jo#J~v-y25YBd%gD-eN0eVJHsc<&0q>&fhh*<3Y{hJl0_jhGPPD;y&(B z`xRp;CSx^bV=O-7J4RVUO=CA!SS^m@FaH)~LLOuQ;a@D}VMo?vMc!l~_Tol%R8IzE zI-b`!nqf;$mY@OvWHsc)T!PzGjE9_IR;OW^z_%XAWkKp=Z8~Vrl+jDt^~-_Gfu6=X;)Kf&S${#^-|eW@1KY zem>@S251Q$=!lkRluYP_7HI2zXoqHESLWl1?&fwb*;npW1ESu4&S;2+WP(=dm3AwB zK4^=EX?xaC|A^^g=7~^k-9#qV{^Hf>1LkO!_GoaP>C%1ang(g37V1-GiiXZVXbvkM>~EHR%xTX^qb2VWzjE25YLuLz~v;rFQDGMr(L3VX2<#w%(LKF6yLi zX?I5Is=(>2M(M3iX|{IjzOL)P-shoa=ex#hz0PE{7VEGE?6y|yss>(+2HdnZ>^-jM zkAC5=u4$rP?8YYLI+pCuMr?C_>b!>FP;TeL-t5aZ?bNnxIi3`z25qR$WYQMt(B9$G z#_YJ(ZO)c!g$`_L=F`{~Zk;aa#;xtWw(TaaI?ZO}%(mstmb8EX03rDV1rz}O04x9i z002M%Kmh;<{{ZjV6G*R~!GQ!5B2>umpu>d_A4Z(mu%JbX7vITiClO;uj}=3L9Qkph z$&x2gs$40OWy+55V#mL!7$bt&{`&!R+!B29Xkh6YEHO|jTVj!IWphLlo@Bf+;_8N%$`9v<{Y}T=(3~}pEkW(_2<#4 zS(}yJI<{+$v~z2&{MzwujkfXDH|Hn=qx^?5|$D0RVJpA|a?}KhnKYzV?`?EoEcYnV>eB1!4RUdr>`o~{*`yGg% zfd#sAo`e4_$e?fSDHvNi12(u5g%iT|V1xlmsG*1gCRpK#7N+A z<(Cjj8KjkDdRZorXU>S>np$@0W}0BeDJPm@vT5g>aGIItn?_QJ+MRnwN#}!r25RV_ zdlt%Pn~NSQsGgFI=_sO;QmSZ8Y-Nema6HgS!Oyao~(-M>Z-41 z|7hy0v~G!Ns;a{3sfW9+I!PGwJLe5vBwfOtgx*TyDYHJMqBK(X);M|ve@=% z8MDrIEA6-7UTdYbwUTQtvf^f|ZJ5n^>n^*{q7(1D^oDEivghWj@1^^itK+}@rYoz! zxwiZ6y|I=9ue=O5jPS77bYrf>0goARlNVEr@5Od{vn0m>103?mBonNzcC42C@HZKS z5B6_0MTvymQVHyQ~b$bJuP6-Qn1B$t+B-|I#-q zfCt{m(4Y)nIN^qWqIlwqH|}^Nk1Ni|n7rjRFb)okAOs)SK-5_xb+F5z>}prL+@0fO%8J!8 zCKw0_l8%3iBcJ$wMm=X-PZzatVJ>D!!y0Nr6WaSmA;!lyDgc25DFC7n|4~3h63BoB zLrkIylo&(`2+@g2oIn$yXaOctv5HolA{MpCMGQ=F24I9j7{y4&6EZM?XEfpRE?7cR zwDFChaAO=LaSKCG0*;30MexWro)Id~41gpd8WjjR74k5K-Ls($6X^^bCNdL{j3f$t z*Z?OU@sgKR!w@jZ$r5zZlb{TxC`b88B$CpSow)E5WmHWgPMut!w$E3$x0s4 z2%G#SIJFRiQks*L=S(L$)w#}5zLJXdgP%O-$v*qp5`ZLJ0~{fE|I9N0f}eo|<_AGY z%qbc32gxiW=9-z%XF_jp5JhD8a*;hnT2zMH)FvTn(ai&R(hA{}!8k`+&XSsxo#`wm z1x||6cA9jhr=);4Svk`;+%Tg!y(vyL%F~{6haA+jQrwc~D@NN{v=2kO$$pQqR1YRJ+5WE>qlIEbTEOn_|K^a$= zc2bp8G=W{|8dDSG00jpqfF-@xNEi|}tcNYE_b#ePph97&0c9R!k&0BM5>$Bg0uof0 z`97<9_OqUCW<+^-z8J!AdpU%mP7Axyo@&CB06422)Ea;S|CAG?;7qF%bjw?}g43_Q z1#WPME8LS7*SN?%E(FdxP6C|Old83X9z5X;#Rh}A)!pHBJ%QaDA~vylNT5+qCkpcV z!KjoCq-DppmZm<88@~YPWAg_J(E0(ilbD=n%SYAsY15kE3vebh*#s@<)U~f=Wo$*N zha3c;0h`<{Zl!w#3imd`6&~q?jga9C4}roQ&aG~Vo7@tcn7QCI>tMYggPS%It&QaH zi*G35T=15LMnJ4%0V;}Em?C!NJ>7ZH8`)|sb$F3l)hCeGh57auzmzkn#q>BgXZqtS}sICtk8H^o|VGXlOg`_8)Um*{f$iEoqkrmXEgLxyq{ETOm z9YJL&Lql!#U2+x@0nt{ER?A(EwK<}2J~5NEwq_p6xlJ*b2*t^ z032$i*;yiK2RHngMRk4o7@X|`>q(1{~os^Fm3M3F6>~SZnx4#Jv4C9`+d;- zFRb;wWqwD}wf~)fbM@*S5*3!Z`8EQG5d00vD@8uFm+}v#)vgY!Do6CUmj|KL!NV))@eBKPIbWt zRmz|Nw#_grg)7~s57+5Rcf7;xEK+b9ot2=}sdpKh^}Jd4yEyh|DoRBSUjSLi{Imqu z)~;@7#hi6=U$-)~I#y5Q+wZh)1K(ZkS_oL6=m$VN1+IXI;_LOc2WNW1d)xHnOa9mp zH+qNjI%vmAVVT#@e*UzNKPuRBo{ly-|G&z9cB8_=$gHo^+!uz8NK6oY^#umPXfgYo z&+zs<2mJ2+U3d&MSfv!;Jul>Z3fFYZw|oote8F{CxaC(eHAYd@WDGTd5;$22Re?q3 zHTwb#W*0k`rh4a6PzJMpt_MM^h6=G~EzO_}7V|L#G*%I1dw$n*n)GbE=TnFvVNQfS zJ(mN`H(|c^gUY9TlGlJO=4Hr7L}V0!6ewSkWr0FvfuAr9y+bah_haM+g0q4vrP6x$ z;(p8nI4Q`2WLSnDLx1%HgNKHHY&Z|J1_b+egFD4xtb~7W;DZK;aY6WDBNk$hmvqQR zgd}D?T;>T$SVkC_h)o!Uir9#H|D-k=XocigUTH@zaYd&|157kpwaCmlzhj}Q3jAx6&7Kp!QgoLZ@rI+u2SmIx_b7(&lDGZ0e>{zPB#myD;jjob(|r6`A4=4CragDW6-uBcXI zplcJ*i}{cT1Xp9mw};E;TY{L2x|oaX7!1hgL)s^d#Q2Xa_YBmqdc7kJxNwCJBn>80 zENrJQ)c91#Q-9g0jonBzjqr`&s0=eWYhiYd$#7teM**{_hqefaCP|O>sDLAtk00d( zN5pS+0D=ApkVI8N{!|RS|HC!N_%8=33o$29VlyxfNrG)RCM!695s8hTIEocX4B>{6 zugFuYID@T-gBz7YIoNm&K#0mfPVY#3xR{QRSC46SG)S&m>d z$N+8vxs%fnZ1jwAYK&ZhMEVE;hNzxKZ~(+Np6;m&9%Y^Z$V%&(gYJ0+!-oj) z>1^j2fO?gK`w0QWcb+DIpIwEPZ&d<>We&Q)mb3{Ac3GQ|HB`2fn;5#AJ85NvdeJcZ5!wZ6MW-Ik2SqI1W`h4$fwz9CoBiYKT^90$-|VP7rGX zT22kx3{xPRvuQ!R1X=NQ3Ub6_a;c%#Kziqr zmhgyrx~F^!F(ayi(|U%CNrcgmf_pMhKUgY$K<7%RVMgwdGuh zu!5$l346=PCEbwPyqm70L=ik z@)`h43jnro0a^vy$cE!H zIPsJT-!-&7npl53QIKhW^Fw=kQ;2Q|wR$@MxE8ena8|XI1_1!Jt{VjmV7p+;wKmFU zO)#}y3t_+uyt8||xeEk!i@Uq)yUcq6S}?p2Tf70F2FZ&7u$yL86tJyHMnpBZ5*nuz z{~D~|`?3?NaveIXn-(_FNVAr^CY5`$zz~@F26y|z2j4YY&<1UW#-moMb*{KKC`5aB zfPc8_yw+I(3XlL^RJ+JKz0TFZO>h9Z+rUfIyecrf!aJ|S>%bh`u@lU@QX8XuW&sx5 zyfK=+KtKqDumYbdV1k>7z)GPqJi~AKPtFRtoWj2EOS$b!e{@g|vPQpY2()15zYK_7 z+cRCPX$vbT4r%~&loOQ_<)C2XMK55zSS-D1P`%5W23)JWGHSwNjHApOy=F|k1HcEl zyTx660&U#J3jjtjx&(AAos#LGS|+2(BnB@$IyFqbG#too5J-=U!-gD+51BQ7|Fi@; zOM{)YeN#MJLhEO1I8ms@TJUPBn4ku=P-w5HI8g~eR?GpUya28XwV7}M1hBOOz{(*z zy}!%0u>7{POl#9?3j^>1xSRr2>wjrh3&97>3Xrv~e7CQ)%2R;MtxUVxBmxWI%pVW| z(3}CJER%eE&3)|0e+TAe5jL4IV4Wt-I+Brbz`bgedO_J=y)`U@A zNuA_6H@OErDU?x*2|r?lP$|Fxv&OG@Kme=jqgYD_03Z&T(8{$S0HzAj;t&9=na1<5 z0Ju7@7rg`q+tFU6u=r}R5v;HUjnF>o%se0hK0pI6%LIG;!hOtc!`RK<|18KiJkI1S zt^UHpxByTrolppLchh9V+?8EOY^C3Tu798aHW0_sj0ff5$@&b+H^^W8<6jLkq@K(o}&Ms3L(XURxyr9fq_hYuN!w^$+qbQ%V3w+B;KaXu2~bT4 zN*&w=##1m&WXXL@-Qc;ebpZ`vZ$8{)gk9K#ebYFd-H6@T=6t!3|HXu!oqbRZx&fBi z=8bjP>Y|}t)mPT71qRroEtn`e3pmF<^o>a-*iyj);-wZ%nduK-5kEt++Bu`4cQeK-q_Yvmwe(xYg3EL zd;P86?VUb!JgL~{t3iE6U?|uL#FjX&U!c@#0X?u1O9UO;)+P<}7n|NY^Y8w)m>Ozl$xS?+O9 zY+4)HhR$6#r7OjBLpM6t44c=jpsqgj%jS{bN1z63vx7mTu4->C=dKRtKN;w<-sCZd zh(8^;G(F($BUR$P>xwwYCG_NnJ|I&5REkcKlz7d7*+P`gJyTriTrQ5yFvt0Jvs|zJg{Ct=Kcs8&;ul|?sJ~zuZ}^X{~!i{sXvTJNI3qFV3-BWU~xZ0 zpiE51BF@L%kPq9{l|MRLDSTSy&_jtxS)xFY2u~|KOp#2S=-a4j*lfQSuVr5U_1o^+ z7yo6gCbXUo@Q!)uAP;`s{_WxZg!R5l#j5gqAfYGk?lJ!eOO`rw5A4iy^?1;PpioWp zorngMR7}P$3Xpx&oupR+=wW%c>CMh|_`J{3t9h@UM+AP(oWY3;xOyc~( zY+kU2U6d}JlF!+=S}Ec@p_;JSouy*52Bn&%rVk=CXn+)})hrV(GrH-@lqU@k3vg8w zl%b}XQ-@vkn&CTD)U|2WxD6&ut}wjAhz&1JoK78M$B!jXrVLrzWz3l^XBIW}?2ji+ zqNo7!M%g^nsa5A=?G#<>)`(URf~^Ka{{#*dT3~o6(P0|5z(wQ9V{5h=Ew7#Lte2pE`+APjb!VM7i(v?CT!SOF%R#Ln5Q zF%&C{Y{e5bbMeI&KM61_&|t7}iZ@hyYBktg>oK29fY2qjyhH-YIp4B#>xC4at4oa) znovQcjF6-#oYuwxvZHTY(~=pH%wy@jJzg0~r|vu(%1kzGVzW$@%G@KgT0Bv1FFM%@ zQm+c45U$2C0`)FXF}PB&zC#ft|LTX^3|QcR1}cQq0tzafz*0;zy`T>g6+0|NP}gZk zMN&zH5gN^GaZ#GiKGQ1&Dl*ZL#~;zTtHtpoqv9#yFO&W9QG#6wdLm4o<|Ly8aols^!8^fv@lSb@QFwQ2rtOcW^|G<`tU!_(QEmWXz z^ic{Gh!lcMA%3{wC{?Hg8^J0T++r3po@`^pQSG=UkCr&v-(`WLjaGb0W{sYcvusA? zEt#ybZP8TSoE8@t_wU`pv$eF8SXDzY4WU7HuJ2?`WXh9M_$jeZL5>?_9e$YS$ zl*y-Peo6ij?21JYH@hd56z@x;Z=fs@Y@p?K1_h-FFPra}1$^z=eZGrGa69;b0vx@Y zdWQL4fp3XRG@S5@O+8Wc#INZrlImO{bZ=3E9w=8i%i$w)wj0@=J_ouxfP*b3!$cM& zP#Py7M@XtWl94>9{~Tm(E-Y=inW_r(9G2lkcZ66*p_2AO<0Xwy##;u|V37 zL2-i{oTmzxzr~#Feq%%&Bg`NN4GyO*b1~qoFqc4lK%-|VON|2`xFKH&CytC-AX3!O zt6zCg1qV^cli)@|BNT@!X1Wd{h%%iw$f#R#0#CkTf+qI7X=i!hp#&ZkJ<(qN0&ZO$Cy^y42t0v83?z?wF29O}Lq1w7Sfk#-pkJAGgQ zx4fZ%k0a1;P9i=^SrB+IjA6c@u$FIh11F&j9vGG&oD7(fZgvZ({8ULqu!IgDP$*X! z4wRBO%pwh&;$$u*dRmqEa4maH3$N| zz-}~30j0_kJc24QU$t;ew&I!ugJ6P`k8*(f8j#BQ1r~2Hs3qQZ$X2n%Qn3SSkgQ0@ zmnK9}|7Y{oF^agU_hgB#}{ zh7Fdwz^88Yk4)JbaPAT>ae(kNY2fN4ZKqX@6@zeD9F{s7o=m>5rqcnHUifV_(Fq7_ztl4xJ2YrXbCihaA@ zO$bB)vo*X84f0I}_ypmrV(a3N9MxfNhI=e!xrKcJNOIl)Y^gQ7xy=VA@SMH940|zR zIbSYVI@Lf_r0`jRRVZBo+)e1XFeKgJln<}E#$qR%xIVZ$D_5r`>S);_mIxV$|In_x z2@d2CZu>0tj(N`!C;C-wLsV7_>ME&CM>r+~gBEa>$xa&r3M-UR*}h3SR7ZoyseYC-YOo4d zS6mfxM=K6%(G7`iTmV|o(gXkifI2GR@ba#JuYEm41^802Mv)5?8$w5d=vO(^#5K;P zXKm02HrqG_`kMil8E-3i%Af4=FWY(U4sY#RNTGc_sn>?t%ljzgloD5`|B6-L*t-R+ z)5t91b}t2C2X)vo7o;WU3}&SLjV@@yfTrxYw4XW@sir%Sy-)F3{7(F`78|-~fg2JQ z0009Ra~=W!00tDIEx6g?#<9lILtVv|;#dMIf>eP#kd~1fK=nbGm-@WW3%$|%xt|L< zZ4&O89*Jls3dT(D&#wmXIY49 zArQKP9c+k+?Ncn@@g%U|7F6RTNNBYk8Im9q2Jz#&B#5v0T7^k_5CVWd)LEY=(IH>! zA@`xOFpw`wdx|C)AwLm7$NRzsD5U}vBAEKG1mw9;dB6c%LpE%~|I&lAV4Fbq7=h{{ zxUVt?JHdoK^RT8W4r{R%xwtnx5r?Db8Fzvb3rGlf@t2cG6y-a?o*6clu?t&Z3mfFV zL6D&;8$V2}10W1CibDy$J2f0c!VB`l#_PYWk&s1FhE9+|biuRUL&8$b0!T{)mQtHb zpsW{|rZ6J`eUbrQGc%o=fCXG524p5Te8U9WhB#cTe@Qgo0>SXfDq%s6=ljFGx}bC7 zkJBPVWrK$K8G>U$u?4b-l8{5ivWUKMi+*Ax3c5u2f&<3d#2zF?PYl9&3kvP~9a=;} zM%Wtuiva&qhH^*&f-JZhh_4WMJD)kMe@qZ9!!OV{7j)r*|Gz5)Y&1K4QUEP7LjXI* z!cjv5)1TCf5of%@Tof{_s>EXfD^EDD2`C-0qDc@Kte&ApV=OqU5CnZofx3IA9ty1Z z11Hq*4oav+ihQf31cQD&Idyc2UAToC^v8Sz#cx=JQW(WiEX6HY#{$ZSC5Qm*K#C1P zsBoyr=JF?5^2(yfN`5K|98yam*uj}AAe)N{oTE07Jj2Z*NoIsV)Jw^Eqyxgs5~zsD zm+YAdaJL!DGsxQ+s~aqvvmF663#@Xgx0pH~a=P?UO8im~?kgy%^hbJ(N*=7r?t?yw zAQMy*zpsqQPnZp{6aiGI8g#G`W*7pf-~b#1zH<0X|F`VJ>SzjS88c>6f;beQC~>0B z%Q<39Kx6z%z*NS-JjohT$>>TBXjz$Hv>9Ykmej$S7@$Oj$eOP*N`oASKG3&xxw9p( zk>!dUBWugh1jxcPHGJVje{J}WjIW{#O3m~W%+tU< zL^u5#fVhwxdIP6F@Xe+4wNgYZ19403^vZvngL~ve)a*fFP^cfAkFUH+X4nL-Oi(0^ z$cV%V9DIo*>;a`2g{O=ITGEZ-!m7F0nGiL_|G?8CzKj$EEJiiksbpMH6;+j!B&}y0 z(fCk?daB7S!XTJT2$;+#VKhLXdJx)(zC_rIRq#J0c~o>X3E|07{QSpx+=9{+(}9x1 zO?3mjv#KAA13H9*RoK055DF%>R4x6|Bv6{zj1vXL0wtY;Sv3PcC8>i1C;(g+h&(J= z46~`D0u9PF6xFPryD2+$K*8LpJ*_NzDnIwS0M2YO-KZ#Ror_b#nhX(}QkaKVq_3+? zC~_?Yv^YW~Jq1>^&mX`)(fm|HamsE9GAQu01z|}d+@z&!y;l87|Mb#+MM7OX%C+=S z9b(9gR2^b1AU$Y+_IXn|J*GTG5fgDU|7Hywj2%pkO}!TlSZH<5n;feXm?n z=PHCx0E<_cOSItvi3ANTGm>+ap;leHj6A+NfK(xyis0!4QG3*%deTP?2fTZVqd_RV zo4A(Dk9=K)ZrBf~mA|}{0*@sxS)J8@y^nBBzOigCgdNnIJ5ggg*6B=EWwlrfBaw`q z*l%M4nDE%j(l{`Dlshn4hD}eV-M}5;DCQxKZ^%RY6xggBwH=JxElsxW>dj9KLaRkf zvnkrf)dXpQgiZiQrUgH}n*~Cw0j#7~u+UmpBjN+kwp-Isz`v&A;F zty92UTib2h{yBpqm0Pg=0N?e1|1i`g`J)1=DFhT~D<{|wO;9!~(9B;72eVt;mDRy> z4M3zVwM?Z&D){w*VM#p%rnTR0 z=mgp!*H#-4FksryAlIo}U)pN{2?$kE``;Yo1b}r_DF#T^fD$x4RX;^e8)4HD{YWK? z-GGYVWyM`G7P^}BoNK+B|AWO~HqNLzIl)|Dxe-8!hoD2Wx!xzp;^6XC>V+FT%!lb! z*Z*yS{8-#vr~wR+kCwvXCW+%uRKZj&6ZK67*d*d8g|%pyJy!F=Pk@&Y_+QwR1X6V3 z*vtT0(9mAh#nqk5)P1}oLdZ#xKPUL&GK)^xrBevLSZXL^2=vfe_OHfkV+;O9=mp9` zm}7*Tm z!gJn(^9_ebC}&8BQSL>@g+Qh7KB{bYxzwm zvtVD#W$HyBVlo9{?~`nT&KG;OYIKEF(&pzWZUIo}Gm~38LR{H%NkU(?0l#|Pg^`}L zCdRdHYh7-`|Bm71jcLyY%V>;t-MIV`OVJ6yq@xEo`s_R#>uzzE%hkn?TFAnEf1-jwewWsE!h3L+IaqjulN%S>5bjS#E8kHoIA{Tu5Msb>4*d4GKlT@fs=xOrFY8 z=!P8+uSdvNdOQV@fZ6>10xWHE{f^B;_z10tC7Puy;O%19ebWP%YBM)4QN({kyZcrrs9e5lTmvluD@K^r5 z>eR)nWV3Gl-UR?nP{Vd|NZ&q3=Q~GNgh#JcPS|u#hl3-4a!Chu{oW-gqVl`&uA@=m z{E`S*F42%1@Ukr?F;Z~ALnCXM1>h&u4bzo;FVXsNzPV2*iKgX;BPT8h% zIE`9qc4()d+Hr>5q;464GPh9ncfw{vuWsHo7o*SwM!#oF=hR*3Bu!6HbH8+UZ+CMC z^{T#V(d}e&$ffqJyH=!+jxv{o1FPy8fMb7kwC+m?0CVd!u%rTkcWb=HTli$BN!Mj~ z|3X{%VmJ2nlXwB+Lug+OObFRrx=CbeDzn@!mdI{^$|ZPU89^NMs;`=5^$B>- zi}i}f8-qW1uD^`efCh%Ax`_{2cT;$?AN#RqcrSm>i+?v|Cx&uBsvoHYhX?s|;Pc=- zdVPMLEO!77z&=t%^q9x$6295-u7gLX;fO1D#s_-G?{vf_ZSp#TFctdjR)ot>HKm)n z4uy`@t=vzBuU~UvIBMlb_~m~Ntk`xzkS#jc6O6%|If2+ z_Obd02!7!07ZTqdKCF9p;}juD;!lTi3r4!--RjczbW)*1i{akQm{BVAPbf<)O z7eAk0cgy!*)y!@r5OpOG4n^()5JHVRz(0l*0SBh-HrsCsfp zfkKA~no`10B?}iW4wF)=4y|IQYR{v(;e_oO<@YaOPP+hONqRKYt-H4h|1-AgSVR_c z`C<`SnuS(SIw6|634^w>)(;UljILl>L1_sK23&}u#kHC?Zx9J>hRIe#iU)Tu?EMfU zaBI4Q2PcLp%?b+?D2!}w@}$ZD2n5WG>ESwd1U^7y=x&3JiwxmG^aRp`(T~Sq!O~t; zv+k_;b3tdl!{^fd`&~A#S4szJE`^}79}bXoGml%}>}4Q=;p8JsB;7dZU?+kN78r)1 zC^Cmqq%!~N#lk3JHY z(U2G!mq!&Am}iR&C6NRjl+a1HQgts`XXK@i1cj$$se-}_H5%m>B#^KcNu-d)-4=zCLttW#6Hk&f zQj}C?iKTU28hb3AbV5eL1Q0avY_k}kCar17EqCpG$=V60Cjo9d4kS-85m3LW;z_R_}4| zDOji4ZET37;l%HG(6%)mP@sMMXJ@D_cf@czKODnzZ^Vlc%4H1s%aUs^_~4|%RI=fT zE3VWG_#n+Sj?qf)>-K#Eg(Eno+HxaWpTmhK6)E@$|KK2Ep1z70B-AIPy{=ubTU26t zFosglhjvoS-2-#yhDD%{Yi+PV8#1Q1_7Kl_$wQvcJ=8GQPu5tv_Z+iWuR57T4&4hgwho1>W$x z70^bBguooxKsbgFuB~k*B;g66H$A&4$#=>DBF%=_J@45t5NB8;4lUD%Ozg2`l6zzf zB^eTe$tX5PGzSyEgoCPm0TTF7S2%vbE_pSD|BR19pc!2U##_=-j$K(}9Fz45tieWw zGH}fbW=4ql;Rs1HSQ#{$b3EwEXRYpVc4G{I!HK@zfIhQ#I}6R8bM5Cb`MY$2}lSMck(1;SqF@JFxCS%k>jpD2YuO|gwzH?Q&HTV26b~LXG)e^= zRh_E6Z+a2}RlO=ZL$nFSWDe#h6k+=5jSv14e1a|>ZO*!H$(m1soq z^@x;vpN`BegTZL-6e!~IZLsI z`HZqfC@|FthI+w*!$fn&Sjr^@E%2dSa0Z^6ykG_3jfI7p*t2p!!6R&#^rR{t4m~C_ z-bIHe$c}b&Pdc{f|AsOpfeZ2`mDpqVRsqO1a0mKWn1$bhwQ4Q6Y!W?@z9F)j(o@aE z5jm?w&m|bi7e45QaNVdflXc8uj#OAbWNVZGcRj7x@G$Ant*)6RuZ2JX9|BZ^DM(vD z<(hWaQlN!y!5I~XK6Gxl|Cy0)vZCC)Ft;(b$tqSLXTEuG=}wXqW%jY}#xR))Pi!*O z%+jM7I8Lv6qp&O5V8DUB?GH0{c&jo=`T+Q>+zA5Q)cN9sHIK_vgK$LaVP_cFXMUHj zdp%Y%qnWLw2sVY&JR)SLH4!<}L(rTk*IZb+OfsNr^{R&)1&LiYf*IE~qeE z#pEA)JKPZ?_qj_rb`2Ie-f1V+lixbHTSp4ZJM)7^#OY^*D%75zo%5O3S@D9fGtcXS z!jvkqo6xm&eDZA5yHObVU9(=G(@bQta|4tbtU@*kk*yRe^ z3D`$K0DKu(m$=Rm?A8%L0pt7u07O8$zY}}_9rTn>)qxz)L1bx$uU$yqr5uGRAM8Ef z^O0GHR3Fy#O!uXL@lXNxS;4e%-^_(W;?3DC%$0IU-sBxpq3z(%U7inm6HZ)DWhDuD z;0mzB-~82I{nZ*1BL7A7#hu)Jpy0j8l-1LGIh;>$7gZ5~0~{FQnAxx`K{5!O=ERR+ zd6xW$K!kl+@|cJl!o?e2AA+r!NZE^T$qw~#iuDbW7Nkp=2!azN$eAb?oD@QU9Lgak z*S_)K`Gp(N^`QC1<`BVmEtPrTL~2!{ zD%#!&Kn4b80ooW`)i6L?=)%gdRAGr08fJh8;)T!%lv~6H_uXP!j7~5bqq>xW^!43Y z#ARDX0w6+; zAdxO;oK^(iQQSZUWR@nx$U{ctu8;#odZ1j0Ui`IX0ptKd?VT>orR%`L-W^*?w&4WH z;q}4c6Zj=yh9F$9;U?H5Bi>}??W9g(-ZxneYo()A!VLGsQ+U)6$Hf*BH05O$MGK6M z=Q*aNjsKEsgeKsmqDZ7cA4w!LU6QUTWCSqeSUP~fL1aX}(gG~h1}p*}K-s+Y=4vg2 z5-f^v4ySO^9SG`WXUHDx3FdM(r*jgfVIrnsCkqINw841EOrwN_PD;iEm%Pq~2(Af)m208NwwRKSxMni52=lpWk=%H^gNu%&Mn zs7JshTCCyzg;5^Dk-qF9gevEAPAG+XNOVpohF0hLts8Ps6(X2jt3gtEp@gNiqlp&F zd|IF92+~N=njVJdS854MoEB=inh-$%NL9&HcpzE1CVV7-0O%Nh3J*QtiW;)r5ja3z z5dSEXj)WEfCviq;yKG&AB4>qO>4aWrbfw>xCg#v3lIK|vydfQOgv9n_;z)o%1|nE? zWfzag4>p{kI)GN)7S*_C6DH1mF^{$cB+ML!!?M|2z9Anap{+mL&EgIxut5-v1+Tb8?3%t ztX2Y&)aZzMl1aFmO+W)WxEHFy4`Xdaoi0-y!Pf#Rj#jWt7t$!Ca#u;HkLn?g2Ur+a zInWKYhRhYJwtruA49n8Z0v$-v>`3&7~-d_ubg9EdjS?xDs7223f! zqy7!s*>qtKaqGL$!)??-Qm7svGARTs)!j5e#lEGwUMkbjnMn!XM#kd@aumS}12Siy3duuusD@TzJytkiHoueJvwk|v}I5o$_?N5L8@ zO{;@N4WmZup_&ppY!U>PY$WuGfnJMhoomZZD%c8V@d@98MPtq4>`6kQ&i1S~`s}F= zR(6R8;kGHHDS!)1OM21Rt)dn?2H6r+Lc%Sr#!-iOeOeTpim^H@610FYqW{l0L{)*d zdrrgMAPzl7=lADdZi<;0k!`1H!JN-T;N&fPk{)$Kal5GRtW~Y41j<9fqxy3NP{I zY$(hty*i@uYUq(nq8cER^qxSgje2z?YM3CdeDM9jsGg#){JF3J zSPr1-z%RzHYl@`^1gQ`ACn$J@{>r8rjD*|H(>{Xe#1=pse8MV;?^l7LVMHMv<|P$Z zGVwa_7EADTGViK(sIJ7oBgxxi)!URf35l-ocdBt6FD2&=+jV3DCxK*K)zuInjM+}X z24sa38j!WVCM>ta!tp0U#KXq%Zz8bmlxgQs=H3zC&968?DM$c)R+!jI3SLUF0$Z}Y z?(73Y@IYuXCxn@OJB}&ca|JV(K6BE5 zAfPT6XVm1dShfUe4ry$l!#b#8k~S(4>r-;zt|8Bi%EImLcIA{xm{HvA-6rr9FEHO? zayNssb*>vpmo!IU5KNt`jmA(J3xL4Nqbj$tJhvVsXNBrj2~K;@nrT2IbKEF|84lm0 zCuplWz!eZA1P3G~L-Q%vOf1W~=Mpn=-7ZwrjG2R`WHx&=-EnhBd$UN3^GJWO?+BPW z8m%e2G-bjg8o#qlxAHFMnus*=D&s&dx3%;6ML>t;GLeQC>swH*6gDU{PX{S7qoVFr zp{%czNBrd!9eE9VzXgU?c604@A7N;E-@ zhyvJl8Bkkm6RdYLFwxZM8_3cEQ>fE1yJZ+J)l;+V7>5LNnTR990fPxfG&XneTJ|Mp zb~kG_SdVmtUl2H*G*G^9Yl(JxmNIpGH+qN9T+20Ump5J8XI*oNw4S$%!}JR<7FuVn z(d2UQJ(z3u7kI;#Y&J>!1t7VG@p43TRA0?MAO8UaQ8n~EH*^=Tgm?8fSGf9l_Bi)} zhU<|iIa-I4(Ckzz5IH%7{bA?n9f5P?*m`084D1jsOI z_JWP)-)*D#oMf-rtY`$%HDRZ3K3z*`68AGR(2*lK8{%o%R5pa`?R1AUm0PDu<#Xp* z@X{F}_5RIABnfzI1si1G5|lcPSM3?{Th|sVQN(pFr(FrK0Ikn@xi*lvWWkCrtWsuTMK zOYi#`_vDN*f%P^qR2SuT8lzpnO3jZRQ2*8xAFFFimJF4$dJ5GHheLtU`Bw&Lt?@Y? z+ggfiK#ZSByfaoeEWuz8u?BYxRnpsmMgp|&TB&&nHdnhg2WE3(`}t|RIo^gpuDqEhc*O-j${EZD*++ogDxCwJl!ZtgA3KBhrjQ@ z6R>VO&VYS5LTv-PBa1KpK7jtvChYdX)YrifjO$bXH~~BIDXC@>* zu4eA=K%nE#qT}5q7fYG?Ygc9I~6=+Hiy#`XGNsCh4*rl&5 z<{4+6OYZolv>3E-X8*=!W@`cnJZwP47F#O0trAR{VWy5&2zrtf3rU$JoYrgva*2{? zvvHB;3Ob0o>BJOsqB6Gw22C^5T$7hRc#%P+ArLX*E1DQM54{3*3V=Njq>^ta{3O(2 z(EJQN6vIOu_+S7&8-0{LDH@v(v`HPjG|~O~f{~Ls0GrAXK(>GngFig{@WTq?D-q8X zT`jX0vt_mUPu8jz$Zch6|_)&5&ieU zfE5jtU<9}zqW@ojGhMi|FZ|k6kp?B1nA99mP4xjBBw!+iSM$u%2uRTK^b0d=8;OxL zveZf!b95OCgJYF}Ey&uCq$M_K2l35~GMI=1VHjj17M!A8HtrKxPI<1QaK)8qTyLN@ zS37jo1f~x)v}_~1^1PNw-g?Eh$`_sDXhM3~Js|u=A z#SYWfPdZMM0c?-sJyA0cIz1&XC4AtF2csmHU;zhWUNWf=jC%ZWGwEX(;-XXQL;DXbpoA`m@UO)uOiDJ8WhASN9 zVZ9dwYX3;CgF&P??3ww&E#x$zLW*pAmF(2N$v@wG4`&hM1HJ}nc)|?zTfVR&o2jEe z$HN#s^!`KVVFXj)gC26BI+v}cXi0m7>s|*g*m*;C5wuzwC@=vaunQ($6A$lH4 zY+q>mliZr;pweif5Ge!^3QZ#+CTxa;&a)d}_EM|7eaaAf+rR^CfUFMbh+X%%!xOsU z3G89uc#BHHBKXHSqOflRnj4+kq^Q5n&>%E6LYVyKH=oPlD1S}#-x$#lzyT8QR|s?& z>lpYzH%hQ}wCm0fzw?F+-l+lRu~WPRRFn`BYZBJZEu07ZfDO83slY z4*!GRg&?Uz4fL&ih*8H7i+DW|?GTiD@l>ax2ox}IaaNvLSPSw&qi*~_m0UE6vA!2Z z{*h5s|JxNBok0d8v_W+*Ys4$s=)gBZO==K9g#iw{KsM)m<@6kBw~5*ChE&NRp%?@gsIWvOCm_iO=wO8uiU2R6 zq}3({s>*=AFQ8I^gA!oziJnbp21U$AIiO%gREjSPF!0&_M2E}JnNbhCB*O}FcfC%y zsF-d{=1P^hOlMYPBw7;>O6*yn3%sj?*EGPM*dsij4u%LpJ(y6}kOeu=VGTlk9RE1e z&`K?Y4XP-DKol@B4^G08d6;r1JZresPX)CoJ_TqfH{eC*QjkJ$GiszRb{gsla(G-+e`ijkX7n2n=CGh`C}KlrYPf_DcVmbPo+ZqM1DXBW z380t+%T1QgnO1>nWm7X6e0s=47PUuVkmQz?*SsiAuX|OxQZ`Nb9kI@EPZ@6Op*+0d z5Z`s2TZJM$R}QqqV)Mpr;BXL(1am`tHlpSrmwH7YJ+pbw2Dq@jkvEUqwIZ}{f7_pi z15N!eJqd8ZoYm2d(7gTaUl;S*v+jbIAO9hhK$c?i%n(%g&8>tn zC*{mF70KT&Nh9m53`2(U2rvVz^D>(#GEZaTm{f!Oy( zS?UpF$TFcpjCNT-21U}!f6e}D&${iW^Wi3NC3Zuy3Ur|iBc9+%=IuFHg79`@)Otv? z_{PtK=j?u4^Jl~7k<8oqOJLXfQ2F^#FP+p*H+AU+9(B~S8nQ1naGR7wK?=lv>=mcB z?Qick5q`7x#rNii1^K)|8~@v8Xt+?L~dZIO(q8B}GusOk=ykL>)*2zw6R;_aSBqH?Cdel8FLK~L#;W&?BSe|(_RG{Mvi zXf>b?Ax3cOO3(zarkU1n{o1ecj%fapZC>c_1p}!D^H2XcY+;Bm`7k8_6Q!S2>j6OF zD-^8{bte-R4)JV&0GaCs3IpDNU}dmnBr?Fl@++gFPzsHV3aju6r9jw757oLv1XC|| zs!kNGV-&3hEGeYT1Fmf|774oS zf?&D;OHAxgK;WqSFtm8=Kt4cNE=bYv&HKj43jeS0F={JyFw6_X1PmL33`>y|#SsPD z?>i>LV;&`&U@;bD5t1m02LtaG?=Ucq#|hpa3O zb12dCEb$VrPy|eeXw2b&!ol>s5y~7W9H(py(FNpmp}%yBkKB+Q1v0B_krsuoc;=BF z^Dz@JArtU%Z)&Ty-lO|aQWhyCe=MgzkSbuhOChZ&ts3$gdxm@XEL&)5n7ZW2ERtL@ zQX|Px96Qni(nGUAazNs2ByZ9N;n5~9#SY`K1M&+K&cg8aMjFRbZMyRC9K->bLBS^M zbi`sIYf@n5k|99{5QZ|BCaV#t>VA-FApcSdJ4P@%oRT|EFe4a^KCf$Be^fFIRTx7uaKDzsn$+!19`g--f-iGJA5Dch4`hqB5*Oof7XJ?{FX_z) zr7Q`hQSPNAMYC-_p%(lSIr2a?{(u_mvKoU@D6bI;s*7gufMz10OPm8tPH!VF^0C6P z3=fk(tL``1?-bjGIOB!+I%`BeE8}8HNl2?DvB#urAT#AqI_0oJRx(jE-~g@@3Fc{U z)Ui9g6Fq%^(!yvbE9xiHb3KRh6aPemMO(BK=F?m5^Bco3KWB#&P3u2%6hOJdB{c#Q z=&Qcak@kuc&U!J27j zGMMLL|qe2*Hlg0v_1#0qRq+&1(~hSYuSiSF*o@5%uyW$CrtK_M+tAY_snkJer9wN!Lcdd6zHMUa ziZRGkOq-QcpLHSC^HW0=RR5*rO-VJG!m+VnMo(KctnTzqUsVBORZkPOP}l5s%*@LI zG)l)52OsHCA(bjVgy`UhVv?c{m?1QsRmd_G@LJPTr?pz8Bt}7z;GEJ7&BPa6fnghV z5+2qRjznT1_Udj2QUJB_($2@=4rl7>Hq1%VY^nnGw41i=U2*6Jg|%3T2goP^Cbi37 zaexYjgAbN!K|pkoPyq&@u@lA$MKw@h2liTLv_{9!VHfrUk&O*~trc-K_l82~z=sk0 zQfkRZmKc>XQC3*9R$i;rVY*f=TapBDL;|X7)Aa6VMyKCkU~M%=6JKf5l0j1T4sE~B z8iQ763}I;T(`Z>i7XSYCX!jFcB(`an);N<0xU}zk!sl4DKnqQ(mkMFY_LX1pE@dNT zWv^6gvomH*Xcfjbq>#fk_#g`lVP*@c2F6ozbKz|ZCIIxbIA*6>kSNFwIX! z4YqF+)^~H1a1GaR*KwQ9rfhSe4Hm%0QeYMuzzSSf6Axf@F}0o8;1gh|(5TZ_KlG?H z=b)MkGb@)5RL694VFa9@e0|WL3J7`Sc1N^Oa%-Rj)+cULzzN!SfF`kaZTEHwHh1lF zcZt^EdiQq=*l#C+;;2o5T@V%{;10r50!%k%&ewHAfPy{gUz_J~sbF&{lf-5QJnkUf zMBsscL}jMPD*s$e95`5tfF`Xd7bOqo3H$?v17;R#Hf~uUg;HS2;&&Bt!36jqb?rAC zU~6WzV2JA%eN&fp@yad%tA7hNfCadvOwfQU;efMPi$S1pqY82F?|7j}cJW{Y9Fzns zcy$qBd%;Nzu-6loYzly)ja9+W_~I8Gv<;?GpR!aNfrTa)Bn3K&g=+JM;n(Fdvxn5E zUb*NHOf&=%K{TLX1UdnH)@Nuw_pM-nMTxi;=$8pnNNy>(2I_VSfi{Zw7EY`9DGB(B z3AbC#ZUMjeX@RPQ4FEnT*?IFfog}vqsF!Xc3RtG=IQXEthGQ9agU}Q~?OZ0LTFi*~ z#D4O@7XS9=ta?y>CAlbtrgY@Di0>|-X6rO_kKl$>j`(#OU}i}m1xUc3?l%X}C`^fGe2*iT{%tr4C=O;$ znU{c>Q$hyXC8HI9LD@oOqi7b+I3|>br0--E@W7iXrAD*@n8>+vnbn*tO%U+so%uJO zrDUFu$)2wmpSAdp?QfQKuW}Z}fd|ZHm-)N`hNJsJc zr~lCgP^K$=$S0%*n_biw3a&YP=GLvP;&_j`hw+z8F?S87>e4Wa0TP2GLNg%AnEA+Q zJ4u%;&%#R14JZbn0v+_|zFC|G2Ygm@v0oe1s;fO^`)3kCFX>unN0o}>G_UVDi~Aa= z5mYNsfS`+;xW^|4=%l!n=l9y0sU8}6QK5*xp_6MEACJOk@#82)XtOckUgE+Cd4f+= z?vx&&`9PCks=z^m0UvrbKkz^zx6rDz@KPU2ob zTgqpJ60HzaplqT@y0Y2t=tdLM7+9Dn4j|zhfQ7m>p=UcYSYLQp=a33U;Or7l9sdJh zrZdF6l@SxNAm>&b>+H%>4yC#px{^~ngJC-|VfTzq`fVFIO??Kp>$<dqQ+s&m5693J6E;6lT z3ndPM85-}~85+TDw{WQ+xk}zx7aVYs9lCiVV8;H9&y#@<;7YDP*?rSJqGh8SPU?qO zp$;${-@POhI{gbjJrw_YaLdrAxtrAod!-*u)^|(=r}n@*&c9f?R|e4|MV39uc$$#?>gqXib=5P~_7 zx19sDum%KL9GdrirGNzP{c_kvYKQ$;gkgq%SP;e+-ywlYPC6X$z3QXB8-zh=tW|e+ zo9aZ}iuJmcft$rm6QeWKv}0=k%0u3!_753cPZ@DBnC*UahYkvbbN{Ep)gHK9}plpGKm;b(fBZs1Y(FV*x4yX$RFi!GG zf)Fx?oen?o*Bn0ajxa=6Wu~vyT@n$X!KIn?@&Up}2u>F>KL0NY;|<- z0C$0d0Rje8Fksh~vSyi7ShxTI#9zRHE#%3gB8mzEj1@4RQsF^dx0JC+s@zsexSR&X zY#6}|A;eA}+Uk%rCqtGu;}9W65#uaMzFw7)A&XY5-we zGeFFef{ot6VH$WiWb%zoE+v6O2$5*%OgAD_rbudXZ08yfx+tl|8wVW5BO-<{=UWeW zU_e}+byW);uv`_!~Y#h#>wd6R-UwQmsHpkpnFg(KEo`Sa!`{; zT68gzm$=7wG!6MbLa#A{19b4L=v3{#qLSbLKq646|ZPMXldnq>FZjH z)Pk*SO+*b7h@WE^^KM99Z7rG__@;3mlk1b`Gm8^`SE4^1m5|wCt1Qj1GYZHNKc`_%QjceF8#o0+3YBQ$aoK$_xi5qmr%BA$W1vuvk zRF&3qsFTd+KKqH(e^TO~QG1pwXL-p+?1GNw%QJ3SU$aI@Miz%`9Mi=yZB#m~k}Yg1i8$8^c@>2CelPL|qGBf7W(3Q}N%q9GESjXT9pg9_c~5*44LWUONi z(NAGDHH!mWy&mT38zjcCMAot#Eswk%(Pt1e(!A?FReSL<&{9(@iXOtxL-5E;X_O?XHI@ds)nK zQM_FgTUH?o+Vm2P74ALld(CoMv#NJ*d`M~EUhsz;+p~9Tkb@i!$%s2-!N>l|CUA_n z$0FvlN;oXd2_1!Gl4)cUi zFR)6P_N7>~e_^pk+xudSzA=t8Za{rIfItP-hyy?9>S0dMwhEk04W7tHEp)@cBhWw& zdPox^K{2A1nouPsYBh{OxwCrMIk zTZ1{wVm@V=8~e)do!O~qHj0QtoXa+=&qbY$t$7O_T3gI-ZY?Gf8{H^hKBraJZ&3mb zzAX@vvCBSHD9&*Jpn|UUNDJ0Xw`lR^050*aBD$#0q7>ONzhXKZeHE>s;NX#wpacq5 zURah#77B2Rpb^9^1i?FbG!?E;0=vHOn14M*VMq15M*;7#Kc4JVh#W@CcC#qlnXNg? zZ@n#cqcM#n<7d*hzui_iD!>tpO&5TI7DV?S8a)q0%##m}4o0JC5%1J#p+^Z56TXYy z27kx73I9-M^(Z(2lk;d@g<$Xigl{W5%hmg@ptRIY$NuQU@>)h0|22mLI_!gTd~3s~ zd&oy#a+5ps#9MXuJfd)yunHdAFy~Po)-_T({u#PWKogVvLINQ8+!^bB=L3H31lg3D z;$3pU?J{5LQ-d%CDKx?aV!#Z!3_6dcZUZxX(3^(aDuuXN;MeK&PDY^J=m?RBA}O+a z(KLD2ZO3>kw+qX-f4IXL^0@Oses{g+z1i*4mL3v=w+yzNe@5E*$I_4^b|HE?CKc)n z(`4?6J!v~>Xo4p%&-%=hW-?PiR}2tV$On+V_4toh1^8ea*x__nVjm6gLk(toPiAC? zL;p0M;D8U931cEM^zneaS8or8Se%4uy;gj{_IAX!e9NaI&u4cf=nLZ|eQab3CKhM* zMQA_(gDU`ot}#FD)qTc6B*?@h1F&=NA{Qb6ZnoteBH=k8fB*w97qXQeFc^eW*M5r8 z2J;s}4zL1$kSJ}EdQ+%>R4^FnqIw(f02QDZ6F@-&IA!D%GhDX;x0i5a0~*06d?2I+ zd*N60Q7N1RghGT72FG@!lsKKhG_Znk$Om`eQ*0!Nh$iS!B$s!MI0@7@8Q+%^ltV$7 zfEbB3OgE^5nP@KS*LZ0VYA_WZ+$K+M6^08SF|>s#TG&<>bU~{)g>7Xqj*$glc>jtz z!T}CsE`7$#(_*w z2N2>QK0{{T2#%1YaUe*J$&!eQh>o&=f_TS{>?jIwMj1dzYQiJ~U3h4i*nP(|80kkr zYT{^VqXwZUit|x3SdK)Q4M=&8ra4~Y z1&`h4V*xmZCjf(1**~)>n@2KCTPcIu@iBcNkRL>7>6MYF7y(si8?6CpohMssi2;p4 zaT3sw7z8AQG?&(K7g`vLu2`3L=_&8gZ=Mj0!LchfDTVU}DGW9dH4t$3#AuUQJVV2t z(`cD!_nELqLO(NVIl+yO6d+|rENZ5FhuCqbDWICg6;<^`s~H|kSpNZp#sSXx0?Wyq z-FJgHc!WUMOn^~@In*enVi1SHCwbB)Wbke}1cnXDmIUXVDe91Ez*@FwK?T49eo>38 zxSW2Wi`oev!+9V^_ni_Kd?f`!;h93YB5@edk`9s}L~{|V(-B4Eq&ep^qok9aKn1h6 z89X^XW%hyGg=VJ-eMy;`Oo^c1aZh#`7Yy2poXClGQAT}Wi)KnxakeA4QEJpF7zaaK ztwEv>FaLxSvZCB;ffe3CtIlz|*)&+uC22v0jN$L`lD*qZP;ZHwl2|~$yyJ}fp z`lU=MrVrJEWn)MYkd=dhtW*gEA+UG@nT+)|Io5FjbOQiRa8qANq9Iyony?Cj>UpYi z9%SHj%<2I9U~oV01oOjW&dml6s9cQAfPE5f?!L zcamI7Iw&PkrO6n1lSv1GLj;RZjj&oPJDMOqd8-t%r3OWs&G)Mvt89$el#uvkBP$n* z!ha`QX^YXUqSyg6plKdjE|mj;?#FEaXadLB5BG6TCQu0c&<`e%v(3pV`|z_js}Y>n zC96^bGJq(g5<-;*t=A|84j7E~YI;JXY7tSG4woA(@&6{S7ODY@2k1Gl?|HDfkSL>? z8=n`Y68jRfY9K^oJr}D~exR!bMVh`kpiB9&h}9c}dxFKK|c zQ#2>zIQFUuEdziC0zx}d3kJxo;R+?pI0UWcyAII<3vedr#i^v$IU`UCvR4z3z`O~@ zG`3o1qM5ftskf)ew|y(J!)j9Nr?`4Es;dI1DpUwHW{}?7xT*uEg9e02)&ntmvp_2h zqI<4B+Yg>_D)+<>CQ!N_k(M~nu6dvg_`rD}k^ch(;iR%F3H2Ja3Jeqa%BlOIx*-LKE48an{SF*n5r`@~hk1P~A(z z?%^P=QMjclui5v*LEL48R+cV^YgL>&c_<$vu?6z65BpF9l^n@D5Xq9tL%EAXP1ax? zTyHpJsasqhX>4URX>UeKDCjB57x=G4fd8*E3%fOQD*y<)bIVUET!`1Ju_oBD&Ihb0 zIKzJo19LdDfQpRQsJbkvzU$k9XQB*%N0#u?2Ul4Lqq3}|mET13j%FwID zzC*{m?19PW%d-T`!K{?Syn=optj`=OmYD-@QlvJlzsx$Uuu2D{C_AOHZi-<8Y;a$% zQ+Z|Dl?b7ykt_`Fc2k)wDhJUcoSQl+pf(Pe(~goI7Kpyhu}(11fVG-m6L`-ITgI$2 z%i~+73(>*nnsu{)2kMF*a@)qYs{dh*V8`Cot1rB_6#ZrwU4BH+mkvnP4w76qtX7(g zySYU}ONIsi_m78WKld4x`Je{!R0#Rd8kD!cZm^p@l0eatTk;h!w3ER|WS*4OGJVPp(=W8?;>vsMLpkXc6NEyS- zX3>B9$CQz2+xpbqXkMfIwMU z8i`%lCh(+AyQof@9kIi{#Y1aXT?gL$aD(I}Op_ox95X$1faXoPhk&Z`;mwt8jShNT zKBTT1j3AS$3B=v7uKknXDE|sy{m=sirV>rtwJmbD{mgx1C3y%1d}-l*n#JuM6Z74t z9PT@ZoQ#Ko*Ua7APEs1=`Y^UJ1U#DyiOmn58VsMCzx{y8)m;NWd#=vNd1iCIu^|H? zYX0WdDLk)0{#)gQZ3X=Sbu3ptHP54^YL@q?5VYK%WvD>P2>lb@a2IJ z23?Kd32tl=J=kYY5O{NfVO==hqVa&Z8;fCG)v50+HuO4t(iZqYsKfnWiNe zo??s^GqoHN?}N)&!v8WDpe5jX)d2|DkWn+JJ-37-3PK?kZ^i`$?nJ*(=CRGO0xHoC zUeP?&f-dmoD#%7(9&V)=PYYM-Sc{<<`ht8eC6FvA82|!kU5ioemcB?_Xj|P(z{Esr z5HijW`|#q0t-!fJDV07KEnyB!5j)0uq5g3sO<=rx0U5^HM*M>cgtjzQ{sK)v=r&*~ z3qV|NObCDu*iK&A;Lrys4FqbFo0}2w+0*Mysg52e?Sj=$UhU9kZcu4{OXu^keq01_ z-jvXOKhhq!ZdtN>zT6}Zo6KD*+z!noE0N|75t@+LBf#kW0NC^3XpCOn1u*Xg=#~M1 z@0osY2m4xQaQ|nD@rIJRY2&g6FAzYNfFJ)|5=A&JUBv;S0w=+-k(at{QN1S@WL6(R| z=#oIdGov(eU_xISJ zs;N-MPJ-&tSrf_CvbDZ&4dYb|7_zz4w%IwxMp79=fciSxc8zW}YtRUD!**ElV#b8! z8hadBGGxk=SzX3lDf4E|z5HM~qCfZ&1y4B!o^CV&%ewiD8bA`>5+1G79a%OfZmWe$?U zyAc}NCzyyNYG{P-G6Ii0=$cW7BzjEx0Hr_rNI<>>2&~|b77W7g1mp@Fl)hS4z#|+g zsGMgfK~DgpDkpxzfP{I_0fJBnx2jMD4g1h7i!GnhV?+_9wf_}a zbjwBM`M3PwwFo15K@OQj!ex+X<2hkN!glIVJf;JZA8y zg0FT`?uIZuceIcK z3`~FuD16E#g$Q^WbUxrNQ7RceDhNs$qs#>%VJ*f=NT#PoE^fkBImJV}G;DYw3Vg;f z=qD-Sc*2bj+oCl`94E;uj6nL5q!$;1{k3VP?Siqc$_Ar)>SL3=nlol!D-GIci)_tW z4WMfx+uuMQz}qON3U|R71b9FI2!~2{-7(1|r0h-kC}5!x?gji_isT%jhW`Y$pdyoS z7I62uLx-}V%RoI4;X#&bULqw=p1gR#k;kR*Ad44-JaUdh*ER*k)u=7KOc0?nq&^5)@u}t5Sz&<%vPX# z1Geq{%-_a^Tke*Is~d;A$3(ogArPW>AuBuqd|$%X{HcZ&f>~Tl)rXRSO(MXY%Kd;o zGZ+xc&}6y!AaDi?!U#evrYf&Mp=LQ-;Qs&!1k`~C28DYam!R;6O#Ivs>u!=p-qz>`BJ&ID*S(-BvpG93XAhK$!hWJtsqgIP?wa~ks7|$@NwsGP zP-0bn`oODSb*QWodaP^YYI?eIK`C9-LRGSoq7!Y}D=pf|Sc2v?2W^c>*|N5$m9=VEMHGwQ9sjE}zisG2l}1BbMKqOfg)4FA ziculX@~#}!tC5hc0won}N_Pe~%DE)1FtwW?Vi!AwNB|NT!JQ{+5I^*ymy$S8 zpeJ7T3C7jYk?&NOYGXRFF4VW2NGL-cFWCehFm_!dG{_Q+s!ST3tQp^+g+Jv3R)_Mo zx6XTEX>byTK%QZ^#Y%2m<=Q;vKCdlyeeQ3m_qAzNE2Lxdt93)A0{Q?arH5s3241>j z^@UEpV$4WorH~OI1G&8R5zAFj#*t-?g$$dMuP!=K-p8pdDt0U+tG|y7G`B#H8^pv@a-4_LeI*U~pa(f~ ztJ|x+Fe-iyG_~2<*5r}ZYKK-d)|_i=4WISVtcl8wX_L6$@CJ%ePzE$`lesK+8oaZ} z#ZEih*_18dy!t+Ms*?=DpD^q`HR#m*zF2`v+QH!s&c_-UY-VTMp*G4UAXse2=FH)V zj!jt4MeiW&h-Wzha`=?UU7(dq+?mVs4F9Xc^=xfFwO(29}R4%JG#-Jlq!Z2y+iO(1)s|#twnaM-yV>gJjdQgDZ7sI|vGr)Vo8&hce$^$^W(5eOmwn{6m2VDl8`hs#Hn@T!=Xguf1V|R``^m zPabO+y>0t8L<72G2!mwOv6U#gB#+XwzsPW_07Q%%8mmDoy$d_NwOAIA zdMP0YG&IaT2h)R2DWQ`aEfjzfZ_+yFt2ftkED7MEaVeZ2Qy*c-Hx`U6!vKardA}I} z;-jW7YQd9WK0>URv?~W*XbBgvIGg{0f!CvfyDJK3OFVN~8J0K~q!5FXxPzxSLR?Ei zK|(on5TTBH3P0Q)WuP7Z&_brd!dk??oby62BtxGQqCLZ?Gd#KlWW(GeydMjv1_-Aw zqQHYoGA!#rTN}KYI6gTE2SDV*TrdDM`a=%vf(l{A&4B`gD+CMq!-5MLT}z-T{4bjU zg_RIOxnmDZ1SXnc2{#)THDf>Qk`QK4!XhE&bfyv1CE&1{QJUF0^2j2eq_xRi(B0h4Nst6uBF?y!&hE&GA9h^z~gqz#p%OBButrO*hK&q%6) z4yl2yiGx1u&^jV5(8@<&Gg1CJ3#tOp8nsW+_)r_w(Fonq`vg(2a=^+IQh0jL8tqT= zMAF}q18ZZ_)5sCgLoVBd8U|fZ-Kyle(t{;2G?rHaKm+YLOx09q zjaF%8uHLlPQoYt}4c1^iR&Djxwp7$_6<1;nrEw)!PBpz^g;YW#P-m6WDV^4MH8Ee5 z*G~1;Va3;NHP?OB*M1dNX~|S{#Z7uOv`1xEN<~X}eHA8>>SBkY* zx0Kj`6r^Poh?Oz0!-#SeE6~maW*B<=B~B)ReWzwR%|4 zYgUoPp^^XPhLg=%pbc7Romka)*&->|=n9&oh1rdD+NXuusC`<8Em%pd&5%u48u>!U z7}~BCTAMxEqHWr-66HQBap*R$N&tliqVrCYCk*peL@nhjgDHCvZm z+rI@|wPo8dq}qcu+-9v?Us217vfIJ+SiwzOzAanHrCf{+T*w_Xw*1)5joZWJS#JPc z#a&#^ZQOd*TWXEkES=oQz1-Gy+|ngo5%b)u#oD?3TG+ka(?wm^4P3J|UDpka+;!XG z?cCXwTdoz};lE3m-sG(Y;?-HKHQwV@4D2Oc@MT`z<=yfH zTkHQ7-`L&W?B!hUjmURR%k)K7`5j;L4O@M1-rY5lbC_D(9bIMB-T-#r;vHZp#oI*l z-vqwi>4n@Z<=@Mt-^W#7cGWoow&28Vj029~4PM}qmD&f6*Nyc#^u1jP7U1^E^7UL9lIWH#q1US?Eo_XdM@Z$wqssqW4vwWZw_dF2Iz?P;aVnWiZ@Kc_!(5KIn87y)Fi3c9!OkuIQ1D>5`u5ljdgc9cY&J=zRZnX+wr- zk`8F(l4)B`X*53NqGlQq?rBeE>X(jP8ZlV9SXPOy=9;$YZBFTKeg?~3>aGsykIiXp z9_n+p>X|<3vv#YQUFWa%YQlBrYo_YBF6+78IkZOVlI~`;UhB5zYmpvnxxVVV&S;4i zT>&KDz3$|2c5A0Dw7(AQ$Hwc(mh59*UWHEVzJ6=Kc5KKF>dD6FA?{!<)?m!$X`2>n zx~6Q@R&CE#Y0w^RFMjQoZtBI}?A7jU%|>Rew&WJBY_^VV(k|`W7HrPG>#|1axAob> zwrs=(ZpJR>;wEm^KI3Ax>a6{2-qvg1mhHBtZojteuqJAZ4)59??f{F_W_6ZfrDkrO zZf@FU%I(F6!Eb?*fnS`!;V$o{{YS zZvj{A{>|J2Pw?P2Y5S(|lFo4H&To}=a0uV--}dj*wr~I+>J8`cb5QW(ZEdc#Ush)C zrG9bR{^>(D@f(-#>E>+|ujS)L9&JF3fB*m?`2+)Sfg%BS~JUEeJLyH(IM%1`5Hs&yIpbOyI})AzFfHK;mo5aC(5pRI_ubxOV8fid2{H9zi%UKKzQ6zdJmIHbUjk0_AAs{6$e(coB3R&q5Iz-Qehwz6 zAcGZJI3YP3c6i~38DgkmMj$q5Vu`l}#@Tf#s_5Z~A)-j(j3bH|olY#~h?R>ivc=3w4lu*jZVUoTvsb!N3O1aLLH1dXJl~j^h=7D44 zxFwfMnHi>&Ub-pgnQ^YT=AA6D`Q~$TKG`Rof2Jv@l6V#xXq|fkdg!8xqB-cJj0)Hx zb&(=!sXLc8s;QrwRto8-k%4OGq!N0X=&6}@YAUB@imE1|1akjrD5|Nh%HWl*vT7@L zxaJD$ta=8k>abuE`=776CVT9o-ZYEsv(SzTYnR5>T5Yk|Vv3Hjy($}Rw@HF4?zqmD z`|Y{Frn{@Q+m@4UyYRXzueZ8l!>PUG=Bw|%`1We9ivI!}*uX#XYcRTc4I1ET)N(uK zzx2Kv@5B^SOb#=C_KUH`_Cm99#~+8hF)hz9Q?kh;ldST}ETepl%P*Hqjm$LHY|YI$ z=REVy1o!N7z&``+FTzYVtZL98^UQHKOK;4yGT}h|P1I6XO?5I-+cFC#Thp?N*QkUI zir8d#5=+^iq^-8uYq#Bq+l<65x7v}+UH9F1=dJhNj`06&cQ16S4R|brFEfnc#UQ@T z;)!45xHL-Zd=9Qi7fjQ>!dh#iH`|EvOW%up{(0!6mmas-U#FfrGhJgnH8`-3BaZB} zzdrlz;MAfz*u10dJKMm!9X#A`|6Tm?br_$#9D6U%{PT-IA3YtWPfvXu)@P6XBhz#5 zed*nUFZ%f8173J9oincf$A9CJMFo!S^$-y2~EWA z8=2VMEbuS|AfR9c4t!t)Ss(=pO0WbMgrEvCNP-Tgz=I&Pzz0PbLK2S9geOel3sjgw zD16X`Gk{?XXXrvGT+fC#tey_*H@_b8Fc3hbUl9L?D8wS3f{3rUg%W*Xxy-dHe%&)e z{iugGD!wgj{L7;Ma#2782Cxy+-=*k&r~<5xKVsQ8ZCqnu1~+nDB&7 zb|DLj%%KhM&JA?Y|rBiPajw2Wge_ee`#_R^QY3}zn% zIZH!Mfq}>rL^74hOj0&;d5K`8^R_2H?43c9QpDydEO|*MN-Sd6(c}*}`AJYN5|ro! zB`mwPPFJon7`pI;DhbHSalG=Le5eI3Ww8H74DPd*L$IX`{He=d5_F&ky<{?F$Yg&s8lg%0zX!Dp%l2P zRrQEaVNSG-R}E_)*Jy$`9-skMg=aj&n988G)r@eBt0p=z1(BZAq(Kj+TnSJt!X;5nA2L6@qf;E>{`Q!QCC(Lg9HEo1<1hJ0}!ADP<1LUZ%JCw z#-OaIO|4T;%UadC_O-2rEoh}G+p_;Qpa7_)fKvTgR6P*2sA}ygaM@bVwYE|UH#{lm zrk7Wg_ElPI$-alHh4 z-Q#uFyFk{iI>qbE@SZmxNKUenZIK2zX0)<-(B~oTYk>e4V5tZOuzxR{;4WvG%U)(M zfyE5lfZi6vBmlr?D?9`Pev$tQRM5kTK@8_)jM%7Lbb=Ub0AIE8p$_RZh2cOUXc)(M z#(cH$jd6S^9^Z6qKV}6mYE}st51GhNHlUI>jpQc7SEm`(=Ys)oU;kd&z+2Wbn7JMvJ35E89TeAiuR_C zpX6xdLi(_isI!K~_c*BjsW26_(k ztxNsl*3To(aIgz8+;k_rfKOL>F6cT2ISj*Gm*a&iD4uPL2XW(z?)Y@ajtw4TH`&_O zF(bZ>ZcVei)8#%l%rB~oHWZWJW%Vy=dp>4a}K6H7zB| zZL#ukyZ6`aY$yK)4gh9aUm>z$yk+fC-ShR=-u>*Kz29Tt=dx1ug)}6r{ykaF{`3F; zHg!$hBs4kV3(D7ZlmtZEP&7j^ZJHN-z$0nJQhnE#ecSg_rI!!i_hjM6Kq1Cj2M`SE zg=8NHd+`Q;D9CfRcY?Dgg0yo5snr59xPJjygNWvTgC&3*Lx2;RnX{&}q zHi(0EwT5gsSe^%Q1n4YUcYx6|N!RczMpJ~=cZ3jl3CY%kfM^SW=rPZOdUUsPJY|Mz zKv9Osfm#1(1Y4M7x`1I}ScxgPf-LxVnOIticuPx=0%@p*p@>p!*b6%-G!o}|o?}vX z7=%MeD|<+C^O7;v18F12gxcUSy10u+$UJ`lZm{$XOV$jE$cV#-g>5izk%(l?zye_S zh|f50m?(dgxO1H-P&H(1qPUGWIAa3{hn0g2vM`R0@N6pOikf46&EP70cxjbDGK0vA zEAt7yh>T=ZdM*cbuta=kl#BqjjQKE$sfGxbsD7h0jnvp`*SJr>bqch-Wkv)i43&@e^_=-?Diyp};Q|TwfkUd+| z2TRy7CfSuJscem)l9ZnijB|HD0@+^_;8;i&Rb-fVyk-JAxo|rU+dU%70A zW@KYIeD5`y6U1bgpq9r7H6sX(%AjOquvCaZPz|7kwE3ELiB_?>StS4gxmlYHaGND? z1%A|7p>Y1{tTA+S!BlXsLl|aHT^aI~ls6 zAF6WxX{O1jrfgcLW7={m_g`P?2_?{z0lA1zRa94ML(@rPg0)vk6p@BHNsRw0MTu#n zk#h~o*IY%}e9z*eWalsBxtZa0I~&D1kk&HOS6&2%q{XK}sG6z{GzH*?g{oF_^u=cd6DO@|C%J58ZH;} zi{oI1uww&;n_1o4 z0N?w!kvqP#TDRFt02zz9^lJn0xki!uUQ0ECY&wIeONxi8sMa~a5Gh2oTe~DOU4tl| z5obf)R}MwiwsQ7@au-)U1y^RvQ)M}N$9Q&Bs-?^br7Khc)Vl!ns{-FExfoEsi92DH zYrcy+zcLH~d3yqm8>K1yxG$^)`W3?@3_%t1opd|`3y?qopf%AvvxM*sL~G4jfX(J0 zwAP!l81Sp#yuBP-0OR}(iOaJfI{;v$&dERkJT(A+APxo)%(XTS6#xYCY^zHk0UzK2 z{_Fz)EdoR^&;(6RpM16Knp{7~Vhr8T4o%9XtboWDhp8MWt*kGxJW?>Af6jwkfGG|H ziBVZ&Vr2}>ff>0BfB`C<323@bbVWAri8d-mN-6&!%~2E0;egZd9JGIsvUiZOOW@5y z-9f_|0Y$BW1c1~^T?+&NHBH?NPi+9)7Xh=d%~h%ZSDmI=Km!JS&|cktv4G24y{_N& z&`xaD5S>yJP0djjHs2QQ$;7l1&YZ3`FxH5ULzr5z5A z-HiQX1Fap;6qO9I?U16ZuA+#UOa=ri?T2N3)@Ys94Nbe1)7AhRF&BN@8U4Ro4QSce zw!~}3z=gJa9h&3?)2l$((EB@wpvW@i*nLpUQ5FSgMB6j=)k<8;AbYV|V{%&j+raJHrhEp%&DO*XDqV}k zPMF0SO@roH-)PIf#5>)f28;@X-PC>9S}{CLWf~v(a_~XxmU#%i zXhmj|1vj4JonTYFZI}=}%ek|(bF5K4j^MS8;HH?Am-OI87}u_hD;}8*$z9i=*xYa? z(itw@@#ISJm84jtQ~6WaB0fCNg6sSHLSxl zSTx@SKIUWYOtnZ;EgI+C1AS@ZqB?Zu(gO>o&gyaS>VQ7$fi6<{*XT3!+tSyzf1X1u z&bn(&=)x}7i5_uJ*yf(pu;(UJ2{ef%mCYo6+ZFz$83=BI8&sxIq~APknE>a@*I= zf`yKVj9v%9{OrugY3nud(~j+>7dxgYHtf}!3lz^Zd8@(W?b}U{jMVP- zBUl*80~%oRf(7g8zUtfqMWO%f>$JY_w(VT7z)9*opO&>^T}=mJn_`0PWVRq&2~^}) z8|((Z^i4kD37?CJUI`8F@z6f)0z^;oX~ZoFnjF9E!(8pgn+cpOd?7#bI!;qTB=0HZ zwCs8VH^B1b-s*7wJu|PxIJLC4Zr&B`cw%bGYdn5CTe=j9V7QOaqt#Y~DBF+s<440@*uswyHlw3yL{789sZ zg=R6=%o8R|GE}JHDkBP4Whn{$oC?%zpfkO$P0Pm3?OSqi<;I0Kw{AMTdDYR?%eSvz zZGi<3ZU%U;sF*kvL+ROsr!8|TCDWu_nX(!nkW)4TQXv9{3Nw~Wp@PTs*F32)*=YL5 zR*ckY8Xvkt+w}jZ+m|lYzKyXFQr?UjaioYQteg-p%Zw(@SaXTA;L>o-sVHoy z?KLCX0K!EnT5M4>Wt_<;8GIa=Ir&vWSqX+Hh&H zJBl1nG0`%S4GQ!aT2G<&;G-cvsag^UiWE4wPXRh9pwEH{F3>YiJo`-H&n0BEftR-a zTJX>aAx!_ULJAvg6hjTYa}!GxK@_nt%eayZt1008$y44`OAUk{4hd9?HuCTah=EwN ztrIF{>W@EVg8Z>abf&l{I_DmeZrAGyx(*~_sWfv7C6axNB>{1|Lk~U7YY7G#mco&> zHjh0>BcWj6_9ivigENBr0x$pqbnzpB0(RSFw}f@!b+->h>s`>oMGb?-QGWN`7o$kQ z{ZkK1MXXG-5rtGK6UPLEGdMzE?BfEAtd)(c-UM+qRPo-(gvcOwiWre{C27MEd{`kYT}Y*N=h4Q=CzGA41;It6%DV2oS;AT6%@O3NWob{rqtNoGLOYuV*uLtkdwpylkd z0d!GM9Rd~LT`Fj-&dYX>CB#aGk4AD73wOxFE$I$&7bswOtySOJ>#yHMo?>dm9{c&S z&qjNCV%W~&lUa5|P40s0#s^ar5!oBgzH@8>XYJC+LV~97^cr_4TGjkbA|VULDyaCd zrAWx?pstQ?aD^6`^9fIwpd67TNr6&f$mo_-tX24AH%n291d7Ke46Y7!Oi{`bF82Qd z_#`AUX{f*(DEA}!XyR#2hziY;B{-{<%Xk_*UJcX0mgwbBder0Lz8vwAh<#;y7Pyhm zEOig>K?4&d;?DSZ7Pa^x;$tMk%u;xy9iYK2ep}p%#|&q{Qt75E_-MiZKtYI1RO~k& z6V?Mex4@}9f(gQa*(*HMn9qbDcr#=j3R>64)xF1bC#b>_LRbnERv-#1fsEl^02yWg zsD(ZdWK2X5L;UcOh8fHty_~1R9fq=pX?w;senPfA)+ zcrnx@Cu_*A8-h}y(;Ma594d!p-l>a@D%f&=4v=>!Vk&TVFM4`_m8IE>f{aJn%9XIxD= z&k4xo@kuKieCPF;0ScwOuna_{&4+B^tnBF`ABF@dB}E{VOX7#124!nI6I!-&D5zfe zYG^}wF}gK@W>3OQXGN>xvwhhkZ-cdmTdGirHdv)9A6>~Ec3DiTfF=JQonVSlRr51m zO3jOi0G{Arnktv+T} zJ6#aEHk2-3<`P%x))l+gedtIVmQKc%)UUIe2Qpkyi6d9S4le&-vU(@yjQTc~eE5Y^ zBVBYAYWc*Z_665es1-#Z1ONh508@;_OxfkofDw197A*=P#hS%ds6<^Y6H@REqe=#; zNrl4?+}w;b4B!J#<*@4VbLiq_wZcz<53;(@2|OpxS>MnES$5XJOGK6m@rtPoS2yFX zxgZKS%&}babzK|pxO%JJ@hPvh=@(DnyGEpycJRRpCMUoJ3E*{;*PF}mqLj)vnKB4_ zsop6YEh?*cLsv|=%r6c>C#V$xBiQ&JSc!=P3M2_OvB?4p2)#BfsDjok2hU5gAO5~e6cMu);%Utz?WDjdmeD%cR-UWkuWstG)ccMnQ8M3*wN1Po_A02h<@ z)}`2WCvet+6+9JLN1UvcbmqJI!z>1_-QNu8q;O(F)w_phP>AnV;-%KH#V@Y5i(dmK zX1HqnC~4s@qK`W)CkAU6uxr*W(W8F_DS%x*?8LQ-JOt^T?!C{2B19!~u4JBCT@nd4h)fkWrW z5BB|tX;P8}BmbV5e9Jc^scvAT*zltiU=iI&pPf;R7D9!ZNDYxS?kvhb*mZi_w%V7A#R4HZy6(zt-)QzaGPf#zEZ1pBR=NCwszz809E z6ZjDVBSCZGHFs+`MEf*$I;eK(J-$0XhU&Y(V*}t)f}{B-6^Jaa(ghP*iTWeC{j&<4 zbCG5NlRL8=Em#37fQ=YvCBWeiRA8|*S+@URARPROK!%Z`V7P}YP>wSJE}Ba)s_Ve0 zBR6r|GgY8E>EpLC$S;)(v*07A1-KF7f1iDL*AlLT{+J^;?^5SOfPXF@h*KkDMdpR&MDl2>q z3MfM%Q><~wp3&LE_2>l4BcfW62;Q(ibeI7=jEXwL#s!KfmoBM2r=b?L~rT}Jvk{>VlMGBLUutyRb)j6@jD)R z#d(yzCpZLV`vWZ~4Jv?*TYSSWvH|~zavzflIa>^{0)$A@NVacGB{Gz@1Hc6S^BD)a zg*|0|kmnjZYwgU09BxYRQ&Nj+^8__oF>8lN<*$_)hE-q+-#;Oq&;w%bixJ(jAf!w=-TtX(aI9S}OsNjjY z!5Odwy60;$b@LNj1hfMf0$ZazMq52BvakU#Gy*s&oT*7gNKB*TH%O$UoU%$i?6z3k zNpAGR@Q5HfLZCiGmc_&j%#wp<`aKb~$%)m3*z=?3nZzQfd+dxw2NzOD%Rk4Z>KqW0eM+x|iHkgMg zphpQpIuk3znXm;}qsBNw|ozk>yM9; zBFwllDYC`9>_Yn+MenM`VB<0`BP(|rG!Pq_)S!cz>rU@{LgAcZV(c~DIsm+fv9Lct?N3n z%*}vo7g2*A3SG~=Yktk!;igLlN z7dU~v`U9ls21xinA`K5=OosE2qoO2KZQWGU+k~DZN2Y^=Rj{l$Ft?i70!XOVN6^fO z*j7tDPb`?yS9s4%tyg=Ur}N>c$*Q(lI2_E1O?M2UbwL*^sI;*J$hOkE8q3vPwa_)i z(8bhLPuwZ{@C^Tp)zf3;3XKF5@;R$VP#O*08DS#Q);w3vbclELftGl!N}7~JP>7x^ z1!1V1sF2xLK!tQA*GHHipu__@>V(Wh2vaGQI{3&=@TZ7f$a&oYl#SPvUC2b(#N#4@ zIuMek@v z!(=e3m>Ji}asm+S4;g(|!fjf5z1P##DoSjj50luQyDG1sJNrmhSxp^P5>Ae?4;8Z7EUwCmS9^;3CVB>M}2$zOWz0+*T~J>;+)Y-jaQy@TGVAstCS{<0gA)3iK>*si~T-K zs@*cJRfgr=-ECNeT-)GvUB7M2p3|Q~AV4aJ;9Y7;DsYA|)DE)91w5#Mqw3!_iQen= zTNDsR7KnjBMBiMff%@DI{o-6m5Z`aOK5-B?AhEsiU7C54VN?0W&b2JiO~wKYkE*QS z(uKX$HR37Fn_Dc}JJ_9ZVc;K3R<-h4uq8s>B;W!*TQ1hIJ~)c>dfR*>!3LIMOyl4b z2+RLkNMlZS1U#pC|VXOfP+m1SjO-S2bW*V-@ zg}NCIuACLdQfP=Tpy5N9W6q@nK^Qj~76&P?-XcLzC03aRQ)1L55lqmJa=_V^d%KA> z9j_&USnV*|Z7VG<+lA`lwG6c8x>#PeSWj$V2qc0MR)!iVVKr!gel=R|<310bVU@j| z3pTjFtVAC819a?sDM-~9e((2}L(_E#2WH^Wh^&JOuJ3T22($ljnK4`2= z{bc*S*W`#-#U$l&#G@0yn^odc$70~xNDWvs-VjxRpDL(=icnm(P%x%_ZWhn<>`nP?mfM&p8 z=yOwr3p)?hWrS5|T8`+uz|~ZA=mWN^tdV7lJr`1G+qPw63;VYlby+;A9CD2B`1i8&mKLMX_d=p~2lPmWzxF@U$hNJ@M@- z{y`Vb05fh`C~ELtI$BGuxk)12^k&Q_NXf=lVQT%d55&D`rGvzj;Z^AA$b~Ryg*{=2 zy2df;6E_4^z2m1A&eC3Q41@}dE<^+eU>y%@{TAz5{%^ENlycY$bdaUHDe`sTg|+su ztYOx#xGJ9*4cvezCR-CJ*AXnoN@|{`(FUMKLZipQSrL%mdxq}%O_u+rsmGe`4=E@> zqz!_PejC+_hILkiIzR{I>oF;S1Azo~ zcZFJcUDJ1=2Z5ol@}MVrct?6(X7a2N0OiwnB=jDj?=>KFfr3w;UI7HBy>{r!_YByI zP5@YZ)@F=nU^M5CE^zJoy*`;%&rqf8n#fU)4|$QN`?_Ct3Kwlc7v)5a2%_{T8;F3B z4hOj_)zwA{U}s2ZiD0Y2`PnW--|l(yL(}g``ggy4%zyWzXR=)$LflkkQCB1)<#LHhO$63Lxz zVCl^Hs1(srN|!R_)RZI^jBsvks(LCa60A_QN~NPmSH5p?e9T~m$Xq^5OkS2j(>6?q z4-V!oAYk`y-31CNL>PcU#frd!WyCO&=CHQJ*|`5PCfE3}9T5(s%;^V2wb)1G*mMcF+(zHbM|fFxjbc(8lj7h#pSdX#DsSh~HTB z5EEz7leI)*#+`2c`pV-cJ$bAlWkkkO-DZU=--(H^aOA%&JIbv5&76{&(w{^1%howz zT9MktpDRoOLY2impGZg69R{-Hk{fSXCzl86lpAi70YLB4#bJ_#%sW_+|kRYK*2DjyMKzS8K3kc$;jq2{|Ng86aS!kr&dy#gf_y zsF4)295TWs@F}B9M0H58k_f9@`K2`TE8k21F==1Q^VshZV^t zD^4beVnnG&8jWh`n+9qPr|Q~Wt;RaBtrf4EYrGBY#lXf8 zSom?r8aI#v5bkc*9UgeRq{%q5P%Hn=c(cqS3=}i*Q3%XdmQq!@=K6`%eTQC*(Nzp0 z8c4fkWwP|B^76&;3rj>XHE_8F2gMp^^fHY#o&r3u!3B$baHuoJ;NcJGK(-g zliTy=>i!4d&=#dD^l^{8wS=X%Xj0=Ft*aqJ9B&P0QzT;ny^-uV7|*k1oT zHfR8-G*w8TB2WV`X0T9;(B*jJ0ef_O0x#O*rgj{NY^A3v-J^3|8WXsfdc%SC8 z(auT8nlyRTN#){1o19O%gjoNSe4J~p$QSxU3DAXZDq$d9fC4oLH{1aqXVO9&Ugw0@ zdB|D~fx(h)7acd~E>pnk9jAylJkAu)ctcQK6v8(?@$JZMZ<|%Euvb8gw4fXFIE%jI zv;q}QfP5SCp%2v{r?8w4Es=hO}*0mCWurk8?W5GaUHrI*9 zV}fI$^j`EDq80AH9P?D?!NR@GN(;oD61N`7;fDP?s{~{_;bs_L#X4KBRidKf* zaEK603(N$}hn8mCL2nzpV;uBI4M1*_o0bXH@L(EKndYGqg@j!mDml2(lrwPhL}%P$ zXw1W1XF90F4pG_p5+vpY11hX%PYVFj6k;-@90+QJvR6^9#9V+=~{V8SQDg_k(tlWCMEP;RI-; zPv(gxpHYB<1LoF9fvlf`f3?Dj>F^@hS?;+s%+A-RVlQx+#Mo zc6)`&4jvY?a$Y__~Mtpa+}i{A03w;}3HFKm6fG1u@#Hw>`_KJua8kxF*I zn2Ibjxl0UjdG=z66S4h59qF) zykZbkfSmvPqO~viu~<26NR)`B8&AjmO zw97<>v~cLL>R`l#%QSY}{cGpCc0oB%23i=%lfsGb$}5>~8{0O=($d6D?s5-(+%uJ_VUPZ=*sa@*?qx`% zSJ*ZpOUyP&T9W!gW3|q%dcXvokAyh@Cp}QD2`R*^Y()c3kFC3&2VfmR9+CC@pJ!( z982itCOW|iD|)5V6ceL5q3Ov>-Wf@K>PM8ts-p&0q!vtP7^8kggRBKDy+v3R68r&^C^ zR+JQ%vTgI656Qqh`_nJza2x%Py{ms9Nl#O?D{QlNU@t?&n(Ivu>`e^yq!X^C*;@rd z8a2o&O~-3J&kjwT#3|1T-InYv6&yIAuO(Mjw97zwA8{4M^<~%sY2OwY&=zo5vEhn; zupr?vo&%nkGQiyWrNALbS+86R9?TWDOwi2L+_l6~=i%SyX@)nAo@XGP=!yS|4Ume{ zxtEuvhN`Fv)!9Y?R!^B}l~~P(7k(k%aR3a!mNV@|jet+QiH2W5z+pgGnZY1d{D3VW@1D2_b@vPz~Fi9Tm2sUG#_##0`XL;TG0n0tpbC z@r+vx#TaY>4RAmv4WCX?pxQ+b$3y`QG5AP-4i`UoRDc!3o!{6=9bkll5W}JT1mn)B7qYn2xm#n5;)b64WB9!;|W-x zgVBgyl~NNZ$_9EM4lKZWe1ZcAhui@eI!we!Y#1DPOPkQ6mEfdKvKXD5Tho~-yJL|WA0aa_WA z$XSfTB^9IvD$f%FkzI6%rL0<%<--y69asuf1ejJc3BU_L!z4wJ>_|i)Qp8AD9t#{M z3M3{8lz?J3=3+YLV?L(K)xjYuPz(a4bO>c5cGbQv{Ss4#x{vCw98dT)x0yB3BZ4r+B)hicZ=rKtXa1%2;g?W@aYt z#AkyDB97*$j*?jP{lYd>1CZLMkmjd9%1CT=2vqgh@h$(Ud=1%vsss#upl@#B2_oJ} zTot1Pz?MPHmuY1t(S;ahK_JkIhVI%8L_iib0c>2S>p(-nWk87v6t<-(tkG%1tO68( z5q)e9j|K&M{%L~lXrK$S2JKA7v) zjV#Fq2fPKDr3rwR=0+cALy~Qpe(~pf-Iv^j#zd0kbwr@6K5UK{&s;DgA@U(_fr1)v z7QwRV!LnKsxPfcM1|RIKoknJgHh{|v-iyL0)}B$bVr)Wbtk+84q5f!)dMs3WkTot( zj{TdYt!<C0|f6byimKujMvkU4m0Z>~$YX+f>gPVnVxRb^HQ z#oQ&NQbd6mk+iJAwgEAy10KwROEe^kx?%zJ)RIpBZNyTad7>wJs^HdU?ALBA$BHf4 z_M_4zMt<1=z&#r6HX4s%D%{3x+O{mr`kRj&$vl;V-2(1`F@wV47jfdNohn3WC`{@s z+^q5%z(VN)qyek-s=|YwLpE zDEfvSxEs6KZ`i zzN!VoB@x|FC=Ki5VuZlu)X`3EV$`h?Fs-mE!PA28#F^nR`qX)-py-xv2`XSOI;&uD z?H-^m`%2&Y#;@xFBz`h#VVT|&_8$PM7gloru4!nhG3MSv22~c6oxH_c00RbR-K?2` zqcB#$3;fNNA|!a0hA^I9rm;^4_vUDw3Idz}(o(MFK4+))fD<@X)LNDr5+7_H+VHdL@Yk;HFuP>g5vfD)wUkg5@?zZmQDeLL6c`5BPL$=TbnIvR$Q)FL(rP za+%z)u^AvIvoiDWIT$iOJ~LwoF-1cvFW`a`Qh>q~5?#fnPE9f@6B%p}C<0#cI-j&B z6ADFiz~vSKb#>+~i>O&HK`w)s-wEq<+L{0~0ZECd8rW?Wv{r61K|VJx)UJ*37^LvI zY7>=2m1gZh2MRX|h59!1*E+MJA~GUZv_&hjS39btN>Xt$fW(~>s<_@Muhu0qBshn& zID7IsZJYz(4M_rqUB44vPeCM5CclO=$rZ&wApmuXlRV4xc48s5L}=nRtO5M;^9(dV z3o~jobu&s7AR5xe*k*OnakVs;c4?G@>Y}fSDQ8_umUw3!Br}Cd{G)>*XB<>DQrh+0?u|VTCH5mwsiJ2 z_1+3}_91Y`SR3G$M*#x>Rsu0p!kc2juqwB!E+#-rX!A}DKMw&bo(7(Z?tWV)(WqxT zZuiD^cT|2Xc#Ah9yQVoKXL>sf#Ef)r#&$}#_`2BlfzkrG_Jv+V$~AaE7c#{=pAAb! zit@CV^V0O;xt0&;fh4E@avm3emQvx)?r}iLcveeBwRUxgllLRf z+okre+@`pz>;P8bnp?{^ zv^~z{xU?Yf7Bf8m!EV1bG^#6>M#vJdN*fsZp}QS9{WAqLc|oiC zR49d%t5sEdwpBxW*(%|*dy}YqYl!YA2qUhce`If*Y84QgX_fqY$O)^RR>mvLCEDz& zr(_6dJJ8+)o7=Zg(#A~<24>HAD zx>v*QqOGlx@~_`4xR_P!=Q5F;vp1>KsXQ> ztg|{7=K0RsMwFQ)fX#-2WAj}+oMRJw&sJ@3F-5W?P}8rk4uhlPm!oN>C+pH@mVb9K zI4adwJ=<>o{*Hy~)^|G>w`@7wZND*D0stTLt| zp0|`HAygpW9cs>!%;^@$wFdNu>Z zfdUK$0GVeCQNCFc!mYRwq(?-F5R(Mq&<#6L$gEejYagj{d+XOWY1FvYwQCLUw7hri!sBZUa52Hah!r+$2XW#_ zRTL`*;sU~nD332dzQ6!;g`t}{Bjq?S0gVihm&BS$3pMJG%&I_k?ZQUvD0yfTTJmtJ z$uCRdsmX&hGXWJh4@T$)WkZpdPK$tWN5~U{&#OJOEXgU9%G^_%YDga8&g9v&)0A)W zWhIWELe)%2`W`s&SwEJ41*0_tMIT+VJR!p4zW@bXi;J`jOe>O36kHIMRX$ndty4si zORsG9vhc#cGUR5k4v`_uFlaukaKsV~>qCVJfFOhr3o46EvlAeIk4BCH$RGkPynBHE z#~vN%aWou-9Fj;5L{pBr8}2x6#*PRW0Lmw2gg_0Qj6*I-B%fjg#vf^zVToTvTB)j& zvXf*xmtfj}$MMYb%{(918%PVFXpFD3f#&0hPgK6qj6VliKtaZ}G2tb%Yw1&LfLCY+; z>n=H`%0x-KG@FIAS2*9uRZcsHotLv!&KhNxe3zLAQ7JN+$qE^qNYoZ7rT}993P=Bi zCJH|m1Ew2PPEDzmTOzEZ8;k@&r3qb6h)o4N9)u9pxaM*d<&wLtww0G$U5vsL zEm9Q7f;@|Tuh?TjP|n$)mj$=FF^QH|2XbM$mgxlysKbWum_UWTaFG^wJ2O*KBwRR1 zK{MI%+$9eahj`>+3VG?p6S!oaI0?Himf@q8e44mq&^83-SD8FGxZ9>BYhidXJ0O<0 zVp5`^IJ-C&^v{Oy#&*=ywkE8G#8>-DS;Jdf_8ex<`$%JEMA}mT^~*MUvS}xKzV-lQ zg%2%(0vgTtjGgpVoh{r(=C`NWke+kD~ejQm%JE4WTpx-Gv7RW)^^iSCOJZ9= zYDM@74uz;ipFL;+DBw55s#+DHT7~5hf;mMzJoJ{{(Cuz{+uP3Z?~4`<<1o|UtI60P zb;*sZT(1>Ux^}?1q`}^F4R;4npbw9RonRF5p^PY0Yi9w#Knhk=9K{JM;J{w0oW(z47*imWmLEaXK@3U#r<(--JVzRQ!Z+9;=xfWuO03G3t6ojU zA#j^p-{um-x_!o3`tu@xa8o04*kNn#sa%Yw`sjdd{*Kby#}8UKngp;ypwI+4 zY;TCzlbjq%`p+^iFqI4rgo6_bONM^6!WO>poHNHVb-2ubVQiT#puy)B{~6H8jo!|@ z%!UYrhA|>)20PYK4ty}RH3+N1V<@di6bZE{KdjPU3vfUT5N9FheP()z{K0N==M<;5 zsg`gIiVrN}58M+8=@d|0Jx0|^Ir1aWynNjMUQ610{(Nd`4g60vTQ#dzQ0SXyN#QD9 z1EOTbWf!5Ljz`-KN(e|mwzu7FZztfb;1+jE>E$zU!v-2m&Bwag&F(4);KWfHpDhj& zYfNk44(2f5Q9S(u3^aj$7r~N{hg@p73Gx=Jc5hCbBDUMuQP2j^Ot=}K-R_L}Mcj)` zH|!7s(^Lt-W+t`~omz6(f<-NDcJo3D7U!~R1LiT;u&l^&ZESxV=Q-E;x7qfdV?oIU zdFZpc+wE=y2;k^RPe6-5V!{g~ZLa}>ck5`VfF~Fh2H#-Hs7YO9BZrfl4iG|Z9$wBC z?+64jG|F2C&VwqKC*vtn!DDFA$ONtb2nZt2q29T2>;@}QUBK7h;hsZ~3#Yc=&0j z6&I|~{`Mu+eeZuieUPd67*!|v@P6jo=)Ufl?}j26xn>D$2tLR_4=$s+eRoCFe|@3e zfAkX4gH%o767HFxWa<#0glqu!UMRA3Z$@~Jp29=-G-@)02CL@~y-~pa!!myA7v(N)S@c9JKwp{Mz_QwpdFZ)7gxC#b!jG!69hw`8& zf#5I>>Cpddu&8j52c3!afG`LL@D|f!ouK13#AmABh}MiLKDb3$Y%k+7E}I-60^R^8 zE@1{UZ z5ev2O13%yi?gokkj~V`88@W+)zVQt>%X4k>KLS z>tqrSbFn4*qdzQ?TOvWbAS+A)at=WzVMq#}zU0ehpzcI$ClO2rSR)!MaUv`Uklt|?y{RU-$+~9a{#-&17-bCr zss3zdq$RYvB)yU8VxWaQ zir>-_zFzXBBq%2JkThxW;94*2Fwz!E?$k8R7Vr5C$%maugV-Fvm&^NZYUo4G_NW@=g3q z4b`x+LK7}2lr*>V>$sal%q}z-g*S-faEr9KsC~7MQa0y-X&^e zb2QB}LVfT-!|srp3kR+=J+f2)wv2jY)RE744=5>pgp6;f4IXJHv2!A;!) zPUY0MhJjXVwN}$DAGAR>9K-u2MT~q^PXp~NpNI7RZPISjaG2p8=mGQAtq;-;T2*J> z%)t}?SZgoqkUO=L52F;xxP^u&;?%gMBg*V8XHhBKaK+&DrB2MZILHS_wwUTbeXskLmX6DB7$VI?$MZ4%#PtPJYW$QqkDls|q0&*{dYPEDts`WAAO0npR`cH5QAiW6^?YtJZ2u21Z4;w@4P| z3d0Uw12&Mw1;j1wSk~JnZ;hZtI?zws7$6E}mS)!t9a;lyVMBS23t>f5VQ)}ET~A8? zu||_1auZzdvo5jrcGR8`W0B@CO_XD&_G%|LWVO~C0JK(o;T&9nb30daKi3zy zuUB0Ffl|>Nn=2@^4a+7aJiJWA`n3WK@z9h|NM=!C`?g_&b|P3}aAA+wbk`+XAY*c& z4w|q#ICKLJ;Sl_*c#W3>BX@G6S9&YgaxYU}LsxWhR7WL+bX9Q`-;izL37%-6gj6V# z0@r*Cfe%D09x=6p`WASx)p!3ELQ$Y0oIuLDAUDzMsdiEXYV&`wBvfl*eLXG{2x|Wh z6M7}LEE?x)UUV|y)q1bjbF+7Qxwmw`x6?fHfy4()rGUKX<|*6B2?mBx{)7qtc4%pv zbZ_amj*2!mT^M&#;C^!f1KWuTh}VWSA*%qTgq=Z}f~W@C$uL7e3z7kcHQ{_ZVb|m{ zY8kkJ9r%IaB5NghWGC2yvp9?2?Q|39dwWl$l1$Tls0LDCA(Sl_SU`Up0Qxk+3JK0W z?d5%amO+<{T?cB z%ZZ^_dass><1~WF5OXP5i$A&JGWZHnAVwIX&mri zw5~%ptS19Ou8(-bHpru%GF5oRp%6kKkC*_zyr#frGhAv@C$}s~q`+gv_*S+V`huhB#a$srIwqC z8i1`O5w~wUptaj2HwBi67>INGdtl*s^_YBZ!nl&B^F~E!{T76)1yF6oUmlukPNNZ6 zYJ`R$PBy5sDx6b&r?C^5r1Y-7zFVo!B)kz1i(dw+LD{tb(>qy~z?^c*U97;x5xFK% zpalRs%={y}tp*Bm;*j&3h*zP%=P3>-%Wq;=cBiU5hybtH1P@Yx_A(d%iZW9vB?|6< zr9e7VcH*%U;e0Jgo38tJ9`}gqK+MNHz(z)DJ)E7}S>{e~yh|Krc9gVjYFX^LN5+c; z81c?mpacX)UPUCB|vT2QeZ=a~g& zVIq)D(v(gq9TOj%v37>}&5HZN9XG?14{fBVp@15|G<%&xysSEz#N9l!x$ne%?^Bg~ z0YD02l!t6EA=qy(+q?u!CW%{eU4no*pmX98c!H<@bbG#K@-%<&^n8NJRr1N1n96#j z3d#c#pJnb4=3pP2Kc?W2Yk*7W+puqZpef?LRmeeHeG&3&mCkgj$I6OluFd@i&g1zQ zl%2_%03pq1qsXn>hBXD8=5Kt%(i-O7p-!(CiVAd@$788#j=O7|bVsNl+vAN`D&5B` zC<(VkPz50FTDrfUOdOiGc~teO90e=4&cDtVvNvYQizqyxz~1QQ)dMUJa(pSqoSYyb zEau$|6vRzSK|ix&8T5TdlhbqV^sLBxv~jfi1m0hLqn15Ju!?2=nDS=8-c1N)9?Ic} z)j*X1u)`A_ha1J{Bbo`C9hNd)7Evu1ZLCfINC4!|FPNaEaE`2RbO9$ zY8OPG`YM1?_eL`R36E$*mMV{2R<6&4H(AaEiL0Rt9* zsBj7-iqt48ngUt66sne>wt!%&vc`kXoRV_hFq>pVom|dxc?#!)$~U%cdGt6-wrr8R zTj1Qpf%k*m!GSg0c<_KqiU0r@AH`}EfD}SXf;iA1AVZWqb_O6G(8R_B2o=EY1dJNB z9&vRRNcmQX1PQr?LX6N=+jh8QBT&q(F(UX6-v&?W#Dh5hkkz zsScD*;rKlFetqDdR~?v(kP2$Nrx7AVPpI5Ou%> zMj`bTP?#*pV1qwdRlrgZoM1rzffy{10fvI@b(UUe9U%cEZ50CB2Pt$DT{2>9kOXRT zP_ls;DwxxUPc3N3fmc-MK~-giCAOGj0aVDqfs)mRMvD&I@f0W+B38j0dDwzTS7ZIR z!(FSjHpOmjeHmadn)skwEx{!;;u8{_xJn9VI>QGG%)t=gcT#9q3U%FBHxM0N2s7S6 z3JEl5d89zJ40`fmq+WaEJnCLX^DPPteU{FoU#6OFnyG*PUFJXl4?uQ;GNult0BNbJ zswz+`P?dm$2ONOMg)Cx8)`xdNvDLPJ$VR+2d79NAqQHVix84HJr&}H(5 zS2%0Yh5}o0HLW7n9;R4h**?bD3Y2vlG+`3Z;>1)|29afIY+C_NBTnbyrdmG$=oXp$ zj_}+J3ml+HI8#IrF>oyeUPG6~6+F-n2rK-=cq9@e;w*wl0cbA5Ow7#Y@Ow!0Dn_P1LFDR!B>JrXgz*WpMPbgZ=sZJR#%QlR&ncqJfixv!n5{4QR>~5IX z+{;cd^&0`0JI**<81_Z>T=Rz2kltgJw2NDO$Wu2{2epLTZ3}?|ALD!?!3T5%BWB>- z`e3TwyJ2u*FjhQQ3*$r%PJy#R3gkpL#vK9*3iAuYvJ;+zY)(58ti#X{Vkpr0LUa`& z9W+V@!qZIyb*gKf3H9?j1Q=#ivMWFuU`8Y&RiP3eU(25i8pj^Y(6 zr6DZjI}0)sM#*qe(kyn&gh2+Eh0ru14WF<<^K|e9LyS*-Ya{~u?($7CjmtJqkQdwP z=}*Dfb1+JHoH!x(I3+4{4+&+cLk9sm4RX+fr5v3I=VPB$o{%OrN#P0;upJ1zVT%fo z882P`p&ziM)R(1snJtLPvsc3I1q`~U+xF1STtQQwOrU{)g5!b9?DMBKr6&}1rK`6< zr!CEy<2|)@3YB zWiJcOKt^n-O1&RdJ2cK}eHMws!z@?-dD^G;1*~LkZCPI{2~na>t+ZuprBs*Ew{pr6 z9$lFl^7_&NxBwc3{p(-#!B=AVG75)HY@1~IQN}P}t7C~Q9p5RwrfJW#a}+Ii>DfR3 zrE<259Kyn6;kk*|k@iaK#j1NzYgMln^sB6Otvp>T+YyekwjPtz>59=-x3cc7y+uGQ zf13xr@}dfRO@MJTWsFMTuDQirY+VHafCx~xLzO)*Xe(>f&rUVOnN1CC=*L-Itf+Zd zpd%&J+urYLHMLw_ZGG*F<5z4@WcuZ=k8LYp-D*m%uB_{Up+{1{I{1bTUUC)s`V!K- z&{QzZ#4$hhnGAPzFV4*FQA-Ts4%bL#do$rlrjb9y4G_8MDPoUkp(bg6C z&?KGVUjMq};i9y#R+F?bDg8>C{sh+Yqv;aBNUuh?u@jetkAC^kK*^oP z;tds_a#4)^Bml512qwLmX7jAYB4b*s*3P)zbB=qlYhDYy*Z&ptpb1S^nG!qEtY9>g z|LSN4|B|^pbZ!HnE0%To#HJju;}(BgvjI&hm{E9a@}Q73<#ZyC$(zB|w%JuVm=h5= z6M-umVFa3!ffgJ948VZ%r2~Eg3amz|o9n&4EWqHKxFSM7Z9Zdt+xjN|{q}dRhjCW6 z@)y7XE_iMaZo*;DfDjizbWwOwT;n?U$xNOV5S+~cXtPM4K3!kOPZfplg>B?CGI)Ed^h5kJTILK>VF@HCmx@V8DwvHVO-PD9pK&TdibARu!2E6foi6AfCC9B zAkn2R->T=?>ahLAjEUOxXDMHZbzm245(qszBpS!2b{Pn1Ll85+a{(%|X(4z6AOHoMfirq{ zBvmv}-XuFeI15j*0S>T7R3jDJ7gW0U2WgWwqfsitNUAb(flX&f>? zj|Ym6XM8vqilqotGH8AGG<5`6fLj<*6XAsk*kfrGi(^>-hPP;jyHyi4fqOEAhg$Iz zdT0YP6MSxn6i^{ECtx5L;!|}eXCm-)O1A)yp?jr46fNKaMP-QKA_LtRgN3+_-53LJ zP!5PERJCFOO`(V@L>1|10c+E4d_yephIxa~VXxB&D5j6}qXk)YI4Oo=#nDy#)QsrH zZ{aisi$FP9fRL@og#cHJJQjun7m>JVksBltxwwlPISKtyXyv3JCb&C!fGWl)Gs5K* z!}u5$7%r693~Hc%b|^mJ*n=!bU86FM3@{{AK#00WB-SK1G${cMfPxT!SRc4TGfA>mE}oD$wIVS6I8@h>bGcyu0%_(ZHW>nwku;h>1p`HOtYw7w zmnV7#I$fwJ5-AO{2!<4Cix-KJ8L5#dWeS?2UV&(KPEjDjs5=doL!wa>-*W;^(G0Z2 zTrjy|GMRhGCn`}f0Z2iIM-fI$(UXsX0y(35G>L-G5;H8xkbTv0pa;7hnYhmx0pv!m{IYVFF6mAnQ2X@Z8Y!$PbdLIa5-ai zihyw$D7XSGGn!8_B-W&v9{?&#kzp1_E(1XS0T>`E8qy_NDFw0tH}Pj(%A+*q^9bt0 z9Tadmw~;ubh)*-Xn^|d0fisqQgaY`NB5lx9Y>6T%Vk+1nn_4KH2xy~#R+j{Koja;q zLPlf>M|ygQGgZ-;xnm&}K%N$OAhHsS24QJV0fM;VaZuBre{*1IBNR=5Jo}j{67rv1 zS``BtKh?;eMj@1?8J-PFpa_(6hDV_yxLHmpG;p#e&LKEt`FX)%O<#GUWvQY8`AArp zNoT2exllxK=?bt&m(;mmq{E|++M_2Ffkmb_P00ZohN(?yk`c10I*B1nkvkwDs;S6c zZ8Rdzk}W?(2GyenW`mRbLzGhRBxB0|pCsTy=BRu75CDWJJOs)@pekG+_!@XnAQO{` zTXQv~sGC_;5T4+9$Jv%?xgq=_CRP9m$=av=_cTfssPq-B?$?|YlZCH!fH{hnKUR@s z2&rJ#i&%zlUs@V5WQ?;SE9^vvQHoj2P&6FKs{}BPtKoQCI-c&xE#^oX-qQfzfU6C_ zlLx|sG)b%idPYy#d~b5BFv0`jnqp?ttW>ZS2jrx93YIkxC-NweEN3SwimvLKu3ra%obsdW<*q{MQxT9V*Z6$bIG!uO1NWM5QSx{% zh*;t&05Two)wh3BK$6Wei2G#!hVbjQg?DQ5(|p#>O_atzd>ZHbnUU=T4I9@*hJ5an7pd#(f5 zUl!4`>#Ch7RA@u{Q9db%FDSZt&@8fwadIZKQNp#5VH7OOk0qD3d0+xYU;_PM1qCov zB+C!|u)Dc{czLuBSFpRYJGQ9N0I?AR-Y7gL78`XEx0^6}Zc%x|kvBHEq256PQpuBi zs~p9l5W$(P-b*#HVMyAMxaU*|(@`CYdqh^I0nahJz5BZxU;^XN4;kDK4{I0upuwIu z5A=xx`_LW#gFZe7CkwhqN0o+IKFAV z7P?3ztGUmyBnOJ`(Oh6zzogifU4S`6iT3OKNTtMqwG2`I$I&Y`enX%ej%3!~A|yn3XqEBb!t^nM6YNY|lBpG>0xm$T_ii=}+ZmQuyJ z7_?aYDMY-*MQBY!`5nt5R6<+JSEY29N&`OAQ?H51jAwqru@9$+&~&^H39Znk zZUf~{)T*pR7f;We&4@s!Hpwe>+sxCd5X@XFgpsT;N(N?WdXdCPoea6=+{Ebo#OoZz z6&a}mwvksX&zW4$3>%?t0?D9A%l!OZOMpHVps6Z!2co(fYjez~;IWbv9AF?Ena<~4pYHvaz}Z1dYUyNrigAhkY=HHr{>4ZVltM~T+N^;ts=dxPxF&TbLZ;KJtk{M@1On63Kv)ZUAj?%k29voKcm!rCvwXTR z6$7GlxU?0lI%BV4cy$^aYh69UAslo;e&v^VteV=%6V#!|0$#Pp02wC&tI2pBQ2Ymw zMnEuDXh_>?!e}|Z3ZWME<4BR~*Lx<)GmW{0eVtQ`zgC=Z25e|fr?~$cwUGUI#X^&R zHxOyvB68be9B>JcNs`wF+SHQ|c>sbGs6}o(0tsCM<1o8Q<BmV>8H89lSt+f$;%{585R70}Sp~WQHmJcnR&$B#hR^OCsB<0)=54rx-~V6xCU-+AuIdHh!(N{N-Y9<6Z9MXA&n- z9m^=1x1T34mUDk4j3*(kXMv4YGySNi{La)Z3M;PI@FgHnaGUNIp+61Y>1*gUzUW%8 zBbeq|$RDw^wKbXVSdCgV#daFZg9WKQG#(Ge;T9-G8a2gfQryB-r;O%|@?p#23;~pU84vu_tiMgJs8g?&?w9liLPVFt^^8PA4 zB9um0gK!Py{NTY~a_V+m-%<{W;V@k^3FDZ64_a!x0G*hL#NizW&@b?dOAhpx8H5JyqC@$k}!=0Zy#4;ZRWQ5BmV{ z60qc7(zW22Mdd&x=+L%V9^^)kPSOFZm#+4*x>-};CPZ%O!gQ6&5r~yeMfo7B*8Fnn zlnEtYg_dvz;`C56Z{ii@h1Qw#eW}yk{_`9e^sIc#A^@D=@)V(; zlA=#~F^POEj*2RyI8D9ZcD%d!Kz+f)_53aY1pp{uPphHw8t7mP(9jG{f2n=o_78Um z3t&!Z$pBO%@n<0M{cEa=V}!%s6z^>SBbwfXz}`ytGJIpG`2v}C5E?F!4DWqnxlqpJ zktlq*64&tOuNF?v4*8M~LVI5M@LbDmkoiBq`B!qf00D$b9$N)75kiRnFwMe03Z~F; zu;GS8ia0u0yqK}#Mu`rlvR}S=`0@oy$L zQTc<2XlpfF7)6G3>dMNcahPRi#1P#vF8|K4`8>eRNS@pm^GZwqm%-A@+*d%Dc|Emq z>g4IuU{DW35xQWbDg!#1fr?~MkN}8smH@D-CNjy2DgE983xpT)$cF{9Q1A(zgD700 zioOI>&>T!KLCyxZUaC(kJRm7dl*DpdO|sXlam9~gGQ(`MUbdMGHPeP1(zVBGG}1=b zaAT6m-F(yGH#Ul^gv#P7ixN2Ieq$~{zaX?AOfe@&3pxfK+Gz%6)G(?)a9QkuyMXK)f{l>+c{Kmecmz+@d}5|9W3H9&bGKoBypVZam&Oo6DclnM(| z`x;1Lo;(=jln{_`N+DEy@Tw{<`$$wq)ILbHp$`_5DMS$eDHe$%4o5m$EZ8K6smzid zGnz2E0!)(0gnoiq$(erMD+%HM7Q-1~I|;-gCl6wpS%>8c9MyQ_ z6Yx+o89X8|fZ|Ch#o%QroI39WbKX+>f}%=(Zi+L<0f*KmydvZ;3DwFZtP5|Q<)J6c zaAS?3Wm#HDVvhG^65-f5RyZJ?HahyIi>4kn>y5YC+U#%3R%>s+20v_ZxAfBE1I`X0 z-E^&cD~-1c1;MQ<=yp?IOto-AD+$os#{|Ip0pQ?df*!gUVIbc~Tz4XpEBV=?1s%DV z?m88P7T`e>TERe6njk?C;9-NaxF8iSW`NHr0AFk&kO#dgVOr4CT?pbrUG*w> zeqapoiZ`v-$O2o;3!Y|xB|Rb*u}IfLV)d~9r@igjrh8fH7T3HMzAI%-Y|Sa#`Pikt z^_dBC4}ja`7__DF)lF;znNAG;v_JItEr5U9he-%n0~3&-5c!ap0-GWo6{Mj}7_m^M zxWb?l2BQrC>lq<0c)^hUV1q1FQ6>^*$dlaRL)1ZpE;Kn3*QKC#n5fGXK9fzxJ5!coT>j3t2z)ExfC@&YY&=Lbk3f(W2c zO`L=4}s7&B?Qr_?+iJQPmwN>}m#%^j$zti9;GSu{GG* z5w|Ymyj9w=mY)IXEsL1MsqK<`NbsH%rwG0(ma$w}Oy-!R;zcMllTu*WK{TcLorjcQ zAh}}WNy^y|zX2wkBfx_JQDDIYP7IBHlvmyK_<<4pjt&j&!a2P&H>6{V~CeVwI6P0C9w z43k^ORSZkP7lte`wtSC`tV}7Jod7UrkbNjCc<85ED$p?p?(NS0YQP5*grtrki7y@X z+upk1hrLrsqZEF?S|xzNjN-Bg8eqHC^)}cdy7etv#TrZEHnFbEJIX|v3z^tJmzI9L z-gI+W8^Qu71BNv$V!PJMD;l$1;SJ0A*hgOOBqgWnMeu1M@s~-WFhM;AOfG656hQS@ zP!cI5kB|J|Q~E#%2;RaLAo$KF+zU<$Zd4b(df@$85*1D8ft9EH2b@I}xWSF94lP_P z!zNd`T;OQbJltH8&Q-+!P8zJXT(F}Qt5|tB{>^vK2jj__NxX)^v5RHhcGDZiA+(m5dG7yCz#^y8rX4?dBgxARTmhJ6v%y@YDPYy zdgbU$a2}r!bF8T&5SFBOZZ=p&9|~R$T~jp!w0-mNR3nNMp%}w%Uaqj_tZZdByUxw7 znqc$X-6EXWp3Pe`Si>BWy>g0s^(vc(#9Vfi&n+t#L z?}Ap)gh@1w8}e&Q1>{a@8pOp8 zUE52=;WPt$G~@SX2Z2|noDX-g|CZ?O8~1Dyo$Uqm2Rk%Cjf#=$hGVXJia?DNx7@=8$Xegh9jJYBy=vmVispZ zKPObVCmAO9b3FM2It&0IJ+L_jQl|~-!VBcQa1%gFOO*(TuN@1C{aQU9XuWzfx(VEX z*;A{co1Aih2Dexq3KG7v3b+?cky!~iQe(jnF)Zo8kUHQFAZS4iEHej?r$zj_fMW*N z0gGAxn+GuKzBTYTBJ?#lBR?g4u4XZf3X`)Dt0*YEmiEgr>sq`-1G<`f6;SCNqOc^)s; z5X=g@EGs?=*$ap}9kikc69EwssTr>7L#;E%h%>0MunAKFsCaZMu|l)wLBdcx4N)Ww zQtYTD;m5#=!X25yRlK-?{69zZv_s+$q2iDMA*fqiBthWCgA_MS@F|l3j4L=pXUN4G z*arl1L}Yw7)Pp_Gvq)V@0o&o6+408q2?r^-G7(Iq7#y&RAObbOMm8XoZ$Oc)0)?0V z&^jLE$u8h378DglB9E&$s~LiwKUk}I%qJn>#Er5?Wl6T_`A1Yd#l-WPYNN>cBghu9 z0xuCT4iTX?us~SJq$!xluRK6@8!`%jw6~QpTTt(!B zlSV^?LLdW78%D8&x;0FgE0|8cgu6JbPA`~CB6y@g(hJHwtu;7^7ckEaiZw<5Ljrvw zr0~oFE|{CR%Y^*{i%*!$`ti%F5Vuw%Mf3 z2lbjc=#3J~P29Au3$4&734t*n$xc(K56wvR%uwY-PUiH`GgPulJH|?@G-j;MO3NcE zlTp;n7as_b8C9+@(+DMygM+%O-};W0>%ALovN&+E1WN+%WQ6y$DjPD*04>lTl~MvN z0|kQvN2p4V(#NM^P~vgW1^rS84b#JrPzueLGsPD)MN=?A*n_ptcReh4Wms2o*oT#a%Af;Apx7Ac(Tg>VL@0&+)Yv*u*I0{$khR!} z#n_6)SUQl{i1oOPUD$?oS&oC*U`i}`y)(qC*SRtt1=Uh$&{tId>_>lvO(dz+ngv*a z)h?MmT0bM$nWb5yU0OU#+Nf1ps67ng@Y15iRl@*LK(4>zA;R0)=|K&Dgu+tP+Fb3| zpmkcQWm=_G+n8Njv~}CGg+M%UeT>9B33TXnt+YlsioDp z<({@ZTcVv?xoyz8i} z<{jR_CEMri-Rbo+;AL9T{oUx5-p(!Fznxslr8DJC(2#1HX4zch|Mgzm6ajo(Ao*S!|5jpI&EZBaWJW&ZMK(gk zDBd{^u`9l0)+J#{USrpc;7SfR_11Q=4DPcBpzjH#^ELYFmm8qN{$}>U1o0XW-qQ5 zW;SAR&eA=eW>OYmZLVhhRcCT`XK)VOJci?MX5dFYxnM5lWA5W~24q`aXPyPu#>)~g6uH}jD z*^FMcab9SRMrn&SXYVDv+70Q5?&p#=KbBT$gl=h#|DNTS{^;2yX_uzynzm`z+~}Ov z>3L3Rq+V&CeqE>L;*LIMYyRV9kzhAYYJ<*d9kyGT7V4jt>Zb1LnI`J7F6y(!YH!AA zriN;<4(qpu>phm~p{DDRwri{2N1xs4wdU)%uIsk;>%TVXyY}PS=xV*zYPIfbo)&Df z4(!GrY{C}bL1yc{w(NI??3w;q$z}yKZOLPV2;$?che^;}Pi22JEM9P0r5O)^2SxUTVb_Zh3a= z+^*c?UhS!VF552dXF%lGj_!>vZR*zT>o#ig=tjbQhG^$rqSwac?*?h*wr=D$Z}_Hf zS{CG1#%Rx#E71n;`S$Pf7Vr8dYV+;x3f^!2#_Su8@7wdt@C|S9Q5Iy?j&KSW+6Gw4ST=2%m@#L=R?4EEY-tiGyj2dCU~Q1LbHu%c1t z>Qizae{K{GZ6~jBAgApz59azF^5s5q5D#$?*W~$@-3A|@-V8Me36DnNDkYGcD4kJp82k~J&iWlW& zw78MuM0_7Zb{t94V@Y@@PnujQQlrb4EK|yaNwcQSn<{7OtcbCt&zL}e@|=h>s8Enc zb1E&mbE#3GO`S$%I<+cQt3#7Y?Wr|u*R4;#dId|itXQ*XuVP)x_H5d)aJj~H8@KJ- zx@zy1wTlxax2bgX$`y>a@L0lz5$9FBxG~nii4R9+44Ja!%a|SWl{~pK=gdeegTDGy z^XQ*SA;DQVmC>ekKLAc+66G~X&g#k9GVLKZ-h#-d)(np|(Y^AdqIVYyL z(_ad9_#uWvSt#R(gUx7ThJLA7;)*%4*rJR0k!a$LFcwLqk+R{ay2Fz?Z^l{YojY>rW^F`` zw`QAmiU}s3e-dh_piLH1d!^(TS*}lp0s*rH4+MsiK@F%IK#p_W3BJUUjPJ zm^GS;>8Y!->Z*jQ#{ar$em(*U)k(I#I;XB&&C08+t)>a=u%Is5Xs*7Jn(VL3&YC5$ z&^GJov4b*Ot+l8!7VNg%Mw{%j;Cic5xY8nv8n)#sD`&Rsc8e;x_KE8*y!3|aW4-dG z3n-RR>Z>j~<+MAmq}bp~u)zo)oUk_xH~iPb5HH-R#QE-PF1iB?tTDhGckJ=U85gXu z$Rw9+GPozFtn$h%N9?l0_P$JU!qf`sp|Ue`+6|>U2mFoC;Q%dk&_s7*3o}U5lC;uG z(}IfAP)GeV)XPjA%hR1y{R!4rcfIx3V23TX*jo3pwb@mp?RDB+v+edW*TfR{+|OK7 zw>8w@ZTH@K%m3_m&MuF8ag@@$?Y7y!e3H1=OgBAC(nlww%+KS9L;2;HTP{v5t6b7K z)}Mo}wdh%cUi#@{r>=M-thdg{>xsZF`|Od_Zu{+v$dUW*bnx!`@4)vCJS>h7Z~QOE zCyw^=YHtJg^Ug=lH{X9pJovwwE)&Qj+s6Tj_uvaJe(arm?)c-62VHsQ&aAJ?HFtC9RgrEc)2n7mO@PZG- zpawTMz77tLd%gpq2*cOC5}MEwBSfJJZ}AFCAn%1ROx!uZWwUkN3{uETp$Erzh|<{) za_9?(`~NgyIZfn3en*^K5{XE^Owwr5t>duZI%;WBKm_kyV@Oyrg#5QU*DvSy06D53t z7!(OUJYGRcqo5whe=nPNH50jpAq7zGCgf(hW1~EWoDu>WNR<81u zsywAEV@bI377fDKf;Y(g~V767s&)h z&hd3w$fh;R_rv;4E`FH2BqulN$#E(}4}1t^AvAHy157anPJpE=b(un1StN}Nzj5C)S$sM=Kku?1!kJi3=e&1F9=yI8`f|_-auYMt$9J$>23|$ zR3sGsU_W(+a)>Am=MmSLi%%MKomf0T0Z?fL8T9j&M#zCYVY$y;n!u(#)#pEh+D<9P zfC3FjDhZ5WNh6lisZcc{OV>$NbXK*X3}q%nLCRI|onfnBeP+!pdc%xy2pf^;9!H63 zhdZ|Qqg}0EH$Az@Z+;S--|$2xJ4uMEj?$(92mn0EAcF$j6sI{&>^y&ZN)sHGlsktpZ zqM0fyw5F*|w#tDGZw;;^-r5BznC_rXwJSKxv0QL)?hR^4Cn%AN2+aaOuw+1l3;+Ou z#KzMLSFnP3oiN_={?rKJJ+F7ytJw6icfIPBEMnuUO8ORHril%#O|1|NJ?J#Mon$Is zTbc`|YC^nMC2(t98&Zf4R}>J2uxw`w+d$mbSmvT(ud0MruvSoicX+TL+B%663-^Lp z12O-Sm=8_5xW$`{ZUGPM*S@l|1Q59H5VBhW0&upxKnCx3-%Dg7ySK=TO|pEIyx0cd zx0L$rr!I29UrzqFlQk$rIQo|h>FU)9vnT|AodHh;^Wn8;h;S-Q!DbUrnE%4v`-Szq zbSpwZqZDeD1BiuNVpiaS7l5ura7P--{gIRpdVMjBUu=sgsDTPJwsAoX%Yk{?J;4WuZ%ZQ%pzVYBmg0ocDi~#iA zpiwtpw0XqArtmLjh}&o$n_IBp>J+jzZD)&2W3hPap-Vc|Y@1vej{k-Dz;d;1u@ zwZ}Rq=)|IR>f+KB210Ng@S(V$0kl~7l zL!5-fuBfpuPNh40kB;{fbcunXDo{{+t~NG#w#(5+V<;?1o1aVD1v4#wA?n=Nb$R~&o&A{g&LfeNvwg6!S$ z`o@nYN_Bcd?b!wbBGTaY1gqhuN;Rx|-}!EN_x$#?&%NMbNqfhtF{dk!bycG(PD?st z6;R;9B9KvbtLHAx*3kO3xK7r0Q6uBMdzzp@feUDF3z4HIdH*Jki26mVKH#PID-{f| zR5Z>M@!Ey#&Szfgyq{a=^Y?kGbBg0DeA)L`^gloFNCPNR8+CvL$Uz8%Olfv`2O|rJ zGYP^bXVN!)W<@8@uz9oqVXOcR);BWEfF-ezO;eM8<5vsi2V*JcZ*ON(o`V4_AZ9-` zca@__o&b1%hjaF4bM%*geV2QC7XygkK1Y^R1&DwRrd3L~Okq_$Spp1Ihd6IWffaaB z^TIn$<8dFT4P1yaA!l4@he;{eVk_8!!iQ+0MuUsSgKT(zJLrac=Z3n6d#t2rR{%>% zn1_1EgiE+pVg)mRm=4z9a1RGQR9G<@W`zTT3>v6~Aph8f1;cQipl6&Xewnmr&A?nR z#&+FiRW4|Tt4CfF5ChrvXm1FIrx=GmSW|pQUqEPwgY{^8h=hFzXMFgFkeG{II1CVI zHY5~%RY)cV<1-`Eh}`!q43~_`n2fITi3hi6f1rsf*oncXPUAI2rO0mCg?Dy`ib42p zKRAa(rf8v zQ#Y_vg5rm5YnKMpNL}7$bfcJ!d4~w87>;zfisEQ$RsJr%??1BoI5DUex6iN{4y2mdCGYR8G^$B;@yW1=Vnn7C;Zxq~~V zis0yq7rB2mXoEBrPh19d>o}Dk`HRS~aD6kA4)=WSI1M@T9{I8>)E6+T0t_w*cH9?} zkw}vQ8EBtilg_w$nTQmO##{>7lhkE{&Oiq3Rst~QU$J zxM`W#3}<$P9$8@^SzBM_k61~PS!t3GW|@{C4GxHiD#rHAdkKXRw(7n1`iZGq;N8;M*a7eEg5Mm8XzHZV%b5PZs|o4fgc!Qh+D;Gq394yh!e6tI^R zpjU_I2?yYRG#H^$HUt%Vhy4|z5PAg#5TVfcN<-9<-Fbk#)rY6~ zogxW^S6QCfkPRn!Le&tO(?Tro37_#?VZIBPp<)*eaWzHp(!2ks9BZTvSr*-IGYzn9I zAg2-<41wCGRxpHZa0P#A1&LZYc>nrhgtbaUC#c(5qr(OZs?ZAGSz(qMNO`uSz~G}l znw}NLo}zh=M=CH%x-DGjH01+7s{m>C`J_@KH7a(3Ql$wdSU^}xpt}l35#*$yqfYbF zVr%qHYX<=+$Zryw33-|a`2YlI;HL_?Nh*M4%w?x!HiW9ztZo_wgbDEh5U$j!2dS4vQ$$zuFsYLn2Y7a=#uJ{%qX?Mll3?YjJ=&IWBMKzs2Hiudr8=5r znW|dClB^0p+QxdZ3O1M^H8#dM#b>edQwS;-o6&`n1{pe%qi-5I0G37ovzZ3~pa%J1 zvMaj+Et?MtzyQw*0nZu$oBw39%`gCy697XfSS#zT>{qihE3_!v47vaSwqOoOtF$UR z0N@(57N7t`o3gf$2Ubf6nI=;Qpl|U?2a-um>iG-bbE95m207$MV#|c%NuF~fs+4K4 z2Wx?fqOiz_Xo?1z=+g%U={ZQmK26zNP3ehF6tcP6LvqD(`EXA77PF!iSmH*q+@%JQ z3jhXyvL+i?gkWj5Pyi`w0iFA~&^oygV7a8Lxt5j(Ire&=d!+ynx)vb043G(ws|T2C z00D3T3DCGCn^YURn+8a!#&f9z%Z{f|yqJow4;FzqY79i`o~O!n;X=0}cn3n1ZH2~> zVAr>oL{5wcuiQ&|CjUrPhfoSz6RcL7ZCM+&x*JE7y8@IO1?~I3`Zd3;Yr8;j0rlIu zD!^U%3xn}nx%8U`r^~r%fWJTx0igQ=`paGJs{jIA0Z~u^5*z_WB}IR$2EPlo2uQZ! zX}o(7o*hiQpV_?5E0!BmFR8k!SwIUW7qP4}dM*WbRyAG8)kGLuwg34!^6S7;T*Ovfy8;lwO56YoJVAW`nN5(i zFV+Orq`|bv!60mzX}rejIY{m4EfBGYx(I?J7zcB?iF-y&EtPaycB|&32k#TO2NyZ! z;Al8}y~1Epc>gf5r8ff;{K&am#Q^}oMw|z>YsmtD$pxIipxXqks|8&QyON6mrCYg| zEK^v^vrmk=Sv&<200Kaaj=I_hlEw^joVnS|vzyC$f|avTpbRDp(A63N;)>1~FahcO4Oj|pTPwgH z-~bSv0U;m)5>3%GaM2l@(P4#W^f|A+{L$Nki^4pNuM)k*?1kV%0}9wcf>6zh>t)-E zxPn~BvH!fKb{VYYd$>3G2l=BxF#7^Sd(h7s0YZDgQ)>>WV6&N^vzse9Og#XzAPzih z4p60*T!VbP#5TYIjts%qtHU~dXys#QJS|KGyv{BQM>IVTcObQq z{Rc#iwMcCMhbDoe_(9jz)L+q9jhTAt!x{#$E)?H?<2B!HALH4tB!rW zIse_wjQ!XNfrZQktN4d~5p+t%I?tB`D$v?TQn_s!Er!`k}Y-_gir zx?PpKjmt&@#tLu(qO?HKz2FSq;HTgVS;^e;Ld;o0%y(_ZM>t0WcUOP=x7sO@mbln& zDdHTfx8jXKDu%sJYPciLk*;|^#)Z}hcIkp$l2J?3M+*8)VZ z9fjs2;)0q3%1&z|nYyNQ;2#a!$V7 zqw3g~ajs6UrY=5p-pkK-gg?*%U%TL+K2opM(QY-1Aq}?(v)u85=!%YvF%vf_{R6@T zQ&}En(siAbKI!J9xV9SJ$#A!eEXKAF$O_rrp56&2K8dMLJBXw>j}YpmZm*-xkLSL5 zyYuR@{>y6)?7fBWLk{SMNU#hg2zy?Da8=xS4w)T`;tLesUfaR0K~$USi(W9r zgPH@VWn=1M-!z6gS?$dAP1^pKnxsF3Ao9;D?&HqlwA1P^wC-tjJleAXgZ~h`sm}89 zUh^(r@4k4pOhD)NPNM_PmG@J%PW%E2i3$U+!IdsQZKnn2*X2^l@XBrMRDbM$lI(LF z@mhaDx^a?_=4)1R808pOu zz=P^{AM5uT=lL^R^ci-^7xXS!;IEfdq$lKNTuu0}THh{&&=pS>Yu$ga@c8QRJ@W99 zhxI4TjLkk$;$Ze0Z|y7Ktkph^!us)faHabF?MAz8Z%^j?^EGqd2>t%Sb3XI?3I`z6 z?tni&qTWKM{tCxmO*y&>A3p_1I0efL4#?+WZ%Ox+>IO%S!?;Vz6aVY@Wn9KUP~ft! zkcSH5M`ME=+1r*^^%FzF&kc}}cr;m`iy6q<4%_jJX8Ngrw>|?9`R19+_Xm$bg9OVd zYj&YYLUYa(M!D1IV8uS25Mt!`X%VA3azcVE1&P++irp%Hj zf2rE3^JN^MSqKi4(j_R+KLY2?nsKz*A6vHM7&RltYDqUxXg$0Wfx{1n%wRAfTb8U1 ze8D_Ljj=*ap0{b>Die5V$rHR)_3rI;Et@rNfd!KzT-fko#DmizocD9xT&ZN|YGJ*P?R&XZ5?Cb1z^s+*nJ&0L2Np=MHl?5$Z<$#f#| z&03w$1WGqsYN7z)kAcAf0NtWX7bsdN??AnV0ZcIl8-&L}2pMZ^LJ1+G>_QALyljlT z&YI!FC=dy49MMWcryLVeQ>{P)!RiP$xtg%yw%mZo1hqtbyXc#6(qjk|ka#>VJ>))m zNhOxBgKm-bvUrY3C0~gW7DL*r36+R$T!TKN(y)cZ)%H3eKlL~?^Q1GmtWQ3(UYi1n zH~%A`H3d{KVzmY?@K4Va`0VqA6si!(4q*ObEJ8#BTmOeb3oEoRGDtIwG!iorbWy|+ zK`X7U(rl5T8Kq)0Zqz9_f@`)YGAMP0QfuVTA{`SFNTPxwi3Bvd=80vbCu@Q-$t9t4 zayme!G&U44;7Ha7Io2w(yb$$Rt0=H&V1S}D3gm*z?aIPxK5f^O&!%bt3;;j{(4}DA z3D{+Kg>_YMx6notRg4`+A!C%Xs3c7*q0+b`1*+gI6{A!8 zsN#iA=!js7+}_Z_;fX`e=%6gu!UvC!%pnsKWp+jLrfy*ZwmM&X^;JsjqFH8_oY3&$ z=bxeQM4&%ggn>WrJ|s(o3LM5{s!B2Iq#IO1>i;6XRjMAgLpC|oC;|fp2;kg2)s@%X z64F+C?G=c@#@+|v-uJP&{k8jTNI?n;EI8Lg8etMOJybgJWsaxltfUIbV|(tfHCE+?V9&B4h(W&wp7@GssGg>w zQkl&F^~X2sAh8fFM$K%^PQWf;PUd587wy*Lja~&_B86LSy6ZixUr8Yi_>)U8(u$9V z-pIsYe9|Ah8FWT7d7x~ShSoSfvP{ng7(RH@g4Wewb)?`0Fg+v+20HFwp8-aMAddH-!TxB^iX$8(aSpN<|802;w@Ja*J^*Yw1(1ht43)5^Al`M2^ z51#8rK_-xgmL+8#`_Tjpch|L`T`g?NBR~OA=sdI~F<#ZX9`@>%ue(K&GSB$l(?oS6 zJ7Efa2g65X+{ZBZA0;Du=3w?k^J zN11sk_si^qqHHjl5Aki2~8UH=Od`=*cCq9vvDMTd`RpEmbD>z6(X7CB%%h(2S z77aR}ArGEORR$!mfFVi}UTbTmYnb+gi{TKHO7NCeY{8m7Y%&>t&=WzXP)eZjfIjzZ z-tv%`CwKL8UbcM9IpXF-?bVWd)^l6Hy6~zKqyQlf(PH2RvJPP)%pTmx$nTaiG=k*e zq05}#eq?tZzwKvn-=XHP`m{|M=*f0BL~5I41ca+OjD}U`AU+Gz{@si5O@zkV}QAAR}ax2+ZVzd=7C=8l|N4 z;Kju3eH5gLHOxr6M^f)0a{qTIOzBG7hCWQxBbe$7CO6Je1uqqF9LLn@2zW{rm7Y;9 zc>RGERA#jp>1nD@s@+GCj_CyO(O^4GyfQXEC%S40_b{*t7@lr2=wH2lIsQ> z%n_dpa6s57d$xVPE~#a6WOvQ7tU?|QhRmFWIMM)7xyt91<|1ri(e_Gj#5bgho$sMU zT1&UZ!Vk)%6UJuK%MsB^8@zzWX7AxyWCWCD3dAWgjaFMcobiR`h+!Tnf>WqkDl-@$ zo*vFSLPqw7OTo1YoepqZbXrw8c;V4E3UMD{jBXRqm5T_cI{*W)aFrUhmRlLWcJVx$r6hnc>bEg8@D1xiOBN_h}tha<}1wihF`q27>NE3d{&cF2~A72d)mu zxzlM1%c+8Vwo+|M+%-ZuYGo<|`>~P5GKQ#l8AeF%sd_qq3R>!&*rvk{E}(V-T#x{3 z+n@?*gmX@hyv#g5Q`_LwHmT+JXJ>z`mj~3Q>?Vl_M(hy|Gicx+jNE7uAosbd0E3D{ z7-8>tHO9{^+W(E$D_B$(F~>b70%~SsNK}X569UnK9lNI)+9MQ($>y zqwJHD7-d*8w#xGTbz`&qaw;xkOR z0Jo^Y004kH9SsnI?t%=xWC>;z4&d5yZt?<|5Mka&i`#9YYvgG)xnXbEy%s$vAsTmXdwpQ81xJICvSXmR}l9*$YKHjG3lb54?gJXz0dgvV zh~j$TK>IF!PS8n@KLvk`RBs5ffT!SkI^m3O{H{?h>?zxK^0mY~=4HO(zOp$NGnvm& zi?s-bTH_y~8#N?Frx)vwBiu}wi7D4JAqvC3LXf7>LRi90s|TdJzP7VOK5=x z@{fqXpNQ%%Z0j_ON+n)P8}~vi{L8`y`Iq~$p8tD^JxY%D13VElKo(Ss^f*8>{G@0q zy#GIof>1fAkC`2KNkbCZn)*lq=!2~hd<2hEgU|TCOe>2RG>fW&LFH3Hi8#Y8v>=TPS`VsHyE(~aa*fFBwNtXcGF_VL@ zm7B#b^uI4Gr~qWN+hQ1J8MC1%y=Z!h0xGlAZzh&(I@Ctwu_ z$d7)zk)ewP5=1rSa05fEDil;PH<+45R0{%fL`1AAf?CIQ6oF1iqgn6@s2GP!kVju? z0wlw`3*!S*1I2>?mn1wzeFP-Qd7Z>lI9P1FN{|#Niac5jDVR$$%~O`L_>BNkDgTgL z3*X}pj_kFr(1v))jdn>OXZVG0sSk_jpQeBVLeM2N3IsO*M~a*w zFeJp}OTlqm!E#)MbW|}(WJe-o29p5`r38m=P{qnonHIu3$jJjXK}v$M38eEY3e3kJ zyp_oL7-_*Gg&VB;JGqwALbc?-DUwKv3>xRNG;sL1gVZPdQK4RPBShdND|o(zn7ZAQ z2rq#EDl{U)(FJdmggFqG^ioHxbdFIl$~}OzZ`4WV2+EH8HaewjBlthHoGoB%#tkuuVq;i@KPj8 zXo;V!lAi3wa9q9?g9IXzx2sCAqkPTIpog!}i17%4@}Wn+%E}M~D^n}aW!R+U3?CH1 z#*lz0d`zkMn?i)lDEG3$c9F$ca>#}xPGifmTfDac0V9i>#(B!fZ8}jDO+!u+5Yifl z*;+LGdC(u|#HyQt7qUtqGtbIQM9UOM^6a`@5Hc~q3b12=Ett$w5QbGaiFCYAcGOBr zNltBxx7~TAfocz*+&ZkH1HsFqma!oQpr;mE$FmZwvYb!~)hG+S(Enju%MC5gh@7jp zv=8Ro%e~~t|A-cw60-o~B~PoYXz`K{WXvKfC=B}1&qU7hWXBncOh@%hvglDr=n<|m zuMyA=&=@!^HH61p79sf5MFrI)SdJ}7H7YquMWBR7rAW-dwI;|15c;wTxPz_x1r zSpHcnXyqh-+7m#L4`JxDcW7ykesUQAvILo zr`)_&6A^})x^f*wXNadnWZQ#f!Hz=#(7nS+16^)?K1$1)aPTD28m-6W+){woRx8JH zT$)Zrf;q@nr>$SAO}01r+v1qY^5j?(lfU`%RbR!>VAWm9+g(_K*vP`(L~UTe{XrIB zq(3=`JQ)EB?x6@Cj}~ASXF!2M2%IbZE?@HA>owgE9uE@G1i{SQ@u)b#1h^$7$5l{a z7UoaPJ%q6!+6xpCII!H!^^V|38lbUXBp?NJBI0rV9dEKdE-l*Bg%RevHeXb zW6KORU^&Hn1; zJ-Hn+72|Cwvd^Nk+} zpWPd)62?mU;R8?b4rb2{T8Fi>Z21yB)SwM|Ci zJT6_O&gUmph80yPhMHANl_;Rmh25kl56jgZi{&L^NQJ&-h7Rj2s||=&rSSu1Vg3}g zpcV_{=t8K0jqVW+)8P03s~|7~CJ8(~odu0RX~IR~yfrQC8;=5FT+M)F8ctXpPGm>? zf=w7-k3Jcg;E@<(;c%E#&jmY+4S}2HhT_@w} z_iEFxz8)&t;wiF)N+@fyPNmim$p7>Lw8Ez0mUae88}3|C+!R<;ij=*f4V@>Tgrn8;@Y;|mb0EJ2(_&)n<+kk;0@j}?wY<^ zd!p7UPVll+7a(6D+9luub`0BYtO_}i3GusjF>qc*q6gXvIC|nL6^Q_fa6t&38MoKc z1%y4O)p}DuTJUSAonItw$N%6cirwmwD^Mwz_3RL0Ynd6iR6Sa+Q-qzZ5-Qn*ARUA} z@0mv=i+l#{8(+}dD`Ge>r3=Y(fBc4;?rWbnTnYL6hF^E5B9b(Y8Riw8}jj$dtZS95;gE^f^xQ;zwe%(LfvMibRlwEzK||z z`V6>te7}6lzjvm0dZ<^te4;x67@};ef>=<8m?90;JNkrrvjU1C#bi(U4Q3D6fZ2C| z{X`khR{8oq`~L>A_=$60ucx zr#3+W2wUaw{Ye1Ag^L6R6ZEM91YIy^%|sxnn8f16j6#%b+!z6sL}nu_4myIdBg!5r zIWpZ@ROg&flD=%gq7lw1 zp;_lx?dtU^%O!+t<--SsNEcgU^NH-}CXA1`A2uv-FrY4hyLb&SsK9rEiWL+D6ONHa zO%zPp7XLH$=J+vWa*`*fqs+~6qY3}Tq*k(+afIui<`oL+_s!!+53m^b&0dx^` znCoG-h6f3<*f`i{bQOfXZB$896B`GxFqMbsG?QY>IB1FRupI>D^yw>^H<#$7#*>js z%q&Z0{u|CZ%#y7Nhx3th^UL<^YHI(f)UQ={O5Jpl`Ep$lG)7$GeFBOc1&;R?6AFg# z!3Z3*Fd+s9RmcHeKtypEA{}~S&OGxpLrZ9vbyitqC}OtaXC-phqKhTw5nNzVtj5}D zu(cK&Y`Ftdax(L930Hx7EMC~_ZEbn3U&PE9uX%9q!{vF1H#=?S0& zt7t-n6IvLdt0q4jDuM)B)Y1mAWE5i4Szz>;Q6e)r!`26M#OdBQv$S{Swc#9#-YZMd zX{T4MOywoHv$7d1@954aNaD&X+sT3EP+r=3ViZXLzz#y~ME%Y@t z;SVct_}uQMyOPSzctM4mFi0WMu~?or%Xu)vSxYU#EHdZM-UdB197P+wMYnvrv9uhJ zBBJ3Nger3=~3;BOWL92W@WJF^83cpm&6@iv3Dw5g4J zY9S5Dpcg*V=m;-&vmW*a$Pw*<<3j?;hcZ-%xV)?odQIK2{=mXR9f|?@9w-B~JWCmB%Z|GU z)Vpegkb@n(qrmDIBND=GdB90w_-@p~7h1*U~g&Js*4ud+&2$BMZG}#J@U_c5ax`m%B-oie4%1?f{!UI&GqW?_)ft7N?7$622 zVrgRVqaG0PyoJfYgf|>f9V$qJIP$I+(wK%GKZr-;l}%E|OPg?J7q@K!QUMY(5+Q9f z#zc}6J66*W2wJm46e3OqK1|_ho;SE9s2~sGz=VP{f&-KAvt0bVna*Z`E}>CHC9B%C<b;!+-vCvf2V-;4=kS;ielsbwlr-L$dI7n_&H42j_0U$sedc{hHx^EV6X!5{{KBU2=)~!T!vzuLqTulw~>?Yv^0GktJvk%QE3wq4~ z5l2D97NPO0XtY!l3^?^74)|U?J+(ulrs1;;7(gHCg2>W_t3O&%C|6ml;8)VNstq1s zSie-1qeQ|ca5^G}GrX@(m^Hb}{V<3ZLl1nErN0U}G5?BFtl|})7;)g;?`s6`Uxid5 zz9G7CdU05V6#K;n<`oDcXfXqbHmPeLT>=bfDoGp-?gWA{>IEdkQ$G!Gi&@m_eY#4t ztp-ql4*sg1B7BuWz~U8Urb0NxL=&H6DT2CiGo1Cs2gX7(njyaPi$+{x7V|lsJm&F> zm3)90`xyjjfChBiRYG9db|tk} z<=;RL48SdSxs^pTFoYjG<}!DSembMVgvewCTx%j2ZK#10-kbxd4I9{=P=m4G35Y}IUMKO5TDLoP-Ec zeBur_k*fDDF4VZO2(gfoii3$azZ(l)4p8@Q_bGP%@Ti+rDg%0h&dzeE8NEK{K1o7b zxRef1Wj{OG4QOgDx|-EowklJ$ObJp`IV0ngFs%c@E|B01&G#(dn0O1h8U()YtLQit z2ruVZteoW^rg^|1FLRduk!_gBE_Q=+qyL_l3g|ao)dTRBq@#mi^lkl=VZ#;iWvW1M z@e=Rch)Q)Rb2iFEqMBH0S%f3-WGM#zz35!|_R*aJ%L+$3={PZJh*R3`6t6ge_zt;7 z-O37qA34VPV0gqU-rlsFYX?^@`SD8%F@RxS=C`|J&eN@?un&6Z!})p93%IGy=g(O2 zpTYgpCH7WVDsfyuJShYuWg6%`$Ud1}Yr$4Q!36{U$7`vY(@j{~RUnvMAXxNL@%>5O zF^J?)4)={f&fH20;6dM=9L}%|_nBapFk9m}9yO)Z+!`CR8IM5_c{5VD~qzG42I7Oc@B(j=j2jLdgk-Rvn(=()>n zn1om9M`9cX4Sa&vRmO-pS0?bw2N=THkwVuTmoC!5F7_fXCPEw_MHigKku3sJm|c|} z1-j@IGb+%T6yCmMSevO?wJ?H*6(R@O*&)&#^6-Mfc){x&4$oy>Z@3t~fsd(hf&6iV z4hUf(4Iw;6)Wey?C)|%5%>O_W+~a;dp$;m*a5&);<`)#071!LOpDf@>JcO1&2xz?G zrR^0cNI)5uNBz{oH&7!+rkON~QZhat(##|Ez;Ipluzu) z#!D89W3`C~0E!wiL7K1tCvg%9fF%lqC0UMTS(3nDpe0#?96Mql@U_A+wo5aHByT~& zK$23)<)t9*C2!GE$)%E3#Fa|U6-&-wOM(VWnUMMM+e{ANYSbhRXh1u*3Qp=ICc0q} zEJ29$)p<38OzEJBz5f$D{Tq(qkfU^jmHCMV#Lux&Wpwpb9*_ksIKWeu&L?OAB<7~i z&DrYsUVXgfZ<3WS1}h9ek|*&K84Arl`CT*E zsCiu!Be_Eq2>*&KB|?A_=p-@Oj#>dMph5qHPAD|!PCY0{N~nc?DVR1VC2Z)2mZ^tQ zCocelnyzV@PS*+eBeCe{i3*zM2?71N%KqJ>o=O^@?r9b<2v6S9kc^L%;@746iDU^_Iz~nBIbO z%9V7Q>G<81r8rhO{FQq})|)nFdscwbvBJ^GL1F0}J^ffST;zJO7kmj>pn`>UU{+?S zlcDy|1vCK+ah3AT9Y#C}1_;0mxEF^csd@bk7R*K+xDAvt8N?7MA3VZDDd^fcsyrem zCswGc*8eO1k?EOYBjia2J^(|u{cE*ho4_93Ohyl(S%{CRgQ6|kIi*iN#@De$uZ85PLvMb2to;*Yig8bryfABl!@ALh0M+@&5m8E z&MUp0Cm(;>ubJ2Ge=@YqntD+Q(k}^ZG;LYLXrip&cFhOD=_zf!=p{nhz za`Fz&>g>+`9;|8C{ z{{P1b;Dee8Wy?Y$X$sL04cyt%C=U!k7N{+f(t{n03AxmYxfZHfwjS7)ZUNjaqXsI? zYMFy}Dc}}w;F4A0x~k!_+#%Wk8(iD-1_$)!VAd=t=^@?s2-<)G9e^Fy9Y8}jHy}DZOFnd+lFiHqO8w3p?$q5;HdVJim3#EP2$a+ zW`NZQYrrB|07k0k7F-h0z*${LuKGo%_28)Fmr`+X2mgWCG1QP|f~1PT+FGSORsRAo zs^RWAKqRn$+)_X!5e5RwKrwZ#%Zd`HLPhYx8|e^l1iz&O4=!{r@8M>!F@Q!IuW_3i zLD*tj2#c_8a_@=;;Uo^p0V0Y0StdIoiEg~!T?BFo>##a4LB;Hij6#J)tY$I*rCl_^ z4YI9wsurdm!tEaF)dj$=8YODz!GTubQd}WJaVZ#&v8uKb1)FgpqG_A9a!Vy*J(1{( z3oov)G5Ps*P?(!S5Z*4rm>>cWk9!@=_fn-);BCoCn%paK=( zh*xrQ++=4adny7xAl!vZaFTL__NFO|@$aQFVcr=UW3Vg3vKn~8lMx4F^8cU?bFVs1 zW(wmeAJ=S*Z))fD2rldLXvOGd z3Tg=a02I71F;>t}i9C)Yi} zTy#~?eDf}-(r}itN#oy1|LjVyvn#JPD-1!{CgMClF6RN=8Q^n1zavkdjuo+J-Qk9+ z{54=##W5FjR5;wDPC-C1bTbggh9s&5kn! zqiR{-Yx1hJv#E7UuXQo4=j-Th_Trlq>u3-9@m=HfKKHaQTVf^Nb7gJE+_jPrL#lA4 zjA*3aBurP@bP1+vKmb_sj;iHQa-F%RkyUdu-_ES4jv1>CFKGYmX!8d;n|4~GHe0tb z++e3%>*|b^a0Z0Ickc0R$M>w{^`aV=S?LI2HA5hHK&csaw#p8DLcxrHW_LBEEp9+1 zQ#6yoH46zdM}IOh=aj=yLRc-R-jVfpn{{~mEH+Ap%fa+ICyd9e5en5_!78mLPfSi1 ziB!usea9$Y$2g4F_tH5vY|Q79^fy)Y(+Zbu(s36E^Y^%jfOAjJBwhdUU&QwAdKO0q z2fFdrfxI@EDT`eXp@V9J_T`tEnjF``FC%)1(Wxd zKUWQ|03vjX!@Rb7dsDdG@?_o=4Uous)aqWjSFOHgLtLs)zpp@l9dEhBy3D4GlNKXj z4U*vFA?bL3P_m%bI_qK35-bV1qJZgs3X{`xR{O3}ql=g+N0obbSwp&bPv@k^;PISM zWX0xh6n7rru(XP%k41rDA%q>cps1ghh^Q^MudPC_uP;|Dz7_uxBZLD%zc?EFIf4&% zGAlB{@we=BS9i?KqB3`Q~ZjL9(Z)3A0O6cM@a{+qmzQPNBRu%Yajes`Encf zc6@q%JBoI1HVuU3UhW{>g`>SQ=L*>xcEBGX6CpbxWO0CcL;y0!DF1QHX6t%zpTs^r z0DdxEO>{^(%s_4kfLYnJp?j^e|Hl@S)3cK^;Gz=M3*)3^Ma83aF|4(S?b+gfJ)mtm z9XobuqFd0uUm%0PF+GhKWVfxo&JOp*U##Lhq<|E3g9Nk{f@T85BS<3OICTN-n(hW!I=%oaQu4S#m?r2Pz~2 zLD(lv(pWwgI)Fl=KmiD9CT5I!6{}SPTeB9T6Qcj7$uUgE67eD8EDR`hpolR`mIje0 zZ{NBd>jv+hynEx2?KBJUD#3wEJ!(ZN)URvRtTlGb4f5m2lPOOwb{R7=V48__e)gFf zXwjYxTCf0w;{?+-R6}5$RV(ZO000cg03wVA-4A&8?tL40h6KV91Ru`9Y!RQzd)UHs z0RjPv(y1RfVZcTvbSXlP{K@?14>@s(#~r@|i_I@PDM{-2{RYsWL=V<)qjW*eFH#1<~1via$_k;*vaOsRWb`FJWdGItDRBuM$ag zW0hn&sY52`G9l!}Z<-156;*CgjIqje+_C?%%#7g-$RIcSkr&d6%%hJLTx$an6;!hb zBL$=kusQ>36TkrAcKgkPE<4x~Ordu35HQQ-qTB za&m9EEtH5)DEg4J56S#^+b=6BxeAbv8uuJU4t&O$=Z-^ad2musY*~UR46}324mKw7 z1CtpeISf@BqX`6-WLy>G4o**)b=D|ijRVzQw#hLY9^vt|MmObA06qi~VKj6qz)#!x)| zST8|V5mBxBUOc%&jxE7avX&$J71+p#6_RFTk44I~7*abR+9(LcmrCWwF;Wr>4@I8Tcg1pH)*PO@Y1)|r#_zMh@?MFA`5K?B@8iy-mS7rRf5XC9 zo+fk6$(eut&X2txe`JvYLNy~i;omfm0uWf3LXlOf6Gf>I9t1~)Dg42PFr=YM z6qYa^B<^}D38J|2gBn+LAue!1!(GU?7qWms1a(kDAvCc*$URPS?zAb4iwQMlsM@W^CrigA zQU>1WEjnHho8}UV1g4c0umn<0g)}5jc2~~d!LEgm90Ee-VnS!E!aLwv7#uc1F1-Ls zJ#CvK6(TebTEM_hL!_hKYy~)1asif1e4;G#h)W}sfDG!(V&%5z#a|Aym>E4%6r72g zqBsef%8TY2uS5VMwhoR8JQ@S>dCii9gb+=zmb#>Z0bBiZ3gHx|?#Ri(b22AZ6?l`I zhWd%OtWXXo4a)zW^3+2+buvCA=|qU6CWXG`gBj&J<@JbgiHafdpb2#%6h+1)hbB{^ zx_qBSgXs;ozSS}q)o8LdDmqNG&;`zW<}(XWQsSWWTW1(Z;V|T(4$&u5nncr| z)yfth$!R>ZnZX;(v!{y4m%f&x)Hve6DV#c$BDX*}B-Y_iM+k)2zLyrZ*hQ;^Givx~ zE70@Nf@F1BqF9;8(9zwJj|XUM2mZur?fLwk+o;W-Z zY8|>jE%N`7STJmW1NktAk}|P6L)fvPO_;%i*AlBs%=W2UP3;hM%iFQ~wwAZN<#4sg z#p7O-X6h)`T)ztq7k#%>o%yb`x(OgPogh?tyG4&Zjhn$_Z)aKi-j|%4U8H^;KnqQ1XfDfe(jwdiw>PPQY{=0_9-@_;1JBJ? zwQ8Y+GcjA(no^)#tt}J87KX4Q?!zF)<)FbqVoMa)mlxH>qT!li@hZ3q>f+X_&69Wbam|BlCEoI zmjp0SI2WQA4h!bmJ^1DImi1eyq%^HzY8WJMK5e=vA#Vjt@mErwO>4OwLsNDDe#X+DuNnk^lxGw`&oVy2x&`Qsew=fnWci z4|c$~rm|XFm&1J9t@QA21BXXFh?~%c))nS9hwkOhl@*cC*yJgn20H{0HT=`G!BRbG zfmF&4v68KH1yXt!umJ?7pVh<7rO)e~l#oF{JH#`~lZjuM%Nw|s0{{>=TX73ZafhDB zb>oA&<9&?6i?n+W-}h|)eb0kOsGx8DPU2Sd+}YRuxx+Y`$8QG4%QWJwl8&~{Zx{XP z2W&Q-3Or9XwZ8hotnF`4@j5wx1_o!l@PIFMj|(v5nnnWzAYMWTI+RcOcn=RW1r-MF z`2f$9cFWtyM?7e$`mC?SdTIMMaQl?Ym~a6ECGSW;r=$L7fNV_vYQvJ!Po@7_5KVAr z_O33V(5r#wZ`Z6W20P2k3Z?g8q#XK9N#L%!I*QK%h2)HkAZ|*g+=~HO@0$G1wgiu! zCeVqbsQJbX=N3-`cP>~w@cY8ANnFE!sv>P>N=;NS0kn$+Q_2PFhz83g;yNq-l;&Q{ z4hM7aCo(P}LJJ7~NdSV%ZZ>b$+JttpfaIVDy4>v!YRLhiFVA9*3Y`cnHo_A`Y#h{Q z925`xI56F`Z)QFb@*=O8nkx+X#uE(a;MPPHu|<28|xE%lPf11{!bu+0%4k=v%w!^FZ6xABTpq}+6l6F0JpKv5jO3t3Lc z37iKul1?345&d3grQWgiHV7Zdjvh&i7MTEJF3a_lpe3NOzFyJ&uC5K2D&?TyC|d3b ztx*aiaW537365+V%b|U8P9wh&96j%8F?SG*BbcEH*s>`$;HLh7rO5IfB|t&6V8kjB zAE=-ji_$2M(l6rx8uy1OC(1_5O#_2ODj~y1s*;#^j1*xl?j-Z=)Nl=hBU`|SIs0sm z=x{7EGdeeu2c*T-B%!4$z&2jsIpb2C7T_Lq!3kFL5xawNPZMZO;9V1Q$vvxIf3K!F2+F_v@-#wamENPanCeeayq%}x}v82 zwgtOB6hxB-_WJQe(Gwz96c8dY<6;O+WpqY$^G2s~7=BYypuq-2Ww7Wp^zPL1@{~i6 zLO8z*%?6;GV&fwCBlMQwHe3KAg6!!&;^X|U0;-@?B1g$AQ~ofM4(qT=8>l~02sDpC zH`0J-xRXo0Get@DHPKUi)U!-SgjHvgRpZkzdoyO<6i!R65733^a`otTwdm%p-iqT< zJz`N!P3Ukn0ge%iT8$PX^g_o;Gi6Yl^hibS@oN8Sa#T&TJ4;nt&y!4D^;Hc6ToL0I z*pv(7YFNBMQH#~m{LNT-)m@FX3{wXUw=+_QwcXeiQad6Tf~^s=RTeX~QeBclTW^m9 zBNwj*4w}Fy?jTC#5n8tuV)ar*Q`K8n^;<8tM$rwffaw!?lo)bM^#tNjk@Vg+tyj(O z#+Gh4(%{h+HBMRM^GYuXOT!KzbRIL+EC;q+n1F_G2sE1t&cv=u&x90jl|?5|T%#5-GFD?ZHb)mzGtOaHe8CmE)@!+zR>O8br7m7+>LpkaSd)-&IHhjX ztm7QjbYjn6c`66v5Mbl4ZG6X6$+C)Eh4}xtR3$+x5oW<%2A8s&b4$M8azib!*A>EvLmCTFpAoZ4@1y)zeT00W#s z3VvV~>R<$Hmt+7(iuy}rDzGm3KvNd(QfAj^=YnZf^+b@8aix}OWwi^v@B3t-a;I1F zqETMcQA69YOCn$a+hBqwfQnck7Z!vC(1!&2_GoX`26c8>?a6LimsD$EbPdlz>UVy} z_siaiipF<5n{Nc*B^O3_cr}42qwi^<7G$J0YRT0`+f*E-7i_85KbNlzH}^mjhyhrF z1tj2)YM=r8)~0?ye9J3+b@1AVAYcF9m;PXD4Zdl=0E{R-wFLmfg;O{*RkzCIcNILi z76@P#oPY|PK~rA0ioR?CN8s=TxB)!)e+^fFzYK@~4|$Upfi0GCo7Z_8?SUnj$0nG) z(y?&vselA<8Gj={;jDmvA697`#*N}Y+*;tW zI-wh85huQYdnHN@q+;b70095MR~I5A52WUN^Vf(6V=#&tm_td({7G(m;44m|!ps79 zQR@jR1QICdb_L^|M`qy+F9neJkB(P8NqKqqd1^QDadEUGx7c#Kn0jG0^_~V@pk|iE zSQ9o_3L=1sYoL6QSP&YZinJho-#|u^Y={F`WHx$45LN>2U}6F^$&N7qECBJ+01N2J zcTx#?{#T}F+HSf51-|)V_O%Vh0UO3SMS=45ZpWvINO5Wxpzygb^%>9-c!5tDm1Rbt zS$TSU;glAdG7*fPQ=kUEECz1q%Zj)X`2eVKrw}$8mnxCaguzd)hM7f4r>9~hM@I!l zh#1+LA)bH_Mqp96iLd{;X@(iOUl7+zNR$FB;6AA7JHk}3+qfuSORyu5VyU>4o0{<& z_;I6Jfm&-T@wbRI%f6WTj&vb4kmRHdn{muR z4W9FOQn5f)aAXdaq`^#+6tqw@jHpW&xg&OBn>MMdcwd+rDlMA}F*~!3WrBagvws6a zfLIH<+q;c`yPY79QUkQ{EJ{aOwN;yn7MaQT1aMjiSo_-dSmPvK(F)84*--H=H-Kxn zSKxYM%|_@4;iG$>SYC!!OtTn#pi0va0}odkcD_ z*3uvY)DWUaTN%^=Lj4%=a4C+zbL_-4rz~+~UPVL>n4iU!fhoMgsavZ1Qxt9-P1+X4 zRqYr-7Xix#S4CjTEY1KqoCeohqUv~IgaHd1kh3nchw_$n5sV7j%YrEa&u@S?*e-6| zh4%y?158fXusLwvX@X>2Y}#_M^KxEdZ=BHrz4-uEI$e-3QVjm9FIe;u^1^T^+ecB^ z!fWQsr`i(We9V{fqe@2*&6d61Bn1`#f^Ji|l+z<777irN_x`Ucbiqrtm@V;~_SQYE_S^Qk*lu3Fu?Cr`|6>SQQIg-qj>L7?vD zOya9!VNC|BSoQ?`EaD^^5b&MTUlbBF3%xzTm&fRF0z{y1_O%iTxT|#9Vn*Jq2sd_fDFqgghMr*5X|r}0Ffc48MTEhs zy5T~qo^rH&!UsOZ>}Q}^0VEIJ)nUC$6rM>KUP-7U3IHhepbZJBVo+7cyTRe&O%yHj zydnQ8%-A)7K!TXH7jDKCDXIedo5SMI&lcer08pcyk*D4Fik#zAPw#AA9K<>V(x(Nu zX?)$JZ(x^podJU0Dc85UqLa9u%UTPv`Jqx@xWXYEEL`vD zuW8<$vnQSXIHAc(d{_$lC3r9a(G495qudQ|ery(mejSd3Eg39AxrhORN{a|1TGXlq zWrTz&&I;Agl&c1jf_QS}Dui(rBq+0zRrFZrNkxs}R`E%O@}x#sk4S~OwQQTTXV(8} zX0yr78#$dkwfThhGiOj_M2i{)W(=uPUQCO&&%7p=PB7q50mBR+YSidb0i|-`!VW^HDG(rbYz78u=OwTz#f?2U9&kK? zDT{y&4+h*p^tc(b45|Y=UqELZpAbYWT8H>ULPy;p7K80%@tYIs-WMv<^DrU|9}>V* zq4Ok8{5T#z3JN*$V||p#ge7{_(k7O~R1HlA9@tZY3Jx{XGzT7Kpi>e;B^CcxS6O)B zF@pit6>+T%N1It`n4?THbKNJ-Z1I0AnV z3G=~WhJ3-n61prIg$iwyR+<5NyfKa~K)|tuB6H;E0~N5zrk8CD#Fm=|U+`uIBTL+| zMr#{XphaoOE%#iVP>AyedFmA;nRdwx)Ll3~K#`gRDL^C;cHdo80-_T|a0p0DV95l1 zvrvK3MscjtpA?*y1W89AwW7*PFv)}yf(r&ERI4{V2w|)ff`Z|NwO%F4RvO$`TmV@_ zAtH&kSRiJx3FNin8)>Q0#IYmFuu6@*2Dar9t68f=j}~?EgKBkrB;EgH*iwSzCk+wO z#tNRysD%s)%sCGsSj=1Po6!P*Z@vP^20#Iqt|rMAJ9@B#YNj=*7AO*MSmy`P6(SiP zcvAF92!3++SPD{1+<+d=90KV#4v}zB4Jee%ghC_9N2we2<)_kpANeOtIJNjks;LCN z>MGAZ{|q#&vK}?7trZ$w>xOj}_W-280#V|yD2~N|4OLX(0t6|}n3ghi<>7U-H`Y0? zt~+uxOKQ!X-H{SSj*Fy{!K4+yi|Za>i!SEmlgG8!YJx`tf+H(t1DJAG@4&m>Km{SB zAuU0b7b5qGiZj7%PUFR1A?oLKQmCx?wkag0Mat&k!7~4)9c=`j=9iX2$G0Xy zlE^Hp{6c$xry7-N@Im)Hw9rHseJj#eX*hX@twj+9-jTy-*435xln2(tu4uz!=43*^ zvzIJ9?I)ke!H5uR^U%`z-h#0{xk<|Km=k2?v3=eoMnSpXnEkMx`NW|Pix?n$cyJ~x zQH}^%0H6n~rk1?XLR|iuSJ2#Z7B|K!w4kyVAuoAHJK7{1p)}_q;CW5b zTOOtcf=XD=dRoNVu|NR1^Mo&Kc`Kj`5W4@uMu);JrDcn!JXz|}H<~6no7m%pO&>ORiuRcK8u2Z_MjbMX05~wsHL5Y2*Aiq*6_-FR-Bq00qNykFw6=gz zW~I&RsbMYoQ+f(@pQa>gQ6CFLr8d>6Plak!m6yDRj`X1(_~k^y`qi+yK(tw_%2~%@ zjx>x>EzsobzTO(wcfwS#FqLg#I6>QLYDHk+w9ef8TGy5mHnE2#u3{6rPRBxaxi1mu zK$!~N=u+0QNkQm@Fw4YTHSemvl!IJ8do};klJ=O$KxWm*08+JX_MrxdjSYH092^NZ zezP^(AayIr9px3jw}K~DQS45UtHTp%2ezC+I z4J#(nE7EB-Aq{;%V^?S(5@v9QO(J06rSkZ~z~U2t0bCyg9cWhzjs}CgrQ{Q4(1JvF zG7RVGnC7Msr0EcOzBoFvJ(H|t&kWDr z;F4^Sy=tE%IST8687{Z&PDEgC7NqoWJFp?INZ}5Gfa@e1GX*-o`?#ETu)QOUZ=^~h zmZu&yg{3P*4EH^-Rs%b1MImL!o8{%F!Ts&s*G5P_thJ{aCNm~VkQ8B!-g zo7Q0dbZ_~lZ~WGEvh@FNw6}o6;B{b!drw4x9p+jQ;Baw*Z8iW_!*_8BU=|19Ys69j zXt82*w_eVN7;D02AiyvEqJdAc3mJ zd%MCwYt{rZ00h&QFdSHXAQ*xnm`oNh8zfk4Rx<;gq=JZ1XAZPtD+U@MaELn=hgxGe z%V#5XR8okbc#n7i$Pxt=GlbGt04gAa)lmyS_Gm&!f9~^yhw@t&aRr>C5TIvBRu~c7 zv+MdFDea}ndF2~9Xjkz_Ve2#F(L0u6xy&0$C~kcuWieM(1*9B>?n2RN(8XZ%1wB+lJ|9AkDv%fTWR&qZ0DfU3#a00U;FL=ObcqBJ6v-j$24n@K zh~dyK;)ef14PcN{c^X=PLJ*UXtH^5-Ifb>jDMGR&&cF<96cNlS?WQ$QB~sg4v70Mti^ z^H~3mV!1h81 zrCA#oL8~<|IA)oC8ma=9D5Rq&5Fyu_M*@DLhpc=tFe#9sBQUHv5Sx3d1=JURtYomGpQK z{sJ*0vuP zaUb1URKEafmHLtY_A`{Kt|!T(TINvjx?T&=JyUYAR9m&p#(Xs+7r!fve1&AxVC|H9$IW}+>8p>*vhVnv+wmyJ!1^rNv4!cdB zAh&XRYvf}wZrcxq5HFF3ikZMS3&R9ZXE^$|dhKD5ptx*f(HTOW^Fm!><0=xT;b%e0eVflzxRP8a{SXvYVJ6F5~U zX?~NhHC80}N{g@R11LZOCeXHNi@nrg1^hs_9GppK3$cfKPA2o3%CWci`G0?l1%bLb zWB{Rax~wS?wk-U>1XH1dS_la-F#5rk6Nxssh!TXdA6uD9(1}$3`@cRzk^x+dw{m4W z+Pbf6#X1Wa+3Ktftg9G7s12&M;Pk4JS!jNyG1^LKDpLa>ys#3nLNklT^t4E*fr%6< zr;w3B{uMhdEGQ#;$B(C$A+W=uqn3@398yTL2f;c}fIGUmE$;yzN{nws>u*gQj8L4e zQ;ey%QovRm7b$0(B4zxB^8FZY)C;MPB z`*6$9*2_UfNrTIzI55M9jK`CPrJ`rdEg(<&cgTUWgT-vjS3t`S@C2@yd76L`Ug@m0 zK*YC`#F7lSl2E$+cDmM?x>1~?>#TcK?8%#j88>^weQUO+_r<09o;9YA9JmA(z#JgZ zEJ{)(;%hN=iV?jmWWun`0fx)q%PCzTnRa&xnuid70hh^vI4^9+Qs|*wQ3rp_nLu*U zbHmV-gs0OC5tpmT-K;tOp()}#$xLjnrIya>ytALUC z6>ZpxH=M=y>%4=@1SRThjKyHg8ZT%uf_4)|EAXnykPlT-)m6C<7J;_8ARW9+xTsZA zb!;(O5ZVK~R^s$R+6qddB+RuPyzvoKk}*Ok?8@8ZK_dVKz#ZI!h(QH$#)3-HuYY4_%h#5x-pMv|J=kl0$k_ue z)dx|i&cM*TT)b%859440syzbxP`oLPv5!coPa^+0!FBv4#Hf_NoDoQDX}p`(hRq1&_C)(KI&RR|i5eJR0t2~hntB&(GQ{?dWq z2z~D7X8_YDKGS2k;w{e3w}RNYi&A0027W++Ps9ep-P@-r61^-MDniPMh6hBzspaTv zY+wVRZm+0O&G9AReJZssfu0Fbr^70* zhzMz+gB~`%ED-ukjv=|T>mq7^S_Q$gdHn~_|q8bLs{60xWpq9Cw*7ySZ9_Ms@x4`_*s4SWuqJLN5 z-ykgOncn?^LQ&|CIEN2=?vpBY^Om6Wa4|wjeA4yc`a) zW8Wue7TvHv&0q&uB_(};h+@Vj%vuOyF@9Nd1}A_3thF5>$Bm(mXlE8OiINW{V)FnE zaY&8=U=0qSQ1cfM1&u!iV~)g&(C0tDmqg(T@;>enCg_5lv`AljNnb08{v1uu-A_;Y z2F!+VC_$O97B~177{BSb``tdtre5hWCh*_?UG^!La&xQpZ2xHAz@;D3_QOK=SY>xu z6ZmjO7VV}C(A*99bN&4Oi6^mlZEyVbj1BU;MS~i9a9dh&baG^wN3yWREI4q+vjvY7R0?A6`NRlN@ zn(D>o$Vwd=G=%XpNhZpfH7CWqS)`>)o;(+bkf8+}p%p-i8a0|`o-H5>@DOpZ!2uB{ zGk)Oka06=xtz4yk1^YFbxM0q{)fh2i-w1wfgU!PS*F;->Y72zRmDgDc2~nd4zzYqN zIXMqP&;Y{J#TpkN8o&56BHYsk-O{jG?089qWGVR6;`@#163mE!KIV9XfGhF7qWTRdS6U75Z{?jEz8o1`$4+Sdrp(XxhDh z1n-g%rs@EVXX!|clpgbb{@AhKNWXh4EYqIm2?8!+hX zGrneNW=0b$u%QXmoXACr1~NgdiW2=(=Cj%e!A-{=^Jzwt6{jH1I7bdbjydqKv%?ZU zsw*fEQ?ARdOzzGUkG%2JJg<^Amy;>JuGqkdCO7SLPtN%~)Itq#R$w5%0SMp^0;Mo% ziVqR!bKnFZqM+hHNxM=oDh3%u3qpSSk#L-G=4xw%0!$F?K5-x!0IU`8v4xjr=;>p` z6z%vafdn>dLJI#WbeSar2@=a7SQ)991qB>Y_;sGmf^94hb5vWAgH_8pAx0t*!M2SQ zq>!zT$ACZwwXqJ7!niTw5G6}3H^L4X=q8a4OiQpt)69C)#Fx!C_eJR#`E+2 zWghAlMoyy%-t+j3S6=xNir3V-UxkX|#5i{mn${iMaCpBA1P$xNpZ`G7zPIT~ z2#X^i!DgbTJw*V42WS``L2x{CJg{>Dpn(J_I6Bf1p%8s)10q(p!4VK|Q^fOv?9yU_ zL)Ok_zLN^Eu(m=>X5}y<@WBUqNCiBUk`H`%g2_TDtRLvgl&AY-)Iez(D)p%oOq?YV zdYA=W)sB=<$Yl&+AWU4sWPNP>$QJA7MPufzj0*7%8PABuXHqkbgmVQ-5T~Ai1uTw_ zGa#cTH?`xbKm~mCBYgmwyg^dRonRS(t|-8QJ~2`)HJ}|oM2Hqf?xS|`#6ulgD3$*f zY-@U*=%gqW+5?4BC6%dMs1Q8K10Qr@inxquL#a4SjK)%wxa4SBD4I)>>XM3Bx}xE( zu^jj;(~Dr-k{Rta#%k6Snb<7JHd~SbHXOqaTO6Pq4HiyW`H6ukGH10K*hk?lDr3@H z0wJY(m=1$Ttpq7$B`lJ_f@nrnMwysc+ z86~Z-7lJ0on^jD%Hz$Nw}XC5zi zsK^j8$)iPe>10-eAk|L4F3QC|{3$}2iImk=sNJlaH{n{707!Web0~cXM6=CT438pn zvSm|TO35KLyqVD(@?i$!+8{SZVuJNF>RpD0ypNdLyAl|%POE^StU%CJXd>cSmMzD3Wdg3k zx_JZ*M-6Nh-SMIGg6SbWfN?^{MBZs9vF2 z8xX06oNtf%*FR)f>7Mq6hYZ)XiL3o}!+mLY(CN5C7sjE6> zsyfHGx@>ZeX#hb;yO-hny5Tdvl+m#M0z_xv1=!?bG}97J|DoAv1x!BxCHR=jOYK6L@GG4ta*Y7S~!NQjMG^K5t}l; zgRR;T2G6iE6hc3!iJ8Njid9pTlk)_yqO)&-5#XpR)+>jp*$r@+he)`;KF9~9VS~-k zw%zi9KPZx3JiRoOg9l6=uz|o9xP!3qjm_}AY@`9J5; zMBKr;dqhkWlCPpcWNEDwhyj`Lgf1|y?W3z`sR06DHI$njf1-^72)s;cwYAWYCFDnf zB%a!1h8zJR=}Cr9Sd<);Krb7P|$15_oax?`*RL6Dvu^{qAzs_^|710@MShhN6XW7@rHf13K8PR)R1_kRrj_M#!|f$z&#- z1jiZM8xuUfpxn%%B(KiYra5S=ogmH9G@#Qw3?q;d7*rCIJ0%wIHLqOCz3jx=H$mt${K~tFDVIz14E1(A`SVX6cKQ?h>RgHI1sp0OI~6O$$U<0i?bMV%(o;0 zx@4R4*(el~P7M^#@pQ4aYAaZ9w>AHfCJ_Wm&eRgn^vs~@OwmjncN)!>V8r~aC@Cnv z+;qR(w8yNeGt(qc`YI$;I4#@)LagAgr3_943((VYMIJ~{F4KeqQ-begCMD3%7ZOj} z%&{+`q6{Tb(n*31^Ux8cA_~L3n=DZcrOfhF(aJmndT|J98ly9D&-N5b_&g>bm5y^1 zQY0W!onVRjL{bAqQG@EGChet+gD~@XO9MqS`%IiJD3=;|0xfmPu7HBtfdnw6LGZ!> zFi<=vm4Y#3y~o^AD5$Xe+<^jpDdqjnlm>(L0?}$sEr$XsP0WBflclD@hlj zG_(*zte(^c9o-NV#81k?nZo;%CZPffN2BTpGE&pjQ|6V=d* zLRIvUqQ_LLi&9mJ%1~HkRa3Q9Qq|O4{nMAy)lJ3JLM7BPM7LobR=63;cq55PBZg_h z2+Rz}NFB#V)l5p2*60GeYg?mi)z)n#)^2SNk@!|{brWp;gmPsBpi38XRaZkjS9L{9 zZ*^CAjVy2dQEcjkdVQOFjZb5}R?RFN?qb%R+1?R91k!hGo6XkBz2A#Rl%smy$JEWK~#e>Q{b|Sd5L>d#l)%z1W$h z*_yT4lm*JH^H^scSfBr7RG%G0jXha}B}aqZ*o8&Dm%Uk+W!j~6S|6oWsO?dyHQJ#i zK8wIno((H?Kumuq9e)ZQ8PZ+OtJlsx@1+C0m?Ltl(2rt;HX%{SL9UTDx^y zqitKZ<=dp~+q6YmN&VZr#ao>f*hOW}o(0;5xZA>Y+{fM9!Ij*#om#-1T*7V7bMU~% zy;|)!+{Bezk?mT@{aVt+Tlf^*%cWe^wOrQ4T+_8%xDDOJ72Vn`6WHZk+{Ibf<=x(8 zAK%^E>(JfOHC&LDUBr!ux*gsUd|kbb+=TsI;KkU=-B;(8+~&Pn;_Y0yon6KC+>?D< z=3U>Xe0C13MJU-&$R^UYuN^;4Mc@Yp-~cAzxJ6#?rHJPZ+xmsw1!myrZD0sa;MkSm_7z|Y z9^iXnT(HyN6&_v=MqkZU*v;)<79L;rwc!$e-wHP1#nl^=pkWZ+U=N<*4vyg=4&fiZ z;c6J+&+T5iP2TN5;-VeL^&MOLb>Z~Q*(26s5^mOj#bF)x-V?SL9`4}`-r^`m^CN^O)w%yS+<2)AM^rYf5u3{jb;y4y$unj~mreG#cVL3M8I}X}3 zF2p{DWIg|OWZ~^PN{%!_9%MxJVnfzs5iHzRM?&o$+Xn@{le;#OihG_RSXbsMv_F?CS zu4RnQXNKnJhmPovrsKo%E`&zqi`M9l{@jJ0XOsqMj~>{FwuY4@=~Gtdlipc}#%Y&U zWS##ebP z1|v3Us77O{-snQ6YNWnue4gm7ZsdZ^Xi*01xAwZJF6)lY;iFz@tsZBzu4@YJX{^QI znSSc9uIITnYH_yeyUuEfuIB{?Y`A`Exh8AEHtR$-?6bb>i@@5*7GjxhY{wRC&7N$~ z#_Q3x>(M3c!?tW3{$r9bo385UcJA8d?(YWgv?l1a zwr;nG&hE}u>B25yR-W$QhVGC)=gn4Q^geI;7UAV~@9?hd;T~_0y=~pLX#3u6nr`j_ z2k%N|ZUhhKb9nCtC+^#Z2ICX(`o8Y;PVhihZv$s=+V<&A_DRm3@Bt6;^j>HT=j07< za1=*u@6PMQX5=tkk{%sOx;SAqzr3Tu|wB3LJ03rDV1rz}P04x9i002M% zKmh;<{{Za?yhpGey@3T2DqLs~p+klbBTAe|v7x+o7cExY2(hC_jv7N|6gjdaNsuQ| znoPM8p39dlVakk2ljTg9A9Lzd$+IWMnm%_5wMq0Q(4j_8o?yCC25lmsY*H^-0pMW25#6 zd$w)IvU9UOt-CjG->KQv2ELm(>*17x8^5g_`SRz!okPdXoO<=^*0F1E=XrW@@4cmi z{~y2Ir||RNyQdc)9X5ON@XK@0zFvL(__5#Lr;lI0ee?hZxZi&S7C4%I2)=e;g8elp z;DGHh=-z}z*=3=G5SsR&e;an_;Cu^GNYjO6i8xk?GTG&jhahIi--i;hC}WE-nrP#V zQIV)3Y8V<=<8eK@cw#z1;)o+-GlnSSixCo;q>oHK8Ksm%I$7kESW?O1k6Ly~Z_Qp|C*|t zD9&nYtGGrgYplBRx(=+B?&|BNwX%BavAHq|?6Sl5+U&2-BAaQj)K**Uwb&Y~t+dH* zyXKb6IvXxJ-;xXMxoBo?%z-Fk~3z37&!uetp8`);M(V1sSIFO?`T zwFkSKu)zz<%dmwGs~fR53RkSEt0VqCK?DNh*|15OSL~nC*&PF4B&C*O$voy<2KP|8|R9D@wOAu!b z@5A_ZU9X>0VsDtW4W(Cj)mTm*{Nw&zbie@XmQ6gG0Rv%d-$)yxiZ61J3gEDDV99;=JAbILXjG zeDTAhqW$)G_}%^Ygoh9Q9fFWwejVgb0et%Cx39hwi^K2!`|-;`zx?*+uRk5`^Y1_A z`}?2nGB>~i_Re<*OrRwc2#W^hZgdbVodma0I@Vd^Q!(1rHu#so0|L-;>(id`MtD4D zOivRg%tb9!=nNLVO%o_|ULme`JuRSM1r)FV32s;d9_|nbKKx-2|96-}9j2g&Nc5o* zlgLCO9X zsK)FR^tCkI?to%=f*Cy6zYhYAhS&?_3S~IR6}nJ{EzAWWtv5p%I1&O7V1XkiIY}oN zfs!%EBqn>v$xeFmlb{TxC`Xw@PC}816DT4S+IPP7_3@P-jHTqdct$eT@|Fv{Mjh9c zuU?w2cX|wF7XHV+VGa|0f=p&DoaaJMu+W)^WTEn4NQk=ep%ylbK`J+y$xC{Zo1_G% zIL8^za*FblMI4_x*D1bMQo)^lTmv4@D9=5f!Ho7i;5O#C|I1!RX&D@ZfiPD|zr`7| z3kZdR2m@)&x-oN^iF~F+p_#~P@}Zj8(L_~wp!ogzIuoF$J3J)QLrT)|pQ?2Jv%~({Uf{QxV@~2YmNl@}V z^|Ow6>QooX(3}=ltBagrMR}@;OGbeJWF>=a3lM<5|60MDSGcVg%(~l4=60lhRjW7s z`rF_Nceubcu5y#R+>?%!w!v)xSsf`@PZ)s=UG+pE2|))kO=b$5_XPRut!j z>}1y)h{{?PJG}I#W;xqgP=NNnlc=8*xc3Kjf_HiVHU~Df5Cs`v52vOb!USXxfY|Ec z3ed%^ZYM0;3QM@RzP)a7C9L5Nd-%A_4KavA8Uq3xAfynOE(M$^11n^=22@bwZDs6) zTxh|?GxY)rbzy`Zx3IiZl_2TR>jy_fHL{fDr-UG)G5MZrv-K4neoaw{D(9CG$DBof z&G)@)#xNgX4s)2Bk%u%;;SG%3DK=r+;0fp!|F9SKFoq}0TRGeLw-nCvh)K+65r?<| zCw8rhYr0w&JA)5rF2s#(oZam@VY_;$!8H>JofP86O@PZhtE`}^NeT@8mdY-;e zva(u}*)~CWaeZ)imCs!%Ok)lYRkdDLZB1}oqgmHprfr+e8th^ho7u|NZN%{m@n<{x z+0oYI1Oi~<7+|%x*zIw)C*49EtG2Z#|JMRFxeH`-ySiZ2Shv0xDAgpt_p(mTxp1J& zxBTwgT5^zez`IKDTj%=d2+u_c3SfYS55ega2l-k*TyZ={-QmbKYsD|#Yzh=W<4j-i z$g_*sh{Dk1-&lFcS&mRqR|wWb?!~)9E$?|Z30j}v9r;lGb3WyL38YOX^NQX)gHtot zV9mr3y{=r23Y+Q{M{d<8Px;7CJ@O@hsVCd;RhoX+kP-#iL=n43M_iCy*KmcVjjUd} z>j~#Ncau_)iE351ix%?C+tAHH_%ItD!N9*jS!HklKo=mlEOxxrwLbAXFTeTAw!Fo2 zNbE~eU>^%M>l6suSnl`H`|xaM|ChHI2UEOUQNvJu-0|r4o5Mc#==wYD^BoIqpsQBR zuo5=%H}qxp9?XPbW;TUzBoF}-mwAFEL(q413^-@e7jat$Vh0!lS1?T*Mtm*s0~)x2 z3Dtof_<S;D z5cq(QH-SzVT!>(65=MSk|A>WJ_=sA_gC6N28e^WgoJ2ee8zx@_-u&iVFt#C8Rm${ z2#H7*iAXjKRzreuXo>5#S+2K&z=D6Mmxlav2y93erudEM#%i%(Ke|_obkhue*n_^O z2C>+ANoP^K)rYybi?-;1y?BTah>!Z{gvRJiTf}|J$c#Ugg}v~Mu@i^Ws5^l02(B=O zoWhCN*l&~5J9nTnpdgCT2#(>HcROf~bZ86zmk&QkYwkE<@(5r$h`I=I_qdOe zcUvJfjKsKzV%CW7|I}Z}xQqcgkV|!G8rf<>X(7Pi3z*h=DQGd?;w})`iP@ly^M{c^ zxsl^Yjvl#kci037=XuRja<}MvY4ws#n2%3*fy*V6(x(O{1ZH6gQY%C^J4s&xIhT$w zGtY3B204^Pxo#+U3I*wq({d{==rK(O4HFp$Q(%#mkTTj(is|;4<|aCc>5)2R3o_`6 zn2AxY*p9SzO#s#gS_hIyXj@5mmPsf{SD=z5m71h!nycw-m?UDXd0QMccoMdjWEPh_ z>65&<26l;;Q4^ev33sp{S%C1DN*S25!X_^WFcfo``^7SL&@vi%Zq|8`Ryk`tl?DL_ zL@Hnd-U*&E|2UajDLqptY(_X{k(81vreN090Ehr#SKtX{MV}s*L!J;=2>5glR-gWP zpZD3G*LDR1>Sp$7pX_N}CGei{2?16hg@<-hU1<(~pqn1pLA{Bgb@@8L=`_RnOFxC2 zUUMyESPqGDgC;sC&rmt%a|y{{U*Tvo*O_nX14teT1W|ATI=Z7aZ~-q+0kN2vdH9jj zG!OBKp2k>sMrs7gGXzaaSROYH#b`E-sz*?*`q&tL`{GKbx3G~x2BpTnqdY+`S4X_+5{~G1aA`s zCKQWb`l`A1s`)8kuG*z&`UbM9rL)RoYj>uw+6Glh0c+`|A0=QWgbG}gM{`tHYtv5H&=$koj~vgG#8(f~epsF}kNbrK)l=(+55Cs9ScKFa&yUxpJC- zqwiE;W;U<%YOdueUcAR(CRSU@u!`uY0tQg32SBjPGXQTB03fLVi+E!B@SP940{M`m zA~~?OV6n%S4-B9N8k+|i3j`5cu_B8I1wcIn@HPg}u(;}=58H z|KqIM2(875v)VdTddi`uaF_!rsNE_cB)TQx>Lf|~FxnFiQdkbxbgp~%k)YQ$DRi|~ zI}Vx&uU4gHZHEXk_V5jSKzeYTlgK(Vg zHKMD6zzidYW+1B5gRWaPnO7U1?>e@Gb!a}eNa#>K`4@n++fbg9!2o9tRtrQKn5iKg z!tMLL?hC#rV7?d7xD}uV)BC;zpa$iexP0KpW0eLaEXZ%XsfA3(c#Hvw|I7y*_s4r2 z!dfJrm=FbN&18uwj!n^cY3oZ=*g&T z&-c7ubxFWWi7FXc%KA5)i=eDj%*yT)L)`UXs`YVtaIBZ>JZh!eUReb+vn(s^wFcfisw;MXs` z(pnG!BKrq~eX-)e42LZNAj1Hb908BLiXA-x6>Wu34T;>-SU*5N2qo1~?b%bkkWvGU z#{!t?qA^Je3|>8_;5P-{MLn)uU0Mv=TBT_Anm#K)o;romx;)q3!(}R`mB}F7lpGGk z-3JrPs{KWwZmrx@01nF?-Oas1sM-umP(9U6+^g6G+x^oN|BX#>8N@u<)U>Py!pGe? zSaYDg-l1(z`)t*rtSL@x%Dk}JtlfR;bH+s)VB@7#30;T1O~!H^;B?J5nM>Ew*LWTHK`zdGNag&f7s=;2I5((78&RbL6u0)Jxt7z=Y(q zL`(9g-XT8Tav6dxQ((T#!Q6QX?9Jlr?FXZcdM{p>qKB3}8 zH$!@r0iNO7BYs34K3-IuN$xr2$B}j0&C4NyIL(6HmPQG2x zhaNdY+zY$j>w#XUe-4%ZwmXO3=bIeF0X3IG>;`&-j2gH~oxS2`W|@?1zie*cL9DHs zUbJP-=`&8_ZYb))bjG^C>d`Yus-EhlYHL~Rwc7*dV#}RQyXxio+v{rJRD3y3ZtT52 z#9QD%zFzFXp6H99RD#azi_TQZe%a4Deq7{P(f-wpFuOU92ov36KmD(WHtG7j?cUz) z0sZmO2!{ws<ZYy|10klFJSYx=eXYMe)2#2`IUW7LUaRR?DmYcNyyO+}^F< z-k4i$3uiBmJ|$2pm+qI0#&8e!Z?FO!>-J{`wy}<8cu42Io1OP=@3}5IZcv<|>+@TX z1cV>(LoDTL*!XGK>qZtoTO0NI+hqQb6An$MT{An z0nuU+6GtGI{`_fXYsyGIcQV;xxe-GjH7xZI(xc4e9h{x;z~nhH=gCj7>M)X0bV?+o zrG&bPvlP~*R;?tVF{AWX)2mkLY~{Kr>!Ytimo_EKXr8lih!%EX`=i@gb6VDl%+n0j zl7xA^23jcR6AdDOfBY59ByYk`67!^qVnGEE6^>V=(c{qMkF|%%`q^fU8nC};9t=jc#*sTZ7raiEv|7?N;gPzI!N3V5}VW+C(cXf|+Go7$v^~ZnIKg%ai z%yCAI-m<_X7_@+BYYZFiu!)u#5Yp?ZF{tX#FbsXja6=mWvTlm&E?8gy0w7QT0?IDK zpoJA@NC7h!+xi3)UTQ?`M$>XcZ8aWci>*i81iGy@B5O(b;6C}5P)6GRDA|FXd}K#5Q@3N5U2!v#4EEQ|Lri+}+GN^HPH114}xR0>KR zH8NC3eI_*=TTP7}AcK^3)*^)rEw|iuE6~N0p7a&UD5rd5O*Zc+^jHwjM2RO5qT|&& zEW?`-rZjay6r5(-M6RWBjv8wyrMyDpl~C@~Q>it&0%WgO_IoN^C3uK8gbhkVDIlhI0-L05{|BBB)@598>PMjN{BK*ki&ywt<h9l;hFMKur$om|L@G*7 zihP~LZl#uN(A20Qw#zbEpm;V9&XTqw3Fr0RWhSRK%Va`6-kG9sQiOhg_bIN`gQOF@ zMezILz)f7y`M_CJzKPo~%aE+M2(efDKOC|mrta#TTX=}^QmSB_HWP%ZE6N%~|f~lzS zOb$i41R^NY|0QrNj5=Ln(hugd3h&4cC1mN{x_k$y;2mTc;cA!dP(hg`Y)W|x5S+%^ z=R6Ius6lcV7{Qj2kMA(xKod#B2~;4ZRYZtGlv;o}Q1bBQJe(iXlcRGH>>$e9q$ z0=h^@6#O?68Q{Z&{Xqpw8u3RGnt^vM+=ClR0f@sSg@!eZ%Hdct0}1T#ie@|23V!IE z>0O{v$ur_S{;;AXhh>* z%&;H2;>^n_K^EgYSQ3Muo+EFS*t04Ca4MNL&jA6y4!%or#bFcgUQ z?Bp?QLTD3jmWE&o(+z&RVipT<%pb;-7|nbpG;Mk^YElyz*WjNRdO4;rZPRnKyT>;# zVj5PY$ehEn3?3p=DP)O_GDw8M6FpeZhgc7uI~!Cz89)aSeorlvgw8~tP*&%3vY;1F-0<>`F0y1FNRYGP=V}gJTWB|P&b_g$1^eDMH^hSqO(w1LKsbMx%%q)5_ z|EBJ9umT=Mo#Q>=F+WBHh&Y{Vk|!iq zAi7pkF#HzGw@HO)#?_6m@QpP+b%dJWRF< zyg(OkFcA??u?I~xKrlpP;9A*A1#x_XUvc8(41D`r2aLD8%+qg?mG`BQ0459?|0;uF zLSdMcXqXOBp09?c#|Q1X@>9#z@=L!6P|AAIvj5dAfF0-GnWDK?J)LEUvJ@Ew3sOq; zUrxEw8I8g%y2%v&~`QQ&gU>--m3xh@& zEilZ02!vjkPpJ?T##;$TE^>UYRlKWQa9{{Wsz^*M4msAdP9G}i#WunewnFcnV(Clq1@F7+~h`8#GZH?y+?)+tdK!5C~tra2uEq&bxlM+&<@&k4}8 zV>UWlZ}akNik4o1B5|n#g*KrOxt6>IwUQQGS`Og@%qO%!6Aoba$yO6J{{i_>gIb(A z$DjP}cb9NWN=M)WLMDL<2v9MKUOWKNtM9|}V26NPiXyyB_;Qg0obS+*2f0k7wN`vF zmr~T(V0NXJ{cGmT;+)z$ulDZ9gOhrF1KajGEp08Fj!NWN+$ZXDFd*{hU%m*pQVyL$ zT==k$a$qNgZpJ~KTIhUlqNEUE(0T~rj_USK)JzdZ+^AscaadRwhUi@-`z+9jTM^X& z0Kf!h%XbPiDGI6d!3RP1NaVJfbWis zw7hP8F(rGHy3AnecLKhBE55Mv&Msfp@`oQLMzK^=DgW5qJ zd<0R$zPLk+?YbJZyFvFO0uTZMyNZc#usE&LgnueUfZ+)6!$SGPF;8Ox5s-s~<3D=2 z!suASh8Bw3Psb|BlkZ`O~m}QJYfiGj6mTL|8WR z(yy5lvs-CKonuCm**PO|z-aWSs(2k^TM44MLr=&92*5MwJC(Arf#9&c%JQ{8#3!Gs z77}WXjxhibYLaDOf_;ohOR=68Lx{Wq$f{hsk6OEcyuO0eF-Ra6QEWJkl*s@RE)YIgjKGmIE#e)V$4;tdl%}HAKJ!EF+cVxmAfnnS6q` z8xCthHg1fpo;00QF{NucuNaW6H_6AmFb}6=jOKtIWVw5Si>TK$r?jkPm*u zM1NF;sMJdA!-L~tif9QMaYC9cUgt7dtxm>aqeM|Owj~jdn|13xXA*j+(*hDv2264Eh6D=JVsRfe=!+C)< zjU&b(157ncNt&9U8_iLk11qW2I&8rX7+s>anyvj_K za1^b1n!{2fpP~pt9b33K;8kBWEL~j)CKJ$5P}MsLQ0sy__!@)}(3c!5gzhRrT%?o? z+0|D_O<*Nfd39JN0GSS@Dk5;m|1B)VAOghpj0sfKxIisNnVJS{)mFr0KtjD0VmLi1 z#aLK$7IZbYjfIw{nb{UdJ!2IFVI!DA(w9Psm#mahEHkwg{8Z0mI8}fI&m^=FK-$|` zsAQ-cVR!;zcv{|rJ6ALhX2?yQ=!SOnpB|NqUFFqKV^fITRm7M@4QUh@=*5rJE!P{o z^odcBeKuwq*^|AIy1mv)-_3skc-D!MbD6{TqjkiWaAgjimra30TR?z#G27Ca}+L=%_)c-6I?U{B_3$D1`khyhK0?&Lxk}bz1-31p0+LhND%X zV*-WJT3E$mff@rYo>kkGSK3ts=Zj@q=7BZ97X~n~*I6fH9T9>?M?sHLo-z zUc0s9H%x=jBr<4z4mBp>3K^OK83CJU0P3{Hp%EyA%VH)N6d&%eT*NI&dOS#u-Gg0U z1`AsFOm()CjvSMM9uR|9R^Ww|`IMPCEO8bGy| zQR3`KT`7juI#8(YqF*XjsCnkSdEi=J#Ycbf;smzpxy(f-GF7(3m%@`*A$Yh`@e~*Z zRJAs*|Ba?&X3kre90*JpSD1o1^4cMmr6TWfyyVbtsst#kb?O7cJX8BX4 zzo{7Va^vPH@MGSh-v(}yALn^)3Lw1&VUxp7Yny=>kQT{c!Fuc9fiRHy6dFs zzU9mgV)085KFG;;lNaY+E@235W)N{m=!WjfZk;ID?JI?SOsL~2g&Z%1WpKWs$Z3Vv zZv9r^MEKVXF_z(SUN*)!$eOhVZ}5$F>p4cCcaS43rw7*R^49utFt61qk7n8vCv~8X z|CvIEGlvN~*p~?M#+FE0J!OWk*63m~FNu}-#3(5Ag0sBsNI{PB zC`ZrNJt%Te$5>QAi!?s+Bbf9}h{^<2)o2IzaOZYLf3h&nZ)#WI*}7z*>vWU)gKQ6L z*!Hz*EJMBn01G~FI3^Vvweq%Zt)-ioapRPMe-3ahcnK#rk?!qX=R1MVfGSuf|89Cb z$BghiKAa2$2bAL1Q2T-h0_hjvWeUGRVZHPt{Om#B)#FiN?!)eZgoHRvbaAJInIHF= zM`%k&0ubLsp2x}~>K7$cj5@slJ1lGnY(+>ScwX=IV%m2&-r!XC_stn3kw*A{Pk8&Y z`e!C{A`6wJZw_*^fmin?I{LqYUm^&)1Ou(1CgV-8LV$@2vJ}fB>Q47+Z%%PA`nkkn zg6xEFR}Wi~`ED=wnlJo0hn&5csxOxNkGXGVdMmuU8*Y z;d&W8kpwqPn*zBiV1p{Kb=9u~*01%}$914I9VrLiQSbE!2r{^*a5k8A|HLT=qp|^o zH+vhP2joMLWVidf{{fol;+beT zdGas+A~%q!S}c+n&1Pjz0LOg*Y~%Gg*BO;mRL40)?dbgxAle6r0tW;VBw#QA0E7bs zba0>nK|_ZRCO)*2rV0ofeK5M{Xyap_V9LxjDY7LA%9Kh}vgFal8B8@jcx-vpQV7nR zA7<|4x$@^tnx4evGD1ZPq%Lq&34=5%oH$`j9yRT>3Kpe7!f16{_J~u~U8BZ|B^zoE zSu7{l8aj20R4YYl=dQh3H&4EDP*&3T)J$0{8Z?)p3FCu_P6-?$|156IfHC6%5r9Og ztj0;3JVVXd!j#S1XKtcLlPhidbh^~2No#BE8d_*;)MUeEUHdlf+L$(4=&fL2!QclE zDhxm|B61JPGYDp0WeJ-KAOzFsNO>cYx_j<<&?=K#vPVf((wZfP-&8jCt*5+|B7Nf>{LNAFo~ol9w25Z z6j3mZvq%Vd7#N^9g5bF(9(#)8m4XZkN>_*uUU(=eU?{riO?$Xu*og9lpcthKSbC{q zOZ@fMIOcTvDKnr*LmOy0ntJMwJf><3ke%)JQIY5%A{?zu8Yj_{Q7-3ZhrGUdXkzSP zFu|}882iF+#Wny03M4p7K?#>d`slwk_8lZX|7O4w1clD#~YLwPY^L_W5jZt|E#ew!~C)WW&&Gn4N%~hEm1X~ z!k;kyK@!Y7(Fq|9Rp6!N_1+nfnGtCVcP>^wwK`9ZHyPACX)3LmF#}Z9kGNb|* z{5H_!e_MXJ<&mFo$1IFXCM&Ze2T3jY((eH6t9rJ0Yy!PCsO7Hgc64xy^-4nM#WE-}m;WFjni=Siw z28n#W;fPDEem6yzQ%fc<0&#%M9b-N|16qs}|Ge1w$f2I-J|(AIz@S@eN=i5Iw59HyAp-*#>i{Z9636a$IsL(?ihuE6 zPx3>aLg;~ubud#M?t=$Fm12uTc~lu^L7qTKNI`EQ#VFd?AlcyrT)9KZytHIBEDXY3 zkFuE7h&6=brJ;l;eBAP!w?gQVN`^6{;oGhU8?xo?4H&b*4(Z23@|lltK|Dj%ggC^O ze8NbTkq+Nh5I$lB2Y(f-OeWIMDMn#|ta+HUe4!{zCfB>M9iR$E=KYc?Gfnut0W~e3Z61tW;BoHdANSauPDN#dB z^en_2%SD4(Ap|M(WP51E1>v$m&aFsnT9DiYjxxLwMj@MueChLsagA|)6Oz>fr#P=S zBtTw7pa7kj{W{Vd<`6ZDV^~DKk~%P@CbbT7$im9tbts8E=4CGPDHCt_n7^eo3l2$- za_ot_uasgFs!&=)nKHpQ{e_3>v*g?Ap2O?G=s9LsqCgGtHXA#Q+yf;n`M~c*lu_R6sEn0`!cKA zBz~tPVtrIZDxgV$5K9GE4X9_K@YQ-EN?mB(WkuJTAhr0Bg1Ez|Tt74)x-R#Zp^(D3 z21q(JO}B(#x~?Ypa1=)@BYHSJjbb5-4_gSeywmBY3labV1_-sP`h2fvIr}n%B;>s= z2njl>s9*is;yM2HZ`j0!!kO@nh{x)?nyC#iWdXphg1a$PNaDZ6bj0w2hL88CIP zgykuxcWGQ>eB?#PK<+Vo)7z0CXL`s9ZUB9Md|znTc)$AXgQY8=4FQ2L3Uef)5fK3n z0Vy%7t^Bf*SY*XbVNt4R)~K2*6@}1&);1po|2w*J(NwXHlq974P>FT3qTTN1#k+CAik((+-v0eadhvJbzcop4 z6iLwa-|%evO%ha1iV%sfw+4(b9auY8C|Oq0$g=j#bBkm6=2dJ!&~wRUlgBT zXNge3Hj9VbO5Bxa_zP}^1G4RmXFlp#+Bl;^eXM}ir%Z)^l}l^Mb^`Z|E<{SPt=&z5F>f7b z?+K*S^}o=G!Y{ut-t&I7JKyx}9F6yMZn07}@R*Ch^gPQMR|3~I6fS+CFGP5?$ zWhuIys`62a?$j)N5DPcNd{0z@@;FT7yVYH12JVC_d{)HQ9HfsDgC4#i^qmX#@qP}v z*RV{Up;zl-Nay@Cyc@mGr@Q@>WRdCbJ*27=qQPoC{7<3(^ENB7HmldS`dawU&=)YvK`(a5Z>6`QSw2E ziG79Iw2-*uOkwSZ9u!@-=?~?7pA8;e=b_*Ev7a$O8ZwwuPszZEI6X zU=RKvFq{+j1R*9tfJWRF%^+d<2*wgl$%pJ;|3P7M4PS_Kfd<}FZsnft!OGi#4=SRf zhbWhZt<_vCz|VMs8J?Coj232zgAMSJ%8^F`J%t*DgCHiO0g+p|AY!*P1#FpM3SOUx zpi%h|g&-IZi*4foY2PQn0wbnh4vyoDK_X27A>`oDKLwe5H4!KAQwHe3C(;us){`(v z|KQls0%zoq_Ep@)sa^gQ;U#jR#c;+m5rSUy#0YA~4M+ejw3+~D%JD!GCgj%)G!1DP znnosCA>aWSbR0TX3a#$n<3 zNQN4KgBA7SEP24-of$5IP!kf zph~VJOI}|b9HwC+reZE8V=_~P`NB3-15e6OXb2@xTINrV6E6_u)1i})B_*yY|3pze zWeM^YGLc|V}MbR%iO*F4SDH1aTlk?L!Bk=zrejv8b#BnzN#8B(yLVcQ+$_;8^>{Y$`H!bv&^SfHh7fx{92KoLdLM3NdUZ~*QU z0tFnWgBHXpF@wqwo^$>dbOOSna6vV;Bz5-DVqPbRW+!AyW;u3e+<@nI!W-TQ;Xr=c zdCpc6swY?(P7C4TEY{U~szl*9fPiHajnvR@5tL=|8{pZ%39 z+S(=|=STccJdS2j#h8+I1~dFApaSYS?S+Wx=uS99kM5FPOsS!f~|5el3U;e#(G5TzCCY9VAW7^O*^%$b-8kII&hCTEZisR%qEPk>>| z!Dl%H=MM}3NoDG$rbI(}fD?#nknY>F_>I0%z$%OY@gb?au4-Y1|7ok_@hm??{*!Y1z} z3!W0e^2~%J{3SkiNK>YzlXg$H&ca$e1QBS$1YjxUKue{1nY)gFZ}pNJyUM#gLUncLjOF`w010n!ctzmhX>8W$N?T*lC z;>*=2(Aw*(Htu7hDXr3&zm7&`-iBlGSO#EPQC(U;4J>G4|4x0`*PAv2rDd)E2mnJ! zEC|TT!colFlI=<4V%W+Ld$FyT$r8khoLSPXq<(_uTq>3k5~;C>5_qZvS^>J=8sMr@ z+M%7`J}+OU4VQLlMPUVpGOpvk>X|}rn%YL?rYK^CFT9O!5PFXt$R9_z=%OmXJEh33 zut5eYRl(VTwU!rv^)f04@E(f7SEG5 z11PwxqJXNEDk_*g(LeRo+G^?aww%dE$SEjp(8}J>YHu%auer$(_+DmE3gw`V=NlBv z)=AET1~9`mrBgl$Md;^H5!HFg-4Ez5w4qYwB5aT?|AZ1RF%v78zu-$S)@7RI<{zMK z3s~v{3vW4a$3*Y}yQ(Y4JuiNSv2s3vhBecv%Byj0C>x)!8|$6XA}xxxFeQ#>5vtdN zt*u_>@D3|s{%rstQ>kgv$ZXp} zi$C3-Pif*Tz^fXsanrhLF$CjYf( zR5~jTfL31(bH-(%G51x_WdaT$lu7VxZX&=0BLWmeqy*cj;nLXTuZXIJM)z^CSK6i)Fn18)fVCInYR=k2(B-1*#v-YZR+exFbT7BDo4Xxyfs|M zHBATQ!BpZ%yCXS|v%V!E2xP~4{xuN`_HSovUti$0W=i9PT}7GpGE?d)k)D4=9bN>Z zMn6CZ=s`DwF&7#@NNY|R4_^eX8Hq`PDp>F0sGH!PwQmD=Q4?`uhZRoHLLpT^eU%V4WYJ=m#mP2lXIyIIT!(?0kQ-KT0o^i^c(wY5 zbpg1mF+Cv#C5TG98F$QCd7Cg>!!b;+Fg&xjTu)+PT5ZD&(5m|=+#~nK1 z-#s;IDiaHlc!}ekYq#=>pE)e&f!Z>#i}urtvTwhou1W0j2he~)4V>>XLQ@epw&*uf zxQBa?M<*PDN;oQxOX>qV89!BVftRv{>GOfFcL5AQw6`F3J#ILVo1BvGxD80CtOLvvyoZ!Xz?RWp#aOz7PC%z~IRq7IOOra$ zmN|-3u1%MdIzYp7pYOZB|2x1M5(3o%Na(>}VI@2JHx zCzYt$p8&(a1`0+p90IfFx2H=qqz|@`W0^B%`?dqH69cZdUk(I>ySUTQ%!T-E^@Ma_uBb*=~$GN&rDtf?C68Rd8}f zR8xVEA9XBbEDczSoy)<7KzmwTyihARG)<&3grW~1FRAV`?~!~;z@f4|9FC57%A-+v z*rYKggE0I;^_{w^!_dO)eI%~o%-cNRdv4BKT3IU#Y-vK>< zkkaQnKPJP|M|}xO|3J6uf&~n?GC+evvs|>JrBGvl8lA1`OYmJ#IH}&6mL@i^{bk#i zJ1QUMmXuW~IMpJIcWbBmdP}Z6Yo2KYaZ#O@ePJ3qF8|=`whIU=YMCz6cA7@$BWRKJ zt0{QpOGfydi-BbRUz1idTma}NXop#RZXz`Y91PgF3PRt3L_W{n#}i4!SCw0O}XuwuM609rt$ziK4*5hOB^BS{(y zSqX>>6?rhNu<_CcgbEf;9I$y4X91l%2LKQdAVC5yA37Ax;Dbg|B1di5siG39N;WW; zr~uMrNf9JRxOSC;Pyb6zpaupIusPs@)jZ0YEl5U1!&a?w>4so-CyI*)0xCqow2X&b zNpt4h+C^z_*SZ_x4#2A=O5`+?|72w>VG0JFE7VNbnG-bG&pDZmsQc;CpH5jaO>vz9 zk27VnDx+Frn_!bhRTw#v(lr8AnM{u1UW7M zK>_-JauBTw47hCy3A%x6t{x8DkU$TV5CV%36FY1~4ll#&4VKVrgQX?(6TpMa;G2;~ z8X;Ikj8Syd5&xTf#+gTo*;tf8k9o!s@(#`-%cL1%?wG`sKA`Calvk=F1lD%v8kvN~TJ+K$aOyE6$+@x1dWPLNo^q&%8p@FWfO82A&=vuZS}te)sQ zDvSXUP_(~Bcj!>VM6zy%(Ju&fk*`0oQ6E{tnZ4nxh5uE7-9F|ktN7!XAjT?N8F zDjJAyP(f`SZ2}xgJZza+ew2a`KCEzpNF!_M><%VpX`&fhXtBeONva^`!s%|IE}Cn# ztb-ZavgpDkG9~Nnk#bvUs2c7tg2&A_zZ;LvIprmB{(j3bVTgiI4?7G0QwuDLTU zLHD6-+D&I`IlZ1Y-oHAt7d@KV+gB${)xOj|f(3jVVI3Bhdv1msmh?ZkLB-f^x;)m{ zR8;*wBC$#zqiozJ98kg_S^Z_X)>TYHY}F5ux?ExjjuKenKWb^R1rb6J{c z1JisZCa11?I#_8#^_t2#ZKA7&h`(B>FykQK1)4Qc# z{{Qdlx8j5_!C~%-k}qyB+=(}&_8F1#qu?P-P55NNHP>XI0)EKkpP&BTArn@UPD02Y zjwJXt&|!}8gf37KI*T0;4XFc!0ujiAAD}20uY(;ddT|}vZS8ir%Uzu~$U!{SYX#*q z+n*+IfzZqz{3n_Fujafk9()mo?eGVesHiy~#32%3h(i}J_pQ;5E)Jvh~y1f>)(a2Xza>VMR$9`-n97?6D|M0e|i8faNdBlx8m zK3GNsl_3SV!K{BVW6TZAxVK{74GocUi;?m%1u(3UjY9YUT39j*FV3-!cYHz~naE2r zIAMT)JVYR~cEJpK1D)yIP9YDuyT5#pk%*|51pYhE&0Ms#*hhe{1d<) zWD|jL(JmNL<=|EsxE=j16h`#l1FeFUaFri^g*P_`BqQcfjOm_{?1!@P_R13SYMp(72@lqqY_%0->-bPPgx1`lNikgnSTJyhSnTfzVf?|j406!M zL18r)6G3DZyV&uXK`D=gY%;e=S=sP%sDdbJXFsdXXKYYw+6XW<2LB8V10PtxO{9!- zYFi#js>}fM2^i;IbQC!(!6}Tu#BTAlVGn3#hu&ixJ&ss9BJI`DywmrTV5Xjb!6+1%zaxj8Sx(`R4K>;gMH$47bAvr8cs zl%Sx+S;%ck^*XQwm;PZ^{bE3btYRb!#aPBSrZJAAdkY)?O~>eILJ|f6znl86DyyyM z50)I5J`v&ssaAEkIvN7t_{Fh{1+0~?OvfxAkPbfWCVuam+5aztIm{z*N1mb4XFnsr z*bh54L6QAuXG54q`veLI(x8oUFmunuM)r!U(5xdzCT%F1+ zc%9#01N*2Q^B1VF*)&9IeFzf~)feJtN&KzCa2Ye&wb~Xx7kGGB!*A_|uv5I^WtYX-`CH%wyUUQi#}2#-N%*uhF<*L*cFI4_DSkd* zjGN1=^s9JDQn7NRf*mFtBw_uZ!@I^pmo7uyW~qHj#zjA2dM5`?di?+^FGEJPWLg5F z5FcynvS)bhZ(nhXZDbAE{?+#@EWUHU9%KPjFy1D|Z%IKB8 zukyn0^5iPV;xA$lXD{#$^dhXaAVA9WLp9=0N&hMe;r4>!s6hW>5BAF7>#~fS!imdd zL-zs@?bfLP+eu#DsU;Z66I5XD8c>BONdjGnx+bu0xDR`X&I6&aZ#Iwv+v*E2uL|oT z(3*hrHcVomN;FE)3@1c%7~=tYi{L6Mmlj8lnt{h=kOsYOX&}piRzd7!qsxM!2dSa= z(hdmONeB&)2$@QRlyC_b5h$Fe39S$kH-^%FP7A>=3$>67>CFd3FbndcJRU#H{s-RPztPU0@Nob}UIH2@Ugcj7R z2RPt25RVov;i>SV5zr0F60R2Y&jxQ17bWY8T1RTgt`RN{7zt1qhw)B0;++Z-0d*#x z(q?7U1_Jvj8Yzzx4nzE=amIqKt=8be zQVJl0asY>tAlvD~RIC7|>cN`vA^)K3Y{Oh$fwKVd7wxjZ@Uke2vV8PF)c%tB0y6>y zb9oZ8GOy2}M2ZR@Kng4>QM8gPithBnXg9%9dc>e7*FrY-&=z9NGiNe1lSYB$pdD|^ z!4bW(a5cy8fFZIa*ZPUb@qACtAKH;%9GR1cRyI2PhNS@ZYQ?nCcH9n^vN*bWoePD~Gs zw~|utKp-ic5d_%44r;t;u3M*7Q-cCc zb!JLZ1PB7vt)>u&a?~1|OV?^45N@D|VCh?!MWyPLUWwISXAP$ERbN|{U!OJY29gzK zc4#Ug!=MFcbCyka_9@cL1>Ww@5H?$zE$Vc#8?b@54(qmr0d!%P#@D;w4Y3Q))MkjL7@{1TNFmz}H z@X8*Km2F+NS>Lu_vor`rOG8bUZfO>E?RI9X0B;$QZJy^5CEy2?#X<>);&OKn=^$7O z_b}1bWW7&od6jWV1qzbkJl=w%_<)Ag)itDu3c}%7J|{Dx=u0+tqmqFS(2J&&j4jvp zbF<`X1Zi{yuyj{fb<;00Dq(RC^uIIHWPrI zG8&_I9QfdO5f=_b2DQXUOjvO^GsIVzm_T+F#~xTKl+KDcAUxO3Gug;#h>LRV5JHHK$ckk!}oZWv%3@iu!HhcG~L zGPvCq0DU6J6M$?;7q~#+NDoXZrJT6XaE4EYs{#6CLeN8rvY;rAh~1({kEB$GXk(S{ zEi758*Jj{AxEGAnM{>Zp79gQ^&A0|MH;*QmgQ?flk|ByxfQ>CTkB4`U`4}}tw|uoG zkpF3Tkki+7hc?;rW{2@(V)7@HT}D>GsTOKr1O(v#ve#m`SOhxZkI*uA$&H<MLkVljkOuA+>tR^6u)wBi~=8GbqEkk zozbzO#qun1WOA2Th52|{DK(@;dNs?Fb*q`0ZFV?slTi>kg!JH>#7>FEj>|N9q5t^T zr}bKTDXsg&V5&L5AsS5_kx1aNjlcAWl zhZ8)crj0RQHPyO9V>qU~DZzxz7}hCmByUpX}_WNLw*!~=z|0xfB-bZh6?}yDp>N64yy8I2LJX*7hG?8 zFd!Uu&>4ENt5ZpG)r$`#_`ZVkXuE*4m4?Sl`k2NT6JIJk3dVOEJI1Q%x{v!b5 zvVKTrqp09tSo(57TZR(Br=qBo2cZ;t+RO1ke~23jY1N4%&jN#!K2CLQU<0ODR9r#9 z85aPxG%fosIhaTIhS=!0#(THJ4N!H=m{<8Nlw%ekVZFh3UtJi!<(svO+`hv=qV_8F z{8mEvA}#(F)6GWJ3_vq-SOl2cxXg!l>pb9E?7lQXkImT2Wy6;i7ovn3)~gUWQ{c4# zMl?eF_0$kXpn^qz0xBlpDZtui#2W_clZT4EZ11L^Lr1U5nddHd2LH-S1k|S>&YMcg z*2joS3?2a$QVNSelFL`YTCT*sgBl3vk_y z+!XFxu8iU~?0O!KkPI_L;I@(eD&yS5rC7`Xm-q_OotQOnS>#Ir)HE1OaykReTq>b} z1lIi4*&O0wVzxlQ;>FU8zmdgrfuL&~E?_{F^B0`Ip%5q_jsL?TjIS64;HbOw7Z09S zdv!<{=s+8(4FDpF1%U+&5P+~^!UP~`Y=EHgu!D{wM~=X$ zcmrcaCb@zY>ImaUw~in;e9>s=1{xZ)K0$Hm!NGzLHvbN(xFaQEMh!e4NSP8Ks8E9p z4j2HCVwMsnJvKaeVT0jJ2LfiONs}k*5*%D5c5PZnT%83|z6Co=G6alog?Ok9S8D`^ zRnE?8AtFIaI6f`pTG>m*BNVwrKH{NLCovsJ$yV_JOy>t(Jd!POjCoU|VbLIp(>0R|z}7XJdhw&gTs=Js2f#2L5dnrtxGQ+nq)a6$zVrZJB> z%_J0_QAPb})CF2K1Rev_@j-=5>Xlf;MeUVEQY}xYwMeAjKq2B6`iVivjffB^LskPx z=ieepD59W(H~a*^gWyRxp`cA^y2Tz4agl*U5`gk!6l&PA)vP3fCDsBh%qT$!gS{Bz zj+c$#Km$^efWfp9v_NDHDlY58N01x|hyNRJ=(S2<$v7!UlaSC=q#&B8`^c4&WXa{2 zUVeEmnE0NF=9y})*$0~dr()cj7%8XdTy{dIrv-gJ45)YtSfEuVQGhp&R&y?zk)ueE z2m>>44W`iub%fPT2q+x3+bd%K(n3!Y3 zSBwVHgC662qH-gi+P9V_%_fthIJr<^1W3v%v$bJ01{qSf=YlJ49Yp^G*So6-lbA3> zAF^(foOwpcEuz)CZ{L3B#;?Es4lHotY#JPAoW|zCzy&jZ7oiJPumVj(E-3J)4SGyB z&cxtc#R5+vdUeK1OAQ+bDm5$J)c+H!pkZ@lC#<%j!K$f!Wq1XL#RSIOgSvr$*if#VqEB^ekcS zCenqt;XWZu@$_FdOD9%7g6?MhcXlOuo#}7+B4FV9H8{czOg{%x+_e}Ng>lT|V&NG8 z2&#a=;8^Z*69a;t=5YZAd`=5e*qCxeatBPvL3CdUT5=5GLgz&W2%B?I1MV=M0W`rD zAX0<&M#mMEec)7Jke?B26}0p@<_F5c307zktSx+OA~Bqm502!PrVLRCT4ajKM(_k~ zW$}uGDVti7l*Kq4u?+Kz7ylFZw+J`NPmTZ_1l|N_zyjtGfqYb;A184P#3gP{AW*{v zjNl&|6yzY2I9vuRfk6)1Ku^jDp*gU!f#5{1u5w{ObXP25hw^=YAK0Z4hvNUC1`uDn$Y|pw4+}c!AQyn*1C9+q6nn~ay$x3 zj;a-{A8o5hOA1$%$`y|+)uX*;^HP}dW~Rd#Cru01zz!%Xr*9ZY!z@UQ@VH>HjIDw` zCg%!=O!Bb#{KFLXxf}tss;w>5=vB80TB%~SB6FzdXxaJ|4~A8jX4oRFM7s$oZgg0} ztL<7*YfR2E6Nzz^t8Z1hQZw;wxOXk?aPykLzy8&nOzjSkz!5@LC~~n02&xOXF;B_O z?4IS&Cr;0^v;X^mvWlb)?P(dRRjQ8HtmtiuUTo)u8aRO@2rBPq&pO+*auuZf1u1WP z3qt^>bie@&u5b@b+?FCYx#4u=O`Cgy2v#xy(hX{L8GDZ102Qes^sbzo5&?%qVTBD< z+E$$>+Vk2s#m*ydSGJ`}839jt>TM5u^Sj?1_jjcK74UBdOjiUe7_SRvaDxvQSQE0u zu$2S>FYak!3>yQIJ>~EvNr>5gY7AtH83QkuDF+&u!Bj*@!cJbRV=$z6qBW*v4cKzS z8|9@~rPZ%R@O#@dFVnVr-ffT35LH0?IKTu3ZZ)KVXd&Bm(L_e@OOO0lnMX6%LOFDXY{E3)U*WYVC_xkPm-at^ycn!V~q>1g1ze4$Pwh2_PXvoUH&+ zQV@f9-IIl^Tp|jlK3HNvhr@~lCMR{d6bde7lGxtYvt0Qk=~l^S+LpP`+}16R2VG4< ziyKR>FtDN-eO$d3d9S=tuDby{X_+k9(sd#fV>NwUDtp?V9VXO=TlvZ?Wbp zc@zQ+o<#W06PwF54ontSW^m{$>Vv2{XfQfjpv4pUIDw{^B!#>P0uNam zOP!}ZJP#;o7MA%TZBHg0z9cKR%JjCH{|s*52Dx8#72LWZz1#;6Hk}d%LUmb@X~rgC za{rerlnSiuVUBH4J>(0LcuK&60|WpAoXkg_WDV>*O&lLPuQopFQlNZrKxp}ZyDmyD zq6oZwMI@I(Y1PK^^5kR)&TzT3^BW4;8Z#*quMfys_bW>!%HlltZy199<6F#2o5kQ` zy15xsb&J09q&M=r4+lu*Ixr8_0Y}D)BDtt9wh9%rIyyjYSF`fK3>M z0*HFOpnCK10jx$sD-bFx0)d%UK@B){kwhMS&;SXOZCXcggMl3j&>U1jCygKszW@bL z@N>_XA>1K->o)?lz#=SS4s+KbmSr8~z(jJQI|$eWYY1m3H)lAAcz6aDBS(pohaT19 zek0cx7I%qHK!sHZNVLUDOTdC#i2o42Vn|{b zhE^w8uYv&*asfMpivq_&cE$r^HG#jRW&;N&BESK!QULNHG(Qk15P%9JC~pDq94VL{ zI;4#T5fvuE0%CD2b&&#ZmI$nLcs3@93-Aec;RJ<10wz!wcTro)cLg0`0y&U?A*Dox zP%fW%c_c+8jiE-RxOwJ=iu(72tO$S)36ZY&il$J7%vEr|q6M(VCkr7ee*!Um!dMOi zA&s*gX17B#2t>yCZxhI3SV$-~7XnYQ1WtE|$vAdR00B#vQw6X)3G-}IpmKp%coOh( z*W*g2lN|0C3^^bq0+IvYV2S(D7!*Kp+VW;Nl1wSoJ-)LFbN6#3<^MWc={wzEB&B$Y z{Fh*3!fve?k!&fEYk7KvW06*&2XBx_*3m%(fjJSAITj$2yhuYPX%4rOEA0hv;WL3@ zghk9KG)wS{0mV88afVDVSRe)f6H@?{`2b^QK{>Q@KbcB`H9tM*-Tx2YitkDS((XB!nb!7d`-23@0UE(-BQ#2AS6j3Ym(@SsF6cmT>t; z3M6R;xI9=99okVHn8|9{$(OLFaDdr62L~NKRG4cOMjFW@b2kG3$t3a!JGYc9*J2RJ z_kw+4L9EmvBOm~KshQCtnF26a_y`w~f`fgSiPP6bs+j}VqW?;ch#nijcx@2@Il_;t zfSW?(GrI{^k@1ym6qYpCak}Ir#>tk4=7dGJe-4SF&ncI48DTgHWgHMDJrP1c!fgLpb7I3H5xe)qJ{67hJBWjyW$^NL6X`@Bx+ZrsR9Fa5My0= zOJMYk0T2$(7et2;s;Dpn(itI+$%wh&1#-crIJh}tD*p}@lA7>13$K}`!t$WCMgm@N zBNdv3AE5{GVGlnRq-vBpAvT1E-XCkt0xG01O%vFEEvYmo-@VMTU`}+Cr7e8cU|r ze(C}j(AqZC`diitsK%wOtVl358nRgMbfb8hy`FO1eg(g zf~O+Crj#haYvC}DySk>y@FVi52`mSv*rT`1W1B@1KP%A*mVt6(;~1EcKasGfFI#R5 ziEijNv(Cw_R#TlE%0^)sY6zxS5-oK2Zw`rLYUMf-0pD z4*uW|Q9G);fV+wi4c~CJ`AD_06$jxU3|y-TAk$n+qt@9hGle?`G z>8;m;xjVZ73gDqcf&tpw0ByjLhyaij%m0!@t92oeE7%65ZV(O8pa`#vejyBU17)1r7IAPoF~wIzWceOR0Sgm*abY*qjPUE`&%l}jUVp>6{PhFYe-x-~U0 zz1=G%R`3_a+A_jwHWBLyMlh@AQ@9_xzQTaGf}j`&`Lcf+vkhs#rq^J8ptC@fINUSD z2f-IQ^05Z|q=sq6C04-`T*1-M3bb3h!XUvCth*R|wN4DYH4v~?IkslDhRP$1$rqrR z2)(pY#Jf4MI82E`sHT8Cr8x8)@%f1}Pyc`Yu;1Bz7 zB)=;~V5H2-Tr%O~O59jA5-Y2T+%goaHWIrvhDxB)OvK`6$W{=3-NOfUco>i%U+}WY zLU){dQ@`!JihWcZe&ko=)JT({#rm7d`AjKBOmR1qN(b=D$|}pFOUndD18LA0tVUuE zuoxi#wU7|ZyAa2{Owke@!MorF{-7ARhRn%q&itdKEmW3bB&px5r1iQg!s&xYoghD#VRV7`8yG)j4(vN!-~wM zn|qFmJkX_DUj*F~P%u4P=>HI0@B=fMKH_k2ye!eUI}&gK7ZI%1yBo+_`@14+7UlD= z95)Y{;SBhYg1BUNmPbv@A*t#%C; z4sfv6yF0unLl{&$0(JL0D;z|#eIx6Xca9N{?0L%uOBs$mxJtv&o6EVAB5E{D1}SW7 zh<%@(`-hMXxU`wVvrs>g@zaw%ZkCPN1Jr1m?SG{FZb}{6T*p4d%FhMDn@y#db@&p* zsytmr2RwEJ#8cO-Xa6~1eF$$e+i+~rYHi%XNzuNX!G*vNxlnh%Ef;2CcNYMq67Zon z*x@nq+yAXRN4m^T!rw0e16f!$v7|gHK+dUpM()!hc>1JjkxENqF7o(Zk$v7kUD;H; z-bQ`YmHXNA4Kg8tr&@3rZp!385|iP_Jt;9ZDLUeQP3f!K z+>c+a8AjU?=DWC-vD4-q2$NnqTp(_OM%kgk&X?`Z?>(rU9nZ0d&-%;;rC#dH)Z(d5 z7*lZ=7`}bO9LD#sR z<*_b?8=`bj*`lz{y;hdP&!#)CAA&Fv?~l|7wj19#vreD$DZujD;L7hY#>sl@>UKp&qKk| z=TISaK$!)Q9l#u5^RQ&J0;j%i8z3H}aprQYQr-mb2D&uk!GE4g$i!0kF}^NxA< z$I}QG-vfMXVyU9W7VXBXAo6Bc@@^dRLy(2RNJX3S(%GQc9M54$T%->Qru3tx?5Ftr$?QTfKfY1}rQu zvaHOWMVnS_s!Ef1VmYEigIo<^{J3qK7q8p9MupJrFxSBb!h=;z$)hY&9yN^}r^#e* z)srrQ2Hz-*8An{roWFdYvrBZeX#PwK%QsJTKGUuJ@$;7&-#lmg`0WETLINalH7|ve zQ24=vANW@GSto)p20Y9_P&7E8(~K$MP)JbmEhO|T6@X+C$ftpnd4ybYFO-i3pZ`K6 z$fH|)zGX)zHf7MpbP1%yC@~9ZVh9hM;#fo!pIq^!r&yqZiZ7(xVCt@*7=$J(3oQgI z!>-0EtHZTETx*hJ!YBf;3?{kD6utIh(JvDPD-4D2lu_XX3v8_M##->1E(^;Fi{*|# z;@AMR&Nx#fw9!a2jhWR_bFDShR;!I4e&Eop261L-<{M^$8*I4Z@X0@F!WGszxlKv^vJG2%Q@1nX6W{N-I_`TvbC^X9Y2@TG>(&F8{#TU~xqi zd-a1v7>8XTg_&%DAX#N^yr9b->j;wrRGKTL$RmxML&;r8E6txNR}-e&){=R0oPGB3 zXCK{ALxsy^z#LA1I_@E}oD4kpz&lJT;J}9j*hI)!IS+^k1cULcK#j>T)vX!u4!(d; zaHu%60doIXluk!InF0e_Itruy>h-8VYtG&fe#aH%yD5k=BoI2Q2*vyC`1`NA}~Nm zehse$F@`Xh6Q3m>k9f=alBvN>+VT_`zW_1V2;>516}rzL0m^XbsL~w z%~{plao(9{-$jkR<)DQ}5Lco#6VVP|m)~pYVKoTTffbHrAnK{BPWEGhrnqA;0oe4K zhp;_E@-xx4EgS8a)-Db1e%|j#-MROfB_YW0X(2P|`iSBq=L!>ncUoXD(nY~oo~T7F zbOVC^sVQQ3D2cnMFsJ6^NCkh$1jP{eFg2KNPY&B4=QvQE@u21vlNBD1|I z1VC@6TLO&4Ha2Z>%OBwoMmNG3jE$DT0EA$TQep!D5|Cdl`N51l z;S47P&}s!FUE%;F!aO#iB~w_!dT0`i#U;!ZX+YW2bg{>i%qbI?O2ZP4^(H8=ZVW42 zi>#8z1{v;whN-%t?o_!-9r7@hvI8QBfH(;u67guaY@V=`c)%uF&xzKXB6aQ{fGbi& zi(CA{F1pAhVVuSy+~SsR_Gm8Mo6~FTo;Z z8C1x!S5}mjv2x`_8|ook4Nqvb>O|afvIzo)c{pHCw<(Ps1!G9h7iP@{R3$ z=Zgk2gt3e;hKpOf;Ega$aD`mpEjQK_YBup;9dZgH3XDmEII_4Is+RLFR;`Q_`l*`_ zTvdEN80hMr5Q45+6|4Q!s=v&toDsm_oSp0>TirU>dcqT-3Vj(z`x4QloHC;twP;|a z0ZR?p%A>`jWr^fsG!%_gq$M>eFRRC_JXJ54KuClYxpqkLiR}j1B3l`~7!CB1)(x-8 zW(|Z<&9AC)JY20~11Q%q!c?^fWwosscmUR~;{Ud)W#KlALs8qaBLIaO@(5=;X zGhi1IlQ%DrjqFnip;Gm(Kxz*(BdB~jjLQi|@ zk5!a<4x$#SFa}vkC^v+_Std+%viYc4Zvd=#tdYYQ{DVedj6cfYW@miBze;Oz9*kgf zIs5mK8R4zAU1b!HH|` ziON!j!=SL~ded~sUw{P|{fm<%ZwBBafP`uj2Z_t*vvZq+wL=`ja5WzCayIxln>C4R zlBdKR4`vKV_Vb5~2L=SRV8&BfK@hsrBFse49?<$=DZnwe+be7yNMufPZdaF~H@{iV z6|`M)AJ*qT*E7%+?;<4%#%Ud2FSDQ;-;mqe^jy?8)(NCGvI2Y`{@sU`ptTtQ)nIWo zcy+>oLopIizzVC7?Gm8z5xFa1Bw9wZIh2Q@?d3xb$L435mcO5I*y7}9M~C3|^RXEu zn>Nq@njO*rf@XAtt-_y_bN}VOSD-h(<~E0X4lxQuaaVqLiG2~~+n??vK95+uw=@e1 zz^5`KW9gogLpGxOa>%2SAghGfQLU;@_M2Y`U z7&7!11U$X^Dg~K<1gyila2Tr)c!ZkJI@Jn>*<-ba5GpCK0xI)B1H`>XVg_{}K9Q(^ zv_rwbo4Y3PHNlZVm;b0iM-+l3h>0O=yN}6)U&D?~*b91>td5mbkQfDQwi z6gV=!d33wH!2vcgj`MHDACZwnkSQy4__(%%32>N}Q`7}>z>m73t}}o_ z>T0f4a5t!#C8mlW>TIKsY=EBl^(DCcNOG3c6aug`huq{6{pchQ zm;wGlO?raME09Ic)2S7Gj~k^hFgZ70&R#{1eQQ)F7hNChs)F95_Ua`OLtC z%newC#Q$6>hLAw?3=qcLx(gh`?&Jf|RL^N7O*C^#gW-=gxIxW$w(%<{A8Et6paD7X zhyk65t_ln#c!6(4N11(GV3;=6pQo47cb+QM}ws z>byVv`;7C1u^3HBCQu|YY_&N0y*QfBbqY!tH9hy7vO#LkCy+H)BNCwuQbI(^SQ|Kf zbh&!^F63c^(_tOnIubP7(2IPqCn(5+@`2mhfiT6=i4X%XRYEfr(QX^jq?m+_Bnq3+ znOwXX=u}ZUolX|T)8aXX&mtcm?bALDt~!7qfC4Cag3tL(f?)g81?1Bkcmjc&Isw50 zF#p&LH^_*L5V(wZgxZ3HRS3A>%M`M5)SFPsLH*ApD8^I`0$}_tj8uYGMS=&5Rab>o zFeRT2v(*NZ%P)o1H4Raq+mx8=6j~gtijq?d`AEIA(>qmG%d-nfAjVXcR>fk~YK1Yn zf~&U@J8dl_*!0sk#MZX*Rvyr*8i)gP?IbUY3vtC(CdjQSRYLPTy;TiUS*=&En$=o; zRW-H95$)F|lmjO06ayjHPf(Esxwf*H5H~eZZGe?ztr$}*c6&|*;d5Rp8wrh zUqh%R{jad3~D5mY9Vr|&nky_oM*b+_Astrl3&01d^!mb5{wD?-> zGa|zp+Ybqnvt1EJKwG5USg^&@wq;R_rB27H&J{(PJXKoZ0m+;bv?;X3y3C5JwOYa5 z+QO|^r>)zvRa{47+{V4w#pTP#jagLTs!5((gj|L#oel9UB;bV*hSvT zRbJaoUFL0G=N(>f)7{+-U8)$}=)K;h&ED-rtnT&R-}A@@JP>%TKGLy(GB1MW?JnP zCG_22{gvPdp5XhfU<(Fd2wq?YzQO}WV9;$P`Agv7<=hR{U<^iK3Rd9@USSnJ;e&l) z-`(C0mfq?0V8EJF6Q17~=HU!(VHW=3^R?gF^9p4z*7;u8L19iCz=-rN^Zp*BWkXU=5G8)jH0RwdqJTlR)- zR^VpVVIJyY)Wu~OUSI@C-N=z2;8F<}qeuQ1<2!re;KD=TaW$Q?}z^j%9yt zXJVG+c-H1^ZsKGvU~oQTaRz9GPTz3;<801nfi~cZ9%zEDOX}rjRdQHlZs>rXXohy^ zP!ef}=46bHXo^;8j@;-z-e_SCX^@WTdtT_7hG}aC=vhW-c>lKNC6;G|?#Nuu-J7;) zn*Qjb?&O^QX`MD`V{YksMrfmEW}=qrb+&4EmgtkNLOjlDJuc|3Zt7#sVWghle>UlA zUTLjP>xDJop1xs=UTUWn>w|V;p(bmyp6a{CYPH5`u4d}Bj_aou?1!ytFUIS_9%`xv zY@S|h#%}Du_Gy=9X?r$Yy|(PjPU^jW=e3sU$6oAEmg~^Y=)At;tJZA0M(V%rYkB5u zuy*X74r|q}qsVT)7D$m=4%dqPT;oY(C+Qu zj_sG$=n*#V;zn-7PVVJ~)92>y?sjdI7Hw*v?(mk1W&g%)>|W=?_HE`~Z{CJ&*?w=x zrsw$vYA#l9>qhUI_G|6-?*2At_U7XN7i^41X!E}9-Olgp*6po6ZU1g>0FQ6_CU3Zg zY8*1~1IOXZhA~9`|t*Kkfz=-V|4HbzbTn2l62Q@%V1?ua<9R_T?eBaScD}&faV# zXL9#u?Ja-uFZc2Z7jRa&a4N5IBfs)AUvVE#axDjQ7l(7<4)O60WCd4qG+*;L*K;@D z^BLE1dfsCUK64pH@H@wDKj(89?{h>)rE}===!F(@DaTyAHFPt_b34!TN9Xbp4|E#V z^uAqm@SgK1mvkJr^h+=D8_#r3&vG8`Y)1EUHs@Um|7IZ{bX2eO?7r_e=k-Jnb;)k) zx~6n1Cw3E0=U3l#_s;cRXZBy0bq|K>@-_8Sr*`5tRLK&a%a$)=%7i(S=Es^fJ*M2r)8fjWKXnQP z8q{XdqePJ!U5c}*)2C3M4oyn6>QtpZuUf^*bgNgcQCEf?n^P;;vsulejhFW8+OBWo z%AHHLF1ML#>FV9UOYSgPx6f>}t}fWe;wxJNIqcz&G>W?Yp>f;=_C6R$f^( z^W)FOCXXI``S04MvqPtj{OI)W*soXr7wG%#fW*McGPD;6DiLVh!<(OnbNurita@l2sXqJiOlU2GYrkr%fX{Uv4 zrm5zd_w5NMm|or~=$U@bNobyk>ZzcjizXWAqmVi}sG)z>Iq9X6X3FTMds!N3r43># z>XoCGil?Tia_VBCo_?BDtER&LDr>5$@}}gfuEv@xsJpJ?Yp=9YD(s=B78q=;tRlGG<)s2hHmSPy6CRUuDkHY``}&l zzT0iJ%H~UMu;P~M@4o;CTu!^)%!p0F2WyLP!VKF0O+d20t7OCKLM-vb^*&s&!}4bQ zBgWr${9CLJ4}9%60+&N_GXsZ%vN9?+!_2}j$IJ{ZGt*o%%{bGtOwOyIqVvx?$I?mA zL;GyB&_pMV^wLZh9rV*sM=kX=J6|(1)mLL}bv9dH>~+LnzZCXN919tqsMc&bie@ZLKe zpcCHn&BVu@j5FgTulzU5+nvkuaStCoDtL7AhxOHGul*g|cRz*q+po(^>g1X81f)2UJkuz$f* zoX~0(JqjkH7R;N6GccGuP3QuHx%glRYvDoVk#KsmzylCSFo71b5CtoEAq-g{Llw-h zh8Cbe4tL1I9_sLiI1Hi>|A9Ef3lPzWM|2?)Q!s-jGLeb$!yoqa+eEb8t1pgQkwFV zr|{o4ww0`Xkp>%LB;ym#sK&)bF$=SV!Y12@LORxQjw1Y}CQQf&Lgew7i#&iLjo<`B z4$_%`oMa@UIn8QH^P1SyVK$Qq0c{d-d*BSGI6q0w-y*a#B|rldsR!hZ(4c?~22}?uSUZjo5An;X zA4s`REH+lBfoPqr-gyno(bA}kUF=aeYtt?qqK$id=wF*h%r`7l88OAG9^7gGu3CW% zh?oEX-0Fpk|H9U*xW%nSufSH@UbMHq4eoEp3f$rv_qcN{0A7tif!Ri~uXPW;x5Yh=Lko{N>gHfXrr&@D*kqXWh2> z*0q*%ojts358s)iuP#6X5?WpD;$fjJ#UpkP%jM?@+OIYi1utUp3~E>#7}GXIeEkq! zO7}F#U1<}z$OVkeeQ(o8%`r5n^$Gei^%D3!HI`G|Wtz%V86ZZm5n^52*Gl)_y2dq} z0Z!|H_d3I5{q-RO&|!so)z8Vd*eSOQVNt+TJ#gJKH-TgWF7e z|FT_B&y(r-`K%Ils6>*{wb?(syM*$-GQC-CZ||Bp%obH4sxe^-Blg?p6flugKWSRlsAvx^Pb@S2EOkRF8mmHBC5ls0QU3){B&6c*nH=7 zfChL|9%nDvmVM;KeUpVH4Fr1WAbN4PX{Gme7Q=xp(r!_Y{lNcw;B`7)iK5-Jsp7766_3*K@o-bZ92Gbb3ROKdZR>(+=2|3eJ(rFyG3 zZ-jt@J9mPmhIi3bTk@A>64iSJ2a0IOhDK+DG#GGj7*^y{hdzjhu2^zuV~!Qb>!ANQ~Hq zfmN1*SfzgXaEbONTBzkx(5MG3c!t$TibaQJZitP8$6FY+dn9Iw7_~jD_=DtFj*(z} z=ZKCV8IrirazBQP_<~;VD33JfSW@6O@j{Hy0F(PzbN;AW8I*GZ|2dE)R(BKURS@6= zSGH;j$&hgvgMmkp51EvM*9<}ViEXfP1V~FbrI8ysb{`p&SSgGiIcYm(j`t!d{bGSc z)(e;QSxL5mRVb6$hBT2_lVM0@eb8mia6N)HkgP|I{#J$o*#@-bhP2g}Bt}=pH<--j zkcL^58upigDFas^m~X{@#Ft-MCU{as2qafejb(sWRF06zNNEv<=&zmo#!H#2?kh-wLGpw6QrX#~z7oh9Ic6tJAx=>)@e1@fm0Kj{hK|EWkPCIi@cp0R}p;yHiQ z>4N?@Q4?1Jp5UEckczw(RhJNs;wY77AV!^eQv#pT!K^gAE4CHk)>Cq*if2@S97tLQC{t$99|?qzf8^Tg{mdZy*CFnmiYmgC^>K zD*AsF8C~_K1vi>JF@QYgnS(6IOgTyhK#2%Bpg}w8f;t)uFdC!?IcxLa3D=pB88r?c zRDT@DpZmE81(Ztq!;t_~nnZO5jzDeuNTAntj@f35ICYw3v!Gtrpm55X+47o2xJy^p zVsTTUJd-m?#y!aBj4u^KCorglTBw8ClE%rK$Fn>(|FleSXaPy83_~DLKwt~wumYJd z55%ddH<$-NkfWNwq9aDBCMuttdaA1is@E9|+3A9%N~2Bs2B10tN_u73`9eynq8o3MhYtj-#Y(2P>1|+Y3DZD3*b*<#{|c=J6Rid_L6%6aDTFxJdOYVURYco7 z#>A~H1E`6Ug_-!Irj-fn8nv$`0O|?=9cs1lssd^-TY11$nc%fv+gh2hRfIscK!CMq zdjVam0AFhWgurl63jlGO2NnPTYM=sko3&Z{wiEESWHUA+s_>;OyR60#9gS1XpX0e zQnon9m<;pwa}Y4NfcpZiRRJg9TzbY;P0+iv$8cvG00C>YtyR3UXSH~X0lTZVz?%lb z3%AF+u4$kE#+$YmkhTHK0Nks)v)8cv{}lyTnx$I$KYPHL0Q#lNioX66gpQR?GE1}0 z+9$WuntbTF4>MogAi7jWKF3y>Fr}UjS5*ylU$y|KcxO4gDP=8qsOy8f7+?TPzyPzg zyeq)F4Dh?b8)Cj2y#j!|)2qB9Jg6hA!6qyK8>|H$T)bLvydm46AZi21dAD2m4NI_} zIJ`MvT8`-3m1X3^?)$@>dAZKIB$QEPL})RS)(AMLqS@oWs9>Vd=B;lqSQ1A}wy-jm z>UsxEQ;uUj{@AUq$GjE50TEor5g-BD%e7$pwh<2PY5#O?1P+|LHxLaKH)03{fddd ztW5Zp0?ypb6D$D^(9R+d0`DBpG!V&{So4n6JJeB>dMMT`DpPVkD9E=t^ zax_2(r@TI>jHsoxU2voaAk%03|%f_`0}!2)n#|`?|2Sm#z22?(KU%ZeoEIZCJsVv4tc#-`M}l>I|Bd<*iFCy09yeCdk57l)8;S>44^UxP%zb8 z4rHj)-w*+_0L__-0hjH^;*bJ>rnQ|t$xi*$Qwaqrv^Nvn0^Xy~R9)5l+{vDt)wx*E z5mP`<{d|Bnb-G<>Pz60IRM&qnK6A~{a!tB$iIKI?kHi_qDYV#4&<&}ETF)Kb9~<4L z{S9f5#VOVe&yC&M|NS!A9S-3=-b!oHi#ytydk5CL)jVARCGCU482OXwS>EDFrOZ3;o{8f<2w&W>j zJU2JoP)^sU|J-g09ErNq=wIIGoghF{L^_H->6}yPmwseyzUicmrSue)1W17MrQ&2% zy3eP4_S3^Pj^n#nv*}1AufB)-eNV*%u02=70Lg_VBn~Vi%Uzs;;n1_>>R;$;Uktvq zkq(9PRXUkII&nZa&JKl-Ksmc3N?eZVoa3MQoa&tZim(vm49&~#!{4&h2^X|wTNcvM zM`f(dfN?(O)yL|wKJP?0>$4;X5?7^llsvia=Y7tFb@Sn+2Jm~ilL1+V=XUIBN7s~- zIn3VZqm$rpfCJUedD!kgqEo(`BMj_R@wRPl+urS+9_G}OYQssFXOzh&P3$dhTF;fm z6-zyA|L*SaF7H}G?>L_@_AZ;5+V57S#ai5DKCh{RexiiHG8O7yRg4b_pXds2J&SJX zVicgFqXZA&@eof%xdi1pFV)Nz+LTgP7Atqd2=pSG0Gw$0)z8o?*vP`e!D zx*rAT;P#u?219_f{ss2vEwcfwQ(pH-H7O$**k~&=>5djuOKRkK1EuDN6wTwj1Uio zwbV_krK%*qG6W}1Da?-+{kohv3!TWC8>`}+`Bh^unwiF#g3p;VTQgR#&J@+z{~xCe z8)b^G-di3eU;aB% z^yi1ta%{4s6tY@uVI4=GYq^hIfi&Anu!2GC@@gxMyzzqV~Dzd;OoHwFZm{! zij-jnuumpjt0KZi3(dn&Jlsq)%|21BGZIAvF-6T*8RxbXx@hVT{(@=ZI+ON$>ofO4 zb3?cvkrQ$u#mqWvhJpxqPJ!vXY9NIRFqj}pDi4yaioyCoM!cK21kXG$(-U({Yql|N z4EIQa!6`g+(JDW#?t@dS1?$U-KqwH?v%n1#%A<-3Fl0`T=A;nt&;2kw9QOZ@x;8O!mBH$V`fvG1+VCy@O~}G|o7iy^p%91iCU$L-{n& zEgNR&1d0gS>PU;W_JpvLG@?R;qInKu1f+TGVyv;m`n~MORyLuu!%Q*NbYNJbS!R#S z!0_SWhvjf4A91A2Ex;)1OD0tsM38GOSy*u0fUSPL1aO95!EZCdDG|yL6Y-8+LnE{_RP4+5H1A}SD8Bam1s&s__sAb2I9&8 zEaugTHSxEv&Z{T!Zi`_Zr)ql2TXrYNV z%IKt%UdCPl^HjNONA1IB>U_w_d@H-KzIf{!XKA)K+aeE!MbJ1Bpy23q)jyowOoPG##xsD1|EzQPLA|m<}ntnArA_*snCTCA~#b3aRt>Q;$dd7DjDG81)rDQ%8(mmXRi>cNO+b|o<%`nJzcV;+ZCr!O-kh|rM8ZHY;r5|;6@sF5e)5t(j2FR zU=G1G76?Rv5IAHGDq|BeQhp`{T|>kQmUyzSv_Yh408$#^I8kRlEJTxG*drdJ8YZ|% z9OJlM{DO%lBr;P;hwyDcearN$$t!S=9C)3(oMWn|FTc0Lqd%?1`m{K3019XW7F7EphmT0 zP*s9tTX2Vo1eO`&cq%!`NkOY(HLF@3Q+T@SRTF~sg!y~||IpG;{%8+QKdVPT-5SCw z#1)hkF&?!tir24!WK9w(RX771*bxY@um<@+0tt&lrl3Je&pFw65AayX9@C{iiiE}t zL%&X;fd_0M$gLcJtNmdRgMJL`HPO3=KZZ=I@F}lo0rWsJM$}hkJmYLRsI=R{4!5~= zooCZlIsGAE*GI6(ZU0B_#Rp;l$y;T$y%7iE!oV9 ztQ7GTsw|L^U1>pNZult%NJ0g6k(3CN|KZeRUAiV#vQikL2}NYb@Ro1n;lB1QNFlwP z(q%rOifHn$!}c%;DKH}$@NqE}q=G8ZmTDm5olkXEd_w}iSflN_NED9z%@m(&s@dt<;V9rrH89u~ zc!NWldFg{b{g)J0KC^nM7>?I`10;|*1xq0!9Soc^kmwvXu}Nou352;g98q#=!t1aY zi^B?q5VcIDxYVfL0VHo=*Bc7KZ3^gWW)0R^o+rTPMEd#96DCSeENtOIi*j=o0}Du2 zykb^7IsyCMH;UV^J%oO*I(5}F|8@i1s+JLM3RD635>Td~3Lzp8Vo-&62h@;I%epeN z_LPe>k+NLRbQJ=t@)#EJaojNTRwQ6o=>h=x6Y=}BlVAk!iu5rLWoqUiic#7zL72Q(ZF z0<=r#d%cl}a0zbLSEB03?vwkh(%fmc;b@W^c{cgvHom<3;HDd|^&F**y_p6a6}xCf zlS$^y58QlonXARj4sbPF%zOfis$k?*u%x2yJRFyZ&$g8TJq7aDE6$NwGmXttU+EdN zTNR<~BAjkO>$VqJM}x`k|F_gP?h)X6uyn(H<$~xH&n^O2eAY?zC0G~^%d#*}+q^3ej&eggu6=J7Q1CZUU{6&x?b z%G?54h>Z$xsuox?DUmM8iz@x$EJ5<0rYkoKs4#VtI^3JO+RHtx3lftviQlV=vl+g7 zBfAd-!C(*-TX>DP%MOo_mx(&1Q(`&hk*@1&jWEib?aMyqx*W+Uvx+MNGD8E%LccvI z1Xg;HkYfhTXtF9(0-_QECd?BFxDl_InKx=2Dr6UI+duPih|+>8b%}~U;tg{^5(DEu zTN#~GA~xB|v!|=T|J$QF41}=V;yv^#u);AL!V)`7xU>%wu7zT|610#uh>nI*!8=I^ z3Sb<@Q9I>B5#*49aHyyRf&w6H9EzX_&e()q_(3h=MDx3aT065LEW%v4Hyxa{lDj4* zTtyC8A#nHu3i2A(DhL`my)e7Exxj-TTtzN~K3#l>KbQqFY(ku)0V)C;VlkV;88>n> zH*|YL3WUQrqy}QxJ^8w(!T^lxATV#cM8rZO0+Fb2^s8Xl!xCHp)X|eX`3*JDoY*)k zMPt5Fs3{>KJ-jQ4ts9`p^1&)!Vuh2G#*#G2p|D2(Vo8ulC~oVzfV8{ZQ zMQqHg|B5Kh`SLj;+>mba3jI3?i1f^1SvqlZz&tCmbNkC^yiULr%nReWa^e<8VH*Ys zB1{;mjcAav;Y@U-CKXc#!U4-^GAs&$rPSj>e!NYitV;Z(HLjFOr?iZ{2}l{NILshO zP)x-#<2l?RD$3c5=KMQ|Q7?__M>+602Dl^UY90jvg3#jum2tNEO>jX$J4NhsAjB+t8{n9Sgj9abMgpH{S4Z}ThEs5dC{~($| z%VV$yTqD19L)&}9XuUcQ!o4k7(S#*Nh-s}Bn91mIK}A$dO^8{U#Y(Xm1Yzigl({$q zD=$Ie(^h!cUM)ow%r58>+Vpu0VNjlP6^0QI1m!W;(G$5RxY;PnHFSlq`5C=cVz{IG z0e98X0-aTdrPZ-5%*rt;jk?8El!Nb5%w-HDkW|)Y+#nJyQD}t;j}6(d`UPqQS_PE6 zEeipbQ^anoGa}?#tBul|t$}37HRcpjhTFv}HN0G#2Rx_&idjFLt*mqP$;M4QaS+<1 zCDp1;JfYp2n?2L@o0vkt(z2C=V8Ye1C0m0ml2P-$of|O4X}w_s(FSv{|B(z#yTw~O z)y{kBTfg^=>U3NXvhNWEsUd0;mghnt9 ziLox{L0O!GQ{MeZFMGhc6^bJ*y;A^MTw}kD3Z*AJ*Azfk z&|L*9{!gkkS^=iwg_#AAV2aeWmhdhCGPGDB%#u zVa2a|!NzjY&(m1_TEhxjHlXmn>K_8c}2bW`dl}=3Ikjai9XCV%(!U+^BWm&z;qM?cD!#qfMa88LSittjzA55*0{dl|B=<(q^Kji-k+soUJ|BOt{IRd z6BOm9BD2QQGxJh%8sHveSA_n7&s9~-UEMCR-orjdf zYL$H50aop@ZD9H=u+vTw`-)%>H6x(38UMy;xR&b--tF4r3E@s=1W)i}zK8=E3BnOA zy<1_poN)W5$jU$iOMV+~R*Vns!lhhgFK$qf5n?O2J8|mK=zY|I&5Tu0LV*RyDT9Sg zn6hZF4925h)Xke3XHA4oZTTMCuRJ1BQi3U92KrU(<4!=mw3Up#AYYa(Ucm)u>1}x6 zXzXk^1bzL~t!hzEkbVMgxMKIDuKT5`%gI(~{NS}002vbVG^hQ5Q zKwsp8{&692gsk{Bk@(8Y)dYl9#dP^LL6w=AbGatxOa7s~@8E5Z6?3{_1vIayhhN1!iC^CT{$^bk+-RfK1Eb{P5Y z!Ah6(e{q8^z;sDBcXU^TPDkzdZuj{PD2rz3KA=#ukqv@QvyNbaHx+=H0|47Lq~Wg4 z3$`se)lwwc#*#`E+={IIRW?^6E|5krf7r+Eye$ZSYIcT3lKO5pi$*YtJA0v|CI`mq0Xu?KsIzwQ2h z4q`_D1ONdnc{+i*kFYOxw6l0R$A%|JO~$5O0CIp0SYA_l??5QvmS5K=fM@^?0i)Ag zH<^jdjkl@gvs6l9tsfxZGFLCQC=qF7 z4oYRVIulJ6;@b5|s07@{_1xEe|6ag_-v51E@AW&oXbE<8eos`ke}LrQP29nTj9=@n z@3%&C8-${T53qcuBt|Rl1QAeqBq034j~FOO^a8%h32FR_YlYW<;>%Z`N!RC-1BAPguTC}6bc(WFEH1Q1~H zK?ej;r$U7aqKXX=F8cJr>LP2_D7JJ3qm^L`+J*_&suj|ZnKNgzL@0d3gone2Zufq` z>-TRDwu0BLimRo@k0)+;|HKhCD;eZfZUoLMjB1mrPTe5G+O%sF=+K`-VD)GLsZs+*n=WMF+RgY;B1EMJ~xe z@k3>h)w19n!3FV<6bU|7nQs8XR#-3vN``|Qkckt{hpRx6Og`g)I1gpts2C0;rEx+U zjB~Vx8jUi3*2)|KA`#$ZdkMJP9${4Y7jVcGc_awTy+Kx7(U~~VGv8&$9ZgkI86I}x zVF?s3L!7XJ3`?Sy|K3s-u@@gzX{xCKeQomPqmDn?7Flyv@S%w&A^@kS2z)5x9G;#y z)6EERZV>`1aW#oY2&II$qGli#2%?`KP8yDBqs925Xf@i%o2Q1dbqc6p>;|NeL@2>0 zt46vi!3@oeC95*#)LIZTl>kExuTx(6>z2WGcT!g!sUhomV>Uo0MQ5VP=9+K zjxa+5*(#tdx5ar20k|aGF@va`#0BUR5(s1B2qPx5$}018Qp0A2U>MG&nr<3njW~w7 zW3{}MN^Ov-u5k&%cnCEVM%!kjLVx}xmc@Up;1;UUIj6_FPCdw&!SRMq)rAE@K%00ImY-}ttOnHMG zGTuDd!lfD=0nvsoZ6^!#&ttM@z{_sRJagF9{?Sbue{?n03W;m3ta_b$ZfrTTG;!b^ zxjo*fq92YUV(KbVt@?@rS*qeASnsRv7no5}W6g%XVk+>x8#2i7OvEDHpNNPBw-#{E zDjgr=(%>6H(~oie-F@%vG2dAdGxGQ389UJ7o13pVz*HP?7%f;Lh`jy_QeAMg%f zI{IDYhG!Zvd}3D40n-qcHw0Qij}a5IUiIo`w`yolZ`;dK_rMpF2~1#cm`KCIxOG1C zrO$CjY+vLIm4*^2i6CmwoJW$P0A_V=1&DAGbP8e+8~}z2g7D%O>EIWnJ>d{Ratypa zL_8)4;}f0ejEh1;!O`5pf-gc1<)$Sn5PEHFz~h<^;?O@&WC9WASq=rnVv8(_4~BPn zg7s!7jT+kUhB+h|vB1I{^X#mXh-+3wJ|LA*u1{t%Od=|!wS>}n<#Rui!R995O7ne* z|2QuI;vz4>J}w@Cmmc(GFQJCKXc(evz08`7K6SM?a>|&_bRQV_D7+->0fRwWMCIB> z2*U}bhltRkEuxSyF+j4MkqkukC|ODFT{24U1oW(?X94hls4XyVNvOYc}%eCbVXnPvvf; zUi;d>gh#E+5%qWGnR8TiOYIRH4TW`wSgwkjN5g6BSocr|fhb)|CnXOtdJ zVFPOYC2$9$3buNtj-8^bNky6?QY`nGs$uR)_n1=HsG@eQD5IbX*t+daV5BfK>*+R!gsUhgMb0_DHR6T;UrEPEr3y51)VX~B+q$+ zS!Bzjn^Ej$5fI`mkt!67a6~WU6xwfN!_1Cy$}=DhO>;j=LFWRpqHw@#|L;1}87)YP zNt1f9cVS#08P@{5?`$l2bBvwy=EDZ`ov(4?D`X(+jI$7S4}KG{71jp$zi0C?kWIqc z3^5~PCGK4x! zwJ(G#sWRcV2VuYL9X?Ca?ySJ8j}V=^ItS<$+x zxuYW%F`PL8wq!mR#lE|<*zybmq7gQXl?il>kCqGPdC>|r5M4|c|6P{qJUY_m))F#y z#fx>b+Zf+-H@v$9gz*6}srLRf71uLrep+RErAFA8+pA>bBvp}wl)!Um-9K{>m8hFD z&%7H778}TvA~UT8I8wX1`U$&_!5bK}L6v3?kMzyVb~7BB&E^@Q+1Y7%10V(F20Q)8-zoC^M zuxL7Bx3t{)h%K9~K8AQxyH1k3~O)fewtk4V7?Qaj=#qc_Md3B;g|E9mY^WyU%SE9}elK7a2 zWV*Wg)}(q2^N#O;c)d=!a6KA8 zer?%k!A#=oHf_TM-Om{1v!Ag4n8}MM=#xT)(J}9_UV)GE={~=@oqloh*n8rpFB2%8 zLZ7T-J!U&d>pwMYd`a+s|71(caa2(^T>t}|07(>JSjZAasK8v55?;_4r+lnx*;MzTd48CN2MG`Mc?(v(03(G9YBGfWSgMzoMHIY zwM_xeQHrC9-(!`Z`E_1A71jDdodQstLseb;shaAo|6cwDSL+>L@thJ9v;YIJjN9!D z#f8glEtCp02>$s}G1-+i{Q(ic1zezm8mhwtGTaA*pz+0l9AeBI)?o~G!5x+Y9cmR$ zDaIpUOu-Br^sz>7NZ%M#%Aep173|=ooQwQ8BE~h)i$%hCo!bv`3=jsP=M~GvfmO0x zMdlb8uc^o4ID`{+fDVM=6UGK~2_I@;Oe0(j4FupuAmIRdq9q7R;%CUV{nc?3PRLHP_(sG(0qoPnE6|Amko%KfckoGjm3EYVPbLq0r_Cqc{+ z9^e(mmK+N{(l&OZ48qGHK!G^Ii%v?&PWGf8Ru?*L!!?kf=A|R~y^|24pHeF2 z4q1Q^D9z$(;0{wSee8bBfZ&fX2C6vpICawF`K6HP8=H#VkYR@2_0 zqa}{tbtL6FTILTC;U*o`R1Q%o7Ss_f|AtobqhPcm6c)iC$3CwF$|E%bs=($LXiqPZ=ld1~V4iHT9Ir`mmHD%zh`iY956CJF`yBnihv z2mt=9T@=Np?(y0uPQ-A~fgH>kjm!r@$cs~)ghn<8CZIzU$c-_f0t%4Ha$YDOJm-cj zlkrvO?_DQ$dZ&nrsATrdB#~!Pip*uEpSpz}0a}0_q(Nu0r>N-@?VaN6b-_)QQ|!@= zf(6JuIzVT7=6dj+e9*@j_Fq;$|40)M6>!$(gXTvu@zT6h-Gy?Fb3UKnZK(6fo|l5? zh=wU)x(0X-<%yoDiKeNFGTq1EL}tlK8MaMk`6sAxPQeu?eyo8F=qaDlKp#xtYE9W| znNTKt0taE^X^PSYbk&ujj|E%>Dt6pPO=m<>-S!hu- zK!$?BbSkOed01-5q^m-vn8vE!l_`qWD$*t0jk!Z^RDw7~ov-%luckq;R)MgJld;z6 z`LKs3Ng1l~7^B5$W`QMwd8(l}2&)XKa!L^aLdCQk6G>GnCS1Z21i%smRh&ZTL8RQL z-sE3;Aqtd$0_u+@08jAM|7)v)X^6_|nA%+(ZA^IDD!|^Vq@h-N4cU(!tW>ZX0_55} z;?KS@YqS1O1vmx8I)x3;!;~QcC{Syn5{SZ-TG9|{w(g!;fx>|#MkkO9hwPv5`Go@@ zP;hD_x++F5`PM9uK)&%z7g}f`n2P`TpT3MKzUHgGj_5D^E1IV0IcSHPeup~{EW$>u z)Y9pYBH4SH1dlaqGpO3Csah47!-Fy!$3D(4HNmS1N+GBvL5eDwEab0^MQ+|5HY#cp zm@9Ks&dg@z2vF##f*B_*RD14T&+>r17H(@irk5sW(TeGJB5l%6=9;o8Jyb&1<>}UG zT3C7QkWsB?;@7C*|3?+*3Z-T3Xkmgu(bx9?q#ndV&>ZM#n(a4;o4h2el^Vsu0T{-` z?cf2-_CZ0W4uo(j+9GwTy6#BIl4`d6Z7zO9sGaKWblA^EZ%K6qVpi|s!k*DOF62@s zW}fFc41@RX&_afu9hC3Abtcv6*8o^0%-Ba$*??6|Tz+^R5UBy{CMsLZz#XIkM7_WO zjx7B3X6JWrT25;NgL(~P&4S*`n>JQ)^j`F4?1)ztiYUq|i z=eJJb&|YttHm;c(rP8iwyv18)deaLd;cQ|svwZGV!c5jVRR-{rwbI@%k}VYG1)tEC z=YY}@no0g#|0+Mtjo|HW4+x`gI)DKD0Y%oPJ#fJdOsNvE3Ez4d39JC~5&$jj><81U z2rclONx})fstGeL3X3Nk3+&Q*4-2y>b#P*HfU$gWZo69D;sk)~y=VQ+9}bUZXyWgg z^#z+?OWP@@YYK2e1_jKd70DT&M=dII48#LNFg)17HdgQx->ejg1oLJN;3jW>lJN(J zfGdt*8tYC8xABO+u?nxSWu9ldv73u_A}-ofzWs5Kj(700W?B#uG)z#d&r*)2y#L%v{N&*{%w>)(;rn=g+8hx zNZIryYry0v!yi~==s*nV^}#)$gQ1!NPKMQpXlffH|bI%aq7$+~0x+ha-^ixAMRDUb}cC>F*Hj-BM7CRu>mWEbe z|1;<9CD%0^zTDyFv|fh-P`XA>t1w_M zrC=8e1*qpyW9xqY@mVzW5j}P?yN^Qy(|iyCSFaM3ML?6z!Z3SE1xa-%<5HurpW0-W zLk7udt2C*SU6D5ODJMl>dQFnvbez2;^~$qO*Z1NQQz+0XU)ORS2lgFn<{J?7e{$wx z2LLJhz*0AM`y4ZnVzmiAp|A-f{Q!Y`%nI26V|;ti{vu3YR19@*c;zfY2Mo;d0&pUM z%WkzbT*LMsKA;D&>Mr02-u)o9Xi|2Smh z-&R5RLNg788#vTtck$Jc@EMVELwAYitSmVt7_+8f!z+8sAyx<8ti>{{p1BFr^7dZh zZ|iuA%IIs_0&8aL!x4Dm=obLoFja2_!t`KeU{(Aom9!Aw8%|`P1j?O=fdEB9Bsc&y zKXz!33v%>FCP?{)wsP6(s3SGiNo#pJ?@Vf|_TWmYjJq7FurZqFHjc}&n}-as;B6H* z+;R+f=Qc7DdUBp775SFVX{FYzKtmy!>-?B4 z#6+|Y-7p@QH<^TlRGK&+3`trc8w!4$joY^y)9MPdIzOX?8hXX$zPtR!|9iai@s}C6 z{PKcmMXW(=>wGPbt*q`3>>X+#=q2zk5+`w!HbR3m8p1hvgTqNOIkB{Jbs`Fy#d83( zWV>H3OAo2C6pcFmDc|!#Wba65x!3m|5~#}Wwwm)dOB6#3$HZt=ch}Z@)GF*+*ZSv{ z>!~@}XraRen4PLwtj4X*GBAPF1XBG_z~Vfd7PuA%Jk7HM7b;YJ9fVSuS-O%^`NU&# zthYh^BEmh~s03trALndf7W>w)+>5|m$)EbkcVik|6y5(MYoL3&zx=Pjd@irVJ)jrY zVx7%TZGFjO`U*V)lC-`lI{Xi5=(|Zoa!Ou8E|9u1EFF|N>Q@D-R zZ**0Fk@PkhYze#SOS3ZlsN8J+c_n&mzqO=QliLPsneEzO9VD z#Q&0IF9%7)>@v*t%f{cICje|L2dbACQzat3io5Oc#O< zKuW-XTMG(snv1|^m4qXEi}dLGw=5$@HlAFSkY!1qGCQywQ8a2!s9lo3oV9$z!2=+F z)THtJxpU^JQYaYJG9f9^)T>!*8l+@PY%Hk0*-Ew~94613=Z#Z#dsY*O!u#m1^yxUW zPjLqAasK@Is^E#tb*ptflO|jEk}WIs>eQ`e+os7hc^-Yql*rh(ch3^O%lItw=q!Oq z&0L>GAsl7BdMN=Icnx8N@3?BT{{Gw1|Gyp^;NzmnCi^3wapswV7ERu; zLIx*{)MYv#gtNgmTkN>f4qCeLu9YsMxy8#tO0jODKbBa5u$}I>qoOV|A}_r-c-VyPvcHEC_K zw_yIjpsr@9NX8aSj+=JNHZi(ZvPU9%LM+e-p>;X%-sH$Ua?CwFbUU%KK5H$Q!q%y=3%VMw@_yoc>HI`UFxMnhG0ty?}#8R+Gn1c@JfkNOx ztUQPXr#j!%`sKUv$b|`Nli*3W-KyceI$rw#ouI${_PwwGe*Xcu|P`T3cbW zL6+(UwI!w)nDAI?617ode@S3{3OqYV9CVj$B6*#D`YbLH=0@@}w zA?3w2XhjTbki$@~l!aN!us{veg#S4+geeJJ;3mzO#|U=VPIp355(L%iWG2QU8V2u) zSZtzLnA0Ljsv@HzdIUp5K(oPwu6{od3Q6Cg%7R)f}7dQGjx&BEL!(`F|N4mz6 zn)IYAWhqKqn!(vY^8it$X`1qQ(>)?e01RS~3uv*!W_CUX)7G*T_YK1ev zjBj~Jh{*-$2L^KhbvmyC(8|~596$#ty|`~MMx7t)YF+A9H@n;QrR|9@oM8>Cl5R9hU?d_mI6h}3ntD_Li?2ion?M!Oe425DrO1dH z3&E~9)-Er_m;xUA7|7u*h0n%Y8)vGdiv9^yI?x#kz{;0 z8P&4}LBI{e>QrAj)=dHU?=T_NE_eCM2RD|@gw0k-nHAQ{TiJ^sk!^-PdwRCj>a?l- zYy{8%Ljo9TP54;ZC@*f%!J|RBcVdH=fq)Ql7>#ZE$!?$Lc%wVU3t{VR@5ACd-|K1D zw0PF^Dm0x9PoFH5r~c}%#YLNjaY2@}i!NmJQV{f^XD004Yh=^A)Lj=%Lmi!^AFBh1j|0#wPicQJg|(ejxkrnt zKt4B-8x-#)N0lP?&i5!(LG}Xwb_WGtc&S99539FH0;NQwnFRpsp7_MDDgW~@0DXXB z+-o>C7wMyqqhnp#@I%*&PyWgH*mTP4+LzFq7B>~|*Z!cm&za8ZEci~PRXD7( z(!d5BkK2qW{;G*T5ba@}kLLK~_+HFPH02VwXZtFz`zr79bgXmaEz^vy)6!4G7p|DX-{zG{$W==balW~2c4@9C*3qWB3egG9NAaEq08$6?dX2wD8;%}NE0$a>D z3akP>;{pvJF`^~%tndmy5Cr*03XEoZ>TMPbs|3Mt{Z4TVSFrWy#TE|W*j`a=W{;=j z@R9Pc_h_pJ^-%aw#SGFUbIyRQ5-F&(!pwR~Wj-Z&nrrK1i9?Kv#4bSwxTp#>(F3zk z3;$an3leC1ZVh$r%@mjr>u1DF6xTvB?5uOx>GEOKV!7;+)2 zF=*h8mpG*Pq~w_D02aW*B8jZMGIAs1qc~cE%-$e1^oS^A?G4*;2k8bLZ}A-k(Bxzi z5oogR8id=XK$L3ICi^b}jw=n220L^CAEcn&dU6x{>T}j-H!x`;t)&D*O({>XvHw)? zU7oVOS`YB3(kib~h88oRPzVRNG9F9LpjPrLRlq4);3fZ$EnMc~xbmvjGAAC$jKYlr z>Cz{EvN_;wO};KWJd7eQqA&gOBGE(XY7_OwksNUWH|wk6qJSzdOC4pgL|_V&c8D{K z6FG5aXi$aPq|Gz$kTQGk1C$~&@-gqgrZdqpD}U=av4aqlAudhRG&vD9Q`2+mKoHVL zevb1HQU^AN?EGd^PHaU`==G-Z1+E0;`YS)RA;3nBEzNB+2ehzEYe6H^LC5kebKcCIk74_4RtaPb+$@Hfew+B#&QvC!J(R9QM0s5 zFBO$ARZ}~VJj*joLzGO*^i1<@KKG4Cm+VagHC5e|Q0T8M^sC`=txx}L4c#I@tK$vq zU>XaxwjLBw=`k`FMTndL73APidsPxkQadk{OFy(z4Z%}CRa(n5JpV?OR27S*YBOr4 ztSeG21KxC1cNEqXPU~WIR*M9=$SzGdixOu2R%8o4qgZ%^CASc#clicXa9p%XjAiA6jnYPHffs{ zaVb`D7uRC@0BUP>wX)72Wsa(HfpDN;3ZO*`w&#CxAQ!&D3B1-0>2O|UHXmVfS$S0^ z*~fBO=vjs!9IRjzxp3%MXr!Kng+^eVEVp%0AblPUOaXUj5jJ5F%k&C2^bR+18TV=! z*9Bxy4cE~A{wV(-Ksri85PpOTQhhCC!}|qEf?aadp$vIxkq;M!gjSodv*6Aex!wJ7I=XcfrU442^V?QMR}K(c^g-P zDqjsIS#WsemUN(U!qK}x?ON!sUr zmtdlBz_o7U7k?(+Zg}nJKxJ}feJ5mXGiQj@wgv3c}%u^|li< z0g0QK1TsBe-H2cUs&+YFRCV0f~4vc&ShThNU2h!|SP_AcRY} zSwrQ|=z!i-xZ~!@px`XLv_L50CoWdNjFce_#A$Q?cZH;5-hzyO=0*)}Z5$lI*&tzv zscL{jOc3rCJtsm6I5`ERBYTG~5$JY>lFf3H?SQ`+j1P8j;nQg2q>Lfhj3*dX2Sf>f zd3oBHdKd9}scHn|Hns}En8R0(-H8vr!5Q%2iU0HX6SqhhfN+qBM0^}B0ism)W^?cDrt6$5FP)=!3ex#+WqFpf#g=b5 zd2^Y0XLOgDu7AL4m>YTn;5Y`-xpAxje4ls-w8IEKx0z{CCQqmHYU3N0p=`eZ&gNm8 z=^{U-sY;3^m0AQ6(UEyDhbU5@8=l}S?b$(MrkpiFAv0BA$M>E+u?x7kiwPE513HY0 zRz=aYDRXm|BiIrw#f_<#d4Bnn5rKB|HxxrSn%%ilGIy)P3#4aZEFbkNIx?keMFmzm z3Z8UmY9IpG5MVN76`mkt7|Ed`P-m8;wExl)sP~v{9<9b!NlXJ*pa+_u4cDM`*{Sn0 zaidzQho_q|W1ZGBllY*=GCP04m$SjwrvI9d^LL3G^|Pt13kcCn0#ThKAP_;r2E^$B zgs5M_<^{)0g?k{jfh!6`;8q-pu(~a6(o;Rv^KRVA=5gc4y+?}kh*w{H?gVPx*OYK zv3q!A`dfoJL;A*JmPCO>9J8_RZT}=%C$++dBc^uE$O@iA7<_}POJM_2iR~nldid}O z;5xIS7i_*aXERG@0C6gW9L9QjL|kkZf^6A#l}l}}yr1WuuXsJT1zI+3qeAs4HvwUx zyTSp}!ZCb;HM~Ti>*ih{u2FXRtZs3PP}we#z5>S|6{ZWF0l{~H2OwdX(8#?}x3(bE z0bqa%d|Wm3`-O^J0PqP%dfeuqN)a=(6UIRcdMmH$dXWeRl{Iv@HB^Z5N{``8nuDW5 zf*(0YE!7mNDs|WT!w}_ zL#n=|98N&H?4}G}0a}#g5C4UstaevP_`Jp`w36`drUP9I#O7-}7;c0-pAbFK*Od)a zc6WDwS2RylW1IhvUnnKA7;gkc26#9otuBG$X=C3%Rh+|##w!oPgFY2C4H9gQzu z2r_<2z%!;sUM@f$#{Vb~0){>08^{mG?%EwAlG=C4mv~e#l zfIu2^2ctj$@i?;N6^PQPu$&kJ;uLoXdvJK;2plX&N}J|UXlTYlJaD84XEF7n5MNAF z`oiHSOUG35_+HU~P#cLrNGqCy90wCA1p)wYZUAwC=m8?E`Z)TA98gVY(IBiK%^O4q zX?@-Sy2pn)5+#Jxl%nPcCOjelZ9!4v#tD#}%tUVFXhB3+Xl^A})(dA7s} zi7m#Mj?^UpBm>e3sX}EyM{M3E*Vf)dIQ zMD2xe5_34wu~MJzDN_qhU%ID9OxArfkx41d)Fgc(MHT;OM zl3H~0(Vy&sb_pSfA~H`b3aQr+qwHPh8kU|)&`mggy33I`1_^l5$#?)r@=d6!I+}y4 z*vb@2MJ)o!EJQ&P>nf#`vg;-r`YLSEMwfUjHN_&E?6OYx^)R#ua4;Iz+;kSSV zWFbMB$X?>A#J(*kaDfc09~1Dfx-5x+26D(z_9#_3Bp9MziK`FY&eNJk35q|L3n55c z=fM)Tra=taT?-dvygkTphBTat4LOuN<{b@K&^wl0q&K}jfDdY=xgPekceWrUKv~}F zS|!2pIVng2*c;slpq zY*08j&MAzeh+BYRU~2@S=jKR969Q6@U?`*^UHHNpDsqv?Q(6vnXtW;s5Qsrk7LBY2 zB?nMV1}D%(z@oDxVAx<=O`L=aUeqlMP|O@v(3dBU)HZ0LX(a7OPf zLZoCWUPQ!9+GvwfL|X#727&}^Yo2?lfn{jR=nW{*P-QZ6qa2OqGpAWiYKHWh8_cFQxBp4BZ|?9z;2ftp z%ZZmsq|=DmQ^DraKmZpIBc9um=REJ}3S)%N1*@c1`p)G~2MA!P0!`0B!FU>7D%7EL znnOV?nl~{-bgNUaoInjVhp~>P4^K(l%c`;FW@* z3ICgvYE{ficCu`=V;{9T691eWOej<^ZFdh^R@HuV zy{>)jTHj0C_%^aFxUH{!dAQr?_0~mtbyiZ@30MZOqLn=@aB?S6i6}Ogifg;#jieg| zDB0u_ouHi@T!)GZc0v$xG^P^BXu@3@a)sRG0e$e6fy**Pg-PI2Of)637KlR&1kGw{ z)7#P2wzj=hErV=-EZ_LrmYZ{BUVi1WU%X<{MuYKdfCW2G!Uh$J3_z@N55-t6^&tw} zsnQH)aIYqSD}a6z&p3z^RWj8=550-Q21Lxb)k$oxaK3~=3h0LU;h;Nr_QY&(@CZ=3 z(SM*?$R;RCFZ`k5gO1x0bV0~C+o9{3l;*Ko+ZtqyV*k`C!$9ft-qZu7T;o1|?Fg8E8AIARPP|H2R=({2PZYkW?K7;AsF5oX0%6p+)-%1tsG! z!<((NFidTZQVz)kV+?Y(2T&Anv8mXbUm@t($xay)=|Lz;%r3EN6bco}QWh%hgQR4n z-0))UAuo;TYjS$jaOHPy0$M-8+tT<3GN$JX&QIGl;Qs<2c&SZ2L^>b5O|!N@lA;{tRstzA zs#FF(Ai&@{;ChWOSj?}9L;@Bhso2Nfl0~cd1{q|6+FE9l-}%nI{s9LQjZk@L9x%%` zIN=SSFf`KrcPrSGdl1(J`nov+1_xMR;u6~yy`#Mn82YsWFy{GO!53_C#~i`%*iu@vFdq6;PW z`w*sY_~=HTNs4-~Z=}DBCw;aits4-}YplkM`d3s#Z~5lAoq70_5t<`pdecd-Z(j`A z)2Y9&@;VKG0myHxM{s!+drPKCkZ~C$u>S%Lh7ft}@fA>Xz zV<>xRE0fIc@j{E0w8I% zK`>?XKR`ePTl4@e5NtxChmF;U1keClLjZ23NuIJgFZOvuC?y(DO?-#~uu%hXM**Yd zDJ!-*{PIe-_a9JrD$w^YqZ39}82?hZGm9coVfa^D{P$@^=2H3>fU-c0W@vzE*aVf~ z31?%9<&p6OmzTmvKRqlZ7Q&ceCUma_-Jcn7+VHeAr^xTc!7*4h#cT1 z0VRo(_=ZV>F&yCq?lC*_AwfKFXb)pykF!UhF%v8(Ua!#!4MQ-ib9>M?5vypB@s^7! zQ;S!Kg&t#eyO?jkaC*NOhDcL}WGIZUcXhBgQ-RR}cc6GTr2;4Db&fGUE$NaE=X;03 z2gasYtQ3jg7#d*&3R%E7OIQavXFFkpRV9G|0ssT>l7Qu?j!LB&10X_^IA#Jck2Jx8 zHJ}ACQDZQ`e6eu|(c~ZgBmW3EQG&rxkWum;ke4~G@=6BDZo@Q8DPW6b@B~{48xh4} zyhsTc`H@7nk;~(e1GtywL`kzJVNEbGmXVT)DUIawk}j!y3qU>=R1Vnk8@q;+u3-aa zw}SHM9{n?Kc9&2I_$?OI8YQs=lo%#k1Ot%8El&B47kB|cGANl?Ma-6p`{)Cyk$Hf( z0aJh}ZdejKSC(87LGsZb`Vl%!co8XaVm=l#5)ysO*$oo{GuI>v`-Xa|hg*Niozl}H zF!eMeDFin0WjvLRHV^@W7>7!vBy~t;hf$t><2SwZ0#_p()G}q%hK?2xQo(7-^HFGwVwFN8lWhU_m1jQW6*Cg53n<+pYYVZj+7YpO1MVwnD4nAPoNC53v#rA+QT} zBn+`43?|?VS(;D|dj%istDnfLC71_e%7mXeJLL+e%Fv|qF*_YWr`0+O2101Vd9tG= zqU0)nVPbw(DT^dPTDHRz52Q1&BL(ah6deIP!qcun7ODLQhW|#dmrAwwim!yZhy3~( zCUGWufd2x5H(UW5un|D84z*(YlZT#R7{X8r{s0RZVn-Dl4dDQ>4jT@+TCvqsD^AgD z&t_sC3w_>JZOO2Bgdn(|H3F3wOdh(A1jH#x>k$r7r-k53GGQP#%cofqcgXnxf=U9b zsFsL|i%o&Fwv#gtk~>XnO`-s={br-uX|GjlwfH(&TAN;qMIMDf0V@$4Dj|!ZRkmg; zLhFSb(U7-tE4RX+3jRR1;Sdd5V7$8!4G(*<8KHIzbg}#(i-}N6R%xu9(}Vw^n@A^w zx>}+u$e|1~N0Vy>sllf1XOZ6bZ$AYlbfLs`JtYtxl2eu>AM?|H!44ArE$ux)tPCMaGmjL z!1l#@1&pHv*EF?jyI3^8&=()_VF2uB5P*cid^kbhilp*0t8Nety0QfvEWIBb4Q(vK z(c2*Y;IfCbLUSB?e@H?L;RV95{S>iU{M%0&NPxltytINHt~xq}|tpVW}I+EC0Q- zY`hm6vC8YlZR`&-gA~GG$6GK0y$nI!sERrF2gGb3jSQDk6~k_6vpV1?@%OD!GKxIx zKTwE!T;;xM$(-$W#CVBKlfcBw_o%pJ+`KonQ49Z<#1i7dOnxUKi15qfDwLvj2 zsJa0dkPO5T1zaOHOV9+TD-NBmAj2fKR0%w?%6`G4x=t6)<4jk&6~zg> z$%5(0pG?iS0>Qt#cse{#r%NAT1yU`x0^*=>Y+^wgh7<|?#%>%A`B2ct=Kl|C?7_8S z1yEfFAePk^k!>od)#M?#9LRzzbH}etvXB6mH)%#Lx`MVN$YH`6Rf*Rm`Vssg1%$e6 zL9se=Ejm#;A(2yhvt`Lqd&!k*&Qy!eld#V0Y?h4k&TlA#qA#xA@OCaoVX z3dxni**}muEN3Zs${E566t<$+i(mq@;M(`UQ5?4&=;iIW3Id*9NQ3k5Bb36Y!0y>T(|R}+Y)X9 z{J`9_O}am-8tKCfzdAt`;N>LYoAZ{qg$|@8DdR0+%7%%o_bb@i-5dEZ3olKTMk^j# z-8zKM&A~$l?+UL~0qTA*uTZweaoafC1$@y8ok8o<=TB>xW3=5g8=WT9m-QeI5i{}!~4|ZPY`v4nQ^9>%@WM6|0SL4camm@t@ zeo6r#3?R^>#9W*wd2MQ#+rAhiac5t9pA?t}k}jxLi75zY1KC)TC?RJz)9Jmy@BMyy zQXU3BX5>4qQtaK+m}+p<oUdaPx+cOekh_k@vHp;i7?jB+(#8P>yuuz{G2sl*9ZA<8A-o6i%<1g zaD)K~1pWMLWNrgeKfW=#2vt#P0;m>1-WSM3Lj#Wu1kclEugTwu_M5!+8N=}C1nY61 zFhc*hk6+05Cs6i#)Uu-OCTg0vHxlo1`OkvH6cjjM3j7gv}i(+LfkxB zee4WeQf1yZHJY@bP{t9{9f4IuiYrvB79T;7=)|GZE7r19J$dpfh71@np}ze5k`%_! z(xp13J)@Qlnl-81s#3LzRqNEPLA`zjYfS7|pk-NsI-6E)r9NxjhB*>f!(2Z{+1{O$ zSMOdwaCOWD{9uD%7%~Wf12VzFV#XK#&BPOay!rYvTM#R($S*% zL!D3cyo#t#qZamUbv}Olp6mlNVnn!Ew!(!Q;b5@v1?V0&L$`)7kW4OKI8R`M0}w$P zwM0H1H28o@NOebMgKk+G1TH1Wncvt6m=2GMdiXQ_R5C~zEO9C-3%i2q!mBW(iYx{{G|R2FM0Cp(6MJF9GacBl z%O|`}+-s7${^}sG$`%;mB4t!y00IaI=+VbZDxxEy4~T1`iB3N0tjV}I3yrj0OfxOD z*}_3BOMf&OsY?_1*~cFVB#2F+-<}}O0RiMN2bw< zm@ZI&S^xs5)=(m6__H!mKatZ(C%*tq$|dtbHs}K^Z4xN@k&W)pggbcrGevprai%F zB+Un=NCQ+gn5>ZlgP(J$1^*dj3T6reraSL~7m{R7K!v9xqzT8D5L8E!S&(2>u7QaD7=mdCzRdHUPZI8$OuA8fkW&nWIW;3aCkJtq4G+EJheRSFA$Q^ z^O~iXGe9d^3-r3z+!B|K_x2vrf$I)2PDQMun`EO z5H&!78s-5w91Nx;Z{&#ozz0A&5+MmUyI+;+(67oQugfyomUH49S#?yrLETj+UxQ5wr zl9P_`(EsIusFbEWC@M=?JRQ1ndCZeb2JE6NArA4D`l4R8EXPX*WbK#U^xEUH zK+IxtVtl2U()y^B(rDDcANz=(7gFz>DF!%#+-e3dZ3};XYiH2z&38+Vf zT~UQPN1@&nok5@{A*HHReO8qS{A3{y!2-~_uqX^ymE{aaX~Tr(@RY`LsL_t$Bo+k` zdbR9}^rWTB1Ds$5c0EBaS0b4s&503mAZba>HKp@q(R`*6&6cvb3Ns30sEBj|WZn5w zDnK@pll3W1GyBt@!r`cz{VW(h3tG~?Q;tj}XBGrE)jnEvRXYrki~{O{uV&Ij24!ej zb!*n#vi~xzu#M|niXh9bzIBMOlpYeXrvi?8l&{mXWMIVynv?D#eZRr3`b48e?0S%o zb?9IRnGm(`l1>+B0%tkLOH$ANR3dys?`F3k+W$T94pRW%X#aOFmO-Jk^=(f|P|;ed zg4MTK6-8IQTHArPsJCM!>w|08+ut5oX0Cv!aEs8^wi*{k7qyWHDyxhoa#R7%oy=*! zIYlZOGkvj9W;lc~nk~S%ypx5*fXw*CTm;8!10q9syAZIifo>Mz((D!h3A&QBDZiS< zNl26=6Tz+610pcw9ZDkL)BaI|f`Uwe!AgVIT9~UW@+x`&7*+*UBbB-BRh%s(VXe}+ z!v7b(aBwxu;SOVXb}}*|)H15XY8@aC@p@^~_obV&>;^q-b$Se@fp@owN+g9m0mLO?1MV*D*7mEpb%`A$4zYQ>T7web&Y@i$xIPG*Eq4%A9D(XDEuY2L6-Os6AoKCkhHViP_$?p!)sO1aETjkraJ<}xJ*pCrQSD~cM6@- zh4+)lHt*l$yNn8-G-xudkbE0_;GEuv1ibrWG@7ae7un+r=z;NxpWx;GgTl(`egS;g zLbDYp2R`nAmv!gE$KDZH(J{`%>7~u%i-!zPDJL>X*8Ju;=diQ=7zWvilG;AkErkx^R0ssF?zpPy-fF zo!YCLjA)O_iakZ!J@HXEuhWejlLB(_H#8CknBoo2>I66pDzr zmRPrL@Sp5i2~csr!-y{s=sS;S30dOG4} zi#z^1gEiPf8n8bGv#PN2JkaBrpUD*g9KaUBvpX{u1WdqK@(UywE-QPzxex)#(T_t} zK@#W~KBNKzZ~~6;fzrylDe6EhDz(@M!7tbi$AW|ua4eHRzWg`@82=QI7F0MZz@r8* zH5u$2d@zV1JO~_guPkVj`KXU7a5=yV9F;4a`k<3Fnk*mSk0uh8kN^iN+%G1KHOCW# zD!hXw9Em~FzhX4TFl;bRnuee&E1J!%fE`N+$7-V#WH><_BT}OngySQAfudgu z2R7m(On5oMNfL2L1|+LOL<_}kh>ZLil{JzwiZBPQNhdDIl!y_Pj{Ji<5S&_T8Lc_S zlRQSw>p1{)#-OXSG@M31lSwutH+Bp(ZwwspiY7`)0gBm^N&nG=FQR}zyhJ%F#6oPu z7%-VrTg2TsqYK=P10%a6`&+Zk^TNVJgE3G^7Wz4sbV&n@ zNmr@H$Bax9xyGZsNu10^aNL{>!UIkek7^3Uq4Xe|v`nW2GCY)vgpr(0LBR^*F@|gtBdty-H$)AF#Mfc$wZ*ud8xS#QehP zq)zJm1>V87|Kk;Dgv^=b&dFpJ?=+FV=)I)d#`8qa^#6PU4r&iPY)`u+%J*DE%PdXR zh6X@!||f=mcTF10+S#`HX{_kUij=ML;T{DHTc1n$Qt7&kDuT&(gY` z^U@l%s-;p;Wb{xjG}AN1zlpj$op6z{Aj4%;(KvI_Ii*u+JO`w12K=B06qVrX>_=I2eN^qIJ zRds|_F;<)V1!WD^ziQS|9^N`DIDJ#D(1w=0RW-!bc^y4o-PL|Fyxou+C4UW9V^O?+C0Ky< zS9I;wa@AMId`WeEuy<7@Sv5mvtXBfe*o~dj(lb4T^{|iy*@P9@k!{#oI@yPHSY16= zG<;P$v)EZ}$#}(BmyJe_rB{3P*g7@Yp8rkRpY>UwqmDw_6Orv$$ zoqbxUCEB1}mZ@!3p~YIOEn2OuRiyP=H;t8NM5s80*NgR)oW-rE)mgRO+BT%xw~fiG z&Dyr*T6--hwVl|CRj8O<+L>)kyiHrQZBfBp+q#uouHB27Jlwfu+*~c($FxH2?N|mj-~*=H=2c(|=3wf5 z;Pky*rWN789pD1);PEBhl|6^MO<<=`;S+A)js0M3LtXThV82aQ7}jA7rrQ-}+1)i^ z9iCx|3N5H{NtRbuYE*)DeC6lUQi z_FxOX;w(nvi%lpTrs9YNV=~TO==EVZp5r>E<1)TuS~X%U{y*y-;XhvAFLvW89=bek z<2N?lL?+-jHR0+#<1_|jE&qO9^-Wqq?%+a3WshcQfqrSDCg!1@X`=qT zqDJXJ4qjIU>ZK0p7|!Ua7HOic+^fcE|5ImG(&}ma=b!FsAzoyC2J5)K;s7M;q(13` zs@avEXolM9t#;~&Zda<7>zIaVN49FL9&5M;9u)rT#&&FWj%uLZEy0%R!`^6W2J4RQ zYrbyk$A0T$!yfINW^AeL>CNtJ({^pqe(lPJ>e8lde%{&(&f(UsXSOEn z(3WN2uIjNB?7KegTgDJ5PVL8S>%BH==AP=}hHcai>a+f2pu26R*6iihZpl_@;EwIt zu4}Qd>rzf`W~}yW@?~W2K5g|*Z<@C2!bacgW^LVu=V<2c$ewHB4(8g9Wc}`MsOZ|C z_Tud(aQhzS=!S2&Rz~n<@CJu&bl&2r$ZpT(?ZDmbty*o_-fv7s@ZElJ2mf%==Hu3# z@cI^TIL>elKko)t@!w`|{)X@*MsDMUSPDn+@@DZB|Lh!ZaTr(VW~?@q)$1Bp?eaB1 zfu8aAm0oYY@B;7g;3mU!2=UUU?*E2{fB*m?`2+(R4! zui!v}2Ms1vxUiu>gbX1{oEWj9#f$RZJ*3F7BgT&oJ#Ne>a->O;C{LDL2{NV1moDMO zlu5JZ%$pr;>eShiCCr{7eFlw36sORk_<$n)d6cQsrb?UcG&;4aRhURrpXt=zhK{{|ksxA0uRhYc(4OHQp| z!ighGew=ghe#h4(%!AR zsBf&dc?Z{<*EDY7zmXqbKD;?Q=ef@sPflHVZ|m5nM~9wWwD#)PyNmy44*a=!?&H&c zUk|=LdiCUozK2hLe*OFIe?q z8|Db)iU@Xg<5fo1lw^=X?&xHXKMqNvlvEOF<&k6+iDi{fb_w5=TuvD#m|IR%rk7%( z*(8)>sY9llYO*=zoJhWTW_c3c31^yd(rKrk-2EwNoq+0D;GT%e>5h(lCR(VYk4B2) zg@a1U=cSECnW>`dENbbekWw0IrJx#0VN|7Vs%ngljp{0=uTK9;>a3>Lis`Db;@WAf zyr$G^smRO6;y{=89~D%Qoxmud)s+?X=V$OI5MSTC1$C&{o?mgx-cL?5$a1 z`|Y;qKIaP23tFL+k@4WPiTkpN{=4^8IIrw8%B}u9@z562Xl_{lx+yfmFE=eS$xufvbvOZkL-p0lOr5pJF?YR+*R;ff z%GhL=4GP(xbdvVkYP0QjB5$wF$lP?-ZFeMi=dJhNeAEB^OWl75{z)&(4E{^uh$kNQ zGTSiT_%)C_?sU(cWmEa(m}g!zO-qtHXD@-nBKjhakY36neWwjJ*eNHIH8`-3Bm3;M zi{p&dve)wU?!24r`yaozE&L(F$K8k^$0vXH@~QVNM;*^YPlxo>PanPMagd(<^xI?a z{r9h!KKl5f2VOYgia)M?$B%_`_}#gpffa~iQCDp z5WQQ48m7PkAfP}B4rE{iA=tnbSa5<5Ou+>;r~(pnkb@rtp$A1cLK2!VgfW`q7eUcc#BsQ(TGUgM&~@oCG43Y4o~O; z{oqzVxOL)wC&Oa?45*7*1kj8A^Thwa7{GB5FcX(Rpe7{PfC*%<1Tv^#9H{`uIoh#; zB^+TL_gDf7ywQ(&gkvBD3CKYf@{ovhqaQIq0SIuAkr#NR^_*8d(^b-jJshGYH#tNl z!f#XZgQ6IsNXjbC&6HQnqW)G{i~fm`j9qNyJO*exG3JgBYDAzGD%gNK#?c7C1g0;C z$;V=?`;{4xePAf`eW%1~w= zvycw$V;c#nNHy#Zml)-PKr_lwjd~OhAPuQVrT4^=p24I-9K%Ubs>5D@^G2GY(>S?l z&UH%D4yRm$PInqbcc$^D`g7$hV`)ZMw(@`c%%~s(aDa`v5upYxXjB^-QC|WQs>U<{ zM7PRJgch@xF=*rgo=Vg&*7KD{-KSdD%7;X-lchY3D_l$G(w7!WI{%ufC~bNNQQ%Y# ze)#KO-&zNtzVoO5s~s1^NI06-00ofYvJD78K%**9sdiSXST#XtM=M&= zmiDw>MXhSce(EddBn$6)`+QxC7yr*3y^RE+MN1KMtU>mnB013Lt}ETH(E3 z`0Nz`LqNR0;tSt|de*)51!jD6kYD_Q*0ib(uz*EN0nJKwwt$o;LfbIU8Z`L9k$rD2 z^dQiAHrBTCB&kY=Yq(1ImAJ-*Vjz%vuShKyO2JwN_t=Tfvq^UnF!r#FVa(1uC~%~G z$fzvy!Q&qD*cN$+0yg(a2pwy|jR8RKWnp{T{=#>^^1ZKrr5t4{a~7&o1+ajPPy*Tt zlK_?V;Dq@=OJ{JI%(ZZEF0`P9F@w?n3CSn~oYzc+F9Nru=u3qu_Wa@E(sZRGwkyWa z(vqYhYa1(WF(_Wl3SvNmY0rp7jMP%U8CqR7o8J%1+eHVWh%%cl}bDQW^*Bs3W>LhP0x}}HjVqt%^?1w zeaQ{yp;0MaTsRuVl8)YC4XcbyV;U1e_B3v50p=u6ptH1AwX0X%>MO6i%2I~4tp7{v zaudJ{3YN1c;!ET>4^XXi#-*8adAmCg?%G761~2qH3F`WJr-9Z|I8{8!uzmP!C4&?# zw7qRKUfSC|&O{o7Ep8Gpl>iX``0{31Enk+i+vV+6cg#QgUjWA&p=0&xdf7YSeM{I1 zjP1ET$NY1~&iS7Iu7epIu?&5sB+{;!^ut>!XgvLCJQSZE*<#F#gu&?(I!&d=$Nq7U z=R@S2J|kqi*vJv$9LrdC@~dA?cX-#_<^Y~3&i#l8eD?ONfd9MTzet5AreaXQ;PWv) z%uNy-$|rg4-q>W^=z=w7<4_R$I~!~E9}8Q5p7x+W$9e$vs=ARepLgBg{q7;qT=lS* z^;L!0_I>DAfegf^c6YDc1n&BbZ+%9khQV;qSpwqEe*94)Z@sYf!NVtWRnR!Tc`$bV z>_P@|k++XsT&BPP0wv`C6}Wf>6SN-HGEcwypPc@=$NKdw=)2De9LE-Du%=QxXa4oS z|Ni?wxKbnvK!Y#1z-J&vbyb%w8IyIy<6>PmGF%gZ+Ax8Q;RsT|2xP}f<#2syM|R}J z2PhzCP;~&F;A0eae(MKw@K$~^S9h@|Ro^vKa0dhU_kV;%Q;BwiI@oxWLo|tJfI^gf z8{-*sAx;&yXc(t76u5*;$P9F2SQ^NIBPVW2rAzU30M6iJ&9GqYmxA=wg(%p1FBorG z*kif?cQV9(Hn@gth=A`yF9pbOejs8(IE0Mib&0}&&R2O#xHOOUgdN9SA(si?HieJX zeTaC1SEvhG$c3i=rf!e;g<#l%E?9|N20LbmhA~uwZP;u$_=!-&I6XKt#n5QRheHVn zEGK4p+MtKXHZwu@s>r6z`Pr-GDdiBeUD zo>x$5*oiv`ilF#xqu7KLXbh(K3&@v>(Q+!!QZFIXF|N31k0yb%=xB=rc1>_aiKs?W zxQ>Uoc2KZ(Fl!mC0xcJ4d*B{*%*Y@V2A1A zE)wY|A`>s;2nsrLjp(?Mz~Fqg*pBYldEiEJo;FnTX9dC7jQO~d`iP8l*N-sSZ7L~Q z`2#$hsD=jr>4_p13+}U!Ok;}VbOv>(Ex1yVN(gpPBn7D8gnfW;ve=POxo}cReIVHk zgkTFIla)Yb4nUTDA=Obb2|F9*Pl(WCyX1)bIEf1;S}y5+@CTEiMFDCMgPw3uZ$NA- zl#|kUkUAxRQJImWh>eAFd?f`7F9VSg$%^5aC|rY-PjG=XiII@jHOD5ENGMH})=@$h zK`E3%z(WXLS(0?=Ub+}^y;PSafCyownhj8XO+^N+c?GBGNQl69w%M9t<(jmo2ez4- ze7BmpX#}yEa=aM>SFl-fsa01HgJ(BYCHOy$@K&XCkUd15o>-ZH8JMzQe1Jxcn?!N26_iK>3`$%LwhMYKoo>L`Ik4-qmrhLnxv43nV9WTo<>?P zv2b1C;8E=9p74orm$_Ym7+EN!pDzGOO#lR|pmx9akG9}?z_epLR-gykPcdquQ6M`& zP=Zm=1`?_V`M_o(>I4Rwpn6cD9g3nT%BFBCP;+{wWtyPDP@-n0qFTV5bDB(uz(q>` z*NMN-3Y(ClbM>gCAgOpJImnQmIFofORSeuXE=2l`O?ae7dY*@n3E$v(y>m8TlU*Q% zSY!lnYh*{cH$m7m1sw-$uOmBYFk7XU2D4;4`OrbwcXkuNl?nhm!FT~#0|A6EtiZYg zbLl{FsshGptjgML7ND)RFb@_mtrbeGQ9uIKdaa&ttiiCYW00?^kgtvR3GBn8ew9)xby9~pq+U|0=BcpeIX20FsKAp&Ii@<80IR-* zHM^=$Qi?`d({b$BQDVtpGH`My7Xb!P0YG2?0I-EC+XO8O07V9~wIH*VWe!aLH4i9< zvj*U;FAEMpK(lB>vuO$dCjbsNo3jQ`0P7k6=5PWln+8uyvnOz$77$4j;HwSrO4vuQ z^;(@Q#jik$kUQ!)KdO^)U4w$y2k%^fCw^)jWx2~i+Yjs$K z^|2QfPh7ckq20J#@X0fc+9sAqyMgi`-!ua#=Hg$b#lFiL#MVF$>& zmT<5L`y_AMwy_AejPQ?xcw-EN2lEuNNXNJMw6Pp|OIYcFhBZ9h7nW!Lcd}V3y9{8m zpi8t2;JN5ax)(sXl6$!k@UxdoTbZi`2zFjaroK&p0qonq>{?k-AOHXbz$izx7=W@P zho9XwJ9#jjI7z910KC@8yAup(Uz?C?>n$Nsq0FIov0-(rE z?6M=rxKO)}mYm5cKnO&O2?1~cnJ@sLjB>U>%Ih@?NsG$&cEFjyzaTgOneel1HOaKh zX_IWSG#~;Tu*v+2kOr!RilW2U#YMjPw+{PK~C(6r&5O|GY>z*Tr$I<(r z7>2#p`^UAjUF&$qs2amRRSJNMh^{&X(gXz+FakM@Mg!0cz1B=XOl!|1GtV|V3*z9^u#682fCm9h1^JNAl^f72aMcFQNC#a3 zVO_3A7N-mC#3EYIgg^tNG|a}le}9|KHKk!GUDtMf(kA^1E6vO)wlOfBhtCiR7JP#j zSlCi9y=B_RNXN~tbgRFN&MSn@G)&H!>8h5EV``9QX|&1x84fRS3EyA?Av^Am8>q#>1U|B%=#!-9{C=XaMNkAC1>} z{Rn#<-O?>19c(ca7?f=-Lxo+!-R)g@o6Wfz;#L&iNZo*Hqv3hOS6Pr#*R$d+e!En3 zJd*_D=sDvxURX0;kbBA8XUy9p7t~&ic@5s=a8sV6;6BAA?7S4p$?a?u0*tBgx z3+&-^X-^^!g}w*gXoJTkKA+BmLm~~Nl!P~NfSw}V;%2_)W1cKXjG$t9 zxyz)z3N>igEmmVXPSSQ=yez%s-FPt295iqLyM#uGGUQ}VjxE_Abz#?ss0Nw|vC6$b zB|znXQ6$5$!2RXL(|~D?#q(7a31G>o|B8v z(Zalgb!}tN3qjT;?20A`3qGlXF5!f3=!f3ye#pFWCA}$(JAydrl%5aZ3!vF6-ZMLSu;A*nUOKG~!KRMxu%7O19_!P*PPkt0Hwp_(Fgc3Ji_j1#NciAgAs0bd?sLi?amm?b&U&6{DrC zX>O|7c6-@ge&XpAoo;^W-lOUpa0BcA{s``_?y_$4sGjpPkMA~LNq+9)ZiTyRSj?-m z0<7GXf2LA1fDa~V;AtmbSXt?cT--k-ibmSyNsLfB{#D3cuXoHh?u)X`FAsX+yBR;AQZgn()g`D_rA)Tpwu@pLt>L z?boLa_Ss#p-=^-AiPDu!Zf`1-s3N(r^D^F2@Nl=}Cg0|qyg1*>3| z|2Ou%Znr^iQcvCuXqUtAxCBH0p7}$knol41eK$+;?Wm-`@aS)meMo^@zy4i6@%9%x zs9Noj3sNM%oti_%$c!aj#cuh*(GUKtWe`=?mX0| z^Q~rW4qfZj%HpQZm+3|S1l%0#LVTD_kXbRpi5fYOsW_U;jE!toU2#)GVE;^uSYi7EjY5`}S zZWtl2BrZ={E%`WTAzy7SM zRE=+Hbagdb@UfBA)ZT*PR$O}%QX&wHOaX@C^w1)~C!wrT7uUYbTaBci|js)mCwQHMUq`d*d|OrnoRyAPI3U zkKpobp^Z1x8a5FiEmCgg=6oY$%0+HAsAX*GI1g7f#p|vWnW9B$OucT(cFR7=(k%`4 zR;XYE40O}~%>q5JyZ}ZK*8PpY0r^})zFC~qDxBePjj=#fpxBpR1{-j2QA#N&VKlq% z*85V37xvp>z$F&E@6Kj2`_}4c)mT=>X{5|!kbxst?UM6DXo;`ZMbd~ghV@bABM)+o zzBl=}5w-lZB)aX344)#}p|g_xuA{}AwiAjI0hB0sgw4LC7}=3=g$D zG%5RzS?DTcx!rlUekKXt)*g7_>lS!#zC9fAoXGwLHSoa)FZ>c(pijKG^%mbazt>V* zwpO<6BN>Vt=_gVGBW+FvCfncPmT(ER$cc9+Q^*s#(+KMLh9}U;!!rIfH11@FJN-Et zGnOX*Em&m9AYr7Js^6e z3kyor+qf4nB-V`=5o?(FOw`2mF_A^5I#OIRk%@t5A%5dfP5oSz4k5KBhqd~Zo_5m^ zn_*&DW8one=TaZakxUO)L{c-LlB78*=^Hi=nf*@oGo07)CG7*v__>oX< z7EnauvE_S|BBAx#!icZ<30`uzM>uRyfh90+Ax&ATvKwFj z25V$h$yXpf@dgN_3kJdAoD+-?3mCA82B_#9ChO3R(bVBv-9ghH4tX^HtRfj`D-Y4S z0|w;$!I6)gn+hfwscHs~gl>rySBRiVG?0lQ3TWmfe&ENgX)TCe5dlQEXT%dG5tbHM zA}vj7%lzO{mlI=!{w&3=EFw!|<>1CWlKGb^V4{CZiJa6dhAZ}2aVQg-MH&|}sJp1` z2TpL=x`7X0W#R4 zG0nk>blgL;_|Po*s*qD#%!j9QQHE$HhavuzRxY%d5ZPK*o1fb04QrX`6TGrBmQgjU zVu-+mq06)O=(8Kw**MG>LV*s_(fh+qrxc(;Pm#*zmA)Yn5>umng^l@#?9 zu08uy2yr-rblpfKbE(jcsk9)v{qmhFkJvRHe6B!#8Uzy-VG*Ssu9DaPl|pp#I=3}K zAW`cTY#;jO#Q4gWv6v#pe5pajf~=rW!@K}7Eh}R#vce8*r~(^Y@MZ=i*fb07>1i>f z#di9JjiO*3+tM1@-|`3EnTqinsaPt@ckTI|`+X`_EVF*JA zViA=6hNB>@q7wm&$2&R9Lb zv&h=9=PGn4h(c76&`h?EQxV@Z2ursemKM;o*S`K%^j~wY7#Tq{Ukus+Mvg+}k_d8=8c6@8y z5TU@Fv+fRqPvC~!2LQkTOlJ!MRNr~Cz)fF20KC2WsT0kp<_dIJRJ|{&E(UMm)QI?3 z!0vjjvPAiNe=R=CQ-;{;Yxzo)@E;&b0m})Or8vE7+c@Kx57V2I(=jcg>jqdMAXsAo z8E6arK$cY?7q-#3<_QV_xda+wnR&Pe-h%_b0|OGvE+#k$?ox%~3kNhCt#6Px<=ccL z=)1n_hH>Jv?9-en2mk=U0AFgW4ER1BV*u?+!k05NaS;JVU;biYX_O;E zsF7nYX2?JLdoL~PKmUV?jj}g|n6SA!GbvfXXG#eFy)!xpqYH$eM*Ka!Xb1^Hx;shH-s}3f|_TNIjk`m zglG}jq*EV zjDo*xnP=h{9egp{xTX?AHV8=zs9Q`WjH^>vF0mQ~M|8viS-$O%H%cSQvKWt0K+3EP zFQ{QFMu>toV1@;P8rDJsZaK33?1SN2J|e(|+OVO)`vc?3B0j(z#Z1VvO0Ty$OW=&i zV5>+DC5M$$x&70>EsIMNYopR=2CH0>gdjxGvaJ3h5(Y`I89~vh3^%I0#J2h@`|wHf zEYHbI$M;lE$)wDlG%{w011W$(6qtjN7|bK>%z4zLfz+$;YfW$2f^F%R$gDwkU<$zt zx^FP8Kgb24Q$Zi7LcjXB9wfE@44o8-e3;9l$od134=pypfJ+{gOAORYy}LrY837YK1mhY! zuCvdqOv)Sd76hHgZHdx(j5=3MMcE(-DGVx#q9ND351*o&;*hf61Q_7qLi*}Z;-pht zG|p$hQ-(9s3wbjZvZ_$Kua7Lh>4GdFvH?2LR$sR>;_>#Ge~%Xp)D%WkvQmJhIj0QqC&2s zqS<~#)xN8=R`pk`9mw$VGZnP5rbI}3LBswOmkU@WApF$~y~W}*R<@1UiY>O}?AKGh zE>=W4yNwHTBH2P+3Qjdsk)eA}m?=vK^BHLBeSqLJIIrPt4RJ__lUzIy_cfs27C+agj^v<*2} z63&P%*86i?-^IWGIz`T9#n@cct`G14tiZsPy=pJ*%+%)pd<^@ zGTg13RLZr2NDD!+vpN335~MZX3&l{}UEuxl-D35d-wj^Dxya(x*b1glnx#OK)r1;o2yy!r zjii8k(*my$-0R)Wa?LdC-2>G9KX0W1d#eF`nqI205@(R%sCC~?_10A=1s}*rZ+(e* zWM3qW-#z%=uuFk`Ow57w*P|J${zcpnBw$*Q9%M7qP~0&sZbtQD;D(LhENfdZe%s5K z-9xih-y%Z)3tnS3ULd-Y&EXha7`81a$iNXU;aM2m3qxT_odSF?4HPzn7KQ=PT`($W zfbdPY@f!zalQ_zqL16%72ik;46T8^ST(E)!7+eK3nS}-&-4IBGG@H<@)n6t(&nV7X z1r;dgqS1+Z4&X3g&NHa0nX);GfiCu9Mp4^`g<#;FSmWHeXbm+gY~v(6o;eY>a0sVE zc1EU{O=k{MJf1ak@y~4&hZD$;3?R|rtw->ULFo+!MJ_-a24d8l!EOnmM;-^CWr>V@ zgrOY@6r|im?#EAH;{IKeFW^`HP30(!1WBua8*!gc%7fA>V1vD)lkLX#fRt7Wsa*cz zUiM}GF&^Vzt|4ok8i-aDH8y7K8o!3dgBpP0xqRHm^&|n5xav)Y68;fSUJu8e%~<1R z#H|HbWMBJr26+;Gw=!%wLXh1QuKA|attP5s1=<{1MtmN(c zWN8+^sF2;q{?Y>+hGfiC#~Ie)8GMOKe#FatVNfmw zE;L^8K*y5HZV72JO-xB82tx4d=i|Fd&<+W97U``*YL86V;9=lWRT=mNT zsG1(Yrl7yRzQ*k52|Yv{h}m!r?Y`@Tz{F`|s(}$GWMf+7d^_Tw<^gt`Tz7;6?!Mfk zc0tZ=$hR6GRn*=WQTT=FL_1Sb3mVQe!f&o4bmW8gHCD!19$;T2T-XwZE?!e zxg>+>`5k^C_k}%o!&^%^ePBz+2WZOxkJUM)mo}$A;H7u^eCM7bQuQK4fPY`2#;Flo zsDOB^`J{g7I=FLck4%&Q z`;kw1l{ejEFMK+`XDHX)D+=(1cn^cmTA&`ZK74ParI8fX&r#cd37O7btW8gI@Sze{biZu^>?H zinsWSH&d!^$D#Q38Sa9<@B3z-bHFEg=Em0{opygEp!G1NcjC|6CvXu^`@iaOki$#d z-5ZJ6P|bJC2}T)2vt8An@A)Qo*RTH$xPJ|>e*{q)L}6!V=Oc%~qstk(vYbDb|Z}%;lp+gNctN%#<~wgR!EUGGT~R z{NPqZgt&6&Mi{U_MHDSEw5V~S1`$JBn3#Dh%C^=5t{wm&fB@GHW_Liap*C(0wK-(q%_gC5gm~{Pfw&T_ zNMMF!c2 zATH)vWtnw&Vu>jZmPZ<*0RdQPs>ui&jWyc!0ByGEnA>d{@W>;9LH5U;2vETALr$91 zqNFFzpt1mi3}WzJlNOFc#gJ%OInOs>X~LyO<_VJjlXA+F2g5IhS zaq*^cM_;tu;m0!1L@Fu2mG-M?rU8Ejnhc^D5ts$FDhz?BsnNK=jX0i)YD%kG9PUa_ zXe{Lcw0f+-$02(RsE`)5R02#Ru=GuM&_ZkflbFeXnM+aF!l^BO-L|UIDScubQ6S@n z#b_p(0CBWl1y9(F(MH0>7SvyXmG8d)`fK&2m*LM-vbZ(Y1L zt3OS6$1E+DE%w}XE06+SajR=CBP4fZrmHP%^cz4c}*cKVmxblWXa<#?>PgNP_Jb0v#4B*amqcN4|wG7`X zk#izBQ&RPS#v)K-ZV?u*`26!!q?qyl3t7qlwwD?JD1*S5P7vxklGMO5h6TFLL$QOM z17{~e5)p%7u^M357`D5zJ*-f{6A*Q#0M!xMzC9uP{?F%>|sc^rh^La4VWPhdzp z#*qSaq7xn*_&^YRs7|#?F%;#9!zf@FV&axUmHg=kDRK(M7t~j$5PTvWf0|$B^h6{C z(F8$e&_ad&cegh51%nszU%C{SKm`V(ff0P*8z)G+3UUoJE_y>A5~jz6i40`2AwXSB zB|H(fg^5oaMOX zPIyMKPN5WKI=$7(&4J4ei^#;h^1`%lHed>RsgN>Ip#ohZkd3{RATZ%5yI*do3IZMI z(VW@MhB4p;9(@o<)i#Dj5QJWLXhgg~!3dL{G!jxP2wyy9{5sna|0X}Npaa}!CWXFjpGPngmVTv9obLD99BDbzr#UEQj=!iqP% zg_R8t^<_l)#Ti>rrWtMj9oVKww4oIV22>i|okdv@&~n@ZALt+qGiXts5(dVtg|!}{ z)ZncG!J$EoylL4I2?4uIpb1@Y00@Jc&Y-+0o>p<{JLgo-(LPnFQN5hz@OP9)#KBpG zvk+``A=}v=D;J(v5*h12hqFR7b|~^|hFp+<3iM{U#y#$FX^LFrZnTe+eNDS65Cvj1 z14GoiZiZ}OPT_Q_XtF(BP!$v)w%}Ao&&wza8TpoTUJ{+dl43oX8s7trwyD<2ifK<< zRV{`~D)!65U|$>CFb%lCn#c}s(VEs2;e&c7SnhHKVq6d~m$`SX8+3^~+`%LYyVmWB zaF?6R>Kf}3fS4-(Rog;~aQx+!$G9jQSKzeYlRfJTQs+!Y8j0ME8KlPx%c&k+(99EiZsGUvQO1~9q+tjPm7 zs7K2`&jWj?#IGoUoay!5BFLlYXH$s`Mf!0Xsh3VI&+PZ#@50JlDTodVqMS6aRhch? zD+J!W0JxuwS;4-ZT`_SXY+exC7{LyW*X(G~D&Y8$GB$u~1W@N4;fMpjIoBk|VvZ_A zcpw(7UNfTqfJxC_4i#sDWzo{R%?Vu4MFj1X4HN)KFy@)kVz6X;^Zi~;!&APY{&zb` z3puLHSKkJA3Y(nFN?52G2A$Z*;I3Qb0rAE6CLdjk%4b1X?r0O1HBF+`pWo&}dH(lv~x+c#7G=HZ^^ z1TK>I-QFy#7etaQ0rEK$v##|gZe0@no*cjpQD3K;T9lpcajNaZ9sh#s2j;f8^vpvA z<@jaeIK1`TS5a%*&i3Qe=xxZ!Z1TYqvqfIpR6qk7&S#G*HovQR+8{4T8uTVuIm;8~ zPt3ai1X4Z#9C$2x!TG|Rwjf7O3p&np=GoGg5-DG|571HXb)b*k=TLR;{N{vWh`fT~ z{d73QE9AX272@JaleH7{PUVi57_cA@yvPavsOcA;*(-qMvaQc-=$S3%GM9YXb^z4z zle(Qpo@Pis5A~<7;DVBUIv^o%%b#`x2L*S^B>_B1E_i@%x`oOAfdskbhqi=W?ZqDY z;ht299aga20zDud%oe(&fW|n>l(860fZ4C$-4uKoi;3CuL0`$8ob*wj2^y2qC12uk z7tD3vAN5A$0nZ0j3=O(h&u|?M9ta>H2o225oI%n7oPg3nTSuiq7GPG2mEBd`02Flp z0~i^}Ul~>ueoZ+jh<#<=xQtgEc+p2}p%@6@Oq2|3HI^!!mIJcUE)kJ!DTz-ci3ci) zMlgxXOk7KZ8Km4{2#VmCm0+*|gCFvO8aSP&NL|8Izy)ldbGeYC(Fo0*H6T9gVj>Q{g72+jkqSTB8x#8bPL;x)*Lnd+|(Vn*|+!?7ee5}5g1<_j(|PmYe51mBv|zAAs-f# z$@!rjArt5<-O)XkB8FqkHR2=kSP_VyBu=75RF2MsfbjqZAwU{9s9V#G#u&x_8zaq` zKpoF3<-wngfhB>KDr!Ir90~_Kz;vNoJYs@hNdV;BfF^z5uN0q_)!hcxoe}&BMrxdv zeF=b|5&}Ni?{NVdj+Dz}AWjUUO0Hx|zF1GN7B%Lf9GxId@*%P%Uq%_C)IEm1(-Z^R(o(v0CR*@d$#c&kjC`yuDiHbselQ*6J5YsROAyg99 zO%6nCKvsG}T~xt_kkLY#!#y}8?cqRc@#KuDC2IVNe|Q?$`KGE>PDzp^>tLaA#%1OW zRbRs8UtWV?4kmQgWFJmvAGDtgfJbvBVk61h!wAG9Qp_q9r49C&#z+_f7+s@n=7W7> zBgLQC2ns^u3 z9p@l`A%rTY?#1MDM(2qb<{xgTUTxnBAc7}C9n*=;g`vi4)EN;tVmUJC8a1f=_{qUt zgj?XoC=w`cuFw(o2)>b_J;l=vNS8mhCNq!@B^hBYk|BUv0lNtQsEtO?GW6zeu8)*X zDXI;pgYM;oQs{CbXLF9AmwxGmf+^uG7=KboFc3`Q1=M1W9*#`{_Eg>)97FX)KJK7=Ko?JIKaNOZ~B z?m$-putu+bn4V$6z&aj{Q~^Hx87+xhis@S$&}h2{;>sS z4J&Lys$d;LTiiw>DCvOqQJ2s}-A-DD^ z$&OB>?id(oKx37H6M(C@Hl5eX>;~kQx`x&hv1o%nD9`TfsFrFmm@2*6>tUGS2+~6( z2!x#WM&mxNPI_Z;#aYt6(6kMgxAE1}N}AN3X9bY|LmmLb@fd5w=36Y7j@WWj-7=m6 zKq_|G8`=IC$o{~P-hs8gCT#vy+?*{P#OfHXCIz}v28E+=7IH9)T@RsA}*^Us>X@>jtiXQvl2n$+QB8*OG5lc zC^8#n@*F0KX9!^1AqruLdhJ=N#@*!2+gwlf&`FO@YZ|1G1l-x9vO@*Q0Yv&-l7eeK z2H^!YUBN__rY5f~ZYq8_FT6r8;Bwv1V(9fsCkhu^_vX>AAcFUzmqj521^@5)u15N* zFZfMvBko9=F>Z{~PqG#Y5z;Vg6!E_b%1nL##Yq^jsoCEGFG(QfXUV3+GDL6$KR|#o zo=pL2Jr=KOI05o*YDlu)eyl6Ij<5(16;)7g^%`ypW3LKpBMV<=AYSYbs-^cy?G4A^ z%mq&epyRRP?{0YG5F7FTGTrwq2q2>)6fG|fsOFFc0}TiOel{>MD9{zJEa-ts`_b)2 zDQ0Tufw`({>Lrz1o-rElLG+SBG_0|_nlNH$uZO%bEf3oqyYMHb+|gaE3+6GjQF6+L zpCJD*DviWKY^1FvvohC<6I$^lMQbVlC@$JUU8F6GNCrNz#Ss9HCIfJy$=JIYfEUB; zZq~t$u3jl`8mOUi8o%q#vhfM)A@_3sC_T?}9N%(cD$LQ*qxtr7cLwu!2IV3LawgqR zzDYpuUgVT5^9Ewaz1{CKL-Qv+@&ZJ%kaDC~@)ZXlsRML@7Bmd6jB~G;2D>pOD6geC z+g@A-ncn6gJEw9EPA}oYGSJp?9N#o8bZ83)P$#n|t~QK~?6Hap1dDC z9QbWBM7I|chmr(GfSn#>ACv>q1Qa$aG0jzn+$J4K2WlH6!nvNN-hM$D=huS1G)!xq zg$Aun<8-m%^AL8_9cw30|Fb{~GeP$?B^sO)^ED(UfsQ_P*&=lWpu%4q$@}ucvawvF zP_v=>$X5EUpn9>(USAB5hWr`-0Y7U&7%#8#;@94mWLvv44(i7(P_N;}k%kVgTx;X) zw834gP-l|#IL{5KY%~rF_AnPSF^}Iw|&>( zaI4rGD1!miS)^65aU^%5+KYC5fRW^wN$ICocxoMdbXsO`SX;NEg>o1}H0=$X`6XaT zg7*FNyWyi~@k(4Fhvff}clU2VfES`?e}3+V~AQhmuUQOLhW9bOJf1qrKQJ z#BjhX#X{_ko$C>EoN^xfPKS?p^;d^eF0UJAn+9l?48pyoe)$PqK5v)9GfhWOJ##3T ziH@0~>Dwfd%LeNFD#%wncK8~PY)&Cy&6WRTBKkxd zoCd&6Cl2cM(E^%v043+Z>Y@xkt0fpS2s&Anp_buVGiZ2sISzXHD~GzMk2p5OBD2Y< zx(i3n!JGEYFRq&Zm%CG!o?REhLfDF&TNI436FY~++KBTw+(}&)nmBW?m&CS9qOl)& zQbY2RfQ=l68)bX+3}5z?d+oK4pKmfC{&c#hi?_~tAyLV+(2BZ~Hrym&u{N&=%W1Se~ws=t`~ zIy6`|w>;i^F3j_5pqeiLs89kZgf>j2he1Qso~N|M1_)@I3-9~J%Rx8jfi7_8j2YS+ z9FGJfL(=2_Z{+*9f188U$B9%od~QquGq~`i0ft*uf#t+O4TOD@iwa-(jXdcGa;IK@ znYP;d?M(E(#;_Lfi#v#yJTQ%V?OfZn$?0#%uM9`7PO6!EH~G@)WR3xroxVpb#s{0J zyN=B{c!4aCMSdnwzUMXC1rSg&Wd0N=y}oe(JxpBG27m#OCa??mk>`fnO^B(hK`a2o zk1t;yFhOCUzyTRD4kSd_@L@!W5FuLm!~;hUjT@OH*%$&N5iue|mh^EFWk)n_ShjTe zvJNCcGH2F&bW>HJU)OkA^Z64fx1d9X7BvQ}Skj}&h%s&Yl;|^PQ>9`8a=}6xPghT@ zumPd}!UPo*Czb^u06>BSTs&xa@YX?xws8@Npy46|-n<$1+T{RZ#t(r(jubY$#9ovO z4F@1>Tws6&3=sa5V0|;?h`hC(;fq14GT>H0stUMHvkn-r%<2 zpEO!__udTx?vtwt7n*fQ_BdHSr9y7O0wvt6SyEO_Posj0xN*voO+khFj~w0OMKStB zEA*f}>)Epxv`Xf;X54KdvZX9vw`HYXoeH$u-Y4+j8I#4uE(>s1p~ll8?PYt6{S z1#En*F)5%3ZaN~FycNh|DRB)pMV2%S4pO!#v4ZiaoYu;>g!|IVY>`{8S!L&lgO*It zyd#q}hdA@i7wDX?31;lX(~dp+4D^;i2km6KECh>!U*FPj^iKcX@RydVn##HTa*NpelA=ZvLp7mppi5>Zal69qPfdz!XNaP_$9=2E`kv$Sf zZ*RLT0%_NzlC+RgNEPUyg?55D*XH;_3`d#>Z|NE0T)`9W?ufvKS=fz}ygNbZBo0B_ zYi}eT1S6ytaLhIb?dX2tZ>NC`E|{FR5nh;XhV9OSD+(E^SfY$Wyx2w>Tuc!Gk3(+R z*2Hl&Tv)|lhMBdQUo*Ba+%QO#H-G>5;R8Ajr2^ZsL?^2OFc%5owNqML+2qz!b|8g9 zYstbdlge1>rQE^rnTkyAdlPtDKH*Q`0p&h;?oRDixJpsgSX9I4wM~`(#=JHo99CL) zm<#dlNtVklEya%!CC9yo+%qMSRFd+WA9&%CH=>?DYG(9x*Aq>&@O26*uznJNe8}7Y z1DH7h7D6S*@<`%X*SZtj#}-f^iAu(xI}7fv3s15Se%j;=4{Bm7qZk=Zd*>sJ`B>CD`xN4UK$H%PR`G<*@Gxe^lH9VGWt;ca zhe{Nx5IQ#3jrn09dvQs^BPe*Y-MOzLVC0B%j_?C3*5V>cSl|vd0=wVU;Vu-!UG4y( zI~W{lR36;U9?bzlwz=u&8sBGi1#FStL10WUfJ9QI&&urm8QJ zkjw^Va)QM!w$D6_<1*dQ4Eh3!%7PY@a>X)P7C|D$9_UC6M#HEXx_E>-QWADtLxb(i zBTm5qv!vwO6C79CQkOcj5Hgjb)c%-FZR&KVJ?-XX#TlXIWyLBs&}_w!np6eUC@lhT zC-#;A)IH5-K52uXE&4Ujk3y+*e|t!XHE_C9 z$hsH4+K>tf7lw@m-FHFSXoh~ZQ7(Cz7N|2CozO)8Lp!yAKs=tZZ#mF5u735a3RQa} z5W4qTXI$WK`h;M9Zadbpj#7Q59H?)Tr~x^!_Ls}BZfZP`K~q^%0yjj3E>@5uF_i3! z{CXPJikgR5NYuMQ=59)uVA$~<(7bxArh2PU!h^+EzEPHPWKlR@b>PEHRywd+wwwUK zd^s#&7W0<}3^b-L=F7Zba4*^1OHAamF*T9_qBUL0u$%DaS~QI&!xB*!5X{ky4g>VV zHREk>Ner_wOi6IyiUKHLkde_Sm5~_YoRDY$DhVxgQGSKaAFO%RA$J#p-!<}KEmB@R z29L>34kVPPOl3{^1k3rjhnBfqYg^X^*SEg4(7H+h|8~};3ipY1o86p%>ctrWl}Ig7 z=NW=Egn6MYegqFVsFc}NKE>He~Iu!HHh^sk&Z|vPY`WN?*@7b zU6c*9z|ky#0tjkIthU z>a)N0GBM0XY&O67)*t{CEp*XZQOK11b>Q^*oU8hs_)5peT*+ektUB6M2rK{sCg=mtH^K)|Gn(g%{IrvT|OMQ)d%3FXIZ(uzL|U5 zS--svoO!N>MbC)mKL2?Ph3?7DZBC)EeXs)pP+DDdV=zw_y}zSw8vjQn}! zZ&qo{Utp-$alDTwK85=Ib@bbX_U*#k($kyz)aY$2eOuptgcoS`K4k?sJ!K!sP#|KG z!M&<PV76EUXKv!IV5IAiKL~CLojgeBI3p^=< zLZvHqEZ_P-SG>;?Xra-;G5qEa2LK3#uMmlFW)jf=kFOC~ zB=L@r@7~JVI%i+X%^>zg4q(MJ|M)39Dhasm#|EfMU7|`7M+L~x@afjj6_enYwBqwP zC~Zo?QD%_!>d@<+VzTlOLMp31UPui>$IO^Y;(9Lthfx4!&KL)>@EVakfX^^Eja*2o z_&S5=gd_-E##+EAdtPe4$cK$KG6l_%1#<@@I`DUZ2OpTBPvFu0o~(ojsmgGXd4MW^ z&;7ZC*h?(qF)BKxCGW71#K}JP(M$NzAA>Ozw?`m9Y$uu0 zC%KIo)k+-+K_d##(duL<{~uEEa?$~qVH{)twem+7@F5SJ&MMPzg0M0xE2vR)Y^W>) zU~VE(+(toCG9~8`E${Fi_oPB*@{t%17~!&kR+BZ6s73110GXxmfHE0POhjIwFI8!u zxQncMGusT)2^KSd-e5b@U=<&;IXrSQ)q^F-3+|)?J;Veo!%`Zm%roUtDQ?gn8)iFw ztUmU}G;gma0kSUUj$fV=z<{c0a#A)2FwhQ?AoEhKAa4MO(kS6?A)5cKdEGX|N4N!O3EM$5*g)lKBp}s9??W-R510kR%E0vL?ZMC6i7Q#GA9#X zP@y`G^gF)NKQO)s z{wg#(wc#Fm$psqY;U-nrFoer`uop2E1T2-_(81%_tQjh<*3tkPwj+Xa^j3;7PG_?= z562CYatmzq2%r-acvDB+lp_r-P_J@G3DrQ=<4~v5N#6@Q%QVYGHR2?dQY&@;9z)}* zroldS6OoPQ{}KQu`>4+P^i5^-F6R^xJ0O56DsoQBR<#vZ&DC)PG&y_qSKHNH1qCG) z)f=GH%BDiI?5;1mj4zgT*HDF8Bah2UYaXOkO26!0KX9n5f?w$LalCa~9TpL#Wi=GY zTl57&l=4=^rlWEdP}S9Ap^jbORb&ArUggyW8^jk}A!Svzn^Ga{>>yJy73Y{wS-s5d z-VR>CH0^+HXVopnP?eM%mQ`C-lvsvhBau)2lvwZ}Iy7Npfi^$gNF^eQfq2zdul8MU zfl0~I^`;E==4NFXm1VcnkSf6dcur>h25nXIf3}Z+f(VG7)=%M7(0-CznKG4XX(=-n zh#DXq|L~xh2!B>c{5Lw00CfpX#XLAkbA?T~CC#2&#e zW`A)!g)qim(793|J9r};fYC>L_Uhlp@VdHcNRPiGF4D=C)aCAc38buvQSnQHdlM^qyl{L7bOV@N4F4S z3~&p5WHyK|cWZ?Y3?(}hq7bN{ zc$fEpgL4F|0KF7*A6Q^I^tS;R*a|cuZXp+#1{G_i*K0_DdaKvfuD2e^wsYsN5-5>Z z|9Ws&$dj$~cYN>3I?%TbQUjm$9TYQZXerwQ0%M^o1c zQle|X#(7KZ3FrWEC8#+(BfUT(b_}?Pn${U=pgJsqbr<+=tzdXTF9aIbiwl@^tFqrT z*bt(3a<>+9GZz<2*pBaH;=ET?vZ!M?(LPqrbkFy1L81l@_=c2^Sj52Pcz_HJ7^9d0 z7L2f#)_^$eFGui(M%MI$&ft>_V+8PLMUC`GYoU0t?Ry?*VA}*6#&IKJqLmjkrDQ@1 z1Yrc8zzy+WmPeqOs$-Y+XK{((NNZr1H867D7>?ukazVK3M0kX^*N(w#U&Tj!|J;X# zwa0~PSqdb84^n^xsDO-Hpm`0rh7tKz%*cbh`GC;~5())An1~ZvECyYuT-fprMJiTp zh6s#Tm-iWq_c@E{l$VP0OgLazSO$qc_DAmbld0Bf-FSng_k+8Zj+>eO@Q`zzS!GRu z4cLf}`?#ZDiv_sZfcJ+4Dwu+=z=PWN!=!+QOWCDmCmiBn=DI*QxP7{+Jt*yqbtXwQ3ND< z?q?N%1gD99Ub?KAppyxa17>U;&AMz}VMtu#0}SF#B*twC;RMcYelRu?|9u5Sr_Dwf z)S{k19bB*)on%@V?UIzbXm_=kp%ioVqPdQlQwXcM^PBEutGoI}%kzK$$bhVX zw8`4HsFyYM_ zhcl#pR3o4OehX-|1C56e+AeCsCTv-+B73qe_uv%uvaK4U?YOf&8!o7Z1j^gIQ2@Oc zVhFrB2Y&8#7RQTFwV8s)c-J|ANf{DEAl&4-pyOM1oyfOI;4JhsW&vQhzo_wy6JAI|MG&M)Gq_+Jt$62cy1&V3YZ^>yB$4mJe;ulYYJPC2*w?UK#hx-PuRnje%SQAqGQFv&9KFgdb{bfRwWJfy z>)u1)zR8-3tpq zrHf&M&&b(5g+Ii9CY{pDPTt!NDzKbQG0j#D47NFm=5S*Psf`oSjcvpqt`S3*Aq3^|8q*j>$svXx)QRQvjJQLI`OM$ zY%CswEhVFfF-)z@6`RsHOAcow2iP3 zrA9Po=zZ|0{v?(+2S`_&6CETHE1Sn%nD?3X85g|@IWw|E5L_GFlL0;oh0K|j=X;l( zlR+3T9#J{o&EH()1^61FMj3zT$Ck~!u!lpJ|wdyHUSgJy8vb5}4wP)1Gn$5<|8#%RW+iH6YH!iZXbL-MI z28@^9y?j~q@%uNG-%5lB2Oivu&6gt{J2;jIc|e7Mw!k&ZbV7mV%@8JQ29hZXkTjD8 zewGsPhU1P$Br$%?_(!;0g>s2ow1MQslOq=+gyW%$nvi-dh$f_kkO70bltlrQM3MA4 zNpMIR0U&^O0t^liEU0B6a2~O$++lB)0qiw1jxNwqN1bYXq>x>o zer}gsZeL9IL7^V(pur2>Xdy$C%tU~we8HW$0Ttq`G7^U&@JGU?|J9Jfr6U>f4W^dP zf&m7Z7)1huLp^xZ6BB~r##1#2Gn*8+&XVC&P-Rt1hOdkWmWe1Hi|mWaE(@cKGSaAH zv^h?a7;Ha6#sMKh=3x$%OEQ@N{|GjWd%?F5Jeh|r0)fLGY8|bn8gN;@CX-S3&5HvQ z50)b6Yil9`PDK+qDMKD_1dy&WMTW5fm#RJ3KmHsXvu?=Lm%Iv#Rfv!+~&KvTuIOB$z4W#zJa6qZ>o6w4Zs zoS_hGXguPQ!}Rg3CB|K>fJjaU01zb*+csy8wjwL_#8Px56;&OH&_TovSh%LQP{Ez2 zMrGY_fG=xEjk!?@SV>Xx{|Ea1AVj{02uPqNd&0hH?NsbE;N@c6M?~cm&MH+Yly_3} z(Qb65{433qXl&HdPftD7)>nOX_M&M0+O-p@ae)j)0s*dMc9>TQ`f1y+cDLrd4W+xK z(=dy{Yb=DXA%1hTrRMXh`GH8B-~2f2UEl6;Jcf= zNgxYIS%L>MH!F{X@PUTP+q)ciDB$(YB_U|g3RM_8TA_h>G@Q^QZa9b>?qPU1{EAn6 zn6ze9Pin}TT3()oJ;Z3QFvn0#_qdiV6^PFqmV3p`&gDZ%U zHU`)r7~RrL80E*cJB$X6`P(03q~pIf&XJI58YD64<+GW6g9?Ge(|r0k%t8_}g(Wnk zHLDpzMrzZMknE;6y%IekVv>lPJYpw3`AH}GC^e)6Lbp`FBw$n_T;W>f6>qe~St_hM zx0Kr`7a#xu7$A0Oj3y=WB+O_Ub5Ouss4=xLoA>M{B+X3dHAN{+YHAdl*Hk10XT{BL zigXl}oZ2`u|9QP)sMMV2L}%7Q>AiNA1fEr3g%!7GPZyk{Fy!(g7#{+JQFKG9K22J~}`LfsA%9B@>mCKannW$I!GRXVIn zkdSW(i4pQs0-$IDb&%!i<}muz(2A9ya2zX0PwUOHPLh%)!i&LeocT~m0`;``SUS-NJD5Iz|~+9wy_m;EdLbyzs}aNW$PKx zQ~HM8|KCIxw4x2`XbZJS)T-7H>}{`mCmF=oP9wJT9VcyPTU(b_ORm_WUtLXEMw|+` zp8DWL+L(0Q>@FKJDy-XNRxz2NkPSmcXjwbmEEEOqH^ zaRgHa)77rz%d2n)j0{&)0=Qia>|hzZ1kgd4UTK(z!Aip&S^j}e>^gBUd)9_P!A2K2;Z(Pg6f2nBPN?cjM=QuDb^@&`;#&5Y{fMKOQ zF%C>b0hLG+fU4M>3Dj+j8FkvTpN|kKlx1Yj_#n&~lb9&lR7pC)pVNj;W)!#1V(CiSWHtLnIl5?g9$!W5%0nFHr^ zi(3Y+6}y!WezuF(>gI@^^Jp9tOdtRbsm#I}mZd4oW7}<$K$q}A@@{ODCKV}M20D_0 ze6*Zo-Y%6B1i>LE@MZ&wZmNzA`2@`>bF+CKdLHEnZ?X>0)As&hYmcmNQI|SK|M^w* zv_Odk{u12ax{x_=7p~U?3}EbXqEjX$6GkCbJdSJ#c zb0ZL=jNn|8a|8qDi9y^4y^#mB06FXl9TOH)8=8&4&_wTfVJ;JvX9+3qM=MsAwy}5dT`>~(GG#S z(Txl}zI%QzM0-}TfWBX%U2RZ#;uT+|##?`ke|UrUhZCQcH2@0N89~Z;KJHD7H(Q`5 zd2dWMv-S%p(;QVg4Ruy;Nj^RGs-N$YhXH^A$a?-}WU^p@u=je@B72S@|1Jgx1P@37 zD$sb}Qa-vTcCs`;46u8p5dj>OVu!U*!1Fr}AOZcQGs`z;Q&2AF0)5>B1G-dKp%Zb^ zg+-N!EuaE3umA{jeI>|o>;rt*vK%WA0Mw+3F)5(0Ymt=S&ps++Cd5xkWiAx}dYZHy%qI(s9S%ko6 z9s>mTV@xfi1|#>3W0yb`kQ^R@2X5vYJsAL=2uFk`BCOvzJ zmwH)@)oGpSlrfoi6B>~LC3u-HfS7bBo{D*yK6yYl@rToN2gGqC+_4=fz-P@yi21@3 zLIMdP;7s+Xf(C#Jh&6aWz;P!zS1Isz0)S@?78GK+{~I7Q9zwT-DIkh&xhJ3Ze!EFk zv_Y0-X(&vwZGi@85E40MAfkJNiwx8&LuWjnb_^J)WNSr$FDjjU`DD_9j62qjgb6N( zxg?SShu#@{UxfhmRK%JKmhRRGh54bMceHTbnQ-FNJ5F({os?>oi{@@Sv+KKl%2e?A7CZLI20tw+D3|C+U z13P%(Py@Tehg_S8xq z3jPocH)OB(ny&!cue(sH{$Q}e02J0Y|5>=g4=~e-l~xnZXBnAzo3(m``N*OAF%@OP zux84F$Ev2P=maGzvIU|?DlwZsk+=3jXF?|h7<8XrvK9RpmwtLsHXDH13Z2$4osmnO zGU`&`x~}V*0=cLKsC07;pa~e@x3=Sl+qks!=T#DfwbPNhr5X*uC( zq&r%vt0t?v3$3fK102Aq8x8a-{|x*PB2Owox3UkA04j-aFZ2;P9*7EP6q~XcrfjRh zuV|(nEWN_H6B`kn9c+*%p_AJiY4tL#QjnsGJ5r2mzK>ASN_jdp%V*&W!!aCAI7>Y?tiCvWsFPs1 zJS-les|lWvt_{n`HQ;{wn~bDoOy>w}q%i}H7y@$e1TrBG28*?`JG;98u=!fWt$Vv= zlDiaCFE9%chJ=gE(Fihm|8DWOvH3C{8C+F$>v?2)bZd&1K}^Dio6Am68&)u~LE$Sj zqsNImzUAw3Wz#z@Jj@$lfAwfXh-XzE-^P3lzgIxV=n_GNt{LdhEi63D%*U+v%yt>b(rk?HTe^@Y&_0a4-Rz&SCLP1&)i9;^tL9a!E)2%DO>kP2^>beIk4p$Ji zA+ZlGF%oG?8`bBd8{oN?3Lo=>o(xex&x(k-qo1jbvwZ|+YL|)+4}mOo|1&uOCI-}~20_X3w-bXKI0{}SnaDWUBED^U)5hj#gn?lP_zKnp<|lmh20 z8tF{a+nZwfyT6a+RHdx1-O%LuaO$WYw*TJh`#|3k@XnWDF_b6~pwKOSU`FG*1hlYl zn|d865dkF74K|>}G{8Or`L={-Wd*^^I7mUadRRoqQE( zAEaDP3lMYy&AxC9fGNJqpO|3yL(T#7)a=Pn0|!zLq5=KmBGXh*vWzd|4G36`*DBus z{7azfnic2$NC)J*43v-akPAwf|6bwl^FPn|MKttqEy#1t+yJ3NLzqbgO;s|O|4^YH zKSBg*fcPMyg9RzH*wS?Yg2s&&G$zB>2?i7?6d!P4cIg|e4PR~?c^T8?97VfqM#JgP zU$=QZ^X1bSR42%v`8@68w{OH!P2E(A>ts@*08r@qV4@d`B99B1s<@y5mcxk%t43&= zHNpTMOky)cSi&Hg6e{7QtTng9t&Jt~W?iI$K}|kpU&S`*`6ibdi-hmk0BfU<(Wa}Y z)S(k6j-y_&e0`R+tk`H|z^)OK_SupjNqYAo+V0x8b@%37oAhwv z!;JANj!QW!=FOcyho0)-q&Y<%7Ga}s`gZLMCw`}Bv6d|c8w~_dz!Bu9{}FTpNaWB& z3Mw=*?a!oXb5~A(tBU>xDch)1{CU zQZOft&FJ2Y zjymfg3d4^{>_jL}J-cJ$I`AsOW4!V%7ytkQ)VoIz4CE64zAEHXqNV%r%WprNaN_B| zZV1##n0fjE6;yskcw#97EyF6D1vuQGLRzk}O#)Wqvw=nzJ)9v>|7W0}@s2()0D(yc zF42OD%1oWH$xJTFMUxbUC9K3M#8F0%@p74>y(9@)0*FlJY35qYq@*%5bB!yG7+DHB zhK(*?b7dPW#k{RvGt0DhHZ}F#*G)M8Jy*_$*uZm7gA=l|p$-=QQ#>`wV}O7GAnxj$ z$EKL#fksJOLJs>Tt*)k&>U1MEdHKy3@4UzCSMPcM_P1?-pA(o)|LqRuGYs&i0<@Ng(=&i# z6|-u(Q2{al>kK+U{vzZ`FZD;5PQ&SDmjQ7SCY&ok9VZv1UN&&%6`H}vW*i(y;V&T4 zz$gM%KM1SDEaICbnWuOL1wj3RZKR} z0%GNXYqgUh@O-et4s`)z7}igoUAg^@~@!^oOtgt+8JH3!ng9A)Ufu2T$2KAOi2Cz=i?F zfg1sWd2C^o30^K|k@^_O^fR((z+z>h+fO7{W&(a#p@d6uP={(jJ&PQ|b{wK25WFRV z-`&mwr|bbLiIBGBDUo5RH03J0Fp}hb0G1AkVk^0*LmqS}m%8*N50yAfVXk31Vtm^* zOp&m5nX!IfQe!pz#zr%_5rFN>lN{q#MOTgeDMgPa8?JY98w~Q zq|_G{X(^RmDw4Z+LV=*L4@lGilf8Ud0|FX?|KAO?R;4thI9uX|TP75hs*F@GZTV1# zO4OnV<>f^)n#5tAK@`Yr-)Q6}KWEwxUCFd&O52FHIBL^5l9-cq2qw2V>JfnoTOb1y z00B{^vpf=9)O${ng^n@df*8!;emvL>4+iKase@_>b5W{Zny?U8#cCoPaJ!`GVQgcA z!d5Yg&@JSUqB1N4F3&nvRHBcR6Q!uET7a=DT6BuCv;#=L$R^In&s{WdCNxjlH$t$L;M@}u6gp0aMGl=SP$#h137;t@6rS<4$sD3VI?|GMgLBwPX`3}w zUn*_2P4Q%_tmRtR+KvwL8J=ec0tQgh|7C12fCyRv+A*+-Rh(S810uqzMcqgkiBg$^^2%|*3@Q`rW&=hB_C%Lt27oj84_+{6bJ2YuD30uwJpj5}>ZK+Go zi{9*l6Q?=FT{$5Lx#hZOvuypWeL@D+)P*i}yC^GE+hf&aP?ZPGl35D)iLD1;wLLYs zYA~WQ#mi!Vc4wU)(rS^99|V`f|Aa`xqmZT~{7S}GeXzssjPg(kWZ^=8mP}7-`Bs3g zk~7sEX?hs~SW2%_u;Tq2HTehR_gyo{avU<99P3R<+u=@jko4l->qHrxfFrN=7A7vC zS^S!H!$|G#f1j{aIAqx_H4w%=pkTsM&9GSu))tvLvSr z*-X7{8{nfEc8-r#01aq2ulSUIP|~=a_N~nZp)8IWw+jd{=p~Av39q_qU$a|k3wAWe z#F>U+!Q10a5BA68EiY(7%`j(5+TrPS?*d~q-*i@-#L{(-oo#(<45n@s!XVYI!_jLV zq)y}?zd|@x7y>XXMrH&s|Dm!WtciUT&xM{~S~-Bxv28~%9>z6vIjUfguMCL@vtmZN z<9+QraF%wjVnv`Im5*{T+DW$Hg|DPe^{{GhTOBnZs>M#QGgcbMcEVp_dCVJwAG}}K zRP4eTPPd1XJ>q|Yf>*b;BdtAg?|?}GLN^}enRvW|kcfF%aZm~n)@2oz>@us4ZG{)g z{HiY;8*3{Nl5J<_1c9u`{(M>L{}g}zC&+9B#3m>cT>h+m_))e*dzSZ~rGSu--NHbiGf~qJv9mTn z!!{v!x9wqr|MQEZ10Gv&j0Jp@OK^;dqJ?z(iV1X#>al@B00#u21t|hM;0wND$}xe{ zo3+E2<7+#Fv##ZnyA9K@n4&@IiwLRVzVb1EPVy%w>JbRQDheaN!PBo!n2QcN3ldsC zl@kVFd!dzkg2`)-rYMPJ(*$FYnL;45)PsiQ**f3?C9D9F0{j+Km?+Vcme13WCtwxX z6T1KVKp2rIDDbvlDLQ-@2Vx0}P9OwjNDLtHw^b^^61+73A-Lk}F&1n=<$FO!tSPpc zgkvDW^1Hhs|J*7*Knp$SKJ?opDp-rvnwd-#LQ9-P86q|*co4H$LSdT)voH|eGBy)9 z5($u+DxA6YpvBsXfIC13Ado;8!8#fUu2qVb0=&0mU=O6j2g9(ol(++<3PZ=xv*l_6 zi^`FZkPHbKpT@C;Wr?0Z1jleJM7U8wMBEJ*Y{Y-M0+T4lD<88#vs_p?1aqNTPl7q*nAXXZK=~IFqK!uOwq$h}kE@&tw zxVMu`8#Gqusp~Or`j6}lYOo6CDqhtc{U^c8G z&8}<~vP2^NI?B%607$?p6M{`dLdXIu9@csZn=8#YXv4?*s;|;W;X5NGNCF)2lf2A= zn-ossybj|;&LB`uyC)RY&qnwGCa0B@)%qMV(8-Rm;LOhv4Jc;^(`P?K( zSUC=}OCMsW_mt2bpwJ0*PU`f`3Qf)p?WOe;PR6Xu%vb~i3Oh#ls1pT+gV-yO3b=ur z%+ff|@+8k09jt9yubYw|?Ao^-Wr&lMkD9~`|CG;+%+4^t0wYDzc#5a!{Ld`_gC~W{ z{{+zCbdcx_IV$ZU>aoiY<5Q z@=1CLFB!$tggd@AjZqrS%oN?LK4=a=^$bEqob_4;$U=x%=!8#j)JJ90LNy{t|6Nl^ zwNyR1)JQ#4&BW9{-7DtfR8N&7w|mn%-5>G%$wkc5RaMnKWz|Imm`zpGS*6t*tyMmi z)m*(*TfI>~6(CZj(-w76eJRKBY*oZMR^&_8WpzYHylUU)Yff1L~cbyZJ;J} zyjNxYv}`3;bahu~?NeSA%6Ns>g`L-f1=WVF*Ma?4@zPgsRo|#vwWV8*b=$f9+P9@w!Cl+K z?OVUSTjG1!rp<=LRb0sZTe;m@!sXeXt=h>=*~RrV&ehkv)!U|(+{k5HW~JP~C0NTv zUBd-jb6wfd_1rWG-Ls8Mi`CqxeLL9AT-IIPtv!aW5u~UKzdGUN{GZE#2EC-sd&mvNc57t=$!j*qfDF=)K+3|4pOxZQJS1UPUC| z<1MW2E!J)YU-7kB?$zGbZPw^z-?bkeg-}1#>0oGss z&EMR0-v)MF7KC5pdO z-wO_56W(5drQnHGVGVBK*=62+-P<9)-ySC4MLgaa{@Wa`;3hWW7Gz-`mSPtU;!`Du znH62Kec}OjStTxEFJ|H`hTthCU@D$pD?Z~fj#D-EVya!%0v2N)MnN);;(v&=U0%;k<3^6zMqXnr|L$Tgj$$?@oUjmSi>2;a?u+Xf9?rwp(fD;7v|uryW&)wOcoCW@w&f zWVYs99%pR!N=Q-A8csA$o zt!HhXF>u~zHtuF)K4y0o=+6D0MIPw-l?Hl7TN{R2gl^}AmR&o>SAQ;NaGmI72IcC) zXj#^0ji%&>25FYQW@_$d7~W-cURrO?XO1S}QRZfO|L$IwW@w3qX^Q4)Zk=FJ-swT! zVVwTwpl<1=9_o=M>5k4cPwQV;R_d!>YI1h!r{-#a?&?)8YNI}Bq$ca5&T5|?=(G0f zwPtIeZfgX-Vm7L3IJIh(reTNP>aUh+w|;4+=4-!xYp5P#gVxiOE?k|q>%{(Owf<|g z25iPg!E!v{{Wa{;-Ds@l=g)=g&DQJ3uIR?LXf!Sp2~O&B`3HC2sB1c5T1bF{S&phC zg@$fHUMYlQZl;FJwr;!Z?(LZV?9l#XR4r^tHwGBgrtRA%WfrdBt$Ds7xT=;%Id#9nXo2DqN)Z1^T{f-Z3S&W(To06XULI@kaJ literal 0 HcmV?d00001 diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/main.cc b/tensorflow/lite/experimental/micro/examples/hello_world/main.cc new file mode 100644 index 00000000000..495977e8f6b --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/main.cc @@ -0,0 +1,93 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/constants.h" +#include "tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h" +#include "tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h" +#include "tensorflow/lite/experimental/micro/kernels/all_ops_resolver.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" +#include "tensorflow/lite/experimental/micro/micro_interpreter.h" +#include "tensorflow/lite/schema/schema_generated.h" +#include "tensorflow/lite/version.h" + +int main(int argc, char* argv[]) { + // Set up logging + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + // Map the model into a usable data structure. This doesn't involve any + // copying or parsing, it's a very lightweight operation. + const tflite::Model* model = ::tflite::GetModel(g_sine_model_data); + if (model->version() != TFLITE_SCHEMA_VERSION) { + error_reporter->Report( + "Model provided is schema version %d not equal " + "to supported version %d.\n", + model->version(), TFLITE_SCHEMA_VERSION); + } + + // This pulls in all the operation implementations we need + tflite::ops::micro::AllOpsResolver resolver; + + // Create an area of memory to use for input, output, and intermediate arrays. + // Finding the minimum value for your model may require some trial and error. + const int tensor_arena_size = 2 * 1024; + uint8_t tensor_arena[tensor_arena_size]; + tflite::SimpleTensorAllocator tensor_allocator(tensor_arena, + tensor_arena_size); + + // Build an interpreter to run the model with + tflite::MicroInterpreter interpreter(model, resolver, &tensor_allocator, + error_reporter); + + // Obtain pointers to the model's input and output tensors + TfLiteTensor* input = interpreter.input(0); + TfLiteTensor* output = interpreter.output(0); + + // Keep track of how many inferences we have performed + int inference_count = 0; + + // Loop indefinitely + while (true) { + // Calculate an x value to feed into the model. We compare the current + // inference_count to the number of inferences per cycle to determine + // our position within the range of possible x values the model was + // trained on, and use this to calculate a value. + float position = static_cast(inference_count) / + static_cast(kInferencesPerCycle); + float x_val = position * kXrange; + + // Place our calculated x value in the model's input tensor + input->data.f[0] = x_val; + + // Run inference, and report any error + TfLiteStatus invoke_status = interpreter.Invoke(); + if (invoke_status != kTfLiteOk) { + error_reporter->Report("Invoke failed on x_val: %f\n", x_val); + continue; + } + + // Read the predicted y value from the model's output tensor + float y_val = output->data.f[0]; + + // Output the results. A custom HandleOutput function can be implemented + // for each supported hardware target. + HandleOutput(error_reporter, x_val, y_val); + + // Increment the inference_counter, and reset it if we have reached + // the total number per cycle + inference_count += 1; + if (inference_count >= kInferencesPerCycle) inference_count = 0; + } +} diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/output_handler.cc b/tensorflow/lite/experimental/micro/examples/hello_world/output_handler.cc new file mode 100644 index 00000000000..63aee55c1af --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/output_handler.cc @@ -0,0 +1,22 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h" + +void HandleOutput(tflite::ErrorReporter* error_reporter, float x_value, + float y_value) { + // Log the current X and Y values + error_reporter->Report("x_value: %f, y_value: %f\n", x_value, y_value); +} diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h b/tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h new file mode 100644 index 00000000000..20741993813 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h @@ -0,0 +1,26 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_OUTPUT_HANDLER_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_OUTPUT_HANDLER_H_ + +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/experimental/micro/micro_error_reporter.h" + +// Called by the main loop to produce some output based on the x and y values +void HandleOutput(tflite::ErrorReporter* error_reporter, float x_value, + float y_value); + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_OUTPUT_HANDLER_H_ diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/output_handler_test.cc b/tensorflow/lite/experimental/micro/examples/hello_world/output_handler_test.cc new file mode 100644 index 00000000000..0259370eda7 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/output_handler_test.cc @@ -0,0 +1,33 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h" + +#include "tensorflow/lite/experimental/micro/testing/micro_test.h" +#include "tensorflow/lite/experimental/micro/testing/test_utils.h" + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(TestCallability) { + tflite::MicroErrorReporter micro_error_reporter; + tflite::ErrorReporter* error_reporter = µ_error_reporter; + + // This will have external side-effects (like printing to the debug console + // or lighting an LED) that are hard to observe, so the most we can do is + // make sure the call doesn't crash. + HandleOutput(error_reporter, 0, 0); +} + +TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.cc b/tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.cc new file mode 100644 index 00000000000..c69c9492c4d --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.cc @@ -0,0 +1,255 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Automatically created from a TensorFlow Lite flatbuffer using the command: +// xxd -i sine_model.tflite > sine_model_data.cc +// See the README for a full description of the creation process. + +#include "tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h" + +// We need to keep the data array aligned on some architectures. +#ifdef __has_attribute +#define HAVE_ATTRIBUTE(x) __has_attribute(x) +#else +#define HAVE_ATTRIBUTE(x) 0 +#endif +#if HAVE_ATTRIBUTE(aligned) || (defined(__GNUC__) && !defined(__clang__)) +#define DATA_ALIGN_ATTRIBUTE __attribute__((aligned(4))) +#else +#define DATA_ALIGN_ATTRIBUTE +#endif + +const unsigned char g_sine_model_data[] DATA_ALIGN_ATTRIBUTE = { + 0x18, 0x00, 0x00, 0x00, 0x54, 0x46, 0x4c, 0x33, 0x00, 0x00, 0x0e, 0x00, + 0x18, 0x00, 0x04, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x14, 0x00, + 0x0e, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x10, 0x0a, 0x00, 0x00, + 0xb8, 0x05, 0x00, 0x00, 0xa0, 0x05, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x0b, 0x00, 0x00, 0x00, 0x90, 0x05, 0x00, 0x00, 0x7c, 0x05, 0x00, 0x00, + 0x24, 0x05, 0x00, 0x00, 0xd4, 0x04, 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, + 0x74, 0x00, 0x00, 0x00, 0x24, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x54, 0xf6, 0xff, 0xff, 0x58, 0xf6, 0xff, 0xff, 0x5c, 0xf6, 0xff, 0xff, + 0x60, 0xf6, 0xff, 0xff, 0xc2, 0xfa, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x7c, 0x19, 0xa7, 0x3e, 0x99, 0x81, 0xb9, 0x3e, + 0x56, 0x8b, 0x9f, 0x3e, 0x88, 0xd8, 0x12, 0xbf, 0x74, 0x10, 0x56, 0x3e, + 0xfe, 0xc6, 0xdf, 0xbe, 0xf2, 0x10, 0x5a, 0xbe, 0xf0, 0xe2, 0x0a, 0xbe, + 0x10, 0x5a, 0x98, 0xbe, 0xb9, 0x36, 0xce, 0x3d, 0x8f, 0x7f, 0x87, 0x3e, + 0x2c, 0xb1, 0xfd, 0xbd, 0xe6, 0xa6, 0x8a, 0xbe, 0xa5, 0x3e, 0xda, 0x3e, + 0x50, 0x34, 0xed, 0xbd, 0x90, 0x91, 0x69, 0xbe, 0x0e, 0xfb, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x67, 0x41, 0x48, 0xbf, + 0x24, 0xcd, 0xa0, 0xbe, 0xb7, 0x92, 0x0c, 0xbf, 0x00, 0x00, 0x00, 0x00, + 0x98, 0xfe, 0x3c, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0x17, 0x9a, 0xbe, + 0x41, 0xcb, 0xb6, 0xbe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x13, 0xd6, 0x1e, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x5a, 0xfb, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, + 0x4b, 0x98, 0xdd, 0xbd, 0x40, 0x6b, 0xcb, 0xbe, 0x36, 0x0c, 0xd4, 0x3c, + 0xbd, 0x44, 0xb5, 0x3e, 0x95, 0x70, 0xe3, 0x3e, 0xe7, 0xac, 0x86, 0x3e, + 0x00, 0xc4, 0x4e, 0x3d, 0x7e, 0xa6, 0x1d, 0x3e, 0xbd, 0x87, 0xbb, 0x3e, + 0xb4, 0xb8, 0x09, 0xbf, 0xa1, 0x1f, 0xf8, 0xbe, 0x8d, 0x90, 0xdd, 0x3e, + 0xde, 0xfa, 0x6f, 0xbe, 0xb2, 0x75, 0xe4, 0x3d, 0x6e, 0xfe, 0x36, 0x3e, + 0x20, 0x18, 0xc2, 0xbe, 0x39, 0xc7, 0xfb, 0xbe, 0xfe, 0xa4, 0x30, 0xbe, + 0xf7, 0x91, 0xde, 0xbe, 0xde, 0xab, 0x24, 0x3e, 0xfb, 0xbb, 0xce, 0x3e, + 0xeb, 0x23, 0x80, 0xbe, 0x7b, 0x58, 0x73, 0xbe, 0x9a, 0x2e, 0x03, 0x3e, + 0x10, 0x42, 0xa9, 0xbc, 0x10, 0x12, 0x64, 0xbd, 0xe3, 0x8d, 0x0c, 0x3d, + 0x9e, 0x48, 0x97, 0xbe, 0x34, 0x51, 0xd4, 0xbe, 0x02, 0x3b, 0x0d, 0x3e, + 0x62, 0x67, 0x89, 0xbe, 0x74, 0xdf, 0xa2, 0x3d, 0xf3, 0x25, 0xb3, 0xbe, + 0xef, 0x34, 0x7b, 0x3d, 0x61, 0x70, 0xe3, 0x3d, 0xba, 0x76, 0xc0, 0xbe, + 0x7d, 0xe9, 0xa7, 0x3e, 0xc3, 0xab, 0xd0, 0xbe, 0xcf, 0x7c, 0xdb, 0xbe, + 0x70, 0x27, 0x9a, 0xbe, 0x98, 0xf5, 0x3c, 0xbd, 0xff, 0x4b, 0x4b, 0x3e, + 0x7e, 0xa0, 0xf8, 0xbd, 0xd4, 0x6e, 0x86, 0x3d, 0x00, 0x4a, 0x07, 0x3a, + 0x4c, 0x24, 0x61, 0xbe, 0x54, 0x68, 0xf7, 0xbd, 0x02, 0x3f, 0x77, 0xbe, + 0x23, 0x79, 0xb3, 0x3e, 0x1c, 0x83, 0xad, 0xbd, 0xc8, 0x92, 0x8d, 0x3e, + 0xa8, 0xf3, 0x15, 0xbd, 0xe6, 0x4d, 0x6c, 0x3d, 0xac, 0xe7, 0x98, 0xbe, + 0x81, 0xec, 0xbd, 0x3e, 0xe2, 0x55, 0x73, 0x3e, 0xc1, 0x77, 0xc7, 0x3e, + 0x6e, 0x1b, 0x5e, 0x3d, 0x27, 0x78, 0x02, 0x3f, 0xd4, 0x21, 0x90, 0x3d, + 0x52, 0xdc, 0x1f, 0x3e, 0xbf, 0xda, 0x88, 0x3e, 0x80, 0x79, 0xe3, 0xbd, + 0x40, 0x6f, 0x10, 0xbe, 0x20, 0x43, 0x2e, 0xbd, 0xf0, 0x76, 0xc5, 0xbd, + 0xcc, 0xa0, 0x04, 0xbe, 0xf0, 0x69, 0xd7, 0xbe, 0xb1, 0xfe, 0x64, 0xbe, + 0x20, 0x41, 0x84, 0xbe, 0xb2, 0xc3, 0x26, 0xbe, 0xd8, 0xf4, 0x09, 0xbe, + 0x64, 0x44, 0xd1, 0x3d, 0xd5, 0xe1, 0xc8, 0xbe, 0x35, 0xbc, 0x3f, 0xbe, + 0xc0, 0x94, 0x82, 0x3d, 0xdc, 0x2b, 0xb1, 0xbd, 0x02, 0xdb, 0xbf, 0xbe, + 0xa5, 0x7f, 0x8a, 0x3e, 0x21, 0xb4, 0xa2, 0x3e, 0xcd, 0x86, 0x56, 0xbf, + 0x9c, 0x3b, 0x76, 0xbc, 0x85, 0x6d, 0x60, 0xbf, 0x86, 0x00, 0x3c, 0xbe, + 0xc1, 0x23, 0x7e, 0x3e, 0x96, 0xcd, 0x3f, 0x3e, 0x86, 0x91, 0x2d, 0x3e, + 0x55, 0xef, 0x87, 0x3e, 0x7e, 0x97, 0x03, 0xbe, 0x2a, 0xcd, 0x01, 0x3e, + 0x32, 0xc9, 0x8e, 0xbe, 0x72, 0x77, 0x3b, 0xbe, 0xe0, 0xa1, 0xbc, 0xbe, + 0x8d, 0xb7, 0xa7, 0x3e, 0x1c, 0x05, 0x95, 0xbe, 0xf7, 0x1f, 0xbb, 0x3e, + 0xc9, 0x3e, 0xd6, 0x3e, 0x80, 0x42, 0xe9, 0xbd, 0x27, 0x0c, 0xd2, 0xbe, + 0x5c, 0x32, 0x34, 0xbe, 0x14, 0xcb, 0xca, 0xbd, 0xdd, 0x3a, 0x67, 0xbe, + 0x1c, 0xbb, 0x8d, 0xbe, 0x91, 0xac, 0x5c, 0xbe, 0x52, 0x40, 0x6f, 0xbe, + 0xd7, 0x71, 0x94, 0x3e, 0x18, 0x71, 0x09, 0xbe, 0x9b, 0x29, 0xd9, 0xbe, + 0x7d, 0x66, 0xd2, 0xbe, 0x98, 0xd6, 0xb2, 0xbe, 0x00, 0xc9, 0x84, 0x3a, + 0xbc, 0xda, 0xc2, 0xbd, 0x1d, 0xc2, 0x1b, 0xbf, 0xd4, 0xdd, 0x92, 0x3e, + 0x07, 0x87, 0x6c, 0xbe, 0x40, 0xc2, 0x3b, 0xbe, 0xbd, 0xe2, 0x9c, 0x3e, + 0x0a, 0xb5, 0xa0, 0xbe, 0xe2, 0xd5, 0x9c, 0xbe, 0x3e, 0xbb, 0x7c, 0x3e, + 0x17, 0xb4, 0xcf, 0x3e, 0xd5, 0x8e, 0xc8, 0xbe, 0x7c, 0xf9, 0x5c, 0x3e, + 0x80, 0xfc, 0x0d, 0x3d, 0xc5, 0xd5, 0x8b, 0x3e, 0xf5, 0x17, 0xa2, 0x3e, + 0xc7, 0x60, 0x89, 0xbe, 0xec, 0x95, 0x87, 0x3d, 0x7a, 0xc2, 0x5d, 0xbf, + 0x77, 0x94, 0x98, 0x3e, 0x77, 0x39, 0x07, 0xbc, 0x42, 0x29, 0x00, 0x3e, + 0xaf, 0xd0, 0xa9, 0x3e, 0x31, 0x23, 0xc4, 0xbe, 0x95, 0x36, 0x5b, 0xbe, + 0xc7, 0xdc, 0x83, 0xbe, 0x1e, 0x6b, 0x47, 0x3e, 0x5b, 0x24, 0x99, 0x3e, + 0x99, 0x27, 0x54, 0x3e, 0xc8, 0x20, 0xdd, 0xbd, 0x5a, 0x86, 0x2f, 0x3e, + 0x80, 0xf0, 0x69, 0xbe, 0x44, 0xfc, 0x84, 0xbd, 0x82, 0xa0, 0x2a, 0xbe, + 0x87, 0xe6, 0x2a, 0x3e, 0xd8, 0x34, 0xae, 0x3d, 0x50, 0xbd, 0xb5, 0x3e, + 0xc4, 0x8c, 0x88, 0xbe, 0xe3, 0xbc, 0xa5, 0x3e, 0xa9, 0xda, 0x9e, 0x3e, + 0x3e, 0xb8, 0x23, 0xbe, 0x80, 0x90, 0x15, 0x3d, 0x97, 0x3f, 0xc3, 0x3e, + 0xca, 0x5c, 0x9d, 0x3e, 0x21, 0xe8, 0xe1, 0x3e, 0xc0, 0x49, 0x01, 0xbc, + 0x00, 0x0b, 0x88, 0xbd, 0x3f, 0xf7, 0xca, 0x3c, 0xfb, 0x5a, 0xb1, 0x3e, + 0x60, 0xd2, 0x0d, 0x3c, 0xce, 0x23, 0x78, 0xbf, 0x8f, 0x4f, 0xb9, 0xbe, + 0x69, 0x6a, 0x34, 0xbf, 0x4b, 0x5e, 0xa9, 0x3e, 0x64, 0x8c, 0xd9, 0x3e, + 0x52, 0x77, 0x36, 0x3e, 0xeb, 0xaf, 0xbe, 0x3e, 0x40, 0xbe, 0x36, 0x3c, + 0x08, 0x65, 0x3b, 0xbd, 0x55, 0xe0, 0x66, 0xbd, 0xd2, 0xe8, 0x9b, 0xbe, + 0x86, 0xe3, 0x09, 0xbe, 0x93, 0x3d, 0xdd, 0x3e, 0x0f, 0x66, 0x18, 0x3f, + 0x18, 0x05, 0x33, 0xbd, 0xde, 0x15, 0xd7, 0xbe, 0xaa, 0xcf, 0x49, 0xbe, + 0xa2, 0xa5, 0x64, 0x3e, 0xe6, 0x9c, 0x42, 0xbe, 0x54, 0x42, 0xcc, 0x3d, + 0xa0, 0xbd, 0x9d, 0xbe, 0xc2, 0x69, 0x48, 0x3e, 0x5b, 0x8b, 0xa2, 0xbe, + 0xc0, 0x13, 0x87, 0x3d, 0x36, 0xfd, 0x69, 0x3e, 0x05, 0x86, 0x40, 0xbe, + 0x1e, 0x7a, 0xce, 0xbe, 0x46, 0x13, 0xa7, 0xbe, 0x68, 0x52, 0x86, 0xbe, + 0x04, 0x9e, 0x86, 0xbd, 0x8c, 0x54, 0xc1, 0x3d, 0xe0, 0x3b, 0xad, 0x3c, + 0x42, 0x67, 0x85, 0xbd, 0xea, 0x97, 0x42, 0x3e, 0x6e, 0x13, 0x3b, 0xbf, + 0x56, 0x5b, 0x16, 0x3e, 0xaa, 0xab, 0xdf, 0x3e, 0xc8, 0x41, 0x36, 0x3d, + 0x24, 0x2d, 0x47, 0xbe, 0x77, 0xa5, 0xae, 0x3e, 0xc0, 0xc2, 0x5b, 0x3c, + 0xac, 0xac, 0x4e, 0x3e, 0x99, 0xec, 0x13, 0xbe, 0xf2, 0xab, 0x73, 0x3e, + 0xaa, 0xa1, 0x48, 0xbe, 0xe8, 0xd3, 0x01, 0xbe, 0x60, 0xb7, 0xc7, 0xbd, + 0x64, 0x72, 0xd3, 0x3d, 0x83, 0xd3, 0x99, 0x3e, 0x0c, 0x76, 0x34, 0xbe, + 0x42, 0xda, 0x0d, 0x3e, 0xfb, 0x47, 0x9a, 0x3e, 0x8b, 0xdc, 0x92, 0xbe, + 0x56, 0x7f, 0x6b, 0x3e, 0x04, 0xd4, 0x88, 0xbd, 0x11, 0x9e, 0x80, 0x3e, + 0x3c, 0x89, 0xff, 0x3d, 0xb3, 0x3e, 0x88, 0x3e, 0xf7, 0xf0, 0x88, 0x3e, + 0x28, 0xfb, 0xc9, 0xbe, 0x53, 0x3e, 0xcf, 0x3e, 0xac, 0x75, 0xdc, 0xbe, + 0xdd, 0xca, 0xd7, 0x3e, 0x01, 0x58, 0xa7, 0x3e, 0x29, 0xb8, 0x13, 0xbf, + 0x76, 0x81, 0x12, 0xbc, 0x28, 0x8b, 0x16, 0xbf, 0x0e, 0xec, 0x0e, 0x3e, + 0x40, 0x0a, 0xdb, 0xbd, 0x98, 0xec, 0xbf, 0xbd, 0x32, 0x55, 0x0c, 0xbe, + 0xfb, 0xf9, 0xc9, 0x3e, 0x83, 0x4a, 0x6d, 0xbe, 0x76, 0x59, 0xe2, 0xbe, + 0x54, 0x7d, 0x9f, 0xbb, 0x9d, 0xe8, 0x95, 0x3e, 0x5c, 0xd3, 0xd0, 0x3d, + 0x19, 0x8a, 0xb0, 0x3e, 0xde, 0x6f, 0x2e, 0xbe, 0xd0, 0x16, 0x83, 0x3d, + 0x9c, 0x7d, 0x11, 0xbf, 0x2b, 0xcc, 0x25, 0x3c, 0x2a, 0xa5, 0x27, 0xbe, + 0x22, 0x14, 0xc7, 0xbe, 0x5e, 0x7a, 0xac, 0x3e, 0x4e, 0x41, 0x94, 0xbe, + 0x5a, 0x68, 0x7b, 0x3e, 0x86, 0xfd, 0x4e, 0x3e, 0xa2, 0x56, 0x6a, 0xbe, + 0xca, 0xfe, 0x81, 0xbe, 0x43, 0xc3, 0xb1, 0xbd, 0xc5, 0xb8, 0xa7, 0x3e, + 0x55, 0x23, 0xcd, 0x3e, 0xaf, 0x2e, 0x76, 0x3e, 0x69, 0xa8, 0x90, 0xbe, + 0x0d, 0xba, 0xb9, 0x3e, 0x66, 0xff, 0xff, 0xff, 0x04, 0x00, 0x00, 0x00, + 0x40, 0x00, 0x00, 0x00, 0x53, 0xd6, 0xe2, 0x3d, 0x66, 0xb6, 0xcc, 0x3e, + 0x03, 0xe7, 0xf6, 0x3e, 0xe0, 0x28, 0x10, 0xbf, 0x00, 0x00, 0x00, 0x00, + 0x3e, 0x3d, 0xb0, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x62, 0xf0, 0x77, 0x3e, + 0xa6, 0x9d, 0xa4, 0x3e, 0x3a, 0x4b, 0xf3, 0xbe, 0x71, 0x9e, 0xa7, 0x3e, + 0x00, 0x00, 0x00, 0x00, 0x34, 0x39, 0xa2, 0x3e, 0x00, 0x00, 0x00, 0x00, + 0xcc, 0x9c, 0x4a, 0x3e, 0xab, 0x40, 0xa3, 0x3e, 0xb2, 0xff, 0xff, 0xff, + 0x04, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0xb3, 0x71, 0x67, 0x3f, + 0x9a, 0x7a, 0x95, 0xbf, 0xe1, 0x48, 0xe8, 0xbe, 0x8a, 0x72, 0x96, 0x3e, + 0x00, 0xd2, 0xd3, 0xbb, 0x1a, 0xc5, 0xd7, 0x3f, 0xac, 0x7e, 0xc8, 0xbe, + 0x90, 0xa7, 0x95, 0xbe, 0x3b, 0xd7, 0xdc, 0xbe, 0x41, 0xa8, 0x16, 0x3f, + 0x50, 0x5b, 0xcb, 0x3f, 0x52, 0xb9, 0xed, 0xbe, 0x2e, 0xa7, 0xc6, 0xbe, + 0xaf, 0x0f, 0x14, 0xbf, 0xb3, 0xda, 0x59, 0x3f, 0x02, 0xec, 0xd7, 0xbe, + 0x00, 0x00, 0x06, 0x00, 0x08, 0x00, 0x04, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x66, 0x11, 0x1f, 0xbf, + 0xb8, 0xfb, 0xff, 0xff, 0x0f, 0x00, 0x00, 0x00, 0x54, 0x4f, 0x43, 0x4f, + 0x20, 0x43, 0x6f, 0x6e, 0x76, 0x65, 0x72, 0x74, 0x65, 0x64, 0x2e, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x14, 0x00, + 0x04, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0xf0, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0xd8, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, + 0x48, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xce, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x08, 0x18, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x1c, 0xfc, 0xff, 0xff, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, + 0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x07, 0x00, 0x10, 0x00, + 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x1c, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xba, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x16, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x0c, 0x00, 0x07, 0x00, 0x10, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x24, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x08, 0x00, 0x07, 0x00, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x10, 0x03, 0x00, 0x00, 0xa4, 0x02, 0x00, 0x00, + 0x40, 0x02, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, 0xac, 0x01, 0x00, 0x00, + 0x48, 0x01, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, + 0x50, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x26, 0xfd, 0xff, 0xff, + 0x3c, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x18, 0xfd, 0xff, 0xff, 0x20, 0x00, 0x00, 0x00, + 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, + 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x34, 0x2f, 0x4d, 0x61, 0x74, + 0x4d, 0x75, 0x6c, 0x5f, 0x62, 0x69, 0x61, 0x73, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x6e, 0xfd, 0xff, 0xff, + 0x50, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x60, 0xfd, 0xff, 0xff, 0x34, 0x00, 0x00, 0x00, + 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, + 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x34, 0x2f, 0x4d, 0x61, 0x74, + 0x4d, 0x75, 0x6c, 0x2f, 0x52, 0x65, 0x61, 0x64, 0x56, 0x61, 0x72, 0x69, + 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x70, 0x6f, 0x73, 0x65, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0xce, 0xfd, 0xff, 0xff, + 0x34, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xc0, 0xfd, 0xff, 0xff, 0x19, 0x00, 0x00, 0x00, + 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, + 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x33, 0x2f, 0x52, 0x65, 0x6c, + 0x75, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x12, 0xfe, 0xff, 0xff, 0x3c, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x04, 0xfe, 0xff, 0xff, 0x20, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e, + 0x73, 0x65, 0x5f, 0x33, 0x2f, 0x4d, 0x61, 0x74, 0x4d, 0x75, 0x6c, 0x5f, + 0x62, 0x69, 0x61, 0x73, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x5a, 0xfe, 0xff, 0xff, 0x50, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x4c, 0xfe, 0xff, 0xff, 0x34, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e, + 0x73, 0x65, 0x5f, 0x33, 0x2f, 0x4d, 0x61, 0x74, 0x4d, 0x75, 0x6c, 0x2f, + 0x52, 0x65, 0x61, 0x64, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, + 0x4f, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0xba, 0xfe, 0xff, 0xff, 0x34, 0x00, 0x00, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xac, 0xfe, 0xff, 0xff, 0x19, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e, + 0x73, 0x65, 0x5f, 0x32, 0x2f, 0x52, 0x65, 0x6c, 0x75, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0xfe, 0xfe, 0xff, 0xff, 0x3c, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xf0, 0xfe, 0xff, 0xff, + 0x20, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x32, + 0x2f, 0x4d, 0x61, 0x74, 0x4d, 0x75, 0x6c, 0x5f, 0x62, 0x69, 0x61, 0x73, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x46, 0xff, 0xff, 0xff, 0x50, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, + 0x0c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x38, 0xff, 0xff, 0xff, + 0x34, 0x00, 0x00, 0x00, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x5f, 0x31, 0x2f, 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x32, + 0x2f, 0x4d, 0x61, 0x74, 0x4d, 0x75, 0x6c, 0x2f, 0x52, 0x65, 0x61, 0x64, + 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x4f, 0x70, 0x2f, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x73, 0x65, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0xa6, 0xff, 0xff, 0xff, 0x48, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, + 0x2c, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, + 0x04, 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x43, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, + 0x64, 0x65, 0x6e, 0x73, 0x65, 0x5f, 0x32, 0x5f, 0x69, 0x6e, 0x70, 0x75, + 0x74, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x14, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x08, 0x00, 0x0c, 0x00, 0x10, 0x00, 0x0e, 0x00, 0x00, 0x00, + 0x28, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x0a, 0x00, 0x0c, 0x00, 0x07, 0x00, 0x00, 0x00, 0x08, 0x00, + 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x03, 0x00, 0x00, 0x00}; +const int g_sine_model_data_len = 2640; diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h b/tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h new file mode 100644 index 00000000000..7a7ce6f47ee --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/sine_model_data.h @@ -0,0 +1,27 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This is a standard TensorFlow Lite model file that has been converted into a +// C data array, so it can be easily compiled into a binary for devices that +// don't have a file system. It was created using the command: +// xxd -i sine_model.tflite > sine_model_data.cc + +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_SINE_MODEL_DATA_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_SINE_MODEL_DATA_H_ + +extern const unsigned char g_sine_model_data[]; +extern const int g_sine_model_data_len; + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_MICRO_EXAMPLES_HELLO_WORLD_SINE_MODEL_DATA_H_ diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/sparkfun_edge/constants.cc b/tensorflow/lite/experimental/micro/examples/hello_world/sparkfun_edge/constants.cc new file mode 100644 index 00000000000..169401dd532 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/sparkfun_edge/constants.cc @@ -0,0 +1,19 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/constants.h" + +// This is tuned so that a full cycle takes ~4 seconds on a SparkFun Edge. +const int kInferencesPerCycle = 1000; diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/sparkfun_edge/output_handler.cc b/tensorflow/lite/experimental/micro/examples/hello_world/sparkfun_edge/output_handler.cc new file mode 100644 index 00000000000..24479eb77a6 --- /dev/null +++ b/tensorflow/lite/experimental/micro/examples/hello_world/sparkfun_edge/output_handler.cc @@ -0,0 +1,84 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/experimental/micro/examples/hello_world/output_handler.h" + +#include "am_bsp.h" // NOLINT + +/* +This function uses the device's LEDs to visually indicate the current y value. +The y value is in the range -1 <= y <= 1. The LEDs (red, green, blue, +and yellow) are physically lined up in the following order: + + [ R G B Y ] + +The following table represents how we will light the LEDs for different values: + +| Range | LEDs lit | +| 0.75 <= y <= 1 | [ 0 0 1 1 ] | +| 0 < y < 0.75 | [ 0 0 1 0 ] | +| y = 0 | [ 0 0 0 0 ] | +| -0.75 < y < 0 | [ 0 1 0 0 ] | +| -1 <= y <= 0.75 | [ 1 1 0 0 ] | + +*/ +void HandleOutput(tflite::ErrorReporter* error_reporter, float x_value, + float y_value) { + // The first time this method runs, set up our LEDs correctly + static bool is_initialized = false; + if (!is_initialized) { + // Set up LEDs as outputs + am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_RED, g_AM_HAL_GPIO_OUTPUT_12); + am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_BLUE, g_AM_HAL_GPIO_OUTPUT_12); + am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_GREEN, g_AM_HAL_GPIO_OUTPUT_12); + am_hal_gpio_pinconfig(AM_BSP_GPIO_LED_YELLOW, g_AM_HAL_GPIO_OUTPUT_12); + // Ensure all pins are cleared + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_RED); + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_BLUE); + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_GREEN); + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_YELLOW); + is_initialized = true; + } + + // Set the LEDs to represent negative values + if (y_value < 0) { + // Clear unnecessary LEDs + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_GREEN); + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_YELLOW); + // The blue LED is lit for all negative values + am_hal_gpio_output_set(AM_BSP_GPIO_LED_BLUE); + // The red LED is lit in only some cases + if (y_value <= -0.75) { + am_hal_gpio_output_set(AM_BSP_GPIO_LED_RED); + } else { + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_RED); + } + // Set the LEDs to represent positive values + } else if (y_value > 0) { + // Clear unnecessary LEDs + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_RED); + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_BLUE); + // The green LED is lit for all positive values + am_hal_gpio_output_set(AM_BSP_GPIO_LED_GREEN); + // The yellow LED is lit in only some cases + if (y_value >= 0.75) { + am_hal_gpio_output_set(AM_BSP_GPIO_LED_YELLOW); + } else { + am_hal_gpio_output_clear(AM_BSP_GPIO_LED_YELLOW); + } + } + // Log the current X and Y values + error_reporter->Report("x_value: %f, y_value: %f\n", x_value, y_value); +} diff --git a/tensorflow/lite/experimental/micro/tools/make/templates/LCD_DISCO_F746NG.lib.tpl b/tensorflow/lite/experimental/micro/tools/make/templates/LCD_DISCO_F746NG.lib.tpl new file mode 100644 index 00000000000..899a504ff76 --- /dev/null +++ b/tensorflow/lite/experimental/micro/tools/make/templates/LCD_DISCO_F746NG.lib.tpl @@ -0,0 +1 @@ +http://os.mbed.com/teams/ST/code/LCD_DISCO_F746NG/#d44525b1de98 From e9bea60124a7a57ffe945e786fcd9c2a0c866028 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 15:05:33 -0700 Subject: [PATCH 304/332] Fix compilation and generation of tflite_runtime python package. New package file structure: tflite_runtime __init__.py interpreter.py _interpreter_wrapper..so interpreter_wrapper.py Cross compilation is fully supported for rpi and aarch64 targets: TENSORFLOW_TARGET=rpi build_pip_package.sh TENSORFLOW_TARGET=aarch64 build_pip_package.sh Python version is configurable by PYTHON env var: PYTHON=python3 build_pip_package.sh PiperOrigin-RevId: 257690020 --- tensorflow/lite/python/interpreter.py | 2 +- tensorflow/lite/tools/make/Makefile | 46 ++++++++--------- .../lite/tools/make/build_aarch64_lib.sh | 8 +-- tensorflow/lite/tools/make/build_lib.sh | 24 +++++++++ tensorflow/lite/tools/make/build_rpi_lib.sh | 8 +-- tensorflow/lite/tools/pip_package/README.md | 4 +- .../tools/pip_package/build_pip_package.sh | 51 +++++++++---------- tensorflow/lite/tools/pip_package/setup.py | 32 ++++++++---- 8 files changed, 103 insertions(+), 72 deletions(-) create mode 100755 tensorflow/lite/tools/make/build_lib.sh diff --git a/tensorflow/lite/python/interpreter.py b/tensorflow/lite/python/interpreter.py index 63e54b5fef7..23a654e3035 100644 --- a/tensorflow/lite/python/interpreter.py +++ b/tensorflow/lite/python/interpreter.py @@ -40,7 +40,7 @@ try: except ImportError: # When full Tensorflow Python PIP is not available do not use lazy load # and instead of the tflite_runtime path. - from tflite_runtime.lite.python import interpreter_wrapper as _interpreter_wrapper + from tflite_runtime import interpreter_wrapper as _interpreter_wrapper def tf_export_dummy(*x, **kwargs): del x, kwargs diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile index 38fb48f9703..393a66255b1 100644 --- a/tensorflow/lite/tools/make/Makefile +++ b/tensorflow/lite/tools/make/Makefile @@ -79,7 +79,7 @@ BENCHMARK_BINARY_NAME := benchmark_model # A small example program that shows how to link against the library. MINIMAL_SRCS := \ -tensorflow/lite/examples/minimal/minimal.cc + tensorflow/lite/examples/minimal/minimal.cc # What sources we want to compile, must be kept in sync with the main Bazel # build files. @@ -91,7 +91,7 @@ PROFILE_SUMMARIZER_SRCS := \ tensorflow/core/util/stats_calculator.cc CMD_LINE_TOOLS_SRCS := \ - tensorflow/lite/tools/command_line_flags.cc + tensorflow/lite/tools/command_line_flags.cc CORE_CC_ALL_SRCS := \ $(wildcard tensorflow/lite/*.cc) \ @@ -99,19 +99,19 @@ $(wildcard tensorflow/lite/*.c) \ $(wildcard tensorflow/lite/c/*.c) \ $(wildcard tensorflow/lite/core/*.cc) \ $(wildcard tensorflow/lite/core/api/*.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/allocator.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/block_map.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/blocking_counter.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/context.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/detect_dotprod.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/kernel.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/pack.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/pmu.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/thread_pool.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/trace.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/trmul.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/tune.cc) \ -$(wildcard tensorflow/lite/experimental/ruy/wait.cc) +tensorflow/lite/experimental/ruy/allocator.cc \ +tensorflow/lite/experimental/ruy/block_map.cc \ +tensorflow/lite/experimental/ruy/blocking_counter.cc \ +tensorflow/lite/experimental/ruy/context.cc \ +tensorflow/lite/experimental/ruy/detect_dotprod.cc \ +tensorflow/lite/experimental/ruy/kernel.cc \ +tensorflow/lite/experimental/ruy/pack.cc \ +tensorflow/lite/experimental/ruy/pmu.cc \ +tensorflow/lite/experimental/ruy/thread_pool.cc \ +tensorflow/lite/experimental/ruy/trace.cc \ +tensorflow/lite/experimental/ruy/trmul.cc \ +tensorflow/lite/experimental/ruy/tune.cc \ +tensorflow/lite/experimental/ruy/wait.cc ifneq ($(BUILD_TYPE),micro) CORE_CC_ALL_SRCS += \ $(wildcard tensorflow/lite/kernels/*.cc) \ @@ -119,13 +119,9 @@ $(wildcard tensorflow/lite/kernels/internal/*.cc) \ $(wildcard tensorflow/lite/kernels/internal/optimized/*.cc) \ $(wildcard tensorflow/lite/kernels/internal/reference/*.cc) \ $(PROFILER_SRCS) \ -$(wildcard tensorflow/lite/kernels/*.c) \ -$(wildcard tensorflow/lite/kernels/internal/*.c) \ -$(wildcard tensorflow/lite/kernels/internal/optimized/*.c) \ -$(wildcard tensorflow/lite/kernels/internal/reference/*.c) \ -$(wildcard tensorflow/lite/tools/make/downloads/farmhash/src/farmhash.cc) \ -$(wildcard tensorflow/lite/tools/make/downloads/fft2d/fftsg.c) \ -$(wildcard tensorflow/lite/tools/make/downloads/flatbuffers/src/util.cpp) +tensorflow/lite/tools/make/downloads/farmhash/src/farmhash.cc \ +tensorflow/lite/tools/make/downloads/fft2d/fftsg.c \ +tensorflow/lite/tools/make/downloads/flatbuffers/src/util.cpp endif # Remove any duplicates. CORE_CC_ALL_SRCS := $(sort $(CORE_CC_ALL_SRCS)) @@ -138,7 +134,7 @@ $(wildcard tensorflow/lite/kernels/*test_main.cc) \ $(wildcard tensorflow/lite/kernels/*test_util.cc) \ $(MINIMAL_SRCS) -BUILD_WITH_MMAP=true +BUILD_WITH_MMAP ?= true ifeq ($(BUILD_TYPE),micro) BUILD_WITH_MMAP=false endif @@ -151,7 +147,7 @@ else CORE_CC_EXCLUDE_SRCS += tensorflow/lite/mmap_allocation_disabled.cc endif -BUILD_WITH_NNAPI=true +BUILD_WITH_NNAPI ?= true ifeq ($(BUILD_TYPE),micro) BUILD_WITH_NNAPI=false endif @@ -191,7 +187,7 @@ EVALUATION_UTILS_SRCS := \ BENCHMARK_ALL_SRCS := $(TF_LITE_CC_SRCS) \ $(wildcard $(BENCHMARK_SRCS_DIR)/*.cc) \ $(PROFILE_SUMMARIZER_SRCS) \ - $(CMD_LINE_TOOLS_SRCS) \ + $(CMD_LINE_TOOLS_SRCS) \ $(EVALUATION_UTILS_SRCS) BENCHMARK_SRCS := $(filter-out \ diff --git a/tensorflow/lite/tools/make/build_aarch64_lib.sh b/tensorflow/lite/tools/make/build_aarch64_lib.sh index 054b3daedf8..0ce4089c11c 100755 --- a/tensorflow/lite/tools/make/build_aarch64_lib.sh +++ b/tensorflow/lite/tools/make/build_aarch64_lib.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # Copyright 2017 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,9 +14,11 @@ # limitations under the License. # ============================================================================== +set -x set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR/../../../.." +TENSORFLOW_DIR="${SCRIPT_DIR}/../../../.." + +make -j 4 TARGET=aarch64 -C "${TENSORFLOW_DIR}" -f tensorflow/lite/tools/make/Makefile -CC_PREFIX=aarch64-linux-gnu- make -j 3 -f tensorflow/lite/tools/make/Makefile TARGET=aarch64 TARGET_ARCH=armv8-a diff --git a/tensorflow/lite/tools/make/build_lib.sh b/tensorflow/lite/tools/make/build_lib.sh new file mode 100755 index 00000000000..7fdd262ee9c --- /dev/null +++ b/tensorflow/lite/tools/make/build_lib.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -x +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TENSORFLOW_DIR="${SCRIPT_DIR}/../../../.." + +make -j 4 BUILD_WITH_NNAPI=false -C "${TENSORFLOW_DIR}" -f tensorflow/lite/tools/make/Makefile + diff --git a/tensorflow/lite/tools/make/build_rpi_lib.sh b/tensorflow/lite/tools/make/build_rpi_lib.sh index 1521bb39332..c7edc6755e9 100755 --- a/tensorflow/lite/tools/make/build_rpi_lib.sh +++ b/tensorflow/lite/tools/make/build_rpi_lib.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # Copyright 2017 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,9 +14,11 @@ # limitations under the License. # ============================================================================== +set -x set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$SCRIPT_DIR/../../../.." +TENSORFLOW_DIR="${SCRIPT_DIR}/../../../.." + +make -j 4 TARGET=rpi -C "${TENSORFLOW_DIR}" -f tensorflow/lite/tools/make/Makefile -CC_PREFIX=arm-linux-gnueabihf- make -j 3 -f tensorflow/lite/tools/make/Makefile TARGET=rpi TARGET_ARCH=armv7l diff --git a/tensorflow/lite/tools/pip_package/README.md b/tensorflow/lite/tools/pip_package/README.md index 8190782c39f..adab810126a 100644 --- a/tensorflow/lite/tools/pip_package/README.md +++ b/tensorflow/lite/tools/pip_package/README.md @@ -18,8 +18,8 @@ pip install --upgrade Note, unlike tensorflow this will be installed to a tflite_runtime namespace. You can then use the Tensorflow Lite interpreter as. ``` -import tflite_runtime as tflr -interpreter = tflr.lite.Interpreter(model_path="foo.tflite") +from tflite_runtime import interpreter as tflr +interpreter = tflr.Interpreter(model_path="foo.tflite") ``` This currently works to build on Linux machines including Raspberry Pi. In diff --git a/tensorflow/lite/tools/pip_package/build_pip_package.sh b/tensorflow/lite/tools/pip_package/build_pip_package.sh index 2887ce84712..1cb3866af73 100644 --- a/tensorflow/lite/tools/pip_package/build_pip_package.sh +++ b/tensorflow/lite/tools/pip_package/build_pip_package.sh @@ -16,39 +16,36 @@ set -e +PYTHON="${PYTHON:-python}" + # Find where this script lives and then the Tensorflow root. -MY_DIRECTORY=`dirname $0` -export TENSORFLOW_SRC_ROOT=`realpath $MY_DIRECTORY/../../../..` +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -export TENSORFLOW_VERSION=`grep "_VERSION = " $TENSORFLOW_SRC_ROOT/tensorflow/tools/pip_package/setup.py | cut -d'=' -f 2 | sed "s/[ '-]//g"`; +export TENSORFLOW_SRC_ROOT="${SCRIPT_DIR}/../../../.." +export TENSORFLOW_VERSION=`grep "_VERSION = " "${TENSORFLOW_SRC_ROOT}/tensorflow/tools/pip_package/setup.py" | cut -d'=' -f 2 | sed "s/[ '-]//g"`; +TFLITE_ROOT="${TENSORFLOW_SRC_ROOT}/tensorflow/lite" # Build a pip build tree. -BUILD_ROOT=/tmp/tflite_pip -rm -rf $BUILD_ROOT -mkdir -p $BUILD_ROOT/tflite_runtime/lite -mkdir -p $BUILD_ROOT/tflite_runtime/lite/python +BUILD_ROOT="/tmp/tflite_pip/${PYTHON}" +rm -rf "${BUILD_ROOT}" +mkdir -p "${BUILD_ROOT}/tflite_runtime/" -# Build an importable module tree -cat > $BUILD_ROOT/tflite_runtime/__init__.py < $BUILD_ROOT/tflite_runtime/lite/__init__.py < $BUILD_ROOT/tflite_runtime/lite/python/__init__.py < Date: Thu, 11 Jul 2019 15:11:51 -0700 Subject: [PATCH 305/332] [XLA] HLO parser: fill abbreviated constant with deterministic random values. The problem that this cl wants to solve: Previously, if a user gives us a problematic hlo graph that OOMs, often in time we can't reproduce their issue by running the graph. This is because the abbreviated constant ("constant({...})")got interpreted as all 0, and then got optimized away into a scalar constant and broadcast (and potentially being CSE'ed). This CL changes fills those constants with deterministic semi garbage values (byte level iota), which makes reproducing the issue easier. PiperOrigin-RevId: 257691180 --- tensorflow/compiler/xla/service/hlo_parser.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index 6a9eeb94d48..2589de633d0 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -2344,6 +2344,20 @@ bool HloParser::ParseDenseLiteral(Literal* literal, const Shape& shape) { } elems_seen_per_dim[0] = shape.dimensions(0); lexer_.Lex(); + // Fill data with deterministic (garbage) values. Use static to avoid + // creating identical constants which could potentially got CSE'ed + // away. This is a best-effort approach to make sure replaying a HLO + // gives us same optimized HLO graph. + static uint32 data = 0; + uint32* raw_data = static_cast(literal->untyped_data()); + for (int64 i = 0; i < literal->size_bytes() / 4; ++i) { + raw_data[i] = data++; + } + uint8* raw_data_int8 = static_cast(literal->untyped_data()); + static uint8 data_int8 = 0; + for (int64 i = 0; i < literal->size_bytes() % 4; ++i) { + raw_data_int8[literal->size_bytes() / 4 + i] = data_int8++; + } break; } case TokKind::kComma: From 901bac3a5d428a001c7fce120402249c833ec6a8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 15:29:05 -0700 Subject: [PATCH 306/332] EmitFastTanh uses better estimation for very small values of x. For small values of x (|x| < 0.004), we now use the approximation tanh(x) = x. PiperOrigin-RevId: 257694297 --- tensorflow/compiler/xla/service/llvm_ir/math_ops.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/llvm_ir/math_ops.cc b/tensorflow/compiler/xla/service/llvm_ir/math_ops.cc index 0e115cdabf4..333a2e8f612 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/math_ops.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/math_ops.cc @@ -22,6 +22,14 @@ namespace llvm_ir { llvm::Value* EmitFastTanh(llvm::IRBuilder<>* b, llvm::Value* input) { llvm::Type* type = input->getType(); + // For small values of x, we can approximate tanh(x)=x. For extremely small + // values of x (|x| < 1e-37), the other approximation evaluates tanh(x) = 0. + const auto kCanUseApprox = 0.0004; + auto abs_x = + llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::fabs, {input}, {type}, b); + auto use_aprox = + b->CreateFCmpOLT(abs_x, llvm::ConstantFP::get(type, kCanUseApprox)); + // Clamp the input to [-9, 9]. llvm::Value* input_clamped = llvm_ir::EmitFloatMin( llvm_ir::EmitFloatMax(input, llvm::ConstantFP::get(type, -9.0), b), @@ -52,7 +60,8 @@ llvm::Value* EmitFastTanh(llvm::IRBuilder<>* b, llvm::Value* input) { llvm::ConstantFP::get(type, denominator_coeffs[i])); } - return b->CreateFDiv(numerator, denominator); + return b->CreateSelect(use_aprox, input, + b->CreateFDiv(numerator, denominator)); } } // namespace llvm_ir From c7973b017676fe84a5b4a859599c99877df4a2c5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 16:23:00 -0700 Subject: [PATCH 307/332] Minor doc formatting fix PiperOrigin-RevId: 257703746 --- tensorflow/python/distribute/distribute_lib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py index cbb394af530..629994a7353 100644 --- a/tensorflow/python/distribute/distribute_lib.py +++ b/tensorflow/python/distribute/distribute_lib.py @@ -618,7 +618,7 @@ class Strategy(object): # Create a dataset dataset = dataset_ops.Dataset.TFRecordDataset([ - "/a/1.tfr", "/a/2.tfr", "/a/3.tfr", /a/4.tfr"]) + "/a/1.tfr", "/a/2.tfr", "/a/3.tfr", "/a/4.tfr"]) # Distribute that dataset dist_dataset = strategy.experimental_distribute_dataset(dataset) From 24174643a75e819b8ce01fd70d45d03616e50071 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 16:23:20 -0700 Subject: [PATCH 308/332] Don't fail in constant folding if we try to materialize a DT_VARIANT or DT_RESOURCE. Addresses: #29525 PiperOrigin-RevId: 257703800 --- .../core/grappler/optimizers/constant_folding.cc | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc index 2a593be9635..54ef5567197 100644 --- a/tensorflow/core/grappler/optimizers/constant_folding.cc +++ b/tensorflow/core/grappler/optimizers/constant_folding.cc @@ -1090,7 +1090,9 @@ Status CreateConstantTensorAttrValue(DataType type, double value, SET_TENSOR_VAL_CASE(DT_QUINT8, int32, int); SET_TENSOR_VAL_CASE(DT_BOOL, bool, bool); default: - return errors::InvalidArgument("Unsupported type: ", type); + return errors::InvalidArgument( + "Unsupported type in CreateConstantTensorAttrValue: ", + DataTypeString(type)); } return Status::OK(); } @@ -1888,13 +1890,15 @@ Status ConstantFolding::ReplaceOperationWithConstant( double value, const GraphProperties& properties, const TensorShapeProto& shape, NodeDef* node, GraphDef* graph) { const DataType dtype = GetDataTypeFromNodeOrProps(*node, properties); - if (dtype == DT_INVALID) { + AttrValue tensor_attr; + Status s = CreateConstantTensorAttrValue(dtype, value, shape, &tensor_attr); + if (!s.ok()) { + // Fail gracefully without mutating the graph. + VLOG(1) << "Failed to replace node " << node->name() << " of type " + << DataTypeString(dtype) << " with constant tensor of value " + << value; return Status::OK(); } - - AttrValue tensor_attr; - TF_RETURN_IF_ERROR( - CreateConstantTensorAttrValue(dtype, value, shape, &tensor_attr)); return ReplaceOperationWithConstantTensor(dtype, tensor_attr.mutable_tensor(), node, graph); } From daacf4c6375e6b2aa50056ee4d6df77644425115 Mon Sep 17 00:00:00 2001 From: Ihor Indyk Date: Thu, 11 Jul 2019 16:34:56 -0700 Subject: [PATCH 309/332] [tf.data] Adding gradient descent algorithm for optimization of tunable parameters to be used in `Model::Optimize`. PiperOrigin-RevId: 257705789 --- tensorflow/core/framework/model.cc | 99 ++++++++++++++++++- tensorflow/core/framework/model.h | 10 ++ .../benchmarks/autotune_benchmark.py | 53 +++++++--- tensorflow/python/data/ops/dataset_ops.py | 1 + 4 files changed, 150 insertions(+), 13 deletions(-) diff --git a/tensorflow/core/framework/model.cc b/tensorflow/core/framework/model.cc index 6dff1ffe56e..a6b61148437 100644 --- a/tensorflow/core/framework/model.cc +++ b/tensorflow/core/framework/model.cc @@ -651,6 +651,10 @@ void Model::Optimize(AutotuneAlgorithm algorithm, int64 cpu_budget) { switch (algorithm) { case AutotuneAlgorithm::HILL_CLIMB: OptimizeHillClimb(cpu_budget); + break; + case AutotuneAlgorithm::GRADIENT_DESCENT: + OptimizeGradientDescent(cpu_budget); + break; } } @@ -715,13 +719,106 @@ std::map> Model::CollectTunableParameters( return parameters; } +void Model::OptimizeGradientDescent(int64 cpu_budget) { + std::shared_ptr snapshot; + { + tf_shared_lock lock(mu_); + snapshot = output_->Snapshot(nullptr); + } + VLOG(2) << "Starting optimization of tunable parameters with GradientDescent"; + const double processing_time = TotalProcessingTime(snapshot); + auto parameters = CollectTunableParameters(snapshot); + for (auto& pair : parameters) { + pair.second->value = pair.second->min; + } + // Gradient descent step size. + constexpr double kDescentStep = 0.7L; + + // Optimization is stopped once the `OutputTime` improvement is smaller than + // this value. + constexpr double kOptimizationPrecision = 100.0L; + + // Penalizing step for the parameters after we overoptimize (output time < + // processing time / cpu budget) the objective. + constexpr double kParametersPenalty = 0.05L; + + // Maximum number of iterations for optimization. + constexpr int64 kMaxIterations = 100; + + double output_time = 0; + double new_output_time; + double new_value; + for (int i = 0; i < kMaxIterations; ++i) { + std::map gradient; + new_output_time = OutputTime(snapshot, &gradient); + if (std::abs(output_time - new_output_time) < kOptimizationPrecision || + new_output_time < processing_time / cpu_budget) { + break; + } + double max_abs_derivative = 1.0; + for (auto& pair : parameters) { + if (pair.second->value != pair.second->max) { + max_abs_derivative = + std::max(max_abs_derivative, std::abs(gradient[pair.first])); + } + } + // Maximizes parameters on early stages of the model. + if (max_abs_derivative < kOptimizationPrecision) { + for (auto& pair : parameters) { + pair.second->value = pair.second->max; + } + break; + } + for (auto& pair : parameters) { + new_value = pair.second->value - + kDescentStep * gradient[pair.first] / max_abs_derivative; + // Projection on a feasible interval. + if (new_value > pair.second->max) { + pair.second->value = pair.second->max; + } else if (new_value < pair.second->min) { + pair.second->value = pair.second->min; + } else { + pair.second->value = new_value; + } + } + output_time = new_output_time; + } + // Penalize parameters if we overoptimized the objective. + for (int i = 0; + i < kMaxIterations && new_output_time < processing_time / cpu_budget; + ++i) { + for (auto& pair : parameters) { + new_value = pair.second->value - kParametersPenalty; + // Projection on a feasible interval. + if (new_value > pair.second->max) { + pair.second->value = pair.second->max; + } else if (new_value < pair.second->min) { + pair.second->value = pair.second->min; + } else { + pair.second->value = new_value; + } + } + new_output_time = OutputTime(snapshot, /*gradient=*/nullptr); + } + VLOG(2) << "Number of tunable parameters: " << parameters.size(); + for (auto& pair : parameters) { + pair.second->value = std::round(pair.second->value); + auto& parameter = pair.second; + VLOG(2) << "Setting tunable parameter " << pair.first << " to " + << parameter->value; + mutex_lock l(*parameter->state->mu); + parameter->state->value = parameter->value; + parameter->state->cond_var->notify_all(); + } +} + void Model::OptimizeHillClimb(int64 cpu_budget) { std::shared_ptr snapshot; { tf_shared_lock lock(mu_); snapshot = output_->Snapshot(nullptr); } - VLOG(2) << "Starting optimization of tunable parameters"; + VLOG(2) << "Starting optimization of tunable parameters with HillClimb"; const double processing_time = TotalProcessingTime(snapshot); auto parameters = CollectTunableParameters(snapshot); for (auto& pair : parameters) { diff --git a/tensorflow/core/framework/model.h b/tensorflow/core/framework/model.h index 3ae279350b4..2687cc6e534 100644 --- a/tensorflow/core/framework/model.h +++ b/tensorflow/core/framework/model.h @@ -40,6 +40,7 @@ constexpr int64 kAutotune = -1; enum class AutotuneAlgorithm { HILL_CLIMB = 0, + GRADIENT_DESCENT = 1, }; // Represents thread-safe state that can be shared between an input pipeline and @@ -543,6 +544,15 @@ class Model { // element divided by CPU budget. void OptimizeHillClimb(int64 cpu_budget); + // This optimization algorithm starts by setting all tunable parallelism + // parameters to the minimum value. It then improves current parameters by + // making a step in the direction opposite to the gradient of `OutputTime` and + // projecting resulting values on the feasible intervals. Improvement step is + // repeated until either the output time improvement is smaller than threshold + // value or the output time is less than the processing time needed to produce + // an element divided by CPU budget. + void OptimizeGradientDescent(int64 cpu_budget); + // Collects the output time and if `gradient` is not `nullptr`, the output // time gradient w.r.t. tunable parameters of the subtree rooted in the given // node and the last input time. diff --git a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py index fdd4f4d11b7..af7c4736083 100644 --- a/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py +++ b/tensorflow/python/data/experimental/benchmarks/autotune_benchmark.py @@ -33,9 +33,14 @@ class AutotuneBenchmark(test.Benchmark): def benchmark_map(self): a = self._benchmark_map(autotune=False) b = self._benchmark_map(autotune=True) - print("speedup: %f" % (a / b)) + c = self._benchmark_map( + autotune=True, algorithm=dataset_ops.AutotuneAlgorithm.GRADIENT_DESCENT) + print("HillClimb vs Default speedup: %f" % (a / b)) + print("GradientDescent vs Default speedup: %f" % (a / c)) - def _benchmark_map(self, autotune): + def _benchmark_map(self, + autotune, + algorithm=dataset_ops.AutotuneAlgorithm.HILL_CLIMB): k = 1024 * 1024 dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), np.random.rand(4 * k, @@ -45,6 +50,8 @@ class AutotuneBenchmark(test.Benchmark): options = dataset_ops.Options() options.experimental_optimization.apply_default_optimizations = False options.experimental_optimization.autotune = autotune + if autotune: + options.experimental_optimization.autotune_algorithm = algorithm.value dataset = dataset.with_options(options) iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() @@ -62,15 +69,19 @@ class AutotuneBenchmark(test.Benchmark): self.report_benchmark( iters=10000, wall_time=np.median(deltas), - name="map" + ("_autotune" if autotune else "")) + name="map" + (("_autotune_%s" % algorithm.name) if autotune else "")) return np.median(deltas) def benchmark_map_and_batch(self): a = self._benchmark_map_and_batch(autotune=False) b = self._benchmark_map_and_batch(autotune=True) - print("speedup: %f" % (a / b)) + c = self._benchmark_map_and_batch( + autotune=True, algorithm=dataset_ops.AutotuneAlgorithm.GRADIENT_DESCENT) + print("HillClimb vs Default speedup: %f" % (a / b)) + print("GradientDescent vs Default speedup: %f" % (a / c)) - def _benchmark_map_and_batch(self, autotune): + def _benchmark_map_and_batch( + self, autotune, algorithm=dataset_ops.AutotuneAlgorithm.HILL_CLIMB): batch_size = 16 k = 1024 * 1024 dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), @@ -83,6 +94,8 @@ class AutotuneBenchmark(test.Benchmark): options.experimental_optimization.apply_default_optimizations = False options.experimental_optimization.map_and_batch_fusion = True options.experimental_optimization.autotune = autotune + if autotune: + options.experimental_optimization.autotune_algorithm = algorithm.value dataset = dataset.with_options(options) iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() @@ -100,15 +113,21 @@ class AutotuneBenchmark(test.Benchmark): self.report_benchmark( iters=1000, wall_time=np.median(deltas), - name="map_and_batch" + ("_autotune" if autotune else "")) + name="map_and_batch" + + (("_autotune_%s" % algorithm.name) if autotune else "")) return np.median(deltas) def benchmark_interleave(self): a = self._benchmark_interleave(autotune=False) b = self._benchmark_interleave(autotune=True) - print("speedup: %f" % (a / b)) + c = self._benchmark_interleave( + autotune=True, algorithm=dataset_ops.AutotuneAlgorithm.GRADIENT_DESCENT) + print("HillClimb vs Default speedup: %f" % (a / b)) + print("GradientDescent vs Default speedup: %f" % (a / c)) - def _benchmark_interleave(self, autotune): + def _benchmark_interleave(self, + autotune, + algorithm=dataset_ops.AutotuneAlgorithm.HILL_CLIMB): k = 1024 * 1024 dataset = dataset_ops.Dataset.from_tensors((np.random.rand(1, 4 * k), np.random.rand(4 * k, @@ -121,6 +140,8 @@ class AutotuneBenchmark(test.Benchmark): options = dataset_ops.Options() options.experimental_optimization.apply_default_optimizations = False options.experimental_optimization.autotune = autotune + if autotune: + options.experimental_optimization.autotune_algorithm = algorithm.value dataset = dataset.with_options(options) iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() @@ -138,15 +159,20 @@ class AutotuneBenchmark(test.Benchmark): self.report_benchmark( iters=10000, wall_time=np.median(deltas), - name="interleave" + ("_autotune" if autotune else "")) + name="interleave" + + (("_autotune_%s" % algorithm.name) if autotune else "")) return np.median(deltas) def benchmark_map_and_interleave(self): a = self._benchmark_map_and_interleave(autotune=False) b = self._benchmark_map_and_interleave(autotune=True) - print("speedup: %f" % (a / b)) + c = self._benchmark_map_and_interleave( + autotune=True, algorithm=dataset_ops.AutotuneAlgorithm.GRADIENT_DESCENT) + print("HillClimb vs Default speedup: %f" % (a / b)) + print("GradientDescent vs Default speedup: %f" % (a / c)) - def _benchmark_map_and_interleave(self, autotune): + def _benchmark_map_and_interleave( + self, autotune, algorithm=dataset_ops.AutotuneAlgorithm.HILL_CLIMB): k = 1024 * 1024 a = (np.random.rand(1, 8 * k), np.random.rand(8 * k, 1)) b = (np.random.rand(1, 4 * k), np.random.rand(4 * k, 1)) @@ -181,6 +207,8 @@ class AutotuneBenchmark(test.Benchmark): options = dataset_ops.Options() options.experimental_optimization.apply_default_optimizations = False options.experimental_optimization.autotune = autotune + if autotune: + options.experimental_optimization.autotune_algorithm = algorithm.value dataset = dataset.with_options(options) iterator = dataset_ops.make_one_shot_iterator(dataset) get_next = iterator.get_next() @@ -198,7 +226,8 @@ class AutotuneBenchmark(test.Benchmark): self.report_benchmark( iters=10000, wall_time=np.median(deltas), - name="map_and_interleave" + ("_autotune" if autotune else "")) + name="map_and_interleave" + + (("_autotune_%s" % algorithm.name) if autotune else "")) return np.median(deltas) diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index cb164d7fbee..e61f8df7d5c 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -97,6 +97,7 @@ tf_export("data.experimental.AUTOTUNE").export_constant(__name__, "AUTOTUNE") class AutotuneAlgorithm(enum.Enum): HILL_CLIMB = 0 + GRADIENT_DESCENT = 1 @tf_export("data.Dataset", v1=[]) From de8298d1aeb311ccb9c9c7293731272c62f76049 Mon Sep 17 00:00:00 2001 From: Pete Warden Date: Thu, 11 Jul 2019 16:37:21 -0700 Subject: [PATCH 310/332] Workaround for nightly CI build problem PiperOrigin-RevId: 257706185 --- .../micro/tools/ci_build/ci_build_micro_projects.sh | 3 +++ .../lite/experimental/micro/tools/ci_build/test_sparkfun.sh | 3 +++ tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh | 3 +++ 3 files changed, 9 insertions(+) diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/ci_build_micro_projects.sh b/tensorflow/lite/experimental/micro/tools/ci_build/ci_build_micro_projects.sh index 36819f0c39c..32291a423a9 100755 --- a/tensorflow/lite/experimental/micro/tools/ci_build/ci_build_micro_projects.sh +++ b/tensorflow/lite/experimental/micro/tools/ci_build/ci_build_micro_projects.sh @@ -33,3 +33,6 @@ make -f tensorflow/lite/experimental/micro/tools/make/Makefile \ TARGET=${1} \ TAGS="${2}" \ generate_projects + +# Needed to solve CI build bug triggered by files added to source tree. +make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean_downloads diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh b/tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh index a49eb4239c2..451e21501bc 100755 --- a/tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh +++ b/tensorflow/lite/experimental/micro/tools/ci_build/test_sparkfun.sh @@ -24,3 +24,6 @@ cd ${ROOT_DIR} pwd make -f tensorflow/lite/experimental/micro/tools/make/Makefile TARGET=sparkfun_edge micro_speech_bin + +# Needed to solve CI build bug triggered by files added to source tree. +make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean_downloads diff --git a/tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh b/tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh index c0de76580a3..16b3e3204d5 100755 --- a/tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh +++ b/tensorflow/lite/experimental/micro/tools/ci_build/test_x86.sh @@ -24,3 +24,6 @@ cd ${ROOT_DIR} pwd make -f tensorflow/lite/experimental/micro/tools/make/Makefile test + +# Needed to solve CI build bug triggered by files added to source tree. +make -f tensorflow/lite/experimental/micro/tools/make/Makefile clean_downloads From b42db64f47bbc3fa012c569e4a8b97e11dca4ddc Mon Sep 17 00:00:00 2001 From: Daniel Situnayake Date: Thu, 11 Jul 2019 17:02:01 -0700 Subject: [PATCH 311/332] Readme and Colab fix PiperOrigin-RevId: 257710217 --- .../experimental/micro/examples/hello_world/README.md | 8 ++++---- .../micro/examples/hello_world/create_sine_model.ipynb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/README.md b/tensorflow/lite/experimental/micro/examples/hello_world/README.md index 5731234d683..1de9730848c 100644 --- a/tensorflow/lite/experimental/micro/examples/hello_world/README.md +++ b/tensorflow/lite/experimental/micro/examples/hello_world/README.md @@ -1,9 +1,9 @@ # Hello World example -This example is designed to demonstrate the absolute basics of using TensorFlow -Lite for Microcontrollers. It includes the full end-to-end workflow of training -a model, converting it for use with TensorFlow Lite, and running inference on a -microcontroller. +This example is designed to demonstrate the absolute basics of using [TensorFlow +Lite for Microcontrollers](https://www.tensorflow.org/lite/microcontrollers/overview). +It includes the full end-to-end workflow of training a model, converting it for +use with TensorFlow Lite, and running inference on a microcontroller. The sample is built around a model trained to replicate a `sine` function. It contains implementations for several platforms. In each case, the model is used diff --git a/tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb b/tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb index 372021dc08c..2a5cbc994c4 100644 --- a/tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb +++ b/tensorflow/lite/experimental/micro/examples/hello_world/create_sine_model.ipynb @@ -60,7 +60,7 @@ "\n", "\n", " \n", "
\n", - " Run in Google Colab\n", + " Run in Google Colab\n", " \n", " View source on GitHub\n", From 3da91cc2b14f4e6594f039a6dbcd2c1783a70682 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 17:09:20 -0700 Subject: [PATCH 312/332] Wrap cuSolver implementation of batched Cholesky decomposition in potrfBatched(). PiperOrigin-RevId: 257711651 --- tensorflow/core/kernels/cholesky_op.cc | 4 +-- tensorflow/core/kernels/cuda_solvers.cc | 37 +++++++++++++++++++++++++ tensorflow/core/kernels/cuda_solvers.h | 16 +++++++++-- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/kernels/cholesky_op.cc b/tensorflow/core/kernels/cholesky_op.cc index bcd42dc8d7a..744436c06e2 100644 --- a/tensorflow/core/kernels/cholesky_op.cc +++ b/tensorflow/core/kernels/cholesky_op.cc @@ -144,8 +144,8 @@ class CholeskyOpGpu : public AsyncOpKernel { const int64 batch_size = input_reshaped.dimension(0); std::vector dev_info; dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "potrf")); - // TODO(rmlarsen): Parallelize over batches if it turns out to be - // an important use case. + // TODO(rmlarsen): Use PotrfBatched for factoring many small matrices in + // parallel. for (int batch = 0; batch < batch_size; ++batch) { OP_REQUIRES_OK_ASYNC(context, solver->Potrf(CUBLAS_FILL_MODE_UPPER, n, diff --git a/tensorflow/core/kernels/cuda_solvers.cc b/tensorflow/core/kernels/cuda_solvers.cc index 6e26cc1d541..9abf5439571 100644 --- a/tensorflow/core/kernels/cuda_solvers.cc +++ b/tensorflow/core/kernels/cuda_solvers.cc @@ -387,6 +387,43 @@ static inline Status PotrfImpl(BufSizeFnT bufsize, SolverFnT solver, TF_CALL_LAPACK_TYPES(POTRF_INSTANCE); +#if CUDA_VERSION >= 9020 +template +static inline Status PotrfBatchedImpl( + SolverFnT solver, CudaSolver* cuda_solver, OpKernelContext* context, + cusolverDnHandle_t cusolver_dn_handle, cublasFillMode_t uplo, int n, + const Scalar* const host_a_dev_ptrs[], int lda, + DeviceLapackInfo* dev_lapack_info, int batch_size) { + mutex_lock lock(handle_map_mutex); + using CudaScalar = typename CUDAComplexT::type; + ScratchSpace dev_a_dev_ptrs = + cuda_solver->GetScratchSpace(sizeof(CudaScalar*) * batch_size, "", + /* on_host */ false); + if (!CopyHostToDevice(context, dev_a_dev_ptrs.mutable_data() /* dest */, + host_a_dev_ptrs /* source */, dev_a_dev_ptrs.bytes())) { + return errors::Internal("PotrfBatched: failed to copy pointers to device"); + } + TF_RETURN_IF_CUSOLVER_ERROR( + solver(cusolver_dn_handle, uplo, n, + reinterpret_cast(dev_a_dev_ptrs.mutable_data()), lda, + dev_lapack_info->mutable_data(), batch_size)); + return Status::OK(); +} + +#define POTRF_BATCHED_INSTANCE(Scalar, type_prefix) \ + template <> \ + Status CudaSolver::PotrfBatched( \ + cublasFillMode_t uplo, int n, const Scalar* const host_a_dev_ptrs[], \ + int lda, DeviceLapackInfo* dev_lapack_info, int batch_size) { \ + return PotrfBatchedImpl(DN_SOLVER_FN(potrfBatched, type_prefix), this, \ + context_, cusolver_dn_handle_, uplo, n, \ + host_a_dev_ptrs, lda, dev_lapack_info, \ + batch_size); \ + } + +TF_CALL_LAPACK_TYPES(POTRF_BATCHED_INSTANCE); +#endif // CUDA_VERSION >= 9020 + template static inline Status GetrfImpl(BufSizeFnT bufsize, SolverFnT solver, CudaSolver* cuda_solver, diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h index e4ae1d93227..9679fad09ac 100644 --- a/tensorflow/core/kernels/cuda_solvers.h +++ b/tensorflow/core/kernels/cuda_solvers.h @@ -208,13 +208,25 @@ class CudaSolver { const Scalar* B, int ldb, Scalar* C, int ldc) const TF_MUST_USE_RESULT; - // Computes the Cholesky factorization A = L * L^T for a single matrix. + // Computes the Cholesky factorization A = L * L^H for a single matrix. // Returns Status::OK() if the kernel was launched successfully. See: // http://docs.nvidia.com/cuda/cusolver/#cuds-lt-t-gt-potrf template Status Potrf(cublasFillMode_t uplo, int n, Scalar* dev_A, int lda, int* dev_lapack_info) TF_MUST_USE_RESULT; +#if CUDA_VERSION >= 9020 + // Computes the Cholesky factorization A = L * L^H for a batch of small + // matrices. + // Returns Status::OK() if the kernel was launched successfully. See: + // http://docs.nvidia.com/cuda/cusolver/index.html#cuds-lt-t-gt-potrfBatched + template + Status PotrfBatched(cublasFillMode_t uplo, int n, + const Scalar* const host_a_dev_ptrs[], int lda, + DeviceLapackInfo* dev_lapack_info, + int batch_size) TF_MUST_USE_RESULT; +#endif // CUDA_VERSION >= 9020 + // LU factorization. // Computes LU factorization with partial pivoting P * A = L * U. // See: http://docs.nvidia.com/cuda/cusolver/#cuds-lt-t-gt-getrf @@ -230,7 +242,7 @@ class CudaSolver { int* dev_lapack_info) const TF_MUST_USE_RESULT; // Computes partially pivoted LU factorizations for a batch of small matrices. - // Returns Status::OK() if the kernel was launched successfully.See: + // Returns Status::OK() if the kernel was launched successfully. See: // http://docs.nvidia.com/cuda/cublas/index.html#cublas-lt-t-gt-getrfbatched template Status GetrfBatched(int n, const Scalar* const host_a_dev_ptrs[], int lda, From e18c374952a7895146eb9bf432b87e03aefc00df Mon Sep 17 00:00:00 2001 From: Penporn Koanantakool Date: Thu, 11 Jul 2019 17:15:16 -0700 Subject: [PATCH 313/332] Use parallelFor in GatherNdSlice. Saw speedups from 3-12% on Intel Skylake. PiperOrigin-RevId: 257712497 --- .../core/kernels/gather_nd_op_cpu_impl.h | 43 ++++++------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/tensorflow/core/kernels/gather_nd_op_cpu_impl.h b/tensorflow/core/kernels/gather_nd_op_cpu_impl.h index f587e969f67..ec3865cc3ee 100644 --- a/tensorflow/core/kernels/gather_nd_op_cpu_impl.h +++ b/tensorflow/core/kernels/gather_nd_op_cpu_impl.h @@ -101,40 +101,21 @@ struct GatherNdSlice { typename TTypes::ConstMatrix Tindices, typename TTypes::Matrix Tout) { std::atomic error_loc(-1); - - const Eigen::DenseIndex batch_size = Tindices.dimension(0); -#if !defined(EIGEN_HAS_INDEX_LIST) - Eigen::Tensor::Dimensions reshape_dims{{ 1 }}; - Eigen::array broadcast_dims{{ batch_size }}; -#else -#if !defined(INTEL_MKL) || !defined(ENABLE_MKL) - Eigen::IndexList > reshape_dims; -#endif // defined(INTEL_MKL) && defined(ENABLE_MKL) - Eigen::IndexList broadcast_dims; - broadcast_dims.set(0, batch_size); -#endif // !defined(EIGEN_HAS_INDEX_LIST) + const Eigen::Index batch_size = Tindices.dimension(0); generator::GatherNdSliceGenerator gather_nd_generator( slice_size, Tindices, Tparams, Tout, &error_loc); -// TODO(b/137289929): Parallelize this with ParallelFor and remove OpenMP call. -#if defined(INTEL_MKL) && defined(ENABLE_MKL) -// Eigen implementation below is not highly performant. gather_nd_generator -// does not seem to be called in parallel, leading to very poor performance. -// Additionally, since it uses scalar (Tscratch) to invoke 'generate', it -// needs to go through redundant operations like 'reshape', 'broadcast' and -// 'sum'. OpenMP loop below essentially does same thing as Eigen code, but -// is considerably more efficient. -#pragma omp parallel for - for (Eigen::DenseIndex i = 0; i < batch_size; i++) { - const Eigen::array loc{i}; - gather_nd_generator(loc); - } -#else // INTEL_MKL && ENABLE_MKL - Tscratch.device(d) = Tscratch.reshape(reshape_dims) - .broadcast(broadcast_dims) - .generate(gather_nd_generator) - .sum(); -#endif // INTEL_MKL && ENABLE_MKL + auto compute_shard = [&](Eigen::Index begin, Eigen::Index end) { + for (Eigen::Index i = begin; i < end; ++i) { + const Eigen::array loc{i}; + gather_nd_generator(loc); + } + }; + Eigen::Index bytes_moved = sizeof(T) * (slice_size + IXDIM); + auto cost = Eigen::TensorOpCost(bytes_moved /* bytes loaded */, + bytes_moved /* bytes stored */, + slice_size + IXDIM /* compute cycles */); + d.parallelFor(batch_size, cost, compute_shard); // error_loc() returns -1 if there's no out-of-bounds index, // otherwise it returns the location of an OOB index in Tindices. From 96287812b88419a4641e4e5b62761f2599ceba34 Mon Sep 17 00:00:00 2001 From: Pavithra Vijay Date: Thu, 11 Jul 2019 17:21:34 -0700 Subject: [PATCH 314/332] tf.keras.layers.Layer.count_params() should use _maybe_build() instead of build() PiperOrigin-RevId: 257713416 --- tensorflow/python/keras/engine/base_layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index a80a4c2a072..75dd8937896 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -1497,7 +1497,7 @@ class Layer(module.Module): if not self.built: if self.__class__.__name__ == 'Sequential': with tf_utils.maybe_init_scope(self): - self.build() # pylint: disable=no-value-for-parameter + self._maybe_build() # pylint: disable=no-value-for-parameter else: raise ValueError('You tried to call `count_params` on ' + self.name + ', but the layer isn\'t built. ' From 4cb74523bd4b75edb85c7c4266c555c553e80fb1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 17:22:42 -0700 Subject: [PATCH 315/332] Automated rollback of commit b9a6fea1f0a501b226394431d0377eef0b40c4b0 PiperOrigin-RevId: 257713568 --- tensorflow/core/grappler/optimizers/BUILD | 2 +- .../grappler/optimizers/meta_optimizer.cc | 8 +-- tensorflow/python/BUILD | 64 ++++++++++--------- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index 707d242c2b7..6dc6cee5bdd 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -594,9 +594,9 @@ cc_library( ":debug_stripper", ":dependency_optimizer", ":function_optimizer", + ":generic_layout_optimizer", ":graph_optimizer", ":implementation_selector", - ":layout_optimizer", ":loop_optimizer", ":memory_optimizer", ":model_pruner", diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 057e5a6f5c0..7f1302d6b09 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -32,8 +32,8 @@ limitations under the License. #include "tensorflow/core/grappler/optimizers/debug_stripper.h" #include "tensorflow/core/grappler/optimizers/dependency_optimizer.h" #include "tensorflow/core/grappler/optimizers/function_optimizer.h" +#include "tensorflow/core/grappler/optimizers/generic_layout_optimizer.h" #include "tensorflow/core/grappler/optimizers/implementation_selector.h" -#include "tensorflow/core/grappler/optimizers/layout_optimizer.h" #include "tensorflow/core/grappler/optimizers/loop_optimizer.h" #include "tensorflow/core/grappler/optimizers/memory_optimizer.h" #include "tensorflow/core/grappler/optimizers/model_pruner.h" @@ -121,7 +121,7 @@ std::unique_ptr MetaOptimizer::MakeNewOptimizer( MK_OPT("constfold", new ConstantFolding(cpu_device_)); MK_OPT("shape", new ShapeOptimizer()); MK_OPT("remap", new Remapper(cfg_.remapping())); - MK_OPT("layout", new LayoutOptimizer()); + MK_OPT("layout", new GenericLayoutOptimizer()); MK_OPT("auto_mixed_precision", new AutoMixedPrecision(cfg_.auto_mixed_precision())); MK_OPT("memory", new MemoryOptimizer(RewriterConfig::MANUAL)); @@ -193,7 +193,7 @@ Status MetaOptimizer::InitializeOptimizers( MakeUnique(cfg_.dependency_optimization())); } if (cfg_.layout_optimizer() != RewriterConfig::OFF) { - optimizers->push_back(MakeUnique()); + optimizers->push_back(MakeUnique()); } if (AutoMixedPrecisionEnabled(cfg_.auto_mixed_precision())) { optimizers->push_back( @@ -267,7 +267,7 @@ Status MetaOptimizer::InitializeCustomGraphOptimizers( TF_RETURN_IF_ERROR(custom_optimizer->Init(&optimizer_config)); optimizers->push_back(std::move(custom_optimizer)); } else { - // If there are no custom optimizers with given name, try to initalize a + // If there are no custom optimizers with given name, try to initialize a // default optimizer. This way, custom configurable optimizers can be // mixed with default optimizers in any order. auto optimizer = MakeNewOptimizer(optimizer_config.name()); diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index c5fc23c6fcc..93f43ab338a 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -6455,37 +6455,39 @@ cuda_py_test( xla_enable_strict_auto_jit = True, ) -cuda_py_test( - name = "layout_optimizer_test", - size = "medium", - srcs = [ - "grappler/layout_optimizer_test.py", - ], - additional_deps = [ - ":client_testlib", - ":framework_for_generated_wrappers", - ":array_ops", - ":constant_op", - ":dtypes", - ":functional_ops", - ":math_ops", - ":nn", - ":ops", - ":random_ops", - ":state_ops", - ":tf_cluster", - ":tf_optimizer", - ":training", - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - ], - shard_count = 10, - tags = [ - "grappler", - ], - # This test will not run on XLA because it primarily tests the TF Classic flow. - xla_enable_strict_auto_jit = False, -) +# TODO(b/131764887) Remove once LayoutOptimizer is swapped out with GenericLayoutOptimizer. +# +# cuda_py_test( +# name = "layout_optimizer_test", +# size = "medium", +# srcs = [ +# "grappler/layout_optimizer_test.py", +# ], +# additional_deps = [ +# ":client_testlib", +# ":framework_for_generated_wrappers", +# ":array_ops", +# ":constant_op", +# ":dtypes", +# ":functional_ops", +# ":math_ops", +# ":nn", +# ":ops", +# ":random_ops", +# ":state_ops", +# ":tf_cluster", +# ":tf_optimizer", +# ":training", +# "//third_party/py/numpy", +# "//tensorflow/core:protos_all_py", +# ], +# shard_count = 10, +# tags = [ +# "grappler", +# ], +# # This test will not run on XLA because it primarily tests the TF Classic flow. +# xla_enable_strict_auto_jit = False, +# ) py_library( name = "cost_analyzer", From 21564c61516d581e9a226e36b632e64f0ea1cba5 Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Thu, 11 Jul 2019 17:25:40 -0700 Subject: [PATCH 316/332] Remove NoSideEffect trait from TensorFlow PlaceholderInputOp This will preserve attributes even if the input is unused. TFLite InputOp has been marked as having side effects for the same reason. TESTED = manually (trivial) PiperOrigin-RevId: 257713937 --- tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td index 1ba3ed6db8b..b2fcb01c2d5 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td @@ -105,8 +105,10 @@ retained with length 1. // In MLIR, the 'tf.Placeholder.input' instruction is used to capture attributes // of function arguments. +// Note: NoSideEffect trait is not added intentionally to preserve the captured +// attributes even if the input is unused. def TF_PlaceholderInputOp : TF_Op<"Placeholder.input", - [NoSideEffect, SameOperandsAndResultType]> { + [SameOperandsAndResultType]> { let summary = "PlaceholderInput op"; let description = [{ From f0cd5330b24b6a4f4a43f26d65cdeee60c35eb56 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 17:27:56 -0700 Subject: [PATCH 317/332] Variable order is misleading in error message. PiperOrigin-RevId: 257714202 --- tensorflow/contrib/boosted_trees/estimator_batch/estimator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py index 5ffbb906708..ca1e968fae1 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py @@ -123,7 +123,7 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator): learner_config.num_classes = n_classes elif learner_config.num_classes != n_classes: raise ValueError("n_classes (%d) doesn't match learner_config (%d)." % - (learner_config.num_classes, n_classes)) + (n_classes, learner_config.num_classes)) super(GradientBoostedDecisionTreeClassifier, self).__init__( model_fn=model.model_builder, params={ From 4e6b1fc7050eff3625e05ac7bd61c40fbceca04b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 11 Jul 2019 17:37:04 -0700 Subject: [PATCH 318/332] Update xla for API changes in llvm revision r365796 PiperOrigin-RevId: 257715553 --- .../compiler/xla/service/gpu/ir_emitter.cc | 5 ++-- .../gpu/tests/gpu_kernel_tiling_test.cc | 24 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc index a82b6119258..be24b8fc05e 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc @@ -209,9 +209,8 @@ bool IrEmitter::MaybeEmitDirectAtomicOperation( // NVPTX supports atomicAdd on F32 and integer types. if (target_triple.isNVPTX() && element_type == F32) { // F32 + F32 - llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::nvvm_atomic_load_add_f32, - {output_address, source}, - {output_address->getType()}, &b_); + AtomicRMW(llvm::AtomicRMWInst::FAdd, output_address, source, + llvm::AtomicOrdering::SequentiallyConsistent); return true; } if (is_atomic_integral) { diff --git a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc index 8e66d0aad6a..a12932f573b 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc +++ b/tensorflow/compiler/xla/service/gpu/tests/gpu_kernel_tiling_test.cc @@ -316,9 +316,9 @@ TEST_F(GpuKernelTilingTest, ColumnReductionWithPowerOf2OutputElementsUnrolled) { CompileAndVerifyIr(std::move(hlo_module), R"( ; CHECK-LABEL: define void @fusion -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 -; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32 +; CHECK: atomicrmw fadd float +; CHECK: atomicrmw fadd float +; CHECK-NOT: atomicrmw fadd float ; CHECK: } )", /*match_optimized_ir=*/true); @@ -363,8 +363,8 @@ TEST_F(GpuKernelTilingTest, CompileAndVerifyIr(std::move(hlo_module), R"( ; CHECK-LABEL: define void @fusion -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 -; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32 +; CHECK: atomicrmw fadd float +; CHECK-NOT: atomicrmw fadd float ; CHECK: } )", /*match_optimized_ir=*/true); @@ -411,11 +411,11 @@ TEST_F(GpuKernelTilingTest, ColumnReductionMOFUnrolled) { CompileAndVerifyIr(std::move(hlo_module), R"( ; CHECK-LABEL: define void @fusion -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 -; CHECK-NOT: call float @llvm.nvvm.atomic.load.add.f32.p0f32 +; CHECK: atomicrmw fadd float +; CHECK: atomicrmw fadd float +; CHECK: atomicrmw fadd float +; CHECK: atomicrmw fadd float +; CHECK-NOT: atomicrmw fadd float ; CHECK: } )", /*match_optimized_ir=*/true); @@ -446,7 +446,7 @@ TEST_F(GpuKernelTilingTest, ColumnReductionWithLayoutChangeTiled) { CompileAndVerifyIr(std::move(hlo_module), R"( ; CHECK-LABEL: define void @reduce -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 +; CHECK: atomicrmw fadd float ; CHECK: } )", /*match_optimized_ir=*/true); @@ -511,7 +511,7 @@ TEST_F(GpuKernelTilingTest, CompileAndVerifyIr(std::move(hlo_module), R"( ; CHECK-LABEL: define void @reduce -; CHECK: call float @llvm.nvvm.atomic.load.add.f32.p0f32 +; CHECK: atomicrmw fadd float ; CHECK: } )", /*match_optimized_ir=*/true); From 180092bbcef588ee457152cb9a49001e54270721 Mon Sep 17 00:00:00 2001 From: Sachin Joglekar Date: Thu, 11 Jul 2019 17:45:12 -0700 Subject: [PATCH 319/332] Adds Object Detection stage. PiperOrigin-RevId: 257716634 --- .../evaluation/proto/evaluation_stages.proto | 36 +++- tensorflow/lite/tools/evaluation/stages/BUILD | 20 ++ .../stages/object_detection_stage.cc | 188 ++++++++++++++++++ .../stages/object_detection_stage.h | 96 +++++++++ 4 files changed, 338 insertions(+), 2 deletions(-) create mode 100644 tensorflow/lite/tools/evaluation/stages/object_detection_stage.cc create mode 100644 tensorflow/lite/tools/evaluation/stages/object_detection_stage.h diff --git a/tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto b/tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto index 45d4a6b4714..74ab8c2a712 100644 --- a/tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto +++ b/tensorflow/lite/tools/evaluation/proto/evaluation_stages.proto @@ -23,7 +23,7 @@ option java_package = "tflite.evaluation"; // Defines the functionality executed by an EvaluationStage. // -// Next ID: 6 +// Next ID: 7 message ProcessSpecification { oneof params { ImagePreprocessingParams image_preprocessing_params = 1; @@ -32,6 +32,7 @@ message ProcessSpecification { ImageClassificationParams image_classification_params = 4; ObjectDetectionAveragePrecisionParams object_detection_average_precision_params = 5; + ObjectDetectionParams object_detection_params = 6; } } @@ -71,7 +72,7 @@ message AccuracyMetrics { // Contains process-specific metrics, which may differ based on what an // EvaluationStage does. // -// Next ID: 7 +// Next ID: 8 message ProcessMetrics { optional LatencyMetrics total_latency = 1; @@ -82,6 +83,7 @@ message ProcessMetrics { InferenceProfilerMetrics inference_profiler_metrics = 5; ObjectDetectionAveragePrecisionMetrics object_detection_average_precision_metrics = 6; + ObjectDetectionMetrics object_detection_metrics = 7; } } @@ -283,3 +285,33 @@ message ObjectDetectionAveragePrecisionMetrics { // Average of Average Precision across all IoU thresholds. optional float overall_mean_average_precision = 2; } + +// Parameters that define how the Object Detection task is evaluated +// end-to-end. +// +// Next ID: 4 +message ObjectDetectionParams { + // Required. + // Model's outputs should be same as a TFLite-compatible SSD model. + // Refer: + // https://www.tensorflow.org/lite/models/object_detection/overview#output + // TODO(b/133772912): Generalize support for other types of object detection + // models. + optional TfliteInferenceParams inference_params = 1; + // Optional. Used to match ground-truth categories with model output. + // SSD Mobilenet V1 Model trained on COCO assumes class 0 is background class + // in the label file and class labels start from 1 to number_of_classes+1. + // Therefore, default value is set as 1. + optional int32 class_offset = 2 [default = 1]; + optional ObjectDetectionAveragePrecisionParams ap_params = 3; +} + +// Metrics from evaluation of the object detection task. +// +// Next ID: 5 +message ObjectDetectionMetrics { + optional LatencyMetrics pre_processing_latency = 1; + optional LatencyMetrics inference_latency = 2; + optional TfliteInferenceMetrics inference_metrics = 3; + optional ObjectDetectionAveragePrecisionMetrics average_precision_metrics = 4; +} diff --git a/tensorflow/lite/tools/evaluation/stages/BUILD b/tensorflow/lite/tools/evaluation/stages/BUILD index 2af95378741..6ee00c853fb 100644 --- a/tensorflow/lite/tools/evaluation/stages/BUILD +++ b/tensorflow/lite/tools/evaluation/stages/BUILD @@ -209,3 +209,23 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +cc_library( + name = "object_detection_stage", + srcs = ["object_detection_stage.cc"], + hdrs = ["object_detection_stage.h"], + copts = tflite_copts(), + deps = [ + ":image_preprocessing_stage", + ":object_detection_average_precision_stage", + ":tflite_inference_stage", + "//tensorflow/core:tflite_portable_logging", + "//tensorflow/lite/c:c_api_internal", + "//tensorflow/lite/tools/evaluation:evaluation_stage", + "//tensorflow/lite/tools/evaluation:utils", + "//tensorflow/lite/tools/evaluation/proto:evaluation_config_cc_proto", + "//tensorflow/lite/tools/evaluation/proto:evaluation_stages_cc_proto", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_protobuf//:protobuf_headers", + ], +) diff --git a/tensorflow/lite/tools/evaluation/stages/object_detection_stage.cc b/tensorflow/lite/tools/evaluation/stages/object_detection_stage.cc new file mode 100644 index 00000000000..298d7eece76 --- /dev/null +++ b/tensorflow/lite/tools/evaluation/stages/object_detection_stage.cc @@ -0,0 +1,188 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/tools/evaluation/stages/object_detection_stage.h" + +#include + +#include "google/protobuf/text_format.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h" +#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h" +#include "tensorflow/lite/tools/evaluation/utils.h" + +namespace tflite { +namespace evaluation { + +TfLiteStatus ObjectDetectionStage::Init() { + // Ensure inference params are provided. + if (!config_.specification().has_object_detection_params()) { + LOG(ERROR) << "ObjectDetectionParams not provided"; + return kTfLiteError; + } + auto& params = config_.specification().object_detection_params(); + if (!params.has_inference_params()) { + LOG(ERROR) << "inference_params not provided"; + return kTfLiteError; + } + if (all_labels_ == nullptr) { + LOG(ERROR) << "Detection output labels not provided"; + return kTfLiteError; + } + + // TfliteInferenceStage. + EvaluationStageConfig tflite_inference_config; + tflite_inference_config.set_name("tflite_inference"); + *tflite_inference_config.mutable_specification() + ->mutable_tflite_inference_params() = params.inference_params(); + inference_stage_.reset(new TfliteInferenceStage(tflite_inference_config)); + TF_LITE_ENSURE_STATUS(inference_stage_->Init()); + + // Validate model inputs. + const TfLiteModelInfo* model_info = inference_stage_->GetModelInfo(); + if (model_info->inputs.size() != 1 || model_info->outputs.size() != 4) { + LOG(ERROR) << "Object detection model must have 1 input & 4 outputs"; + return kTfLiteError; + } + TfLiteType input_type = model_info->inputs[0]->type; + auto* input_shape = model_info->inputs[0]->dims; + // Input should be of the shape {1, height, width, 3} + if (input_shape->size != 4 || input_shape->data[0] != 1 || + input_shape->data[3] != 3) { + LOG(ERROR) << "Invalid input shape for model"; + return kTfLiteError; + } + + // ImagePreprocessingStage + EvaluationStageConfig preprocessing_config; + preprocessing_config.set_name("image_preprocessing"); + auto* preprocess_params = preprocessing_config.mutable_specification() + ->mutable_image_preprocessing_params(); + preprocess_params->set_image_height(input_shape->data[1]); + preprocess_params->set_image_width(input_shape->data[2]); + preprocess_params->set_cropping_fraction(1.0); + preprocess_params->set_output_type(static_cast(input_type)); + preprocessing_stage_.reset(new ImagePreprocessingStage(preprocessing_config)); + TF_LITE_ENSURE_STATUS(preprocessing_stage_->Init()); + + // ObjectDetectionAveragePrecisionStage + EvaluationStageConfig eval_config; + eval_config.set_name("average_precision"); + *eval_config.mutable_specification() + ->mutable_object_detection_average_precision_params() = + params.ap_params(); + eval_config.mutable_specification() + ->mutable_object_detection_average_precision_params() + ->set_num_classes(all_labels_->size()); + eval_stage_.reset(new ObjectDetectionAveragePrecisionStage(eval_config)); + TF_LITE_ENSURE_STATUS(eval_stage_->Init()); + + return kTfLiteOk; +} + +TfLiteStatus ObjectDetectionStage::Run() { + if (image_path_.empty()) { + LOG(ERROR) << "Input image not set"; + return kTfLiteError; + } + + // Preprocessing. + preprocessing_stage_->SetImagePath(&image_path_); + TF_LITE_ENSURE_STATUS(preprocessing_stage_->Run()); + + // Inference. + std::vector data_ptrs = {}; + data_ptrs.push_back(preprocessing_stage_->GetPreprocessedImageData()); + inference_stage_->SetInputs(data_ptrs); + TF_LITE_ENSURE_STATUS(inference_stage_->Run()); + + // Convert model output to ObjectsSet. + ObjectDetectionResult predicted_objects; + const int class_offset = + config_.specification().object_detection_params().class_offset(); + const std::vector* outputs = inference_stage_->GetOutputs(); + int num_detections = static_cast(*static_cast(outputs->at(3))); + float* detected_label_boxes = static_cast(outputs->at(0)); + float* detected_label_indices = static_cast(outputs->at(1)); + float* detected_label_probabilities = static_cast(outputs->at(2)); + for (int i = 0; i < num_detections; ++i) { + const int bounding_box_offset = i * 4; + auto* object = predicted_objects.add_objects(); + // Bounding box + auto* bbox = object->mutable_bounding_box(); + bbox->set_normalized_top(detected_label_boxes[bounding_box_offset + 0]); + bbox->set_normalized_left(detected_label_boxes[bounding_box_offset + 1]); + bbox->set_normalized_bottom(detected_label_boxes[bounding_box_offset + 2]); + bbox->set_normalized_right(detected_label_boxes[bounding_box_offset + 3]); + // Class. + object->set_class_id(static_cast(detected_label_indices[i]) + + class_offset); + // Score + object->set_score(detected_label_probabilities[i]); + } + + // AP Evaluation. + eval_stage_->SetEvalInputs(predicted_objects, *ground_truth_objects_); + TF_LITE_ENSURE_STATUS(eval_stage_->Run()); + + return kTfLiteOk; +} + +EvaluationStageMetrics ObjectDetectionStage::LatestMetrics() { + EvaluationStageMetrics metrics; + auto* detection_metrics = + metrics.mutable_process_metrics()->mutable_object_detection_metrics(); + + *detection_metrics->mutable_pre_processing_latency() = + preprocessing_stage_->LatestMetrics().process_metrics().total_latency(); + EvaluationStageMetrics inference_metrics = inference_stage_->LatestMetrics(); + *detection_metrics->mutable_inference_latency() = + inference_metrics.process_metrics().total_latency(); + *detection_metrics->mutable_inference_metrics() = + inference_metrics.process_metrics().tflite_inference_metrics(); + *detection_metrics->mutable_average_precision_metrics() = + eval_stage_->LatestMetrics() + .process_metrics() + .object_detection_average_precision_metrics(); + metrics.set_num_runs(inference_metrics.num_runs()); + return metrics; +} + +TfLiteStatus PopulateGroundTruth( + const std::string& grouth_truth_pbtxt_file, + absl::flat_hash_map* + ground_truth_mapping) { + if (ground_truth_mapping == nullptr) { + return kTfLiteError; + } + ground_truth_mapping->clear(); + + // Read the ground truth dump. + std::ifstream t(grouth_truth_pbtxt_file); + std::string proto_str((std::istreambuf_iterator(t)), + std::istreambuf_iterator()); + ObjectDetectionGroundTruth ground_truth_proto; + proto2::TextFormat::ParseFromString(proto_str, &ground_truth_proto); + + for (auto image_ground_truth : ground_truth_proto.detection_results()) { + (*ground_truth_mapping)[image_ground_truth.image_name()] = + image_ground_truth; + } + + return kTfLiteOk; +} + +} // namespace evaluation +} // namespace tflite diff --git a/tensorflow/lite/tools/evaluation/stages/object_detection_stage.h b/tensorflow/lite/tools/evaluation/stages/object_detection_stage.h new file mode 100644 index 00000000000..ec9772754eb --- /dev/null +++ b/tensorflow/lite/tools/evaluation/stages/object_detection_stage.h @@ -0,0 +1,96 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_STAGE_H_ +#define TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_STAGE_H_ + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "tensorflow/lite/tools/evaluation/evaluation_stage.h" +#include "tensorflow/lite/tools/evaluation/proto/evaluation_config.pb.h" +#include "tensorflow/lite/tools/evaluation/proto/evaluation_stages.pb.h" +#include "tensorflow/lite/tools/evaluation/stages/image_preprocessing_stage.h" +#include "tensorflow/lite/tools/evaluation/stages/object_detection_average_precision_stage.h" +#include "tensorflow/lite/tools/evaluation/stages/tflite_inference_stage.h" + +namespace tflite { +namespace evaluation { + +// An EvaluationStage to encapsulate the complete Object Detection task. +// Assumes that the object detection model's signature (number of +// inputs/outputs, ordering of outputs & what they denote) is same as the +// MobileNet SSD model: +// https://www.tensorflow.org/lite/models/object_detection/overview#output. +// Input size/type & number of detections could be different. +// TODO(b/133772912): Generalize support for other types of object detection +// models. +class ObjectDetectionStage : public EvaluationStage { + public: + explicit ObjectDetectionStage(const EvaluationStageConfig& config) + : EvaluationStage(config) {} + + TfLiteStatus Init() override; + + TfLiteStatus Run() override; + + EvaluationStageMetrics LatestMetrics() override; + + // Call before Init(). all_labels should contain all possible object labels + // that can be detected by the model, in the correct order. all_labels should + // outlive the call to Init(). + void SetAllLabels(const std::vector& all_labels) { + all_labels_ = &all_labels; + } + + // Call before Run(). + // ground_truth_objects instance should outlive the call to Run(). + void SetInputs(const std::string& image_path, + const ObjectDetectionResult& ground_truth_objects) { + image_path_ = image_path; + ground_truth_objects_ = &ground_truth_objects; + } + + // Provides a pointer to the underlying TfLiteInferenceStage. + // Returns non-null value only if this stage has been initialized. + TfliteInferenceStage* const GetInferenceStage() { + return inference_stage_.get(); + } + + private: + const std::vector* all_labels_ = nullptr; + std::unique_ptr preprocessing_stage_; + std::unique_ptr inference_stage_; + std::unique_ptr eval_stage_; + std::string image_path_; + const ObjectDetectionResult* ground_truth_objects_; +}; + +// Reads a tflite::evaluation::ObjectDetectionGroundTruth instance from a +// textproto file and populates a mapping of image name to +// ObjectDetectionResult. +// File with ObjectDetectionGroundTruth can be generated using the +// preprocess_coco_minival.py script in evaluation/tasks/coco_object_detection. +// Useful for wrappers/scripts that use ObjectDetectionStage. +TfLiteStatus PopulateGroundTruth( + const std::string& grouth_truth_pbtxt_file, + absl::flat_hash_map* + ground_truth_mapping); + +} // namespace evaluation +} // namespace tflite + +#endif // TENSORFLOW_LITE_TOOLS_EVALUATION_STAGES_OBJECT_DETECTION_STAGE_H_ From bea83f2f84d576852b0f5e276a581d6d10ccf6d6 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Thu, 11 Jul 2019 17:59:12 -0700 Subject: [PATCH 320/332] Add new training loop logic for v2. PiperOrigin-RevId: 257718378 --- tensorflow/python/keras/BUILD | 1 + .../distribute/distributed_training_utils.py | 7 - .../python/keras/engine/data_adapter.py | 302 +++++++-- .../python/keras/engine/data_adapter_test.py | 168 +++-- tensorflow/python/keras/engine/training.py | 7 +- .../python/keras/engine/training_utils.py | 7 + tensorflow/python/keras/engine/training_v2.py | 606 ++++++++++++++++++ 7 files changed, 992 insertions(+), 106 deletions(-) create mode 100644 tensorflow/python/keras/engine/training_v2.py diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index 1e554b8a6e8..841a871bf06 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -173,6 +173,7 @@ py_library( "engine/training_eager.py", "engine/training_generator.py", "engine/training_utils.py", + "engine/training_v2.py", "metrics.py", # Need base_layer "models.py", "utils/metrics_utils.py", diff --git a/tensorflow/python/keras/distribute/distributed_training_utils.py b/tensorflow/python/keras/distribute/distributed_training_utils.py index 0d269ba59e8..b2d1e0f6784 100644 --- a/tensorflow/python/keras/distribute/distributed_training_utils.py +++ b/tensorflow/python/keras/distribute/distributed_training_utils.py @@ -547,13 +547,6 @@ def get_batch_dimension(iterator): return dims[0] if dims else None -def list_to_tuple(maybe_list): - """Datasets treat lists specially, so switch them to tuples.""" - if isinstance(maybe_list, list): - return tuple(maybe_list) - return maybe_list - - def get_iterator(dataset, distribution_strategy): with distribution_strategy.scope(): iterator = distribution_strategy.make_dataset_iterator(dataset) diff --git a/tensorflow/python/keras/engine/data_adapter.py b/tensorflow/python/keras/engine/data_adapter.py index 95ca3b41dbb..4603166b6c2 100644 --- a/tensorflow/python/keras/engine/data_adapter.py +++ b/tensorflow/python/keras/engine/data_adapter.py @@ -20,12 +20,14 @@ from __future__ import print_function import abc import itertools +import math import numpy as np import six from tensorflow.python.data.ops import dataset_ops from tensorflow.python.framework import ops +from tensorflow.python.keras.engine import training_utils from tensorflow.python.keras.utils import data_utils from tensorflow.python.util import nest from tensorflow.python.util import tf_inspect @@ -53,7 +55,7 @@ class DataAdapter(object): if len(applicable_adapters) != 1: raise ValueError("Expect only one adapter class to handle the input") - dataset = applicable_adapters[0]().get_dataset(x) + dataset = applicable_adapters[0](x).get_dataset() for data in dataset: # training ``` @@ -77,8 +79,8 @@ class DataAdapter(object): raise NotImplementedError @abc.abstractmethod - def get_dataset(self, x, y=None, **kwargs): - """Convert the input x and y into dataset. + def __init__(self, x, y=None, **kwargs): + """Create a DataAdapter based on data inputs. The caller must make sure to call `can_handle()` first before invoking this method. Provide unsupported data type will result into unexpected behavior. @@ -88,14 +90,26 @@ class DataAdapter(object): y: target labels. Note that y could be None in the case of prediction. **kwargs: Other keyword arguments for DataAdapter during the construction of the tf.dataset.Dataset. For example: - - Numpy data might need to have `batch_size` parameter when constructing - the dataset and iterator. - - Numpy data might have "evaluation_split" which will split the input - data into training and validation set. - Numpy data might have `sample_weights` which will be used for weighting the loss function during training. + - Numpy data might need to have `batch_size` parameter when constructing + the dataset and iterator. + - Certain input might need to be distribution strategy aware. When + `distribution_strategy` is passed, the created dataset need to respect + the strategy. DataAdapter might choose to ignore any keyword argument if it doesn't use it, or raise exception if any required argument is not provide. + """ + if not self.can_handle(x, y): + raise ValueError("{} Cannot handle input {}".format(self.__class__, x)) + + @abc.abstractmethod + def get_dataset(self): + """Get a dataset instance for the current DataAdapter. + + Note that the dataset returned does not repeat for epoch, so caller might + need to create new iterator for the same dataset at the beginning of the + epoch. This behavior might change in future. Returns: An tf.dataset.Dataset. Caller might use the dataset in different @@ -104,20 +118,58 @@ class DataAdapter(object): """ raise NotImplementedError + @abc.abstractmethod + def get_size(self): + """Return the size (number of batches) for the dataset created. + + For certain type of the data input, the number of batches is known, eg for + Numpy data, the size is same as (number_of_element / batch_size). Whereas + for dataset or python generator, the size is unknown since it may or may not + have a end state. + + Returns: + int, the number of batches for the dataset, or None if it is unknown. The + caller could use this to control the loop of training, show progress bar, + or handle unexpected StopIteration error. + """ + raise NotImplementedError + + @abc.abstractmethod + def batch_size(self): + """Return the batch size of the dataset created. + + For certain type of the data input, the batch size is known, and even + required, like numpy array. Where as for dataset, the batch is unknown + unless we take a peek. + + Returns: + int, the batch size of the dataset, or None if it is unknown. + """ + raise NotImplementedError + + @abc.abstractmethod + def has_partial_batch(self): + """Whether the dataset has partial batch at the end.""" + raise NotImplementedError + class NumpyArrayDataAdapter(DataAdapter): """Adapter that handles the Numpy array.""" @staticmethod def can_handle(x, y=None): - if y is not None and type(x) is not type(y): - raise ValueError("input feature and target should have same type, got " - "x: {}, y: {}".format(type(x), type(y))) - return isinstance(x, np.ndarray) + flat_inputs = nest.flatten(x) + if y is not None: + flat_inputs += nest.flatten(y) - def get_dataset(self, x, y=None, sample_weights=None, batch_size=None, - shuffle=False, **kwargs): - # TODO(scottzhu): Handle validation_split + return all(isinstance(v, np.ndarray) for v in flat_inputs) + + def __init__(self, x, y=None, sample_weights=None, batch_size=None, + shuffle=False, distribution_strategy=None, **kwargs): + super(NumpyArrayDataAdapter, self).__init__(x, y, **kwargs) + x = _process_numpy_inputs(x) + y = _process_numpy_inputs(y) + sample_weights = _process_numpy_inputs(sample_weights) if y is not None and sample_weights is not None: inputs = (x, y, sample_weights) elif y is not None: @@ -125,37 +177,98 @@ class NumpyArrayDataAdapter(DataAdapter): # sample_weight is ignored. inputs = (x, y) else: - inputs = x + inputs = (x,) if not batch_size: raise ValueError("batch size is required for Numpy input data.") - # TODO(scottzhu): might need to check large data input (> 2G). - dataset = dataset_ops.DatasetV2.from_tensor_slices(inputs) + if distribution_strategy is not None: + dataset = distribution_strategy.experimental_make_numpy_dataset(inputs) + else: + dataset = dataset_ops.DatasetV2.from_tensor_slices(inputs) + + num_samples = int(nest.flatten(x)[0].shape[0]) if shuffle: - num_samples = int(nest.flatten(x)[0].shape[0]) + # Note that we use the full input data length as buffer window, which + # might have memory consumption consequence. This is on the radar of + # tf.data team and they will address it. dataset = dataset.shuffle(num_samples) - return dataset.batch(batch_size) + self._dataset = dataset.batch(batch_size) + self._size = int(math.ceil(num_samples / batch_size)) + self._batch_size = batch_size + self._has_partial_batch = (self._size != (num_samples // batch_size)) + + def get_dataset(self): + return self._dataset + + def get_size(self): + return self._size + + def batch_size(self): + return self._batch_size + + def has_partial_batch(self): + return self._has_partial_batch +# TODO(scottzhu): Eventually the numpy array and eager tensor should be treated +# in the same way. Merge this class with NumpyArrayDataAdapter. class TensorDataAdapter(DataAdapter): - """Adapter that handles Tensorflow tensors.""" + """Adapter that handles Tensorflow eager tensors.""" @staticmethod def can_handle(x, y=None): - return isinstance(x, ops.Tensor) + flat_inputs = nest.flatten(x) + if y is not None: + flat_inputs += nest.flatten(y) - def get_dataset(self, x, y=None, batch_size=None, shuffle=False, **kwargs): - inputs = x if y is None else (x, y) + return all(isinstance(v, ops.Tensor) for v in flat_inputs) - # Do we need batch_size for data tensor? - if not batch_size: - raise ValueError("batch size is required for tensor input data.") + def __init__(self, x, y=None, sample_weights=None, batch_size=None, + shuffle=False, **kwargs): + super(TensorDataAdapter, self).__init__(x, y, **kwargs) + x = _process_numpy_inputs(x) + y = _process_numpy_inputs(y) + sample_weights = _process_numpy_inputs(sample_weights) + if y is not None and sample_weights is not None: + inputs = (x, y, sample_weights) + elif y is not None: + # Sample weight is only needed for training, so if y is None, then + # sample_weight is ignored. + inputs = (x, y) + else: + inputs = (x,) + + # TODO(scottzhu): We should treat data tensor same as numpy array, make + # the batch_size a required param. + # if not batch_size: + # raise ValueError("batch size is required for tensor input data.") dataset = dataset_ops.DatasetV2.from_tensor_slices(inputs) + num_samples = int(nest.flatten(x)[0].shape[0]) if shuffle: - num_samples = int(nest.flatten(x)[0].shape[0]) dataset = dataset.shuffle(num_samples) - return dataset.batch(batch_size) + if batch_size: + dataset = dataset.batch(batch_size) + self._size = int(math.ceil(num_samples / batch_size)) + self._batch_size = batch_size + self._has_partial_batch = (self._size != (num_samples // batch_size)) + else: + self._size = 1 + self._batch_size = num_samples + self._has_partial_batch = False + self._dataset = dataset + + def get_dataset(self): + return self._dataset + + def get_size(self): + return self._size + + def batch_size(self): + return self._batch_size + + def has_partial_batch(self): + return self._has_partial_batch class DatasetAdapter(DataAdapter): @@ -165,12 +278,30 @@ class DatasetAdapter(DataAdapter): def can_handle(x, y=None): return isinstance(x, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)) - def get_dataset(self, x, y=None, **kwargs): - # TODO(scottzhu): throw error when sample_weights, etc is provided. - if y is not None: - raise ValueError("target input is expected to be None when using " + def __init__(self, x, y=None, sample_weights=None, **kwargs): + super(DatasetAdapter, self).__init__(x, y, **kwargs) + if not is_none_or_empty(y): + raise ValueError("`y` argument is not supported when using " "dataset as input.") - return x + if not is_none_or_empty(sample_weights): + raise ValueError("`sample_weight` argument is not supported when using " + "dataset as input.") + # Note that the dataset instance is immutable, its fine to reusing the user + # provided dataset. + self._dataset = x + + def get_dataset(self): + return self._dataset + + def get_size(self): + # The size of dataset is unknown, unless its fully consumed. + return None + + def batch_size(self): + return None + + def has_partial_batch(self): + return False class GeneratorDataAdapter(DataAdapter): @@ -180,10 +311,13 @@ class GeneratorDataAdapter(DataAdapter): def can_handle(x, y=None): return tf_inspect.isgenerator(x) - def get_dataset(self, x, y=None, **kwargs): - # TODO(scottzhu): throw error when sample_weights, etc is provided. - if y is not None: - raise ValueError("target input is expected to be None when using " + def __init__(self, x, y=None, sample_weights=None, **kwargs): + super(GeneratorDataAdapter, self).__init__(x, y, **kwargs) + if not is_none_or_empty(y): + raise ValueError("`y` argument is not supported when using " + "python generator as input.") + if not is_none_or_empty(sample_weights): + raise ValueError("`sample_weight` argument is not supported when using " "python generator as input.") # Since we have to know the dtype of the python generator when we build the @@ -198,8 +332,21 @@ class GeneratorDataAdapter(DataAdapter): def reassemble(): return itertools.chain([peek], x) - return dataset_ops.DatasetV2.from_generator(reassemble, nested_dtypes, - output_shapes=nested_shape) + self._batch_size = int(nest.flatten(peek)[0].shape[0]) + self._dataset = dataset_ops.DatasetV2.from_generator( + reassemble, nested_dtypes, output_shapes=nested_shape) + + def get_dataset(self): + return self._dataset + + def get_size(self): + return None + + def batch_size(self): + return self._batch_size + + def has_partial_batch(self): + return False class KerasSequenceAdapter(DataAdapter): @@ -209,12 +356,14 @@ class KerasSequenceAdapter(DataAdapter): def can_handle(x, y=None): return isinstance(x, data_utils.Sequence) - def get_dataset(self, x, y=None, shuffle=False, **kwargs): - # TODO(scottzhu): throw error when sample_weights, etc is provided. - if y is not None: - raise ValueError("target input is expected to be None when using " + def __init__(self, x, y=None, sample_weights=None, shuffle=False, **kwargs): + super(KerasSequenceAdapter, self).__init__(x, y, **kwargs) + if not is_none_or_empty(y): + raise ValueError("`y` argument is not supported when using " + "`keras.utils.Sequence` as input.") + if not is_none_or_empty(sample_weights): + raise ValueError("`sample_weight` argument is not supported when using " "`keras.utils.Sequence` as input.") - peek = x[0] nested_dtypes = nest.map_structure(lambda t: t.dtype, peek) nested_shape = nest.map_structure(lambda t: t.shape, peek) @@ -226,4 +375,67 @@ class KerasSequenceAdapter(DataAdapter): output_shapes=nested_shape) if shuffle: dataset = dataset.shuffle(len(x)) - return dataset + self._dataset = dataset + self._size = len(x) + self._batch_size = int(nest.flatten(peek)[0].shape[0]) + + def get_dataset(self): + return self._dataset + + def get_size(self): + return self._size + + def batch_size(self): + return self._batch_size + + def has_partial_batch(self): + return False + + +ALL_ADAPTER_CLS = [NumpyArrayDataAdapter, TensorDataAdapter, DatasetAdapter, + GeneratorDataAdapter, KerasSequenceAdapter] + + +def select_data_adapter(x, y): + adapter_cls = [cls for cls in ALL_ADAPTER_CLS if cls.can_handle(x, y)] + if not adapter_cls: + raise ValueError("Failed to find data adapter that can handle " + "input: {}, {}".format(type(x), type(y))) + elif len(adapter_cls) > 1: + raise RuntimeError("Data adapter should be mutually exclusive for " + "handling inputs. Found multiple adapter {} to handle " + "input: {}, {}".format(adapter_cls, type(x), type(y))) + return adapter_cls[0] + + +def _process_numpy_inputs(inputs): + """Process numpy array inputs. + + For numpy inputs, it is possible to be single numpy array, or list/dict of + them. They could also be preprocessed by other lib to match with the order + of position for the model. The result here should be something that can be + used to build dataset. + + Args: + inputs: single or list/tuple/dict of numpy array. + Returns: + numpy arrays can be used to build dataset. + """ + if is_none_or_empty(inputs): + return None + flat_inputs = nest.flatten(inputs) + if len(flat_inputs) == 1: + return flat_inputs[0] + # For more complicated structure, we only convert the out most list to tuple + # since dataset will stack the list, but treat elements in the tuple as + # individual element. + return training_utils.list_to_tuple(inputs) + + +def is_none_or_empty(inputs): + # util method to check if the input is a None or a empty list. + # the python "not" check will raise an error like below if the input is a + # numpy array + # "The truth value of an array with more than one element is ambiguous. + # Use a.any() or a.all()" + return inputs is None or not nest.flatten(inputs) diff --git a/tensorflow/python/keras/engine/data_adapter_test.py b/tensorflow/python/keras/engine/data_adapter_test.py index 8e339fb7388..fd439edb724 100644 --- a/tensorflow/python/keras/engine/data_adapter_test.py +++ b/tensorflow/python/keras/engine/data_adapter_test.py @@ -41,7 +41,8 @@ class DataAdapterTestBase(test.TestCase): self.tensor_input = constant_op.constant(2.0, shape=(50, 10)) self.tensor_target = array_ops.ones((50,)) self.dataset_input = dataset_ops.DatasetV2.from_tensor_slices( - (self.numpy_input, self.numpy_target)).batch(self.batch_size).shuffle(1) + (self.numpy_input, self.numpy_target)).shuffle(50).batch( + self.batch_size) def generator(): yield (np.zeros((self.batch_size, 10)), np.ones(self.batch_size)) @@ -70,26 +71,43 @@ class NumpyDataAdapterTest(DataAdapterTestBase): def setUp(self): super(NumpyDataAdapterTest, self).setUp() - self.adapter = data_adapter.NumpyArrayDataAdapter() + self.adapter_cls = data_adapter.NumpyArrayDataAdapter def test_can_handle(self): - self.assertTrue(self.adapter.can_handle(self.numpy_input)) + self.assertTrue(self.adapter_cls.can_handle(self.numpy_input)) self.assertTrue( - self.adapter.can_handle(self.numpy_input, self.numpy_target)) + self.adapter_cls.can_handle(self.numpy_input, self.numpy_target)) - self.assertFalse(self.adapter.can_handle(self.tensor_input)) - self.assertFalse(self.adapter.can_handle(self.dataset_input)) - self.assertFalse(self.adapter.can_handle(self.generator_input)) - self.assertFalse(self.adapter.can_handle(self.sequence_input)) + self.assertFalse(self.adapter_cls.can_handle(self.tensor_input)) + self.assertFalse(self.adapter_cls.can_handle(self.dataset_input)) + self.assertFalse(self.adapter_cls.can_handle(self.generator_input)) + self.assertFalse(self.adapter_cls.can_handle(self.sequence_input)) def test_iterator_expect_batch_size(self): with self.assertRaisesRegexp(ValueError, 'batch size is required'): - self.adapter.get_dataset(self.numpy_input, self.numpy_target) + self.adapter_cls(self.numpy_input, self.numpy_target) + + def test_size(self): + adapter = self.adapter_cls( + self.numpy_input, self.numpy_target, batch_size=5) + self.assertEqual(adapter.get_size(), 10) + self.assertFalse(adapter.has_partial_batch()) + + def test_batch_size(self): + adapter = self.adapter_cls( + self.numpy_input, self.numpy_target, batch_size=5) + self.assertEqual(adapter.batch_size(), 5) + + def test_partial_batch(self): + adapter = self.adapter_cls( + self.numpy_input, self.numpy_target, batch_size=4) + self.assertEqual(adapter.get_size(), 13) # 50/4 + self.assertTrue(adapter.has_partial_batch()) def test_training(self): - dataset = self.adapter.get_dataset( - self.numpy_input, self.numpy_target, batch_size=5) - self.model.compile(loss='mse', optimizer='sgd') + dataset = self.adapter_cls( + self.numpy_input, self.numpy_target, batch_size=5).get_dataset() + self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd') self.model.fit(dataset) @@ -97,84 +115,134 @@ class TensorDataAdapterTest(DataAdapterTestBase): def setUp(self): super(TensorDataAdapterTest, self).setUp() - self.adapter = data_adapter.TensorDataAdapter() + self.adapter_cls = data_adapter.TensorDataAdapter def test_can_handle(self): - self.assertTrue(self.adapter.can_handle(self.tensor_input)) + self.assertTrue(self.adapter_cls.can_handle(self.tensor_input)) self.assertTrue( - self.adapter.can_handle(self.tensor_input, self.tensor_target)) + self.adapter_cls.can_handle(self.tensor_input, self.tensor_target)) - self.assertFalse(self.adapter.can_handle(self.numpy_input)) - self.assertFalse(self.adapter.can_handle(self.dataset_input)) - self.assertFalse(self.adapter.can_handle(self.generator_input)) - self.assertFalse(self.adapter.can_handle(self.sequence_input)) - - def test_iterator_expect_batch_size(self): - with self.assertRaisesRegexp(ValueError, 'batch size is required'): - self.adapter.get_dataset(self.tensor_input, self.tensor_target) + self.assertFalse(self.adapter_cls.can_handle(self.numpy_input)) + self.assertFalse(self.adapter_cls.can_handle(self.dataset_input)) + self.assertFalse(self.adapter_cls.can_handle(self.generator_input)) + self.assertFalse(self.adapter_cls.can_handle(self.sequence_input)) def test_training(self): - dataset = self.adapter.get_dataset( - self.tensor_input, self.tensor_target, batch_size=5) - self.model.compile(loss='mse', optimizer='sgd') + dataset = self.adapter_cls( + self.tensor_input, self.tensor_target, batch_size=5).get_dataset() + self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd') self.model.fit(dataset) + def test_size(self): + adapter = self.adapter_cls( + self.tensor_input, self.tensor_target, batch_size=5) + self.assertEqual(adapter.get_size(), 10) + self.assertFalse(adapter.has_partial_batch()) + + def test_batch_size(self): + adapter = self.adapter_cls( + self.tensor_input, self.tensor_target, batch_size=5) + self.assertEqual(adapter.batch_size(), 5) + + def test_partial_batch(self): + adapter = self.adapter_cls( + self.tensor_input, self.tensor_target, batch_size=4) + self.assertEqual(adapter.get_size(), 13) # 50/4 + self.assertTrue(adapter.has_partial_batch()) + class DatasetAdapterTest(DataAdapterTestBase): def setUp(self): super(DatasetAdapterTest, self).setUp() - self.adapter = data_adapter.DatasetAdapter() + self.adapter_cls = data_adapter.DatasetAdapter def test_can_handle(self): - self.assertFalse(self.adapter.can_handle(self.numpy_input)) - self.assertFalse(self.adapter.can_handle(self.tensor_input)) - self.assertTrue(self.adapter.can_handle(self.dataset_input)) - self.assertFalse(self.adapter.can_handle(self.generator_input)) - self.assertFalse(self.adapter.can_handle(self.sequence_input)) + self.assertFalse(self.adapter_cls.can_handle(self.numpy_input)) + self.assertFalse(self.adapter_cls.can_handle(self.tensor_input)) + self.assertTrue(self.adapter_cls.can_handle(self.dataset_input)) + self.assertFalse(self.adapter_cls.can_handle(self.generator_input)) + self.assertFalse(self.adapter_cls.can_handle(self.sequence_input)) def test_training(self): - dataset = self.adapter.get_dataset(self.dataset_input) - self.model.compile(loss='mse', optimizer='sgd') + dataset = self.adapter_cls(self.dataset_input).get_dataset() + self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd') self.model.fit(dataset) + def test_size(self): + adapter = self.adapter_cls(self.dataset_input) + self.assertIsNone(adapter.get_size()) + + def test_batch_size(self): + adapter = self.adapter_cls(self.dataset_input) + self.assertIsNone(adapter.batch_size()) + + def test_partial_batch(self): + adapter = self.adapter_cls(self.dataset_input) + self.assertFalse(adapter.has_partial_batch()) + class GeneratorDataAdapterTest(DataAdapterTestBase): def setUp(self): super(GeneratorDataAdapterTest, self).setUp() - self.adapter = data_adapter.GeneratorDataAdapter() + self.adapter_cls = data_adapter.GeneratorDataAdapter def test_can_handle(self): - self.assertFalse(self.adapter.can_handle(self.numpy_input)) - self.assertFalse(self.adapter.can_handle(self.tensor_input)) - self.assertFalse(self.adapter.can_handle(self.dataset_input)) - self.assertTrue(self.adapter.can_handle(self.generator_input)) - self.assertFalse(self.adapter.can_handle(self.sequence_input)) + self.assertFalse(self.adapter_cls.can_handle(self.numpy_input)) + self.assertFalse(self.adapter_cls.can_handle(self.tensor_input)) + self.assertFalse(self.adapter_cls.can_handle(self.dataset_input)) + self.assertTrue(self.adapter_cls.can_handle(self.generator_input)) + self.assertFalse(self.adapter_cls.can_handle(self.sequence_input)) def test_training(self): - dataset = self.adapter.get_dataset(self.generator_input) - self.model.compile(loss='mse', optimizer='sgd') + dataset = self.adapter_cls(self.generator_input).get_dataset() + self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd') self.model.fit(dataset) + def test_size(self): + adapter = self.adapter_cls(self.generator_input) + self.assertIsNone(adapter.get_size()) + + def test_batch_size(self): + adapter = self.adapter_cls(self.generator_input) + self.assertEqual(adapter.batch_size(), 5) + + def test_partial_batch(self): + adapter = self.adapter_cls(self.generator_input) + self.assertFalse(adapter.has_partial_batch()) + class KerasSequenceAdapterTest(DataAdapterTestBase): def setUp(self): super(KerasSequenceAdapterTest, self).setUp() - self.adapter = data_adapter.KerasSequenceAdapter() + self.adapter_cls = data_adapter.KerasSequenceAdapter def test_can_handle(self): - self.assertFalse(self.adapter.can_handle(self.numpy_input)) - self.assertFalse(self.adapter.can_handle(self.tensor_input)) - self.assertFalse(self.adapter.can_handle(self.dataset_input)) - self.assertFalse(self.adapter.can_handle(self.generator_input)) - self.assertTrue(self.adapter.can_handle(self.sequence_input)) + self.assertFalse(self.adapter_cls.can_handle(self.numpy_input)) + self.assertFalse(self.adapter_cls.can_handle(self.tensor_input)) + self.assertFalse(self.adapter_cls.can_handle(self.dataset_input)) + self.assertFalse(self.adapter_cls.can_handle(self.generator_input)) + self.assertTrue(self.adapter_cls.can_handle(self.sequence_input)) def test_training(self): - dataset = self.adapter.get_dataset(self.sequence_input) - self.model.compile(loss='mse', optimizer='sgd') + dataset = self.adapter_cls(self.sequence_input).get_dataset() + self.model.compile(loss='sparse_categorical_crossentropy', optimizer='sgd') self.model.fit(dataset) + def test_size(self): + adapter = self.adapter_cls(self.sequence_input) + self.assertEqual(adapter.get_size(), 10) + + def test_batch_size(self): + adapter = self.adapter_cls(self.sequence_input) + self.assertEqual(adapter.batch_size(), 5) + + def test_partial_batch(self): + adapter = self.adapter_cls(self.sequence_input) + self.assertFalse(adapter.has_partial_batch()) + + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index 584ec1a8a0e..dbcc82e3be5 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -2106,12 +2106,11 @@ class Model(network.Network): first_x_value = nest.flatten(x)[0] if isinstance(first_x_value, np.ndarray): - x = distributed_training_utils.list_to_tuple(x) + x = training_utils.list_to_tuple(x) if y is not None: - y = distributed_training_utils.list_to_tuple(y) + y = training_utils.list_to_tuple(y) if sample_weight is not None: - sample_weight = distributed_training_utils.list_to_tuple( - sample_weight) + sample_weight = training_utils.list_to_tuple(sample_weight) in_tuple = (x, y, sample_weight) else: in_tuple = (x, y) diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py index 8f5c830d6e9..400530e1567 100644 --- a/tensorflow/python/keras/engine/training_utils.py +++ b/tensorflow/python/keras/engine/training_utils.py @@ -1723,6 +1723,13 @@ def convert_eager_tensors_to_numpy(structure): return nest.map_structure(_convert, structure) +def list_to_tuple(maybe_list): + """Datasets will stack the list of tensor, so switch them to tuples.""" + if isinstance(maybe_list, list): + return tuple(maybe_list) + return maybe_list + + def should_run_validation(validation_freq, epoch): """Checks if validation should be run this epoch. diff --git a/tensorflow/python/keras/engine/training_v2.py b/tensorflow/python/keras/engine/training_v2.py new file mode 100644 index 00000000000..9c494bfd92d --- /dev/null +++ b/tensorflow/python/keras/engine/training_v2.py @@ -0,0 +1,606 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Training related logic for Keras model in TF 2.0 context. + +Note that all the code under this module is under active development, please DO +NOT use it unless you are really sure what you are doing. +""" + +# pylint: disable=protected-access +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections + +import numpy as np + + +from tensorflow.python.distribute import distribution_strategy_context +from tensorflow.python.keras import backend +from tensorflow.python.keras import callbacks as cbks +from tensorflow.python.keras.distribute import distributed_training_utils as dist_utils +from tensorflow.python.keras.engine import data_adapter +from tensorflow.python.keras.engine import training_utils +from tensorflow.python.keras.utils.mode_keys import ModeKeys +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util import nest +from tensorflow.python.util import tf_contextlib + + +# The list of DataAdapter that support validation_split, only numpy and data +# tensor support validation_split for now. +_ADAPTER_FOR_VALIDATION_SPLIT = [data_adapter.NumpyArrayDataAdapter, + data_adapter.TensorDataAdapter] + +# The list of DataAdapter that support model._standardize_user_data. Currently +# keras.sequence/python generator will cause error when calling +# model._standardize_user_data, this should be updated in future cl, eg, the +# dataset/generate/sequence input will be peeked and processed by +# model._standardize_user_data() +_ADAPTER_FOR_STANDARDIZE_USER_DATA = [data_adapter.NumpyArrayDataAdapter, + data_adapter.TensorDataAdapter, + data_adapter.DatasetAdapter] + + +def run_one_epoch(model, + iterator, + execution_function, + dataset_size=None, + strategy=None, + steps_per_epoch=None, + mode=ModeKeys.TRAIN, + training_context=None, + current_epoch=1): + """Run the execution function with the data from iterator. + + Given the dataset iterator and execution function, get the data from iterator + and call it with the execution function to get the result (metric/loss). + It will run for steps_per_epoch or until to the iterator is fully consumed. + + Args: + model: The keras model to run. + iterator: the dataset iterator to fetch the data. + execution_function: a tf.function that can be called with data. + dataset_size: the size of iterator, None when unknown. + strategy: the distribution strategy instance from the model. + steps_per_epoch: the number of steps to run for the epoch. + mode: the mode for the current epoch. + training_context: the context that contains callbacks and progress bar. + current_epoch: the epoch number. Used when throw error when the + the iterator is unexpected reach its end. + Returns: + The loss and metric value from the model. + """ + if mode == ModeKeys.PREDICT: + aggregator = training_utils.OutputsAggregator( + use_steps=True, num_samples_or_steps=steps_per_epoch) + else: + aggregator = training_utils.MetricsAggregator( + use_steps=True, num_samples_or_steps=steps_per_epoch) + callbacks = training_context.callbacks + progbar = training_context.progbar + + if callbacks.model.stop_training: + return + + target_steps = steps_per_epoch or np.inf + step = 0 + + while step < target_steps: + with training_context.on_batch(step, mode=mode) as batch_logs: + try: + batch_ins = create_batch_inputs(iterator, mode, model, strategy) + batch_outs = execution_function(batch_ins) + except StopIteration: + # The only acceptable case here is that the input has a unknown + # length, and configured to fully consume it. + if (dataset_size is None + and steps_per_epoch is None + and step > 0): + # The input passed by the user ran out of batches. + # Now we know the cardinality of the input(dataset or generator). + steps_per_epoch = step + aggregator.num_samples_or_steps = steps_per_epoch + progbar.params['steps'] = steps_per_epoch + progbar.progbar.target = steps_per_epoch + else: + callbacks.model.stop_training = True + logging.warning( + 'Your input ran out of data; interrupting training. ' + 'Make sure that your dataset or generator can generate at ' + 'least {} batches. You may need to use the repeat() function ' + 'when building your dataset.'.format( + current_epoch * steps_per_epoch)) + # In either case, break out the loop for training batch. + break + + if not isinstance(batch_outs, list): + batch_outs = [batch_outs] + if strategy: + batch_outs = dist_utils._per_replica_aggregate_batch( + batch_outs, model, mode) + + if step == 0: + aggregator.create(batch_outs) + aggregator.aggregate(batch_outs) + cbks.make_logs(model, batch_logs, batch_outs, mode) + step += 1 + + if callbacks.model.stop_training: + break + + # End of an epoch. + aggregator.finalize() + results = aggregator.results + return results + + +def create_batch_inputs(iterator, mode, model, strategy): + """Create the input data from the iterator based on the model and strategy.""" + if strategy: + # Note that the batch_ins is a function to avoid the tf.function + # retrace. + def distribute_batch_ins(): + return dist_utils._prepare_feed_values(model, iterator, None, None, mode) + batch_ins = distribute_batch_ins + else: + batch_ins = next(iterator) + if (mode == ModeKeys.TRAIN + and not model.run_eagerly + and not isinstance(backend.symbolic_learning_phase(), int)): + # Add learning phase value. + if not isinstance(batch_ins, collections.Sequence): + batch_ins = (batch_ins, True) + else: + batch_ins += (True,) + return batch_ins + + +class Loop(training_utils.TrainingLoop): + """The training loop for the TF 2.0. + + This class has some existing assumption for runtime, eg eager by default, + have distribution strategy, etc. + """ + + def fit( + self, model, x=None, y=None, batch_size=None, epochs=1, verbose=1, + callbacks=None, validation_split=0., validation_data=None, shuffle=True, + class_weight=None, sample_weight=None, initial_epoch=0, + steps_per_epoch=None, validation_steps=None, validation_freq=1, **kwargs): + batch_size = model._validate_or_infer_batch_size( + batch_size, steps_per_epoch, x) + + strategy = _get_distribution_strategy(model) + if strategy: + batch_size, steps_per_epoch = dist_utils.process_batch_and_step_size( + strategy, x, batch_size, steps_per_epoch, ModeKeys.TRAIN) + dist_utils.validate_callbacks(input_callbacks=callbacks, + optimizer=model.optimizer) + # Enter tf.distribute.Strategy scope. + scope = dist_utils.distributed_scope( + strategy=strategy, learning_phase=1) + scope.__enter__() + + training_data_adapter, validation_adapter = _process_training_inputs( + model, + x, + y, + batch_size=batch_size, + sample_weights=sample_weight, + class_weights=class_weight, + validation_split=validation_split, + steps_per_epoch=steps_per_epoch, + shuffle=shuffle, + validation_data=validation_data, + validation_steps=validation_steps, + distribution_strategy=strategy) + + do_validation = (validation_adapter is not None) + + if not steps_per_epoch: + steps_per_epoch = training_data_adapter.get_size() + + # tf.print('{} on {} steps.'.format(ModeKeys.TRAIN, steps_per_epoch)) + training_context = TrainingContext() + + initial_epoch = model._maybe_load_initial_epoch_from_ckpt( + initial_epoch, ModeKeys.TRAIN) + + _update_sample_weight_mode(model, ModeKeys.TRAIN, training_data_adapter) + training_function = _make_execution_function(model, ModeKeys.TRAIN) + + training_data_iter = None + # Only recreate iterator when the data has a fixed length, which will be + # fully consumed every epoch, or has a unknown length (dataset, generator) + # and will be fully consumed (steps_per_epoch is None) + recreate_training_iterator = (training_data_adapter.get_size() is not None + or steps_per_epoch is None) + + if do_validation: + if not validation_steps: + validation_steps = validation_adapter.get_size() + eval_function = _make_execution_function(model, ModeKeys.TEST) + eval_data_iter = None + recreate_eval_iterator = (validation_adapter.get_size() is not None + or validation_steps is None) + + callbacks = cbks.configure_callbacks( + callbacks, + model, + do_validation=do_validation, + batch_size=batch_size, + epochs=epochs, + steps_per_epoch=steps_per_epoch, + samples=None, + verbose=0, # Handle ProgBarLogger separately in this loop. + mode=ModeKeys.TRAIN) + + with training_context.on_start(model, callbacks, verbose, ModeKeys.TRAIN): + # TODO(scottzhu): Handle TPUStrategy training loop + for epoch in range(initial_epoch, epochs): + if training_context.callbacks.model.stop_training: + break + + # Training + with training_context.on_epoch(epoch, ModeKeys.TRAIN) as epoch_logs: + model.reset_metrics() + if training_data_iter is None or recreate_training_iterator: + training_data_iter = _create_dataset_iterator( + strategy, training_data_adapter.get_dataset()) + + training_result = run_one_epoch( + model, + training_data_iter, + training_function, + dataset_size=training_data_adapter.get_size(), + strategy=strategy, + steps_per_epoch=steps_per_epoch, + mode=ModeKeys.TRAIN, + training_context=training_context, + current_epoch=epoch) + cbks.make_logs(model, epoch_logs, training_result, ModeKeys.TRAIN) + + # Evaluation + if (do_validation and + training_utils.should_run_validation(validation_freq, epoch) and + not callbacks.model.stop_training): + if eval_data_iter is None or recreate_eval_iterator: + eval_data_iter = _create_dataset_iterator( + strategy, validation_adapter.get_dataset()) + eval_context = TrainingContext() + with eval_context.on_start( + model, callbacks, verbose=0, mode=ModeKeys.TEST): + with eval_context.on_epoch(epoch, ModeKeys.TEST): + model.reset_metrics() + eval_result = run_one_epoch( + model, + eval_data_iter, + eval_function, + dataset_size=validation_adapter.get_size(), + strategy=strategy, + steps_per_epoch=validation_steps, + mode=ModeKeys.TEST, + training_context=eval_context, + current_epoch=epochs) + cbks.make_logs(model, epoch_logs, eval_result, ModeKeys.TRAIN, + prefix='val_') + + if strategy: + scope.__exit__(None, None, None) + + return model.history + + def _model_iteration( + self, model, mode, x=None, y=None, batch_size=None, verbose=1, + sample_weight=None, steps=None, callbacks=None, **kwargs): + + batch_size = model._validate_or_infer_batch_size( + batch_size, steps, x) + strategy = _get_distribution_strategy(model) + if strategy: + batch_size, steps = dist_utils.process_batch_and_step_size( + strategy, x, batch_size, steps, mode) + dist_utils.validate_callbacks(input_callbacks=callbacks, + optimizer=model.optimizer) + # Enter tf.distribute.Strategy scope. + scope = dist_utils.distributed_scope( + strategy=strategy, learning_phase=0) + scope.__enter__() + + adapter = _process_inputs( + model, + x, + y, + batch_size=batch_size, + sample_weights=sample_weight, + steps=steps, + distribution_strategy=strategy) + + if not steps: + steps = adapter.get_size() + + # tf.print('{} on {} steps.'.format(ModeKeys.TRAIN, steps_per_epoch)) + training_context = TrainingContext() + + _update_sample_weight_mode(model, mode, adapter) + execution_function = _make_execution_function(model, mode) + data_iterator = _create_dataset_iterator( + strategy, adapter.get_dataset()) + + callbacks = cbks.configure_callbacks( + callbacks, + model, + do_validation=False, + batch_size=batch_size, + epochs=1, + steps_per_epoch=steps, + samples=None, + verbose=0, # Handle ProgBarLogger separately in this loop. + mode=mode) + + with training_context.on_start(model, callbacks, verbose, mode): + # TODO(scottzhu): Handle TPUStrategy training loop + with training_context.on_epoch(0, mode) as epoch_logs: + model.reset_metrics() + result = run_one_epoch( + model, + data_iterator, + execution_function, + dataset_size=adapter.get_size(), + strategy=strategy, + steps_per_epoch=steps, + mode=mode, + training_context=training_context, + current_epoch=1) + cbks.make_logs(model, epoch_logs, result, mode) + + if strategy: + scope.__exit__(None, None, None) + + if len(result) == 1: + result = result[0] + return result + + def evaluate( + self, model, x=None, y=None, batch_size=None, verbose=1, + sample_weight=None, steps=None, callbacks=None, **kwargs): + return self._model_iteration( + model, ModeKeys.TEST, x=x, y=y, batch_size=batch_size, verbose=verbose, + sample_weight=sample_weight, steps=steps, callbacks=callbacks, **kwargs) + + def predict(self, model, x, batch_size=None, verbose=0, steps=None, + callbacks=None, **kwargs): + return self._model_iteration( + model, ModeKeys.PREDICT, x=x, batch_size=batch_size, verbose=verbose, + steps=steps, callbacks=callbacks, **kwargs) + + +def _get_distribution_strategy(model): + if model._distribution_strategy: + return model._distribution_strategy + # TODO(scottzhu): might want to just get the default strategy in future. + elif distribution_strategy_context.has_strategy(): + return distribution_strategy_context.get_strategy() + else: + return None + + +def _create_dataset_iterator(strategy, training_dataset): + if strategy: + training_data_iter = strategy.make_dataset_iterator(training_dataset) + else: + training_data_iter = iter(training_dataset) + return training_data_iter + + +def _process_training_inputs(model, x, y, batch_size=None, + sample_weights=None, class_weights=None, + steps_per_epoch=None, validation_split=0., + validation_data=None, validation_steps=None, + shuffle=True, distribution_strategy=None): + """Process the data input for fit() with respect to validation_split.""" + if validation_split and 0. < validation_split < 1. and validation_data: + raise ValueError('validation_data and validation_split cannot be used ' + 'at same time.') + + adapter_cls = data_adapter.select_data_adapter(x, y) + + # Handle validation_split, we want to split the data and get the training + # section before we give it to data adapter. + if validation_split and 0. < validation_split < 1.: + if adapter_cls not in _ADAPTER_FOR_VALIDATION_SPLIT: + raise ValueError( + '`validation_split` argument is not supported when ' + 'data adapter is {}. Received: x={}, validation_split={}'.format( + adapter_cls, x, validation_split)) + # Retrieve the training section from x and y, and then construct dataset + # from it. + x, y, sample_weights = model._standardize_user_data( + x, y, sample_weight=sample_weights, + class_weight=class_weights, + batch_size=batch_size, + check_steps=True, + steps=steps_per_epoch) + (x, y, sample_weights, + val_x, val_y, + val_sample_weights) = training_utils.split_training_and_validation_data( + x, y, sample_weights, validation_split) + train_adapter = adapter_cls(x, y, batch_size=batch_size, + sample_weights=sample_weights, shuffle=shuffle, + distribution_strategy=distribution_strategy) + val_adapter = adapter_cls(val_x, val_y, + sample_weights=val_sample_weights, + batch_size=batch_size, + distribution_strategy=distribution_strategy) + else: + train_adapter = _process_inputs(model, x, y, sample_weights=sample_weights, + batch_size=batch_size, + class_weights=class_weights, + shuffle=shuffle, steps=steps_per_epoch, + distribution_strategy=distribution_strategy) + val_adapter = None + if validation_data: + (val_x, val_y, + val_sample_weights) = training_utils.unpack_validation_data( + validation_data) + # For eval data, we use the training data batch_size it was unknown. + # This is useful for generator/sequence training data input with numpy + # validation data input. + if not batch_size: + batch_size = train_adapter.batch_size() + val_adapter = _process_inputs(model, val_x, val_y, + sample_weights=val_sample_weights, + batch_size=batch_size, + class_weights=class_weights, + steps=validation_steps, + distribution_strategy=distribution_strategy) + elif validation_steps: + raise ValueError('`validation_steps` should not be specified if ' + '`validation_data` is None.') + return train_adapter, val_adapter + + +def _process_inputs(model, x, y, batch_size=None, sample_weights=None, + class_weights=None, shuffle=False, steps=None, + distribution_strategy=None): + """Process the inputs for fit/eval/predict().""" + adapter_cls = data_adapter.select_data_adapter(x, y) + if adapter_cls in _ADAPTER_FOR_STANDARDIZE_USER_DATA: + x, y, sample_weights = model._standardize_user_data( + x, + y, + sample_weight=sample_weights, + class_weight=class_weights, + batch_size=batch_size, + check_steps=True, + steps=steps) + # TODO(scottzhu): The generator and keras.sequence does not work with + # model._standardize_user_data() so far. However that method is very + # important which contains on-fly model build/tensor align for dict input, + # etc. We should still call the _standardize_user_data with the peeked data + # from generator or sequence, and let model compile. + return adapter_cls(x, y, batch_size=batch_size, + sample_weights=sample_weights, shuffle=shuffle, + distribution_strategy=distribution_strategy) + + +def _make_execution_function(model, mode): + """Makes function to run one step of model execution.""" + if model._distribution_strategy: + return dist_utils._make_execution_function(model, mode) + else: + return model._make_execution_function(mode) + + +def _update_sample_weight_mode(model, mode, adapter): + """Updates the sample_weight_mode of a given model.""" + # Add a quick return to prevent us from calling model._feed_targets that + # accesses certain model properties that may not be set in the `PREDICT` mode. + if mode == ModeKeys.PREDICT: + return + + sample_weights = None + + # Get some sample inputs from the data_adapter + iterator = _create_dataset_iterator(model._distribution_strategy, + adapter.get_dataset()) + inputs = create_batch_inputs(iterator, mode, model, + model._distribution_strategy) + # `inputs` is the model's inputs + targets + sample_weights + + # learning phase placeholder if specified. To update the sample_weight_mode + # we need to determine if the user has passed sample weights as part of the + # input. + if not callable(inputs): + # if not isinstance(inputs, collections.Sequence): + # inputs = (inputs,) + # Note that the batch inputs should be a tuple of 2, 3 or 4 items. + # (input, target, {sample_weights}, {learning_phase}) + sample_weights_index = 0 + if model._feed_inputs: + sample_weights_index += 1 + if model._feed_targets: + sample_weights_index += 1 + + sample_weights = inputs[sample_weights_index:] + has_learning_phase_pl = (mode == ModeKeys.TRAIN and + not isinstance(backend.symbolic_learning_phase(), + int)) + if has_learning_phase_pl: + sample_weights = sample_weights[:-1] + model._update_sample_weight_modes(nest.flatten(sample_weights)) + + # Call the DistributionStrategy specific function to update the + # sample_weight_mode on the model. + if model._distribution_strategy: + dist_utils._update_sample_weight_modes(model, mode, sample_weights) + + # Force delete the iterator. + del iterator + + +class TrainingContext(object): + """Utility object that wrap around callbacks and progress bars.""" + + @tf_contextlib.contextmanager + def on_start(self, model, callbacks=None, verbose=0, mode=ModeKeys.TRAIN): + """Provide a scope for the whole training process.""" + # TODO(omalleyt): Handle ProgBar as part of Callbacks once hooks are ready. + progbar = training_utils.get_progbar(model, 'steps') + progbar.params = callbacks.params + progbar.params['verbose'] = verbose + callbacks.model.stop_training = False + callbacks._call_begin_hook(mode) + progbar.on_train_begin() + + # Cache those two instance so that it can be used in other functions. + self.callbacks = callbacks + self.progbar = progbar + + try: + yield + finally: + # End of all epochs + self.callbacks._call_end_hook(mode) + + @tf_contextlib.contextmanager + def on_epoch(self, epoch=0, mode=ModeKeys.TRAIN): + """Provide a scope for running one epoch.""" + epoch_logs = {} + if mode == ModeKeys.TRAIN: + self.callbacks.on_epoch_begin(epoch, epoch_logs) + self.progbar.on_epoch_begin(epoch, epoch_logs) + try: + yield epoch_logs + finally: + if mode == ModeKeys.TRAIN: + # Epochs only apply to `fit`. + self.callbacks.on_epoch_end(epoch, epoch_logs) + self.progbar.on_epoch_end(epoch, epoch_logs) + + @tf_contextlib.contextmanager + def on_batch(self, step=0, mode=ModeKeys.TRAIN): + """Provide a scope for running one batch.""" + batch_logs = {'batch': step, 'size': 1} + self.callbacks._call_batch_hook( + mode, 'begin', step, batch_logs) + self.progbar.on_batch_begin(step, batch_logs) + try: + yield batch_logs + finally: + self.callbacks._call_batch_hook( + mode, 'end', step, batch_logs) + self.progbar.on_batch_end(step, batch_logs) From f8ec9b9929920d8e6ecf6d32ff95371562fb4606 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 11 Jul 2019 18:21:11 -0700 Subject: [PATCH 321/332] [XLA:Python] Stage host to device transfers via pinned memory. On GPU, the CUDA runtime internally stages transfers from pageable memory to pinned memory anyway. It is slightly cheaper to transfer via a pool of pinned memory that we control since it allows us to amortize the cost of pinning. [TF] Expose gpu_host_allocator.h to XLA. XLA/Python already depends on the BFCAllocator classes so this is a minor dependency change. PiperOrigin-RevId: 257721430 --- .../compiler/xla/python/local_client.cc | 72 ++++++++++++++----- tensorflow/compiler/xla/python/local_client.h | 17 ++++- tensorflow/core/BUILD | 6 +- 3 files changed, 74 insertions(+), 21 deletions(-) diff --git a/tensorflow/compiler/xla/python/local_client.cc b/tensorflow/compiler/xla/python/local_client.cc index 4f8a1b0e713..397562592c6 100644 --- a/tensorflow/compiler/xla/python/local_client.cc +++ b/tensorflow/compiler/xla/python/local_client.cc @@ -98,6 +98,7 @@ limitations under the License. #include "tensorflow/compiler/xla/util.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/common_runtime/bfc_allocator.h" +#include "tensorflow/core/common_runtime/gpu/gpu_host_allocator.h" #include "tensorflow/core/common_runtime/gpu/gpu_mem_allocator.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/profiler/lib/traceme.h" @@ -159,19 +160,30 @@ StatusOr> PyLocalClient::Get( options.set_platform(platform); TF_ASSIGN_OR_RETURN(LocalClient * client, ClientLibrary::GetOrCreateLocalClient(options)); + bool gpu_platform = platform_name == "gpu"; std::unique_ptr allocator; - if (allocator_config.kind == AllocatorConfig::Kind::kBFC || - (gpu_platform && - allocator_config.kind == AllocatorConfig::Kind::kDefault)) { - if (!gpu_platform) { - return Unimplemented("BFCAllocator only available for GPU."); + std::unique_ptr host_memory_allocator; + if (gpu_platform) { + if (allocator_config.kind != AllocatorConfig::Kind::kPlatform) { + TF_ASSIGN_OR_RETURN( + allocator, + CreateBFCAllocator(platform, client, allocator_config.memory_fraction, + allocator_config.preallocate)); } - TF_ASSIGN_OR_RETURN( - auto bfc_allocator, - CreateBFCAllocator(platform, client, allocator_config.memory_fraction, - allocator_config.preallocate)); - allocator = std::move(bfc_allocator); + + tensorflow::SubAllocator* sub_allocator = new tensorflow::GpuHostAllocator( + client->backend().stream_executor(0).ValueOrDie(), /*numa_node=*/0, + /*alloc_visitors=*/{}, + /*free_visitors=*/{}); + // TODO(phawkins): allow the user to tune this. + const int64 kGpuHostMemoryLimitBytes = 64 * (1LL << 30); + host_memory_allocator = absl::make_unique( + sub_allocator, kGpuHostMemoryLimitBytes, /*allow_growth=*/true, + /*name=*/"xla_gpu_host_bfc"); + + } else if (allocator_config.kind == AllocatorConfig::Kind::kBFC) { + return Unimplemented("BFCAllocator only available for GPU."); } std::vector> devices; @@ -185,17 +197,20 @@ StatusOr> PyLocalClient::Get( /*allow_event_reuse=*/gpu_platform)); } return std::make_shared( - platform_name, client, std::move(devices), std::move(allocator)); + platform_name, client, std::move(devices), std::move(allocator), + std::move(host_memory_allocator)); } PyLocalClient::PyLocalClient( std::string platform_name, LocalClient* client, std::vector> devices, - std::unique_ptr allocator) + std::unique_ptr allocator, + std::unique_ptr host_memory_allocator) : platform_name_(std::move(platform_name)), client_(client), devices_(std::move(devices)), owned_allocator_(std::move(allocator)), + host_memory_allocator_(std::move(host_memory_allocator)), h2d_transfer_pool_(tensorflow::Env::Default(), "py_xla_h2d_transfer", client->device_count()) { if (owned_allocator_ != nullptr) { @@ -245,9 +260,9 @@ StatusOr> PyLocalBuffer::FromPython( << " device ordinal: " << device_ordinal; Device* device = &client->device(device_ordinal); - se::DeviceMemoryAllocator* allocator = client->allocator(); TransferManager* transfer_manager = client->client()->backend().transfer_manager(); + se::DeviceMemoryAllocator* allocator = client->allocator(); TF_ASSIGN_OR_RETURN( Shape shape, transfer_manager->ChooseCompactLayoutForShape(tree.shape)); TF_ASSIGN_OR_RETURN(ScopedShapedBuffer buffer, @@ -256,6 +271,8 @@ StatusOr> PyLocalBuffer::FromPython( TF_RETURN_IF_ERROR(transfer_manager->WriteTupleIndexTablesAsync( device->host_to_device_stream(), buffer)); + std::vector> staging_buffers; + staging_buffers.reserve(tree.leaves.size()); auto it = tree.leaves.begin(); for (const ShapeUtil::IndexedShape& indexed_shape : ShapeUtil::GetLeafShapes(shape)) { @@ -269,8 +286,27 @@ StatusOr> PyLocalBuffer::FromPython( device->host_to_device_stream()->parent(), leaf)) { device->host_to_device_stream()->ThenWaitFor(device->compute_stream()); } - TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralToDeviceAsync( - device->host_to_device_stream(), *it, leaf)); + + // If applicable on the backend, stage the transfer via host memory + // allocated via the host_memory_allocator. On GPU, this is pinned memory. + if (client->host_memory_allocator()) { + int64 size = it->size_bytes({}); + void* ptr = client->host_memory_allocator()->AllocateRaw( + tensorflow::Allocator::kAllocatorAlignment, size); + std::shared_ptr staging_buffer(ptr, [client](void* ptr) { + client->host_memory_allocator()->DeallocateRaw(ptr); + }); + std::memcpy(ptr, it->untyped_data({}), size); + BorrowingLiteral literal(static_cast(staging_buffer.get()), + it->shape()); + TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralToDeviceAsync( + device->host_to_device_stream(), literal, leaf)); + staging_buffers.push_back(std::move(staging_buffer)); + } else { + // Otherwise, just transfer the literal. + TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralToDeviceAsync( + device->host_to_device_stream(), *it, leaf)); + } ++it; } @@ -287,8 +323,10 @@ StatusOr> PyLocalBuffer::FromPython( if (device->synchronous_deallocation()) { device->ThenRelease(device->host_to_device_stream(), device_buffer); } - device->ThenRelease(device->host_to_device_stream(), - std::move(py_buffer_ref)); + device->ThenRelease( + device->host_to_device_stream(), + std::make_pair(std::move(py_buffer_ref), std::move(staging_buffers))); + return absl::make_unique(shape, std::move(device_buffer), std::move(client)); } diff --git a/tensorflow/compiler/xla/python/local_client.h b/tensorflow/compiler/xla/python/local_client.h index daa7d42d1d0..8ad4c44d53f 100644 --- a/tensorflow/compiler/xla/python/local_client.h +++ b/tensorflow/compiler/xla/python/local_client.h @@ -35,6 +35,7 @@ limitations under the License. #include "tensorflow/compiler/xla/shape.h" #include "tensorflow/compiler/xla/status.h" #include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/core/framework/allocator.h" #include "tensorflow/core/lib/core/status.h" namespace xla { @@ -69,9 +70,11 @@ class PyLocalClient { bool asynchronous, const AllocatorConfig& allocator_config); // `allocator` may null, in which case the platform default allocator is used. - explicit PyLocalClient(std::string platform_name, LocalClient* client, - std::vector> devices, - std::unique_ptr allocator); + explicit PyLocalClient( + std::string platform_name, LocalClient* client, + std::vector> devices, + std::unique_ptr allocator, + std::unique_ptr host_memory_allocator); virtual ~PyLocalClient() = default; Status TransferToInfeed(const LiteralSlice& literal, int device_ordinal); @@ -84,6 +87,9 @@ class PyLocalClient { } LocalClient* client() const { return client_; } se::DeviceMemoryAllocator* allocator() const { return allocator_; } + tensorflow::Allocator* host_memory_allocator() const { + return host_memory_allocator_.get(); + } tensorflow::thread::ThreadPool* h2d_transfer_pool() { return &h2d_transfer_pool_; @@ -105,6 +111,11 @@ class PyLocalClient { se::DeviceMemoryAllocator* allocator_; std::unique_ptr owned_allocator_; + // Allocator to be used for staging memory transfers to devices. Optional; + // only used on GPU where it is more efficient to copy buffers to and from the + // device via a staging area of pinned memory. + std::unique_ptr host_memory_allocator_; + tensorflow::thread::ThreadPool h2d_transfer_pool_; }; diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index e28a115c25a..1a92570917c 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -3631,11 +3631,15 @@ tf_cuda_library( srcs = [ "common_runtime/gpu/gpu_id.h", ], - hdrs = ["common_runtime/gpu/gpu_mem_allocator.h"], + hdrs = [ + "common_runtime/gpu/gpu_host_allocator.h", + "common_runtime/gpu/gpu_mem_allocator.h", + ], features = ["parse_headers"], visibility = ["//visibility:public"], deps = [ ":allocator", + ":lib", ":lib_internal", ":stream_executor", ], From dd8e9cf06b1a0385b5bb5bac4e4ecdfb649accdc Mon Sep 17 00:00:00 2001 From: Katherine Wu Date: Thu, 11 Jul 2019 18:43:07 -0700 Subject: [PATCH 322/332] Additional fixes to Layer call function tracing. -Ensure that all call functions have the same signature (using tf decorator to wrap the original layer call function) -Added code to maintain losses -Added unit tests to ensures that the training arg replacement works correctly, and that losses are properly maintained. PiperOrigin-RevId: 257723770 --- .../python/keras/saving/saved_model/save.py | 83 +++++++------ .../saving/saved_model/saved_model_test.py | 109 +++++++++++++++--- .../python/keras/saving/saved_model/utils.py | 64 +++++++++- 3 files changed, 193 insertions(+), 63 deletions(-) diff --git a/tensorflow/python/keras/saving/saved_model/save.py b/tensorflow/python/keras/saving/saved_model/save.py index 1bf80e6ae13..045386c97d7 100644 --- a/tensorflow/python/keras/saving/saved_model/save.py +++ b/tensorflow/python/keras/saving/saved_model/save.py @@ -39,7 +39,7 @@ from tensorflow.python.training.tracking import base as trackable from tensorflow.python.training.tracking import data_structures from tensorflow.python.training.tracking import layer_utils as trackable_layer_utils from tensorflow.python.util import nest -from tensorflow.python.util import tf_inspect +from tensorflow.python.util import tf_decorator from tensorflow.python.util.lazy_loader import LazyLoader # To avoid circular dependencies between keras/engine and keras/saving, @@ -252,7 +252,8 @@ def _wrap_layer_functions(layer, serialization_cache): fns['activity_regularizer_fn'] = _wrap_activity_regularizer(layer) fns['call_and_return_all_conditional_losses'] = ( call_collection.add_function( - _append_activity_regularizer_loss(call_fn_with_losses, + _append_activity_regularizer_loss(layer, + call_fn_with_losses, fns['activity_regularizer_fn']), '{}_layer_call_and_return_all_conditional_losses'.format(layer.name) )) @@ -391,8 +392,10 @@ class LayerCallCollection(object): """ def __init__(self, layer): - self._layer = layer + self.layer = layer self._expects_training_arg = layer._expects_training_arg # pylint: disable=protected-access + self._training_arg_index = utils.get_training_arg_index(layer) + self._input_signature = self._generate_input_signature(layer) self._functions = weakref.WeakValueDictionary() # Bool indicating whether this object is currently tracing the layer call @@ -447,24 +450,14 @@ class LayerCallCollection(object): # TODO(kathywu): Replace arguments with broader shapes defined in the # input signature. if self._expects_training_arg: - arg_list = tf_inspect.getfullargspec(fn.python_function).args - if 'training' in arg_list: - training_arg_index = arg_list.index('training') - else: - training_arg_index = -1 - - def set_training_arg(training, index=training_arg_index): - if index >= 0 and len(args) > index: - args[index] = training - else: - kwargs['training'] = training - - set_training_arg(False) - fn.original_get_concrete_function(*args, **kwargs) - set_training_arg(True) - fn.original_get_concrete_function(*args, **kwargs) + args, kwargs = utils.set_training_arg(False, self._training_arg_index, + args, kwargs) + fn.get_concrete_function(*args, **kwargs) + args, kwargs = utils.set_training_arg(True, self._training_arg_index, + args, kwargs) + fn.get_concrete_function(*args, **kwargs) else: - fn.original_get_concrete_function(*args, **kwargs) + fn.get_concrete_function(*args, **kwargs) self.tracing = False @property @@ -494,6 +487,18 @@ class LayerCallCollection(object): return fn +def maintain_losses(method): + """Ensures layer losses are kept the same, and runs method in call context.""" + def wrapper(self, *args, **kwargs): + layer = self.call_collection.layer + original_losses = _reset_layer_losses(layer) + with base_layer_utils.call_context().enter(layer, None, True, None): + ret = method(self, *args, **kwargs) + _restore_layer_losses(original_losses) + return ret + return tf_decorator.make_decorator(target=method, decorator_func=wrapper) + + class LayerCall(def_function.Function): """Function that triggers traces of other functions in the same collection.""" @@ -501,19 +506,18 @@ class LayerCall(def_function.Function): super(LayerCall, self).__init__(*args, **kwargs) self.call_collection = call_collection + @maintain_losses def __call__(self, *args, **kwargs): if not self.call_collection.tracing: self.call_collection.add_trace(*args, **kwargs) return super(LayerCall, self).__call__(*args, **kwargs) + @maintain_losses def get_concrete_function(self, *args, **kwargs): if not self.call_collection.tracing: self.call_collection.add_trace(*args, **kwargs) return super(LayerCall, self).get_concrete_function(*args, **kwargs) - def original_get_concrete_function(self, *args, **kwargs): - return super(LayerCall, self).get_concrete_function(*args, **kwargs) - def _wrap_call_and_conditional_losses(layer): """Wraps call function that returns a tuple of (outputs, losses). @@ -530,37 +534,32 @@ def _wrap_call_and_conditional_losses(layer): """ # Create function that generates both outputs and losses layer_call = layer.call - if layer._expects_training_arg: # pylint: disable=protected-access - def call_and_return_conditional_losses(inputs, training=False): - return layer_call(inputs, training=training), layer.get_losses_for(inputs) - else: - def call_and_return_conditional_losses(inputs): - K.set_learning_phase(0) - return layer_call(inputs), layer.get_losses_for(inputs) - return call_and_return_conditional_losses + + def call_and_return_conditional_losses(inputs, *args, **kwargs): + return layer_call(inputs, *args, **kwargs), layer.get_losses_for(inputs) + return tf_decorator.make_decorator( + layer_call, call_and_return_conditional_losses) def _extract_outputs_from_fn(layer, call_and_return_conditional_losses): """Returns a function that returns only call function outputs.""" if isinstance(layer, keras_load.RevivedLayer): return layer.keras_api.__call__ # pylint: disable=protected-access - if layer._expects_training_arg: # pylint: disable=protected-access - def call(inputs, training=False): - return call_and_return_conditional_losses(inputs, training=training)[0] - else: - def call(inputs): - return call_and_return_conditional_losses(inputs)[0] - return call + def call(inputs, *args, **kwargs): + return call_and_return_conditional_losses(inputs, *args, **kwargs)[0] + layer_call = layer.call + return tf_decorator.make_decorator(layer_call, call) def _append_activity_regularizer_loss( - call_fn_with_losses, activity_regularizer_fn): + layer, call_fn_with_losses, activity_regularizer_fn): """Appends activity regularizer loss to losses returned by the wrapped fn.""" - def fn(*args, **kwargs): - outputs, losses = call_fn_with_losses(*args, **kwargs) + def fn(inputs, *args, **kwargs): + outputs, losses = call_fn_with_losses(inputs, *args, **kwargs) losses.append(activity_regularizer_fn(outputs)) return outputs, losses - return fn + layer_call = layer.call + return tf_decorator.make_decorator(target=layer_call, decorator_func=fn) def _wrap_unconditional_loss(loss_fn, index): diff --git a/tensorflow/python/keras/saving/saved_model/saved_model_test.py b/tensorflow/python/keras/saving/saved_model/saved_model_test.py index 7358f431df1..b90b576446d 100644 --- a/tensorflow/python/keras/saving/saved_model/saved_model_test.py +++ b/tensorflow/python/keras/saving/saved_model/saved_model_test.py @@ -30,7 +30,8 @@ from tensorflow.python.framework import test_util from tensorflow.python.keras import keras_parameterized from tensorflow.python.keras import regularizers from tensorflow.python.keras import testing_utils -from tensorflow.python.keras.saving.saved_model import load as saved_model_load +from tensorflow.python.keras.saving.saved_model import load as keras_load +from tensorflow.python.keras.saving.saved_model import save as keras_save from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops @@ -60,6 +61,13 @@ class LayerWithLearningPhase(keras.engine.base_layer.Layer): return input_shape +class LayerWithLoss(keras.layers.Layer): + + def call(self, inputs): + self.add_loss(math_ops.reduce_sum(inputs), inputs) + return inputs + + @test_util.run_all_in_graph_and_eager_modes class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): @@ -87,7 +95,7 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): saved_model_dir = self._save_model_dir() tf_save.save(model, saved_model_dir) - loaded = saved_model_load.load(saved_model_dir) + loaded = keras_load.load(saved_model_dir) self.evaluate(variables.variables_initializer(loaded.variables)) self.assertAllClose(self.evaluate(model.weights), self.evaluate(loaded.weights)) @@ -123,7 +131,7 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): saved_model_dir = self._save_model_dir() self.evaluate(variables.variables_initializer(layer.variables)) tf_save.save(layer, saved_model_dir) - loaded = saved_model_load.load(saved_model_dir) + loaded = keras_load.load(saved_model_dir) self.evaluate(variables.variables_initializer(loaded.variables)) equal_attrs = ['name', '_expects_training_arg', 'trainable'] @@ -137,13 +145,6 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): def test_maintains_losses(self): """Tests that the layer losses do not change before and after export.""" - - class LayerWithLoss(keras.layers.Layer): - - def call(self, inputs): - self.add_loss(math_ops.reduce_sum(inputs), inputs) - return inputs - model = keras.models.Sequential([LayerWithLoss()]) model.compile( loss='mse', @@ -172,7 +173,7 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): layer.build([None, None]) saved_model_dir = self._save_model_dir() tf_save.save(layer, saved_model_dir) - loaded = saved_model_load.load(saved_model_dir) + loaded = keras_load.load(saved_model_dir) input_arr = array_ops.ones((4, 3)) # Run the layer, and use the keras backend learing phase @@ -214,7 +215,7 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): self.assertEqual(expected_layers, len(loaded.keras_api.layers)) input_arr = array_ops.ones((4, 3)) self.assertAllClose(self.evaluate(model(input_arr)), - self.evaluate(loaded(input_arr))) + self.evaluate(loaded(input_arr, training=False))) @keras_parameterized.run_with_all_model_types def test_compiled_model(self): @@ -232,7 +233,7 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): # TODO(b/134519980): Issue with model.fit if the model call function uses # a tf.function (Graph mode only). with context.eager_mode(): - loaded = saved_model_load.load(saved_model_dir) + loaded = keras_load.load(saved_model_dir) actual_predict = loaded.predict(input_arr) self.assertAllClose(expected_predict, actual_predict) @@ -261,7 +262,7 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): layer = LayerWithNestedSpec() saved_model_dir = self._save_model_dir() tf_save.save(layer, saved_model_dir) - loaded = saved_model_load.load(saved_model_dir) + loaded = keras_load.load(saved_model_dir) self.assertEqual(3, loaded.input_spec['a'].max_ndim) self.assertEqual({-1: 2}, loaded.input_spec['a'].axes) self.assertAllEqual([None, 2, 3], loaded.input_spec['b'].shape) @@ -274,7 +275,7 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): saved_model_dir = self._save_model_dir() model.save(saved_model_dir, save_format='tf') - loaded = saved_model_load.load(saved_model_dir) + loaded = keras_load.load(saved_model_dir) input_arr_1 = np.random.random((1, 3)).astype('float32') input_arr_2 = np.random.random((1, 5)).astype('float32') @@ -292,7 +293,7 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): saved_model_dir = self._save_model_dir() model.save(saved_model_dir, save_format='tf') - loaded = saved_model_load.load(saved_model_dir) + loaded = keras_load.load(saved_model_dir) self.assertLen(loaded.layers, 2) self.assertLen(loaded.losses, 2) @@ -307,5 +308,81 @@ class TestModelSavingAndLoadingV2(keras_parameterized.TestCase): self.assertLen(loaded.layers, 2) self.assertLen(loaded.losses, 2) + +class TestLayerCallTracing(test.TestCase): + + def test_functions_have_same_trace(self): + + class Layer(keras.engine.base_layer.Layer): + + def call(self, inputs): + return inputs + + def call2(self, inputs): + return inputs * 2 + + layer = Layer() + call_collection = keras_save.LayerCallCollection(layer) + fn = call_collection.add_function(layer.call, 'call') + fn2 = call_collection.add_function(layer.call2, 'call2') + + fn(np.ones((2, 3))) + fn(np.ones((4, 5))) + + self.assertLen(fn._list_all_concrete_functions_for_serialization(), 2) + self.assertLen(fn2._list_all_concrete_functions_for_serialization(), 2) + + # Check that the shapes are correct + self.assertEqual( + {(2, 3), (4, 5)}, + set(tuple(c.structured_input_signature[0][0].shape.as_list()) + for c in fn2._list_all_concrete_functions_for_serialization())) + + def test_training_arg_replacement(self): + + def assert_num_traces(layer_cls, training_keyword): + layer = layer_cls() + call_collection = keras_save.LayerCallCollection(layer) + fn = call_collection.add_function(layer.call, 'call') + + fn(np.ones((2, 3)), training=True) + self.assertLen(fn._list_all_concrete_functions_for_serialization(), 2) + + fn(np.ones((2, 4)), training=False) + self.assertLen(fn._list_all_concrete_functions_for_serialization(), 4) + + if training_keyword: + fn(np.ones((2, 5)), True) + self.assertLen(fn._list_all_concrete_functions_for_serialization(), 6) + fn(np.ones((2, 6))) + self.assertLen(fn._list_all_concrete_functions_for_serialization(), 8) + + class LayerWithTrainingKeyword(keras.engine.base_layer.Layer): + + def call(self, inputs, training=False): + return inputs * training + + assert_num_traces(LayerWithTrainingKeyword, training_keyword=True) + + class LayerWithKwargs(keras.engine.base_layer.Layer): + + def call(self, inputs, **kwargs): + return inputs * kwargs['training'] + + assert_num_traces(LayerWithKwargs, training_keyword=False) + + @test_util.run_in_graph_and_eager_modes + def test_maintains_losses(self): + layer = LayerWithLoss() + layer(np.ones((2, 3))) + previous_losses = layer.losses[:] + + call_collection = keras_save.LayerCallCollection(layer) + fn = call_collection.add_function(layer.call, 'call') + fn(np.ones((2, 3))) + + self.assertAllEqual(previous_losses, layer.losses) + + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/saving/saved_model/utils.py b/tensorflow/python/keras/saving/saved_model/utils.py index 960b3709273..1f44174deec 100644 --- a/tensorflow/python/keras/saving/saved_model/utils.py +++ b/tensorflow/python/keras/saving/saved_model/utils.py @@ -19,6 +19,7 @@ from __future__ import print_function from tensorflow.python.keras import backend as K from tensorflow.python.keras.utils import tf_utils +from tensorflow.python.util import tf_inspect def use_wrapped_call(layer, call_fn): @@ -33,19 +34,72 @@ def use_wrapped_call(layer, call_fn): function that calls call_fn and returns the outputs. Losses returned by call_fn are added to the layer losses. """ - # TODO(kathywu): Support mask argument and multi-input call functions. - def wrapped_call(inputs, **kwargs): + training_arg_index = get_training_arg_index(layer) + + def wrapped_call(inputs, *args, **kwargs): """Returns the outputs from the call_fn, and adds the losses.""" if layer._expects_training_arg: # pylint: disable=protected-access - training = kwargs.pop('training', None) + training = get_training_arg(training_arg_index, args, kwargs) if training is None: training = K.learning_phase() + + args = list(args) + kwargs = kwargs.copy() + + def replace_training_and_call(training): + new_args, new_kwargs = set_training_arg(training, training_arg_index, + args, kwargs) + return call_fn(inputs, *new_args, **new_kwargs) + outputs, losses = tf_utils.smart_cond( training, - lambda: call_fn(inputs, training=True), - lambda: call_fn(inputs, training=False)) + lambda: replace_training_and_call(True), + lambda: replace_training_and_call(False)) else: outputs, losses = call_fn(inputs) layer.add_loss(losses, inputs) return outputs return wrapped_call + + +def get_training_arg_index(layer): + """Returns the index of 'training' in the layer call function arguments. + + Args: + layer: Keras layer + + Returns: + - n: index of 'training' in the call function arguments. + - -1: if 'training' is not found in the arguments, but layer.call accepts + variable keyword arguments + - None: if layer doesn't expect a training argument. + """ + if not layer._expects_training_arg: # pylint: disable=protected-access + return None + + arg_list = tf_inspect.getfullargspec(layer.call).args + if tf_inspect.ismethod(layer.call): + arg_list = arg_list[1:] + if 'training' in arg_list: + return arg_list.index('training') + else: + return -1 + + +def set_training_arg(training, index, args, kwargs): + if index is None: + pass + elif index >= 0 and len(args) > index: + args[index] = training + else: + kwargs['training'] = training + return args, kwargs + + +def get_training_arg(index, args, kwargs): + if index is None: + return None + elif index >= 0 and len(args) > index: + return args[index] + else: + return kwargs.get('training', None) From 968d511ebf5fe5651752580a13e457498ebab50d Mon Sep 17 00:00:00 2001 From: Daniel Situnayake Date: Thu, 11 Jul 2019 21:07:00 -0700 Subject: [PATCH 323/332] Update to readme PiperOrigin-RevId: 257736687 --- tensorflow/lite/experimental/micro/README.md | 28 +++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/experimental/micro/README.md b/tensorflow/lite/experimental/micro/README.md index 78362debfcf..6100d3af365 100644 --- a/tensorflow/lite/experimental/micro/README.md +++ b/tensorflow/lite/experimental/micro/README.md @@ -10,7 +10,7 @@ detection model, takes up a total of 22KB. ## Table of Contents - [Getting Started](#getting-started) - + * [Examples](#examples) * [Getting Started with Portable Reference Code](#getting-started-with-portable-reference-code) * [Building Portable Reference Code using Make](#building-portable-reference-code-using-make) * [Building for the "Blue Pill" STM32F103 using Make](#building-for-the-blue-pill-stm32f103-using-make) @@ -39,6 +39,32 @@ detection model, takes up a total of 22KB. # Getting Started +## Examples + +The fastest way to learn how TensorFlow Lite for Microcontrollers works is by +exploring and running our examples, which include application code and trained +TensorFlow models. + +The following examples are available: + +- [hello_world](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/examples/hello_world) + * Uses a very simple model, trained to reproduce a sine wave, to control an + LED or animation + * Application code for Arduino, SparkFun Edge, and STM32F746 + * Colab walkthrough of model training and conversion + +- [micro_speech](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/examples/micro_speech) + * Uses a 20kb model to recognize keywords in spoken audio + * Application code for Arduino, SparkFun Edge, and STM32F746 + * Python scripts for model training and conversion + +- [micro_vision](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/lite/experimental/micro/examples/micro_vision) + * Uses a 250kb model to recognize presence or absence of a person in images + captured by a camera + * Application code for SparkFun Edge + +## Pre-generated Project Files + One of the challenges of embedded software development is that there are a lot of different architectures, devices, operating systems, and build systems. We aim to support as many of the popular combinations as we can, and make it as From 7e7db7228f211b368a9c37d40145d48eb65d0e1e Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Thu, 11 Jul 2019 22:15:51 -0700 Subject: [PATCH 324/332] Move ctx->Unref from DestoryRemoteTensorHandle PiperOrigin-RevId: 257742480 --- .../distributed_runtime/eager/remote_tensor_handle_data.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc b/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc index 747eaed1f44..d3a7c60af31 100644 --- a/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc +++ b/tensorflow/core/distributed_runtime/eager/remote_tensor_handle_data.cc @@ -27,8 +27,6 @@ void DestoryRemoteTensorHandle(EagerContext* ctx, eager::EagerClient* eager_client, uint64 context_id, uint64 op_id, int output_num) { - auto cleanup = gtl::MakeCleanup([ctx]() { ctx->Unref(); }); - if (ctx->GetContextId() != context_id) { // This means that this tensor was pointing to a remote device, which // has been changed out from under us. Simply return since there is @@ -75,6 +73,7 @@ RemoteTensorHandleData::RemoteTensorHandleData(int64 op_id, int output_num, RemoteTensorHandleData::~RemoteTensorHandleData() { DestoryRemoteTensorHandle(ctx_, eager_client_, context_id_, op_id_, output_num_); + ctx_->Unref(); } Status RemoteTensorHandleData::Tensor(const tensorflow::Tensor** t) const { @@ -137,9 +136,8 @@ UnshapedRemoteTensorHandleData::~UnshapedRemoteTensorHandleData() { if (delete_remote_tensor_) { DestoryRemoteTensorHandle(ctx_, eager_client_, context_id_, op_id_, output_num_); - } else { - ctx_->Unref(); } + ctx_->Unref(); } Status UnshapedRemoteTensorHandleData::Tensor( From 1db9b2b7c39ad393d130e85d07677dc5ee1640be Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Fri, 12 Jul 2019 00:27:43 -0700 Subject: [PATCH 325/332] Add control-to-executor dialect conversion pass This pass convert an MLIR representation of TensorFlow graph from the TensorFlow Control Dialect (_tf) to a mix of tf_executor and tf dialects. This is intended for managing the transition only, the TensorFlow Control dialect is ultimately intended to be removed after the GraphDef importer is updated to target directly the tf_executor dialect. PiperOrigin-RevId: 257754326 --- tensorflow/compiler/mlir/tensorflow/BUILD | 2 + .../mlir/tensorflow/ir/tf_executor_ops.td | 84 +++--- .../tests/control_to_executor_dialect.mlir | 86 ++++++ .../mlir/tensorflow/transforms/passes.h | 1 - .../translate/control_to_executor_dialect.cc | 248 ++++++++++++++++++ 5 files changed, 370 insertions(+), 51 deletions(-) create mode 100644 tensorflow/compiler/mlir/tensorflow/tests/control_to_executor_dialect.mlir create mode 100644 tensorflow/compiler/mlir/tensorflow/translate/control_to_executor_dialect.cc diff --git a/tensorflow/compiler/mlir/tensorflow/BUILD b/tensorflow/compiler/mlir/tensorflow/BUILD index f9457659887..464dc7ae345 100644 --- a/tensorflow/compiler/mlir/tensorflow/BUILD +++ b/tensorflow/compiler/mlir/tensorflow/BUILD @@ -103,6 +103,7 @@ cc_library( "transforms/generated_optimize.inc", "transforms/optimize.cc", "transforms/raise_control_flow.cc", + "translate/control_to_executor_dialect.cc", ], hdrs = [ "ir/control_flow_ops.h", @@ -472,6 +473,7 @@ cc_library( "@llvm//:support", "@local_config_mlir//:IR", "@local_config_mlir//:Parser", + "@local_config_mlir//:Pass", ], ) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor_ops.td index c15bbd0cd8d..125ef1bfda6 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor_ops.td @@ -250,25 +250,6 @@ def TfExecutor_SwitchOp : TfExecutor_Op<"Switch", TfeControlType: $control ); - let builders = [OpBuilder< - "Builder *builder, OperationState *result, ArrayRef operands = {}", - [{ - assert(operands.size() >= 2 && "tf_executor.Switch builder expects at " - "least two operands"); - return build(builder, result, operands[0], operands[1], operands.drop_front(2)); - }]>, - OpBuilder< - "Builder *builder, OperationState *result, Value *data, Value *predicate, ArrayRef controls = {}", - [{ - Type dataTy = data->getType(); - Type controlTy = ControlType::get(builder->getContext()); - result->types = { dataTy, dataTy, controlTy }; - result->operands.push_back(data); - result->operands.push_back(predicate); - result->operands.insert(result->operands.end(), controls.begin(), controls.end()); - }]> - ]; - let verifier = ?; } @@ -311,7 +292,6 @@ def TfExecutor_SwitchNOp : Variadic:$outputs, TfeControlType: $control ); - } def TfExecutor_MergeOp : TfExecutor_Op<"Merge", [NoSideEffect, ControlOperandsAfterAllData]> { @@ -346,18 +326,6 @@ def TfExecutor_MergeOp : TfExecutor_Op<"Merge", [NoSideEffect, ControlOperandsAf TensorOf<[I32]>:$valueIndex, TfeControlType:$control ); - - let builders = [OpBuilder< - "Builder *builder, OperationState *result, ArrayRef operands", - [{ - assert(operands.size() >= 1 && "tf_executor.Merge builder expects at " - "least one operand"); - Type data_type = operands[0]->getType(); - Type control_type = ControlType::get(builder->getContext()); - result->types = { data_type, builder->getIntegerType(32), control_type}; - result->operands.append(operands.begin(), operands.end()); - }]> - ]; } def TfExecutor_EnterOp : TfExecutor_Op<"Enter", @@ -408,19 +376,6 @@ def TfExecutor_EnterOp : TfExecutor_Op<"Enter", ); let verifier = ?; - - let builders = [OpBuilder< - "Builder *builder, OperationState *result, ArrayRef operands", - [{ - assert(operands.size() >= 1 && "tf_executor.Enter builder " - "expects at least one operand"); - result->operands.append(operands.begin(), operands.end()); - - Type control_type = ControlType::get(builder->getContext()); - result->types.push_back(operands[0]->getType()); - result->types.push_back(control_type); - }]> - ]; } def TfExecutor_NextIterationSourceOp : TfExecutor_Op<"NextIteration.Source", [NoSideEffect]> { @@ -472,12 +427,14 @@ def TfExecutor_NextIterationSourceOp : TfExecutor_Op<"NextIteration.Source", [No ); let builders = [OpBuilder< - "Builder *builder, OperationState *result, Type resultTy, ArrayRef controlInputs = {}", + "Builder *builder, OperationState *result, Type result_type, " + "ArrayRef control_inputs = {}, ArrayRef attributes = {}", [{ - Type tokenTy = TokenType::get(builder->getContext()); - Type controlTy = ControlType::get(builder->getContext()); - result->types = { resultTy, tokenTy, controlTy }; - result->operands.append(controlInputs.begin(), controlInputs.end()); + Type token_type = TokenType::get(builder->getContext()); + Type control_type = ControlType::get(builder->getContext()); + result->types = { result_type, token_type, control_type }; + result->operands.append(control_inputs.begin(), control_inputs.end()); + result->attributes.append(attributes.begin(), attributes.end()); }]> ]; } @@ -527,6 +484,19 @@ def TfExecutor_NextIterationSinkOp : TfExecutor_Op<"NextIteration.Sink"> { // Optional extra control inputs. Variadic:$controlInputs ); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, Value *token, " + "ArrayRef operands, ArrayRef attributes = {}", + [{ + assert(operands.size() >= 1 && "tf_executor.NextIteration.Sink builder " + "expects at least one operand"); + result->operands.push_back(token); + result->operands.insert(result->operands.end(), operands.begin(), + operands.end()); + result->attributes.append(attributes.begin(), attributes.end()); + }]> + ]; } def TfExecutor_ExitOp : TfExecutor_Op<"Exit", @@ -590,6 +560,20 @@ def TfExecutor_ControlTriggerOp : TfExecutor_Op<"ControlTrigger", [NoSideEffect] ); let verifier = ?; + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, " + "ArrayRef operands, ArrayRef attributes = {}", + [{ + assert(operands.size() >= 1 && "tf_executor.ControlTrigger builder " + "expects at least one operand"); + result->operands.insert(result->operands.end(), operands.begin(), + operands.end()); + Type control_type = ControlType::get(builder->getContext()); + result->types = {control_type}; + result->attributes.append(attributes.begin(), attributes.end()); + }]> + ]; } def TfExecutor_LoopCondOp : TfExecutor_Op<"LoopCond", [NoSideEffect]> { diff --git a/tensorflow/compiler/mlir/tensorflow/tests/control_to_executor_dialect.mlir b/tensorflow/compiler/mlir/tensorflow/tests/control_to_executor_dialect.mlir new file mode 100644 index 00000000000..b1a9dd71fc7 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/tests/control_to_executor_dialect.mlir @@ -0,0 +1,86 @@ +// RUN: tf-opt -tf-control-to-executor-conversion %s | FileCheck %s --dump-input=fail + + +// CHECK-LABEL: func @islands_with_control +// CHECK-SAME: (%[[ARG0:[a-z0-9]*]]: tensor<*xf32>) +func @islands_with_control(tensor<*xf32>) -> tensor<*xf32> { +^bb0(%0: tensor<*xf32>): + %1:2 = "_tf.Identity"(%0) : (tensor<*xf32>) -> (tensor<*xf32>, !_tf.control) + %2 = "_tf.Add"(%0, %0, %1#1) : (tensor<*xf32>, tensor<*xf32>, !_tf.control) -> tensor<*xf32> + return %2 : tensor<*xf32> +} + +// CHECK-NEXT: %[[GRAPH:[0-9]*]] = tf_executor.graph { +// CHECK-NEXT: %[[IDENTITY:[0-9]*]]:2 = tf_executor.island { +// CHECK-NEXT: %{{[0-9]*}} = "tf.Identity"(%[[ARG0]]) : (tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: tf_executor.yield %{{[0-9]*}} : tensor<*xf32> +// CHECK-NEXT: } +// CHECK-NEXT: %[[ADD:[0-9]*]]:2 = tf_executor.island(%[[IDENTITY]]#1) { +// CHECK-NEXT: %{{[0-9]*}} = "tf.Add"(%[[ARG0]], %[[ARG0]]) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> +// CHECK-NEXT: tf_executor.yield %{{[0-9]*}} : tensor<*xf32> +// CHECK-NEXT: } +// CHECK-NEXT: tf_executor.fetch %[[ADD]]#0 : tensor<*xf32> +// CHECK-NEXT: } +// CHECK-NEXT: return %[[GRAPH]] : tensor<*xf32> + +// CHECK-LABEL: func @LoopTest() { + +func @LoopTest() { + %0:2 = "_tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "Const", value = dense<1> : tensor} : () -> (tensor, !_tf.control) + %1:2 = "_tf.Enter"(%0#0) {T = "tfdtype$DT_INT32", device = "", frame_name = "while/while_context", is_constant = false, name = "while/Enter", parallel_iterations = 10 : i64} : (tensor) -> (tensor<*xi32>, !_tf.control) + %2 = "_tf.NoOp"() {device = "", name = "cluster/pivot"} : () -> !_tf.control + %3:2 = "_tf.NextIteration.source"() {T = "tfdtype$DT_INT32", device = "", id = 0 : i64, name = "while/NextIteration"} : () -> (tensor<*xi32>, !_tf.control) + %4:3 = "_tf.Merge"(%3#0, %1#0) {N = 2 : i64, T = "tfdtype$DT_INT32", device = "", name = "while/Merge"} : (tensor<*xi32>, tensor<*xi32>) -> (tensor<*xi32>, tensor, !_tf.control) + %5:2 = "_tf.Const"(%4#2) {device = "", dtype = "tfdtype$DT_INT32", name = "while/Less/y", value = dense<2> : tensor} : (!_tf.control) -> (tensor, !_tf.control) + %6:2 = "_tf.Less"(%4#0, %5#0) {T = "tfdtype$DT_INT32", device = "", name = "while/Less"} : (tensor<*xi32>, tensor) -> (tensor<*xi1>, !_tf.control) + %7:2 = "_tf.LoopCond"(%6#0) {device = "", name = "while/LoopCond"} : (tensor<*xi1>) -> (tensor, !_tf.control) + %8:3 = "_tf.Switch"(%4#0, %7#0) {T = "tfdtype$DT_INT32", _class = ["loc = @while/Merge"], device = "", name = "while/Switch"} : (tensor<*xi32>, tensor) -> (tensor<*xi32>, tensor<*xi32>, !_tf.control) + %9:2 = "_tf.Exit"(%8#0) {T = "tfdtype$DT_INT32", device = "", name = "while/Exit"} : (tensor<*xi32>) -> (tensor<*xi32>, !_tf.control) + %10:2 = "_tf.Identity"(%8#1) {T = "tfdtype$DT_INT32", device = "", name = "while/Identity"} : (tensor<*xi32>) -> (tensor<*xi32>, !_tf.control) + %11:2 = "_tf.Const"(%10#1) {device = "", dtype = "tfdtype$DT_INT32", name = "while/Add/y", value = dense<3> : tensor} : (!_tf.control) -> (tensor, !_tf.control) + %12:2 = "_tf.Add"(%10#0, %11#0) {T = "tfdtype$DT_INT32", device = "", name = "while/Add"} : (tensor<*xi32>, tensor) -> (tensor<*xi32>, !_tf.control) + %13 = "_tf.ControlTrigger"(%2, %12#1, %9#1) {_tpu_replicate = "cluster", device = "", name = "gradients/while/mul_2_Da30D05wlPU_grad/SymbolicGradient/b_sync"} : (!_tf.control, !_tf.control, !_tf.control) -> !_tf.control + %14 = "_tf.NextIteration.sink"(%12#0, %13) {T = "tfdtype$DT_INT32", device = "", id = 0 : i64, name = "while/NextIteration"} : (tensor<*xi32>, !_tf.control) -> (!_tf.control) + return +} + +// CHECK-NEXT: tf_executor.graph { +// CHECK-NEXT: %[[CONST:[0-9]*]]:2 = tf_executor.island { +// CHECK-NEXT: %{{[a-z0-9]*}} = "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "Const", value = dense<1> : tensor} : () -> tensor +// CHECK-NEXT: tf_executor.yield %{{[a-z0-9]*}} : tensor +// CHECK-NEXT: } +// CHECK-NEXT: %[[ENTER:[0-9]*]]:2 = tf_executor.Enter %[[CONST]]#0 frame "while/while_context" : (tensor) -> (tensor<*xi32>, !tf_executor.control) {T = "tfdtype$DT_INT32", device = "", name = "while/Enter"} +// CHECK-NEXT: %[[NOOP:[0-9]*]] = tf_executor.island { +// CHECK-NEXT: "tf.NoOp"() {device = "", name = "cluster/pivot"} : () -> () +// CHECK-NEXT: tf_executor.yield +// CHECK-NEXT: } +// CHECK-NEXT: %[[NEXTIT_SRC:[0-9]*]]:3 = tf_executor.NextIteration.Source : tensor<*xi32> {T = "tfdtype$DT_INT32", device = "", id = 0 : i64, name = "while/NextIteration"} +// CHECK-NEXT: %[[MERGE:[0-9]*]]:3 = tf_executor.Merge %[[NEXTIT_SRC]]#0, %[[ENTER]]#0 : tensor<*xi32> {N = 2 : i64, T = "tfdtype$DT_INT32", device = "", name = "while/Merge"} +// CHECK-NEXT: %[[CONST_LESS:[0-9]*]]:2 = tf_executor.island(%[[MERGE]]#2) { +// CHECK-NEXT: %{{[a-z0-9]*}} = "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "while/Less/y", value = dense<2> : tensor} : () -> tensor +// CHECK-NEXT: tf_executor.yield %{{[a-z0-9]*}} : tensor +// CHECK-NEXT: } +// CHECK-NEXT: %[[LESS:[0-9]*]]:2 = tf_executor.island { +// CHECK-NEXT: %{{[a-z0-9]*}} = "tf.Less"(%[[MERGE]]#0, %[[CONST_LESS]]#0) {T = "tfdtype$DT_INT32", device = "", name = "while/Less"} : (tensor<*xi32>, tensor) -> tensor<*xi1> +// CHECK-NEXT: tf_executor.yield %{{[a-z0-9]*}} : tensor<*xi1> +// CHECK-NEXT: } +// CHECK-NEXT: %[[COND:[0-9]*]]:2 = tf_executor.LoopCond %[[LESS:[0-9]*]]#0 : (tensor<*xi1>) -> (tensor, !tf_executor.control) {device = "", name = "while/LoopCond"} +// CHECK-NEXT: %[[SWITCH:[0-9]*]]:3 = tf_executor.Switch %[[MERGE]]#0, %[[COND]]#0 : tensor<*xi32> {T = "tfdtype$DT_INT32", _class = ["loc = @while/Merge"], device = "", name = "while/Switch"} +// CHECK-NEXT: %[[EXIT:[0-9]*]]:2 = tf_executor.Exit %[[SWITCH]]#0 : tensor<*xi32> {T = "tfdtype$DT_INT32", device = "", name = "while/Exit"} +// CHECK-NEXT: %[[IDENTITY:[0-9]*]]:2 = tf_executor.island { +// CHECK-NEXT: %{{[a-z0-9]*}} = "tf.Identity"(%[[SWITCH]]#1) {T = "tfdtype$DT_INT32", device = "", name = "while/Identity"} : (tensor<*xi32>) -> tensor<*xi32> +// CHECK-NEXT: tf_executor.yield %{{[a-z0-9]*}} : tensor<*xi32> +// CHECK-NEXT: } +// CHECK-NEXT: %[[CONST_ADD:[0-9]*]]:2 = tf_executor.island(%[[IDENTITY]]#1) { +// CHECK-NEXT: %{{[a-z0-9]*}} = "tf.Const"() {device = "", dtype = "tfdtype$DT_INT32", name = "while/Add/y", value = dense<3> : tensor} : () -> tensor +// CHECK-NEXT: tf_executor.yield %{{[a-z0-9]*}} : tensor +// CHECK-NEXT: } +// CHECK-NEXT: %[[ADD:[0-9]*]]:2 = tf_executor.island { +// CHECK-NEXT: %{{[0-9]*}} = "tf.Add"(%[[IDENTITY]]#0, %[[CONST_ADD]]#0) {T = "tfdtype$DT_INT32", device = "", name = "while/Add"} : (tensor<*xi32>, tensor) -> tensor<*xi32> +// CHECK-NEXT: tf_executor.yield %{{[0-9]*}} : tensor<*xi32> +// CHECK-NEXT: } +// CHECK-NEXT: %[[CT:[0-9]*]] = tf_executor.ControlTrigger %2, %12#1, %9#1 {_tpu_replicate = "cluster", device = "", name = "gradients/while/mul_2_Da30D05wlPU_grad/SymbolicGradient/b_sync"} +// CHECK-NEXT: tf_executor.NextIteration.Sink [%[[NEXTIT_SRC]]#1] %[[ADD]]#0, %[[CT]] : tensor<*xi32> {T = "tfdtype$DT_INT32", device = "", id = 0 : i64, name = "while/NextIteration"} +// CHECK-NEXT: tf_executor.fetch +// CHECK-NEXT: } +// CHECK-NEXT: return diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/passes.h b/tensorflow/compiler/mlir/tensorflow/transforms/passes.h index d300c014aed..1202d4d432c 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/passes.h +++ b/tensorflow/compiler/mlir/tensorflow/transforms/passes.h @@ -35,7 +35,6 @@ namespace TFControlFlow { FunctionPassBase *CreateRaiseTFControlFlowPass(); } // namespace TFControlFlow - } // namespace mlir #endif // TENSORFLOW_COMPILER_MLIR_TENSORFLOW_TRANSFORMS_PASSES_H_ diff --git a/tensorflow/compiler/mlir/tensorflow/translate/control_to_executor_dialect.cc b/tensorflow/compiler/mlir/tensorflow/translate/control_to_executor_dialect.cc new file mode 100644 index 00000000000..4d9b3ca7ab7 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/translate/control_to_executor_dialect.cc @@ -0,0 +1,248 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This transformation pass transforms MLIR TF contol dialect into a combination +// of the TF and TF executor dialects. +// +// !! This code is only intended for migration purpose and will be deleted when +// !! the importer is updated to directly emit the tf_executor dialect. + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/Support/Debug.h" +#include "mlir/IR/Builders.h" // TF:local_config_mlir +#include "mlir/IR/Operation.h" // TF:local_config_mlir +#include "mlir/IR/Value.h" // TF:local_config_mlir +#include "mlir/Pass/Pass.h" // TF:local_config_mlir +#include "mlir/Pass/PassRegistry.h" // TF:local_config_mlir +#include "mlir/StandardOps/Ops.h" // TF:local_config_mlir +#include "mlir/Support/LLVM.h" // TF:local_config_mlir +#include "tensorflow/compiler/mlir/tensorflow/ir/control_flow_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/transforms/passes.h" + +#define DEBUG_TYPE "tf-ctl-to-executor" + +namespace mlir { + +namespace { +// This pass checks if a function contains only operations in the TF control +// dialect and converts it to a mix of the tf_executor and tf dialects. +// Operations that exist in the tf_executor dialects are used directly, +// otherwise _tf operations are wrapped in an island and the _ prefix is +// removed. Control dependencies are moved to be handled by the island itself. +struct ControlToExecutorDialectConversion + : public FunctionPass { + void runOnFunction() override; + + private: + tf_executor::IslandOp CreateIslandForOp(Operation *op, OpBuilder *builder); +}; +} // end anonymous namespace + +static bool IsUnderscoredTFOp(Operation *op) { + return op->getName().getStringRef().startswith("_tf."); +} + +static bool HasOnlyTFControlOperations(FuncOp function) { + return llvm::all_of(function, [](Block &block) { + return llvm::all_of(block, [](Operation &op) { + return IsUnderscoredTFOp(&op) || isa(op); + }); + }); +} + +tf_executor::IslandOp ControlToExecutorDialectConversion::CreateIslandForOp( + Operation *op, OpBuilder *builder) { + // Create a new region for the tf_executor.island body + SmallVector operands; + for (Value *operand : op->getOperands()) + if (operand->getType().isa()) + operands.push_back(operand); + SmallVector types; + for (Type result_type : op->getResultTypes()) + if (!result_type.isa()) + types.push_back(result_type); + types.push_back(tf_executor::ControlType::get(&getContext())); + + auto island = builder->create( + op->getLoc(), types, operands, ArrayRef{}); + island.body().push_back(new Block); + + return island; +} + +void ControlToExecutorDialectConversion::runOnFunction() { + if (!HasOnlyTFControlOperations(getFunction())) { + LLVM_DEBUG(llvm::dbgs() << "Function has unsupported operation, skip " + "tf_executor dialect conversion\n"); + return; + } + if (getFunction().getBlocks().size() != 1) { + LLVM_DEBUG(llvm::dbgs() << "Expect single block function, , skip " + "tf_executor dialect conversion\n"); + return; + } + + Block &body = getFunction().getBody().front(); + OpBuilder builder(&body, body.begin()); + + // Create a new tf_executor.graph at the beginning of the function. + auto graph_op = builder.create( + getFunction().getLoc(), getFunction().getType().getResults()); + graph_op.body().push_back(new Block); + builder.setInsertionPointToEnd(&graph_op.GetBody()); + llvm::StringMap frame_name_to_loop; + + // Loop over operations in the function and move them into the graph region. + for (Operation &op : llvm::make_early_inc_range(body)) { + // Skip the just-created tf_executor.graph. + if (isa(op)) continue; + + // This is the new operation that will replace the current one in the graph. + Operation *replacement = nullptr; + if (op.isKnownTerminator()) { + // This is the return of the function, we will create a fetch in the graph + // matching the operands of the returns. The return is then updated to + // take as operands the results of the tf_executor.graph operation. + SmallVector ret_vals; + for (Value *operand : op.getOperands()) ret_vals.push_back(operand); + for (auto &graph_result : llvm::enumerate(graph_op.getResults())) + op.setOperand(graph_result.index(), graph_result.value()); + builder.create(getFunction().getLoc(), ret_vals); + continue; + } + assert(IsUnderscoredTFOp(&op) && "Expected only _tf operations"); + + // The operands and types arrays are used to create the tf_executor ops. + SmallVector operands; + operands.append(op.getOperands().begin(), op.getOperands().end()); + SmallVector types; + for (Type result_type : op.getResultTypes()) { + if (result_type.isa()) + types.push_back(tf_executor::ControlType::get(&getContext())); + else + types.push_back(result_type); + } + auto loc = op.getLoc(); + + // Match the specific operation that has a tf_executor equivalent, the + // others will be wrapped in an island. + + // FIXME: StringSwitch + + if (op.getName().getStringRef() == "_tf.Switch") { + replacement = builder.create( + loc, types, operands, ArrayRef{}); + } else if (op.getName().getStringRef() == "_tf.SwitchN") { + replacement = builder.create( + loc, types, operands, ArrayRef{}); + } else if (op.getName().getStringRef() == "_tf.Merge") { + replacement = builder.create( + loc, types, operands, ArrayRef{}); + } else if (op.getName().getStringRef() == "_tf.NextIteration.source") { + replacement = builder.create( + loc, op.getResult(0)->getType(), operands); + // Record a mapping of the name to the nextiteration.source so that when + // we convert the sink we can get the token. + StringAttr frame = op.getAttrOfType("name"); + assert(!frame.getValue().empty()); + frame_name_to_loop[frame.getValue()] = + cast(replacement); + // Replace the results here since the _tf source does not produce a token + // there isn't a mapping for the new result #1. + op.getResult(0)->replaceAllUsesWith(replacement->getResult(0)); + for (int i : llvm::seq(1, op.getNumResults())) + op.getResult(i)->replaceAllUsesWith(replacement->getResult(i + 1)); + replacement->setAttrs(op.getAttrList()); + op.erase(); + continue; + } else if (op.getName().getStringRef() == "_tf.NextIteration.sink") { + StringAttr frame = op.getAttrOfType("name"); + assert(!frame.getValue().empty()); + tf_executor::NextIterationSourceOp srcOp = + frame_name_to_loop[frame.getValue()]; + replacement = builder.create( + loc, srcOp.token(), operands, ArrayRef{}); + replacement->setAttrs(op.getAttrList()); + op.erase(); + continue; + } else if (op.getName().getStringRef() == "_tf.LoopCond") { + replacement = builder.create( + loc, types, operands, ArrayRef{}); + } else if (op.getName().getStringRef() == "_tf.Enter") { + replacement = builder.create( + loc, types, operands, ArrayRef{}); + } else if (op.getName().getStringRef() == "_tf.Exit") { + replacement = builder.create( + loc, types, operands, ArrayRef{}); + } else if (op.getName().getStringRef() == "_tf.ControlTrigger") { + replacement = + builder.create(loc, operands); + } else { + tf_executor::IslandOp island = CreateIslandForOp(&op, &builder); + replacement = island.getOperation(); + + // General case, drop the leading _ off the name and wrap in an island. + OperationState result(loc, op.getName().getStringRef().drop_front()); + + // Only the non-control operands are carried over, the island is handling + // the control input. + for (Value *operand : op.getOperands()) + if (!operand->getType().isa()) + result.operands.push_back(operand); + + // Add a result type for each non-control result we find + bool sawControlResult = false; + for (Type result_type : op.getResultTypes()) { + if (result_type.isa()) { + sawControlResult = true; + continue; + } + // We assume all control inputs are at the end of the result list. + assert(!sawControlResult && "all control results must be last"); + result.types.push_back(result_type); + } + + // Create the operation inside the island + OpBuilder island_builder(&island.GetBody()); + Operation *inner_op = island_builder.createOperation(result); + inner_op->setAttrs(op.getAttrList()); + + // Add the terminator for the island + SmallVector ret_vals(inner_op->getResults()); + island_builder.create(loc, ret_vals); + } + + // Copy the attributes from the original operation to the replacement and + // remap the results. + if (!isa(replacement)) + replacement->setAttrs(op.getAttrList()); + for (int i : llvm::seq(0, op.getNumResults())) + op.getResult(i)->replaceAllUsesWith(replacement->getResult(i)); + op.erase(); + } +} + +FunctionPassBase *CreateTFControlToExecutorDialectConversion() { + return new ControlToExecutorDialectConversion(); +} + +} // namespace mlir + +static mlir::PassRegistration pass( + "tf-control-to-executor-conversion", + "Transform from TF control dialect to TF executor dialect."); From 2e3ac570434bba547f06a3fd67cf8acd35e31a4e Mon Sep 17 00:00:00 2001 From: Juhyun Lee Date: Fri, 12 Jul 2019 00:50:39 -0700 Subject: [PATCH 326/332] Replace Elvis operator with ternary operator. PiperOrigin-RevId: 257757016 --- tensorflow/lite/delegates/gpu/common/model_builder.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/common/model_builder.cc b/tensorflow/lite/delegates/gpu/common/model_builder.cc index f762a0fe0a3..986cbe5d5b7 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder.cc +++ b/tensorflow/lite/delegates/gpu/common/model_builder.cc @@ -1491,9 +1491,9 @@ Status ExtractTensorShape(const TfLiteTensor& tflite_tensor, BHWC* bhwc) { *bhwc = BHWC(dims->data[0], dims->data[1], dims->data[2], dims->data[3]); return OkStatus(); default: - return InvalidArgumentError( - absl::StrCat("Tensor \"", tflite_tensor.name ?: "nullptr", - "\" has bad input dims size: ", dims->size, ".")); + return InvalidArgumentError(absl::StrCat( + "Tensor \"", tflite_tensor.name ? tflite_tensor.name : "nullptr", + "\" has bad input dims size: ", dims->size, ".")); } } From a19640f24a31db41f8809ee55d7c2704cd7be57c Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Fri, 12 Jul 2019 01:11:35 -0700 Subject: [PATCH 327/332] [TF:XLA] Bump open source llvm revision to r365803 PiperOrigin-RevId: 257760052 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index b6a1808640c..4fa98fc0ca9 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -543,11 +543,11 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "llvm", build_file = clean_dep("//third_party/llvm:llvm.autogenerated.BUILD"), - sha256 = "a6c9e2379fb92da8b94315b96bb2c7f569ff9fc2a05de4af1afb23956908f393", - strip_prefix = "llvm-27f427783fe8d884a98276ae7d9b5413c0e03533", + sha256 = "9257e111ae3d5b9d80925ef1329666440460abf4d052e701fa587f5236be6fcc", + strip_prefix = "llvm-df22a5e50a3d36a7b68eea106970dfa5df6d2453", urls = [ - "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/27f427783fe8d884a98276ae7d9b5413c0e03533.tar.gz", - "https://github.com/llvm-mirror/llvm/archive/27f427783fe8d884a98276ae7d9b5413c0e03533.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/df22a5e50a3d36a7b68eea106970dfa5df6d2453.tar.gz", + "https://github.com/llvm-mirror/llvm/archive/df22a5e50a3d36a7b68eea106970dfa5df6d2453.tar.gz", ], ) From 45b32a32b0d3bf5fd4f8d865a3a52f34a2e7322a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 12 Jul 2019 02:02:14 -0700 Subject: [PATCH 328/332] compat: Update forward compatibility horizon to 2019-07-12 PiperOrigin-RevId: 257765123 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 1c1eafe4024..c976e0acb20 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -27,7 +27,7 @@ import datetime from tensorflow.python.util import tf_contextlib from tensorflow.python.util.tf_export import tf_export -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 7, 11) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2019, 7, 12) @tf_export("compat.forward_compatible") From 6bcff21ad866f6388f9be3bdaab232f8180d4d55 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 12 Jul 2019 02:02:17 -0700 Subject: [PATCH 329/332] Update GraphDef version to 94. PiperOrigin-RevId: 257765144 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 614f9016b24..b1976d7382a 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 93 // Updated: 2019/7/11 +#define TF_GRAPH_DEF_VERSION 94 // Updated: 2019/7/12 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From cbe94fd66eb2b77932a8d2f8ab04cdcac6ba7645 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 12 Jul 2019 02:44:28 -0700 Subject: [PATCH 330/332] Add grad_pass_through utility to custom_gradients. grad_pass_through will let user wrap any op (including non differentiable ops such as assign operations) and produce a new op where gradients can 'pass through' to the inputs. An example use case is defining an approximate 'differentiable' moving avarage, where in the forward pass we use the current value of the moving average, and in the backward pass we only use last value fed to the op. PiperOrigin-RevId: 257770003 --- tensorflow/python/BUILD | 2 + tensorflow/python/ops/custom_gradient.py | 55 ++++++++++++++++- tensorflow/python/ops/gradients_test.py | 61 +++++++++++++++++++ .../tools/api/golden/v1/tensorflow.pbtxt | 4 ++ .../tools/api/golden/v2/tensorflow.pbtxt | 4 ++ 5 files changed, 125 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 93f43ab338a..15e447efede 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3920,6 +3920,7 @@ cuda_py_test( ":framework_test_lib", ":functional_ops", ":gradients", + ":init_ops", ":list_ops", ":math_grad", ":math_ops", @@ -3927,6 +3928,7 @@ cuda_py_test( ":nn_ops", ":platform_test", ":state_grad", + ":state_ops", ":tensor_array_grad", ":tensor_array_ops", ":test_ops", diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index e2bbdc7f788..12b4feb68e5 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -74,7 +74,7 @@ def copy_handle_data(source_t, target_t): shapes, types = zip(*[(pair.shape, pair.dtype) for pair in handle_data.shape_and_type]) ranks = [len(s.dim) if not s.unknown_rank else -1 for s in shapes] - shapes = [[d.size for d in s.dim] + shapes = [[d.size for d in s.dim] # pylint: disable=g-complex-comprehension if not s.unknown_rank else None for s in shapes] pywrap_tensorflow.TF_GraphSetOutputHandleShapesAndTypes_wrapper( target_t._op._graph._c_graph, # pylint: disable=protected-access @@ -394,3 +394,56 @@ def recompute_grad(f): return result, grad return inner + + +@tf_export("grad_pass_through") +def grad_pass_through(f): + """Creates a grad-pass-through op with the forward behavior provided in f. + + Use this function to wrap any op, maintaining its behavior in the forward + pass, but replacing the original op in the backward graph with an identity. + For example: + + ```python + x = tf.Variable(1.0, name="x") + z = tf.Variable(3.0, name="z") + + with tf.GradientTape() as tape: + # y will evaluate to 9.0 + y = tf.grad_pass_through(x.assign)(z**2) + # grads will evaluate to 6.0 + grads = tape.gradient(y, z) + ``` + + Another example is a 'differentiable' moving average approximation, where + gradients are allowed to flow into the last value fed to the moving average, + but the moving average is still used for the forward pass: + + ```python + x = ... # Some scalar value + # A moving average object, we don't need to know how this is implemented + moving_average = MovingAverage() + with backprop.GradientTape() as tape: + # mavg_x will evaluate to the current running average value + mavg_x = tf.grad_pass_through(moving_average)(x) + grads = tape.gradient(mavg_x, x) # grads will evaluate to 1.0 + ``` + + Args: + f: function `f(*x)` that returns a `Tensor` or nested structure of `Tensor` + outputs. + + Returns: + A function `h(x)` which returns the same values as `f(x)` and whose + gradients are the same as those of an identity function. + """ + @custom_gradient + def _grad_pass_through_op(*args, **kwargs): + def grad(*args, **kwargs): + variables = kwargs.get("variables") + if variables is not None: + # Variables involved in the wrapped op will not receive gradients. + return args, [None] * len(variables) + return args + return f(*args, **kwargs), grad + return tf_decorator.make_decorator(f, _grad_pass_through_op) diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py index 1b6dcd35bf3..be98f2a6279 100644 --- a/tensorflow/python/ops/gradients_test.py +++ b/tensorflow/python/ops/gradients_test.py @@ -44,12 +44,14 @@ from tensorflow.python.ops import data_flow_ops # pylint: disable=unused-import from tensorflow.python.ops import functional_ops # pylint: disable=unused-import from tensorflow.python.ops import gradients from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import init_ops from tensorflow.python.ops import list_ops from tensorflow.python.ops import math_grad # pylint: disable=unused-import from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_grad # pylint: disable=unused-import from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_grad # pylint: disable=unused-import +from tensorflow.python.ops import state_ops from tensorflow.python.ops import tensor_array_grad # pylint: disable=unused-import from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops import unconnected_gradients @@ -1389,5 +1391,64 @@ class VariablesGradientTest(test_util.TensorFlowTestCase): self.assertAllClose(g, g_re) +class GradPassThroughTest(test_util.TensorFlowTestCase): + + @test_util.run_v1_only("b/120545219") + def test_gradients_v1(self): + x = variable_scope.get_variable( + name="x", shape=(), initializer=init_ops.constant_initializer(1.0), + use_resource=True) + z = variable_scope.get_variable( + name="z", shape=(), initializer=init_ops.constant_initializer(3.0), + use_resource=True) + + # Verify that assign op is not differentiable + y = state_ops.assign(x, z**2) + grads = gradients.gradients(y, z) + self.assertIsNone(grads[0]) + + # Verify that when the (non differentiable) assign op is wrapped with + # grad_pass_through, gradients are correctly forwarded to the inputs. + # Form an input as quadratic function of variable z and check that the + # gradient of output wrt to z is correct. + y = custom_gradient.grad_pass_through( + lambda v: state_ops.assign(x, v))(z**2) + grads = gradients.gradients(y, z) + with self.cached_session() as sess: + sess.run(variables.global_variables_initializer()) + self.assertAllClose(grads[0].eval(), 6.0) + + # Verify that variables involved in the wrapped op do not receive gradients. + y = custom_gradient.grad_pass_through(lambda v: x * v)(z) + grads = gradients.gradients(y, x) + self.assertIsNone(grads[0]) + + @test_util.run_v2_only + def test_gradients_v2(self): + x = variables.Variable(1.0, name="x") + z = variables.Variable(3.0, name="z") + + # Verify that assign op is not differentiable + with backprop.GradientTape() as tape: + y = x.assign(z**2) + grads = tape.gradient(y, z) + self.assertIsNone(grads) + + # Verify that when the (non differentiable) assign op is wrapped with + # grad_pass_through, gradients are correctly forwarded to the inputs. + # Form an input as quadratic function of variable z and check that the + # gradient of output wrt to z is correct. + with backprop.GradientTape() as tape: + y = custom_gradient.grad_pass_through(x.assign)(z**2) + grads = tape.gradient(y, z) + self.assertAllClose(grads, 6.0) + + # Verify that variables involved in the wrapped op do not receive gradients. + with backprop.GradientTape() as tape: + y = custom_gradient.grad_pass_through(lambda v: x * v)(z) + grads = tape.gradient(y, x) + self.assertIsNone(grads) + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index 50b3b399a9f..178daad4a2a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -1388,6 +1388,10 @@ tf_module { name: "global_variables_initializer" argspec: "args=[], varargs=None, keywords=None, defaults=None" } + member_method { + name: "grad_pass_through" + argspec: "args=[\'f\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "gradients" argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'colocate_gradients_with_ops\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt index 6d15fa0c841..33c4610d97b 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt @@ -664,6 +664,10 @@ tf_module { name: "get_static_value" argspec: "args=[\'tensor\', \'partial\'], varargs=None, keywords=None, defaults=[\'False\'], " } + member_method { + name: "grad_pass_through" + argspec: "args=[\'f\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "gradients" argspec: "args=[\'ys\', \'xs\', \'grad_ys\', \'name\', \'gate_gradients\', \'aggregation_method\', \'stop_gradients\', \'unconnected_gradients\'], varargs=None, keywords=None, defaults=[\'None\', \'gradients\', \'False\', \'None\', \'None\', \'UnconnectedGradients.NONE\'], " From 323564b4968f83f40b54ecc8ca91067e4bec1ca1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 12 Jul 2019 03:19:54 -0700 Subject: [PATCH 331/332] Add cast op support to the NNAPI delegate PiperOrigin-RevId: 257774029 --- .../lite/delegates/nnapi/nnapi_delegate.cc | 15 +++++ tensorflow/lite/kernels/BUILD | 1 + tensorflow/lite/kernels/cast_test.cc | 56 +++++++++++++++++-- tensorflow/lite/nnapi/NeuralNetworksTypes.h | 1 + 4 files changed, 69 insertions(+), 4 deletions(-) diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc index 30f754156a4..6d17fa260a8 100644 --- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc +++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc @@ -1729,6 +1729,21 @@ class NNAPIDelegateKernel { return BasicMappingFn; } } break; + case kTfLiteBuiltinCast: { + const TfLiteType input_type = + context->tensors[node->inputs->data[0]].type; + const TfLiteType output_type = + context->tensors[node->outputs->data[0]].type; + auto is_supported_tensor_type = [](const TfLiteType& type) { + return (type == kTfLiteFloat32 || type == kTfLiteInt32 || + type == kTfLiteUInt8); + }; + if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI12 && + is_supported_tensor_type(input_type) && + is_supported_tensor_type(output_type)) { + return BasicMappingFn; + } + } break; case kTfLiteBuiltinPrelu: if (version == 1 && android_sdk_version >= kMinSdkVersionForNNAPI12) { if (!IsFloatOrUint8Operator(context, node)) { diff --git a/tensorflow/lite/kernels/BUILD b/tensorflow/lite/kernels/BUILD index 0e40f3fa0cd..f70ccf3a3d9 100644 --- a/tensorflow/lite/kernels/BUILD +++ b/tensorflow/lite/kernels/BUILD @@ -669,6 +669,7 @@ cc_test( name = "cast_test", size = "small", srcs = ["cast_test.cc"], + tags = ["tflite_nnapi"], deps = [ ":builtin_ops", ":test_main", diff --git a/tensorflow/lite/kernels/cast_test.cc b/tensorflow/lite/kernels/cast_test.cc index 6bad3d6e7b3..8f1cb44f1c9 100644 --- a/tensorflow/lite/kernels/cast_test.cc +++ b/tensorflow/lite/kernels/cast_test.cc @@ -43,7 +43,23 @@ class CastOpModel : public SingleOpModel { int output_; }; -TEST(CastOpModel, CastIntToFloat) { +TEST(CastOpModel, CastInt32ToFloat) { + CastOpModel m({TensorType_INT32, {2, 3}}, {TensorType_FLOAT32, {2, 3}}); + m.PopulateTensor(m.input(), {100, 200, 300, 400, 500, 600}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({100.f, 200.f, 300.f, 400.f, 500.f, 600.f})); +} + +TEST(CastOpModel, CastFloatToInt32) { + CastOpModel m({TensorType_FLOAT32, {3, 2}}, {TensorType_INT32, {3, 2}}); + m.PopulateTensor(m.input(), {100.f, 20.f, 3.f, 0.4f, 0.999f, 1.1f}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({100, 20, 3, 0, 0, 1})); +} + +TEST(CastOpModel, CastInt64ToFloat) { CastOpModel m({TensorType_INT64, {2, 3}}, {TensorType_FLOAT32, {2, 3}}); m.PopulateTensor(m.input(), {100, 200, 300, 400, 500, 600}); m.Invoke(); @@ -51,11 +67,11 @@ TEST(CastOpModel, CastIntToFloat) { ElementsAreArray({100.f, 200.f, 300.f, 400.f, 500.f, 600.f})); } -TEST(CastOpModel, CastFloatToInt) { - CastOpModel m({TensorType_FLOAT32, {3, 2}}, {TensorType_INT32, {3, 2}}); +TEST(CastOpModel, CastFloatToInt64) { + CastOpModel m({TensorType_FLOAT32, {3, 2}}, {TensorType_INT64, {3, 2}}); m.PopulateTensor(m.input(), {100.f, 20.f, 3.f, 0.4f, 0.999f, 1.1f}); m.Invoke(); - EXPECT_THAT(m.ExtractVector(m.output()), + EXPECT_THAT(m.ExtractVector(m.output()), ElementsAreArray({100, 20, 3, 0, 0, 1})); } @@ -75,6 +91,38 @@ TEST(CastOpModel, CastBoolToFloat) { ElementsAreArray({1.f, 1.0f, 0.f, 1.0f, 0.0f, 1.0f})); } +TEST(CastOpModel, CastFloatToUInt8) { + CastOpModel m({TensorType_FLOAT32, {3, 2}}, {TensorType_UINT8, {3, 2}}); + m.PopulateTensor(m.input(), {100.f, 1.0f, 0.f, 0.4f, 1.999f, 1.1f}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({100, 1, 0, 0, 1, 1})); +} + +TEST(CastOpModel, CastUInt8ToFloat) { + CastOpModel m({TensorType_UINT8, {3, 2}}, {TensorType_FLOAT32, {3, 2}}); + m.PopulateTensor(m.input(), {123, 0, 1, 2, 3, 4}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({123.f, 0.f, 1.f, 2.f, 3.f, 4.f})); +} + +TEST(CastOpModel, CastInt32ToUInt8) { + CastOpModel m({TensorType_INT32, {3, 2}}, {TensorType_UINT8, {3, 2}}); + m.PopulateTensor(m.input(), {100, 1, 200, 2, 255, 3}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({100, 1, 200, 2, 255, 3})); +} + +TEST(CastOpModel, CastUInt8ToInt32) { + CastOpModel m({TensorType_UINT8, {3, 2}}, {TensorType_INT32, {3, 2}}); + m.PopulateTensor(m.input(), {100, 1, 200, 2, 255, 3}); + m.Invoke(); + EXPECT_THAT(m.ExtractVector(m.output()), + ElementsAreArray({100, 1, 200, 2, 255, 3})); +} + TEST(CastOpModel, CastComplex64ToFloat) { CastOpModel m({TensorType_COMPLEX64, {2, 3}}, {TensorType_FLOAT32, {2, 3}}); m.PopulateTensor>( diff --git a/tensorflow/lite/nnapi/NeuralNetworksTypes.h b/tensorflow/lite/nnapi/NeuralNetworksTypes.h index c5519b93c6d..40c3ecf3c91 100644 --- a/tensorflow/lite/nnapi/NeuralNetworksTypes.h +++ b/tensorflow/lite/nnapi/NeuralNetworksTypes.h @@ -93,6 +93,7 @@ enum { ANEURALNETWORKS_ARGMAX = 39, ANEURALNETWORKS_ARGMIN = 40, ANEURALNETWORKS_BIDIRECTIONAL_SEQUENCE_LSTM = 42, + ANEURALNETWORKS_CAST = 45, ANEURALNETWORKS_EQUAL = 48, ANEURALNETWORKS_EXP = 49, ANEURALNETWORKS_EXPAND_DIMS = 50, From 43dcd3dc3ee4b090832455acf43e8dd483a6117b Mon Sep 17 00:00:00 2001 From: Chris Jones Date: Fri, 12 Jul 2019 03:48:28 -0700 Subject: [PATCH 332/332] Use current device scope in `initialize_tpu_system` if called without a cluster resolver. PiperOrigin-RevId: 257776495 --- tensorflow/python/tpu/tpu_strategy_util.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/tpu/tpu_strategy_util.py b/tensorflow/python/tpu/tpu_strategy_util.py index 0f2f93deac5..6b62f55b5bf 100644 --- a/tensorflow/python/tpu/tpu_strategy_util.py +++ b/tensorflow/python/tpu/tpu_strategy_util.py @@ -23,6 +23,7 @@ from tensorflow.python.client import session as session_lib from tensorflow.python.distribute.cluster_resolver import TPUClusterResolver from tensorflow.python.eager import context from tensorflow.python.eager import function +from tensorflow.python.framework import device from tensorflow.python.framework import ops from tensorflow.python.platform import tf_logging as logging from tensorflow.python.tpu import topology @@ -48,7 +49,15 @@ def initialize_tpu_system(cluster_resolver=None): Raises: RuntimeError: If no TPU devices found for eager execution. """ + job = None if cluster_resolver is None: + # If no cluster resolver is specified, and running eagerly, execute the init + # ops in the current device scope. + if context.executing_eagerly(): + curr_device = device.DeviceSpec.from_string(context.context().device_name) + if curr_device.job is not None: + job = "{}/replica:0/task:0".format(curr_device.job) + cluster_resolver = TPUClusterResolver("") assert isinstance(cluster_resolver, TPUClusterResolver) @@ -66,7 +75,6 @@ def initialize_tpu_system(cluster_resolver=None): # DistributedTPURewritePass. This pass actually adds real ops that # initialize the TPU system. Thus, we can't simply run tpu.initialize_system # eagerly. We need to wrap it in defun and trigger the rewrite passes on it. - job = None if tpu_name not in _LOCAL_MASTERS: # Explicitly place the tpu.initialize_system in the first worker to # avoid the output node match multiple devices error.