From db4afe53df7d6d37724a5a6d17dd33650dde99c2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 14:24:39 -0700 Subject: [PATCH 01/67] Make convolutional, pooling and normalization layers to work with EAGER. PiperOrigin-RevId: 167049392 --- tensorflow/python/layers/convolutional.py | 59 ++++++++++++----------- tensorflow/python/layers/normalization.py | 26 +++++++--- 2 files changed, 49 insertions(+), 36 deletions(-) diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index 68293aa5fe5..e46c0b29542 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -24,6 +24,7 @@ import six from six.moves import xrange # pylint: disable=redefined-builtin import numpy as np +from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops @@ -1293,18 +1294,19 @@ class Conv2DTranspose(Conv2D): padding=self.padding.upper(), data_format=utils.convert_data_format(self.data_format, ndim=4)) - # Infer the static output shape: - out_shape = inputs.get_shape().as_list() - out_shape[c_axis] = self.filters - out_shape[h_axis] = utils.deconv_output_length(out_shape[h_axis], - kernel_h, - self.padding, - stride_h) - out_shape[w_axis] = utils.deconv_output_length(out_shape[w_axis], - kernel_w, - self.padding, - stride_w) - outputs.set_shape(out_shape) + if context.in_graph_mode(): + # Infer the static output shape: + out_shape = inputs.get_shape().as_list() + out_shape[c_axis] = self.filters + out_shape[h_axis] = utils.deconv_output_length(out_shape[h_axis], + kernel_h, + self.padding, + stride_h) + out_shape[w_axis] = utils.deconv_output_length(out_shape[w_axis], + kernel_w, + self.padding, + stride_w) + outputs.set_shape(out_shape) if self.bias: outputs = nn.bias_add( @@ -1591,22 +1593,23 @@ class Conv3DTranspose(Conv3D): data_format=utils.convert_data_format(self.data_format, ndim=5), padding=self.padding.upper()) - # Infer the static output shape: - out_shape = inputs.get_shape().as_list() - out_shape[c_axis] = self.filters - out_shape[d_axis] = utils.deconv_output_length(out_shape[d_axis], - kernel_d, - self.padding, - stride_d) - out_shape[h_axis] = utils.deconv_output_length(out_shape[h_axis], - kernel_h, - self.padding, - stride_h) - out_shape[w_axis] = utils.deconv_output_length(out_shape[w_axis], - kernel_w, - self.padding, - stride_w) - outputs.set_shape(out_shape) + if context.in_graph_mode(): + # Infer the static output shape: + out_shape = inputs.get_shape().as_list() + out_shape[c_axis] = self.filters + out_shape[d_axis] = utils.deconv_output_length(out_shape[d_axis], + kernel_d, + self.padding, + stride_d) + out_shape[h_axis] = utils.deconv_output_length(out_shape[h_axis], + kernel_h, + self.padding, + stride_h) + out_shape[w_axis] = utils.deconv_output_length(out_shape[w_axis], + kernel_w, + self.padding, + stride_w) + outputs.set_shape(out_shape) if self.bias: outputs_shape = outputs.shape.as_list() diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py index 62f5881f164..1fc2d70f9ca 100644 --- a/tensorflow/python/layers/normalization.py +++ b/tensorflow/python/layers/normalization.py @@ -25,6 +25,7 @@ import six from six.moves import xrange # pylint: disable=redefined-builtin import numpy as np +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import ops @@ -242,15 +243,20 @@ class BatchNormalization(base.Layer): initializer=init_ops.zeros_initializer(), trainable=False) return var + with ops.device(None): - with ops.device(lambda _: self.moving_mean.device): + device = ((lambda _: self.moving_mean.device) + if context.in_graph_mode() else self.moving_mean.device) + with ops.device(device): self.renorm_mean = _renorm_variable('renorm_mean', (param_dim,)) self.renorm_mean_weight = _renorm_variable('renorm_mean_weight', ()) # We initialize renorm_stddev to 0, and maintain the (0-initialized) # renorm_stddev_weight. This allows us to (1) mix the average # stddev with the minibatch stddev early in training, and (2) compute # the unbiased average stddev by dividing renorm_stddev by the weight. - with ops.device(lambda _: self.moving_variance.device): + device = ((lambda _: self.moving_variance.device) + if context.in_graph_mode() else self.moving_variance.device) + with ops.device(device): self.renorm_stddev = _renorm_variable('renorm_stddev', (param_dim,)) self.renorm_stddev_weight = _renorm_variable( 'renorm_stddev_weight', ()) @@ -301,8 +307,12 @@ class BatchNormalization(base.Layer): self.moving_mean, mean, decay, zero_debias=False) variance_update = moving_averages.assign_moving_average( self.moving_variance, variance, decay, zero_debias=False) - self.add_update(mean_update, inputs=inputs) - self.add_update(variance_update, inputs=inputs) + if context.in_graph_mode(): + # Note that in Eager mode, the updates are already executed when running + # assign_moving_averages. So we do not need to put them into + # collections. + self.add_update(mean_update, inputs=inputs) + self.add_update(variance_update, inputs=inputs) return output @@ -335,6 +345,7 @@ class BatchNormalization(base.Layer): r = _smart_select(training, lambda: r, lambda: array_ops.ones_like(r)) d = _smart_select(training, lambda: d, lambda: array_ops.zeros_like(d)) decay = _smart_select(training, lambda: self.renorm_momentum, lambda: 1.) + def _update_renorm_variable(var, weight, value): """Updates a moving average and weight, returns the unbiased value.""" # Update the variables without zero debiasing. The debiasing will be @@ -418,9 +429,9 @@ class BatchNormalization(base.Layer): self.moving_mean, new_mean, decay, zero_debias=False) variance_update = moving_averages.assign_moving_average( self.moving_variance, new_variance, decay, zero_debias=False) - - self.add_update(mean_update, inputs=inputs) - self.add_update(variance_update, inputs=inputs) + if context.in_graph_mode(): + self.add_update(mean_update, inputs=inputs) + self.add_update(variance_update, inputs=inputs) else: mean, variance = self.moving_mean, self.moving_variance @@ -566,7 +577,6 @@ def batch_normalization(inputs, BatchNorm = BatchNormalization batch_norm = batch_normalization - # Helper function From 7453a0fe7e31af6763748458aed21750bc1b4000 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 14:29:05 -0700 Subject: [PATCH 02/67] Fix conversion of string constants with nulls to EagerTensors. Fix testing on GPUs in constant_op_eager_test. PiperOrigin-RevId: 167050082 --- tensorflow/python/framework/ops.py | 9 ++++ tensorflow/python/framework/test_util.py | 2 +- .../kernel_tests/constant_op_eager_test.py | 50 ++++++++++++++----- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 5a0c323ce47..da3757190c7 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -604,6 +604,13 @@ def _maybe_modify_numpy_dtype_determination(np_array): return np_array +def _has_string(value): + if isinstance(value, compat.bytes_or_text_types): return True + if isinstance(value, collections.Sequence) and value: + return _has_string(value[0]) + return False + + # TODO(agarwal): rename to TensorHandle. class EagerTensor(Tensor): """A TensorFlow Eager Tensor.""" @@ -625,6 +632,8 @@ class EagerTensor(Tensor): # https://www.tensorflow.org/code/tensorflow/python/framework/constant_op.py self._id = uid() if not isinstance(value, np.ndarray): + if dtype is None and _has_string(value): + dtype = dtypes.string npt = None if dtype is None else dtype.as_numpy_dtype try: value = np.array(value, dtype=npt) diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index c65816a5436..73b7f821c82 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -65,7 +65,7 @@ def gpu_device_name(): """Returns the name of a GPU device if available or the empty string.""" for x in device_lib.list_local_devices(): if x.device_type == "GPU" or x.device_type == "SYCL": - return x.name + return compat.as_str(x.name) return "" diff --git a/tensorflow/python/kernel_tests/constant_op_eager_test.py b/tensorflow/python/kernel_tests/constant_op_eager_test.py index 0e98afbe6e4..0b4fa60d81b 100644 --- a/tensorflow/python/kernel_tests/constant_op_eager_test.py +++ b/tensorflow/python/kernel_tests/constant_op_eager_test.py @@ -26,27 +26,33 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_lib from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops +from tensorflow.python.util import compat -# TODO(josh11b): add tests with string types, lists/tuples, Shape. +# TODO(josh11b): add tests with lists/tuples, Shape. class ConstantTest(test.TestCase): def _testCpu(self, x): np_ans = np.array(x) - tf_ans = ops.convert_to_tensor(x).numpy() + with context.device("/device:CPU:0"): + tf_ans = ops.convert_to_tensor(x).numpy() if np_ans.dtype in [np.float32, np.float64, np.complex64, np.complex128]: self.assertAllClose(np_ans, tf_ans) else: self.assertAllEqual(np_ans, tf_ans) def _testGpu(self, x): - np_ans = np.array(x) - tf_ans = ops.convert_to_tensor(x).numpy() - if np_ans.dtype in [np.float32, np.float64, np.complex64, np.complex128]: - self.assertAllClose(np_ans, tf_ans) - else: - self.assertAllEqual(np_ans, tf_ans) + device = test_util.gpu_device_name() + if device: + np_ans = np.array(x) + with context.device(device): + tf_ans = ops.convert_to_tensor(x).numpy() + if np_ans.dtype in [np.float32, np.float64, np.complex64, np.complex128]: + self.assertAllClose(np_ans, tf_ans) + else: + self.assertAllEqual(np_ans, tf_ans) def _testAll(self, x): self._testCpu(x) @@ -78,11 +84,11 @@ class ConstantTest(test.TestCase): def testComplex64(self): self._testAll( - np.complex(1, 2) * np.arange(-15, 15).reshape([2, 3, 5 - ]).astype(np.complex64)) + np.complex(1, 2) * + np.arange(-15, 15).reshape([2, 3, 5]).astype(np.complex64)) self._testAll( - np.complex(1, 2) * np.random.normal(size=30).reshape( - [2, 3, 5]).astype(np.complex64)) + np.complex(1, 2) * + np.random.normal(size=30).reshape([2, 3, 5]).astype(np.complex64)) self._testAll(np.empty((2, 0, 5)).astype(np.complex64)) def testComplex128(self): @@ -94,6 +100,26 @@ class ConstantTest(test.TestCase): [2, 3, 5]).astype(np.complex128)) self._testAll(np.empty((2, 0, 5)).astype(np.complex128)) + def testString(self): + val = [compat.as_bytes(str(x)) for x in np.arange(-15, 15)] + self._testCpu(np.array(val).reshape([2, 3, 5])) + self._testCpu(np.empty((2, 0, 5)).astype(np.str_)) + + def testStringWithNulls(self): + val = ops.convert_to_tensor(b"\0\0\0\0").numpy() + self.assertEqual(len(val), 4) + self.assertEqual(val, b"\0\0\0\0") + + val = ops.convert_to_tensor(b"xx\0xx").numpy() + self.assertEqual(len(val), 5) + self.assertAllEqual(val, b"xx\0xx") + + nested = [[b"\0\0\0\0", b"xx\0xx"], [b"\0_\0_\0_\0", b"\0"]] + val = ops.convert_to_tensor(nested).numpy() + # NOTE(mrry): Do not use assertAllEqual, because it converts nested to a + # numpy array, which loses the null terminators. + self.assertEqual(val.tolist(), nested) + def testExplicitShapeNumPy(self): c = constant_op.constant( np.arange(-15, 15).reshape([2, 3, 5]).astype(np.float32), From bd64cddd4773708fc95cfab33d16e7785967d1e3 Mon Sep 17 00:00:00 2001 From: Xiaoqiang Zheng Date: Wed, 30 Aug 2017 14:37:32 -0700 Subject: [PATCH 03/67] Fix bias_add size calculation for half. PiperOrigin-RevId: 167051419 --- tensorflow/core/kernels/bias_op_gpu.cu.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/bias_op_gpu.cu.cc b/tensorflow/core/kernels/bias_op_gpu.cu.cc index e07ca5e0c4c..ddc2d457b0e 100644 --- a/tensorflow/core/kernels/bias_op_gpu.cu.cc +++ b/tensorflow/core/kernels/bias_op_gpu.cu.cc @@ -142,9 +142,9 @@ __global__ void BiasGradNCHW_SharedAtomics(const T* output_backprop, int group_size) { // Initialize the shared memory. typedef typename AccumulatorType::type AccT; - __shared__ AccT s_data[32]; - int32 s_data_size = sizeof(s_data) / sizeof(T); - for (int32 index = threadIdx.x; index < s_data_size; index += blockDim.x) { + const int32 kSDataSize = 32; + __shared__ AccT s_data[kSDataSize]; + for (int32 index = threadIdx.x; index < kSDataSize; index += blockDim.x) { s_data[index] = AccT(0); } __syncthreads(); From 2ce226d4b66bd4b77fede24793e438c4b42c0633 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 14:44:08 -0700 Subject: [PATCH 04/67] Only add reduce-precision ops inside FusionKind::kLoop fusion nodes. Other sorts of fusion nodes can't necessarily handle arbitrary elementwise additions into the computation. PiperOrigin-RevId: 167052456 --- .../compiler/xla/service/reduce_precision_insertion.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/reduce_precision_insertion.cc b/tensorflow/compiler/xla/service/reduce_precision_insertion.cc index 01dbb7e8663..33327dc60fb 100644 --- a/tensorflow/compiler/xla/service/reduce_precision_insertion.cc +++ b/tensorflow/compiler/xla/service/reduce_precision_insertion.cc @@ -122,7 +122,8 @@ StatusOr ReducePrecisionInsertion::insert_on_inputs( continue; } - if (instruction->opcode() == HloOpcode::kFusion) { + if (instruction->opcode() == HloOpcode::kFusion && + instruction->fusion_kind() == HloInstruction::FusionKind::kLoop) { // Insert the reduce-precision operation inside the fusion computation, // after the corresponding parameter instruction. TF_ASSIGN_OR_RETURN( @@ -171,7 +172,8 @@ StatusOr ReducePrecisionInsertion::insert_on_outputs( continue; } - if (instruction->opcode() == HloOpcode::kFusion) { + if (instruction->opcode() == HloOpcode::kFusion && + instruction->fusion_kind() == HloInstruction::FusionKind::kLoop) { // Insert the reduce-precision operation as the last operation inside // the fusion computation. HloInstruction* fusion_root = instruction->fused_expression_root(); From 394214dc5ca7437fa07532be44b6f99193313a27 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 14:44:44 -0700 Subject: [PATCH 05/67] Set the GPU execution context inside the callback wrapper in CudaSolver::CopyLapackInfoToHostAsync, in case the caller launches additional kernels in the provided callback. PiperOrigin-RevId: 167052565 --- tensorflow/core/kernels/cuda_solvers.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/cuda_solvers.cc b/tensorflow/core/kernels/cuda_solvers.cc index 3a8ccfe6b74..5c6b5eec829 100644 --- a/tensorflow/core/kernels/cuda_solvers.cc +++ b/tensorflow/core/kernels/cuda_solvers.cc @@ -30,10 +30,13 @@ #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/platform/cuda.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/stream_executor.h" #include "tensorflow/core/platform/types.h" +using ::perftools::gputools::cuda::ScopedActivateExecutorContext; + namespace tensorflow { namespace { @@ -148,7 +151,12 @@ Status CudaSolver::CopyLapackInfoToHostAsync( // This callback checks that all batch items in all calls were processed // successfully and passes status to the info_checker_callback accordingly. auto wrapped_info_checker_callback = - [info_checker_callback](std::vector host_lapack_infos) { + [](OpKernelContext* context, + std::function&)> + info_checker_callback, + std::vector host_lapack_infos) { + auto stream = context->op_device_context()->stream(); + ScopedActivateExecutorContext scoped_activation{stream->parent()}; Status status; for (const auto& host_lapack_info : host_lapack_infos) { for (int i = 0; i < host_lapack_info.size() && status.ok(); ++i) { @@ -166,8 +174,10 @@ Status CudaSolver::CopyLapackInfoToHostAsync( } info_checker_callback(status, host_lapack_infos); }; + auto cb = - std::bind(wrapped_info_checker_callback, std::move(host_lapack_infos)); + std::bind(wrapped_info_checker_callback, context_, + std::move(info_checker_callback), std::move(host_lapack_infos)); auto stream = context_->op_device_context()->stream(); context_->device()->tensorflow_gpu_device_info()->event_mgr->ThenExecute( stream, std::move(cb)); From f63078d75c06d401edd52f4afd3f9efaf667f72f Mon Sep 17 00:00:00 2001 From: Jianwei Xie Date: Wed, 30 Aug 2017 14:52:52 -0700 Subject: [PATCH 06/67] Fixed a bug that the job was calculated based on wrong master for evaluation. PiperOrigin-RevId: 167053816 --- .../contrib/tpu/python/tpu/tpu_estimator.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 7c883ec9266..485c884bb30 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -102,10 +102,12 @@ def _increase_eval_step_op(iterations_per_loop): use_locking=True) -def _tpu_job(run_config): +def _tpu_job(run_config, mode): # The tpu job is determined by the run_config. Right now, this method is # required as tpu_config is not part of the RunConfig. - return None if run_config.master in ['', 'local'] else 'tpu_worker' + master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL + else run_config.master) + return None if master in ['', 'local'] else 'tpu_worker' def _is_running_on_cpu(use_tpu, mode, eval_batch_size): @@ -265,9 +267,9 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): dequeue. """ - def __init__(self, run_config, enqueue_fn, dequeue_ops=None): + def __init__(self, run_config, mode, enqueue_fn, dequeue_ops=None): self._iterations = run_config.tpu_config.iterations_per_loop - self._tpu_job = _tpu_job(run_config) + self._tpu_job = _tpu_job(run_config, mode) self._enqueue_fn = enqueue_fn self._dequeue_ops = dequeue_ops @@ -899,7 +901,7 @@ class _EvalMetrics(object): """ num_shards = run_config.tpu_config.num_shards - job = _tpu_job(run_config) + job = _tpu_job(run_config, model_fn_lib.ModeKeys.EVAL) job_device = '' if job is None else ('/job:%s' % job) # For each i, dequeue_ops[i] is a list containing the tensors from all @@ -1162,7 +1164,7 @@ class TPUEstimator(estimator_lib.Estimator): with ops.device('/device:CPU:0'): return input_fn(**kwargs) - job = _tpu_job(config) + job = _tpu_job(config, mode) def placement_function(index): if job is None: return '/replica:0/task:0/device:CPU:0' @@ -1190,13 +1192,14 @@ class TPUEstimator(estimator_lib.Estimator): # TODO(b/64607814): Ensure batch_axis works with nested structures. def _create_infeed_enqueue_ops_and_dequeue_fn(inputs_holder, run_config, - batch_axis): + batch_axis, mode): """Utility to convert input_fn to enqueue and dequeue fns for TPU. Args: inputs_holder: An `_InputsHolder` holding features and labels. run_config: A `RunConfig` instance. batch_axis: A python list of batch dimensions. + mode: ModeKeys Returns: A tuple of (dequeue_fn, enqueue_fn) @@ -1239,7 +1242,7 @@ def _create_infeed_enqueue_ops_and_dequeue_fn(inputs_holder, run_config, return infeed_queue.generate_enqueue_ops( sharded_inputs, tpu_ordinal_function=tpu_ordinal_function) else: - job = _tpu_job(run_config) + job = _tpu_job(run_config, mode) def placement_function(index): if job is None: return '/replica:0/task:0/device:CPU:0' @@ -1271,12 +1274,12 @@ def _augment_model_fn(model_fn, train_batch_size, eval_batch_size, use_tpu, num_shards=config.tpu_config.num_shards) dequeue_fn, enqueue_fn = _create_infeed_enqueue_ops_and_dequeue_fn( - inputs, config, batch_axis) + inputs, config, batch_axis, mode) if mode == model_fn_lib.ModeKeys.TRAIN: loss = _train_on_tpu_system(model_fn_wrapper, dequeue_fn) hooks = [ - TPUInfeedOutfeedSessionHook(config, enqueue_fn), + TPUInfeedOutfeedSessionHook(config, mode, enqueue_fn), training.LoggingTensorHook( {'loss': array_ops.identity(loss), 'step': training.get_global_step()}, @@ -1318,7 +1321,7 @@ def _augment_model_fn(model_fn, train_batch_size, eval_batch_size, use_tpu, eval_metric_ops.to_metric_metric_ops_for_tpu( config, dummy_update_op)) hooks = [ - TPUInfeedOutfeedSessionHook(config, enqueue_fn, eval_update_ops), + TPUInfeedOutfeedSessionHook(config, mode, enqueue_fn, eval_update_ops), ] return model_fn_lib.EstimatorSpec( From af20b42c2c926e998a80a6e725d282f7fab79992 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 14:59:32 -0700 Subject: [PATCH 07/67] Add make_best_model_export_strategy which exports the best model according to eval results. PiperOrigin-RevId: 167054780 --- .../learn/utils/saved_model_export_utils.py | 197 +++++++++++--- .../utils/saved_model_export_utils_test.py | 241 ++++++++++++------ 2 files changed, 315 insertions(+), 123 deletions(-) diff --git a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py index 1e68a3ef660..676e1f2b51c 100644 --- a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py +++ b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - """Utilities supporting export to SavedModel. Some contents of this file are moved to tensorflow/python/estimator/export.py: @@ -39,6 +38,7 @@ import time from tensorflow.contrib.layers.python.layers import feature_column from tensorflow.contrib.learn.python.learn import export_strategy from tensorflow.contrib.learn.python.learn.estimators import constants +from tensorflow.contrib.learn.python.learn.estimators import metric_key from tensorflow.contrib.learn.python.learn.estimators import prediction_key from tensorflow.contrib.learn.python.learn.utils import gc from tensorflow.contrib.learn.python.learn.utils import input_fn_utils @@ -75,8 +75,8 @@ FEATURES_INPUT_ALTERNATIVE_KEY = 'features_input_alternative' _FALLBACK_DEFAULT_OUTPUT_ALTERNATIVE_KEY = 'default_output_alternative' -def build_standardized_signature_def( - input_tensors, output_tensors, problem_type): +def build_standardized_signature_def(input_tensors, output_tensors, + problem_type): """Build a SignatureDef using problem type and input and output Tensors. Note that this delegates the actual creation of the signatures to methods in @@ -116,8 +116,8 @@ def build_standardized_signature_def( (_, predictions), = output_tensors.items() return signature_def_utils.regression_signature_def(examples, predictions) else: - return signature_def_utils.predict_signature_def( - input_tensors, output_tensors) + return signature_def_utils.predict_signature_def(input_tensors, + output_tensors) def _get_classification_scores(output_tensors): @@ -139,17 +139,15 @@ def _is_classification_problem(problem_type, input_tensors, output_tensors): classes = _get_classification_classes(output_tensors) scores = _get_classification_scores(output_tensors) return ((problem_type == constants.ProblemType.CLASSIFICATION or - problem_type == constants.ProblemType.LOGISTIC_REGRESSION) - and len(input_tensors) == 1 - and (classes is not None or - scores is not None or - len(output_tensors) == 1)) + problem_type == constants.ProblemType.LOGISTIC_REGRESSION) and + len(input_tensors) == 1 and + (classes is not None or scores is not None or + len(output_tensors) == 1)) def _is_regression_problem(problem_type, input_tensors, output_tensors): - return (problem_type == constants.ProblemType.LINEAR_REGRESSION - and len(input_tensors) == 1 - and len(output_tensors) == 1) + return (problem_type == constants.ProblemType.LINEAR_REGRESSION and + len(input_tensors) == 1 and len(output_tensors) == 1) def get_input_alternatives(input_ops): @@ -177,9 +175,7 @@ def get_input_alternatives(input_ops): return input_alternatives, features -def get_output_alternatives( - model_fn_ops, - default_output_alternative_key=None): +def get_output_alternatives(model_fn_ops, default_output_alternative_key=None): """Obtain all output alternatives using the model_fn output and heuristics. Args: @@ -218,8 +214,10 @@ def get_output_alternatives( default_outputs = {prediction_key.PredictionKey.GENERIC: default_outputs} actual_default_output_alternative_key = ( _FALLBACK_DEFAULT_OUTPUT_ALTERNATIVE_KEY) - output_alternatives = {actual_default_output_alternative_key: - (default_problem_type, default_outputs)} + output_alternatives = { + actual_default_output_alternative_key: (default_problem_type, + default_outputs) + } return output_alternatives, actual_default_output_alternative_key if default_output_alternative_key: @@ -246,13 +244,12 @@ def build_all_signature_defs(input_alternatives, output_alternatives, actual_default_output_alternative_key): """Build `SignatureDef`s from all pairs of input and output alternatives.""" - signature_def_map = { - ('%s:%s' % (input_key, output_key or 'None')): - build_standardized_signature_def( - inputs, outputs, problem_type) - for input_key, inputs in input_alternatives.items() - for output_key, (problem_type, outputs) - in output_alternatives.items()} + signature_def_map = {('%s:%s' % (input_key, output_key or 'None')): + build_standardized_signature_def(inputs, outputs, + problem_type) + for input_key, inputs in input_alternatives.items() + for output_key, (problem_type, + outputs) in output_alternatives.items()} # Add the default SignatureDef default_inputs = input_alternatives.get(DEFAULT_INPUT_ALTERNATIVE_KEY) @@ -263,8 +260,8 @@ def build_all_signature_defs(input_alternatives, output_alternatives, (default_problem_type, default_outputs) = ( output_alternatives[actual_default_output_alternative_key]) signature_def_map[signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] = ( - build_standardized_signature_def( - default_inputs, default_outputs, default_problem_type)) + build_standardized_signature_def(default_inputs, default_outputs, + default_problem_type)) return signature_def_map @@ -308,9 +305,8 @@ def get_timestamped_export_dir(export_dir_base): return export_dir time.sleep(1) attempts += 1 - logging.warn( - 'Export directory {} already exists; retrying (attempt {}/{})'.format( - export_dir, attempts, MAX_DIRECTORY_CREATION_ATTEMPTS)) + logging.warn('Export directory {} already exists; retrying (attempt {}/{})'. + format(export_dir, attempts, MAX_DIRECTORY_CREATION_ATTEMPTS)) raise RuntimeError('Failed to obtain a unique export directory name after ' '{} attempts.'.format(MAX_DIRECTORY_CREATION_ATTEMPTS)) @@ -330,8 +326,7 @@ def get_temp_export_dir(timestamped_export_dir): """ (dirname, basename) = os.path.split(timestamped_export_dir) temp_export_dir = os.path.join( - compat.as_bytes(dirname), - compat.as_bytes('temp-{}'.format(basename))) + compat.as_bytes(dirname), compat.as_bytes('temp-{}'.format(basename))) return temp_export_dir @@ -357,8 +352,8 @@ def get_most_recent_export(export_dir_base): A gc.Path, with is just a namedtuple of (path, export_version). """ select_filter = gc.largest_export_versions(1) - results = select_filter(gc.get_paths(export_dir_base, - parser=_export_version_parser)) + results = select_filter( + gc.get_paths(export_dir_base, parser=_export_version_parser)) return next(iter(results or []), None) @@ -378,8 +373,8 @@ def garbage_collect_exports(export_dir_base, exports_to_keep): keep_filter = gc.largest_export_versions(exports_to_keep) delete_filter = gc.negation(keep_filter) - for p in delete_filter(gc.get_paths(export_dir_base, - parser=_export_version_parser)): + for p in delete_filter( + gc.get_paths(export_dir_base, parser=_export_version_parser)): try: gfile.DeleteRecursively(p.path) except errors_impl.NotFoundError as e: @@ -416,10 +411,7 @@ def make_export_strategy(serving_input_fn, An ExportStrategy that can be passed to the Experiment constructor. """ - def export_fn(estimator, - export_dir_base, - checkpoint_path=None - ): + def export_fn(estimator, export_dir_base, checkpoint_path=None): """Exports the given Estimator as a SavedModel. Args: @@ -512,3 +504,128 @@ def make_parsing_export_strategy(feature_columns, assets_extra=assets_extra, as_text=as_text, exports_to_keep=exports_to_keep) + + +def _default_compare_fn(curr_best_eval_result, cand_eval_result): + """Compares two evaluation results and returns true if the 2nd one is better. + + Both evaluation results should have the values for MetricKey.LOSS, which are + used for comparison. + + Args: + curr_best_eval_result: current best eval metrics. + cand_eval_result: candidate eval metrics. + + Returns: + True if cand_eval_result is better. + + Raises: + ValueError: If input eval result is None or no loss is available. + """ + default_key = metric_key.MetricKey.LOSS + if not curr_best_eval_result or default_key not in curr_best_eval_result: + raise ValueError( + 'curr_best_eval_result cannot be empty or no loss is found in it.') + + if not cand_eval_result or default_key not in cand_eval_result: + raise ValueError( + 'cand_eval_result cannot be empty or no loss is found in it.') + + return curr_best_eval_result[default_key] > cand_eval_result[default_key] + + +class BestModelSelector(object): + """A helper that keeps track of export selection candidates.""" + + def __init__(self, compare_fn=None): + """Constructor of this class. + + Args: + compare_fn: a function that returns true if the candidate is better than + the current best model. + """ + self._best_eval_result = None + self._compare_fn = compare_fn or _default_compare_fn + + def update(self, checkpoint_path, eval_result): + """Records a given checkpoint and exports if this is the best model. + + Args: + checkpoint_path: the checkpoint path to export. + eval_result: a dictionary which is usually generated in evaluation runs. + By default, eval_results contains 'loss' field. + + Returns: + A string representing the path to the checkpoint to be exported. + A dictionary of the same type of eval_result. + + Raises: + ValueError: if checkpoint path is empty. + ValueError: if eval_results is None object. + """ + if not checkpoint_path: + raise ValueError('Checkpoint path is empty.') + if eval_result is None: + raise ValueError('%s has empty evaluation results.', checkpoint_path) + + if (self._best_eval_result is None or + self._compare_fn(self._best_eval_result, eval_result)): + self._best_eval_result = eval_result + return checkpoint_path, eval_result + else: + return '', None + + +def make_best_model_export_strategy(serving_input_fn, + exports_to_keep=1, + compare_fn=None, + default_output_alternative_key=None): + """Creates an custom ExportStrategy for use with tf.contrib.learn.Experiment. + + Args: + serving_input_fn: a function that takes no arguments and returns an + `InputFnOps`. + exports_to_keep: an integer indicating how many historical best models need + to be preserved. + compare_fn: a function that select the 'best' candidate from a dictionary + of evaluation result keyed by corresponding checkpoint path. + default_output_alternative_key: the key for default serving signature for + multi-headed inference graphs. + + Returns: + An ExportStrategy that can be passed to the Experiment constructor. + """ + best_model_export_strategy = make_export_strategy( + serving_input_fn, + exports_to_keep=exports_to_keep, + default_output_alternative_key=default_output_alternative_key) + + best_model_selector = BestModelSelector(compare_fn) + + def export_fn(estimator, export_dir_base, checkpoint_path, eval_result=None): + """Exports the given Estimator as a SavedModel. + + Args: + estimator: the Estimator to export. + export_dir_base: A string containing a directory to write the exported + graph and checkpoints. + checkpoint_path: The checkpoint path to export. If None (the default), + the most recent checkpoint found within the model directory is chosen. + eval_result: placehold args matching the call signature of ExportStrategy. + + Returns: + The string path to the exported directory. + """ + + export_checkpoint_path, export_eval_result = best_model_selector.update( + checkpoint_path, eval_result) + + if export_checkpoint_path and export_eval_result is not None: + checkpoint_base = os.path.basename(export_checkpoint_path) + export_dir = os.path.join(export_dir_base, checkpoint_base) + return best_model_export_strategy.export( + estimator, export_dir, export_checkpoint_path, export_eval_result) + else: + return '' + + return export_strategy.ExportStrategy('best_model', export_fn) diff --git a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils_test.py b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils_test.py index 9e778ab72ad..66bca9c0f53 100644 --- a/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils_test.py +++ b/tensorflow/contrib/learn/python/learn/utils/saved_model_export_utils_test.py @@ -24,6 +24,7 @@ import time from tensorflow.contrib.layers.python.layers import feature_column as fc from tensorflow.contrib.learn.python.learn import export_strategy as export_strategy_lib from tensorflow.contrib.learn.python.learn.estimators import constants +from tensorflow.contrib.learn.python.learn.estimators import estimator as core_estimator from tensorflow.contrib.learn.python.learn.estimators import model_fn from tensorflow.contrib.learn.python.learn.utils import input_fn_utils from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils @@ -40,18 +41,43 @@ from tensorflow.python.saved_model import signature_def_utils from tensorflow.python.util import compat +class TestEstimator(core_estimator.Estimator): + + def __init__(self, *args, **kwargs): + super(TestEstimator, self).__init__(*args, **kwargs) + self.last_exported_checkpoint = "" + self.last_exported_dir = "" + + # @Override + def export_savedmodel(self, + export_dir, + serving_input_fn, + default_output_alternative_key=None, + assets_extra=None, + as_text=False, + checkpoint_path=None): + + if not os.path.exists(export_dir): + os.makedirs(export_dir) + + open(os.path.join(export_dir, "placeholder.txt"), "a").close() + + self.last_exported_checkpoint = checkpoint_path + self.last_exported_dir = export_dir + + return export_dir + + class SavedModelExportUtilsTest(test.TestCase): def test_build_standardized_signature_def_regression(self): input_tensors = { "input-1": - array_ops.placeholder( - dtypes.float32, 1, name="input-tensor-1") + array_ops.placeholder(dtypes.float32, 1, name="input-tensor-1") } output_tensors = { "output-1": - array_ops.placeholder( - dtypes.float32, 1, name="output-tensor-1") + array_ops.placeholder(dtypes.float32, 1, name="output-tensor-1") } problem_type = constants.ProblemType.LINEAR_REGRESSION actual_signature_def = ( @@ -61,10 +87,9 @@ class SavedModelExportUtilsTest(test.TestCase): shape = tensor_shape_pb2.TensorShapeProto( dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) dtype = types_pb2.DataType.Value("DT_FLOAT") - expected_signature_def.inputs[ - signature_constants.REGRESS_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype, tensor_shape=shape)) + expected_signature_def.inputs[signature_constants.REGRESS_INPUTS].CopyFrom( + meta_graph_pb2.TensorInfo( + name="input-tensor-1:0", dtype=dtype, tensor_shape=shape)) expected_signature_def.outputs[ signature_constants.REGRESS_OUTPUTS].CopyFrom( meta_graph_pb2.TensorInfo( @@ -77,13 +102,11 @@ class SavedModelExportUtilsTest(test.TestCase): """Tests classification with one output tensor.""" input_tensors = { "input-1": - array_ops.placeholder( - dtypes.float32, 1, name="input-tensor-1") + array_ops.placeholder(dtypes.float32, 1, name="input-tensor-1") } output_tensors = { "output-1": - array_ops.placeholder( - dtypes.string, 1, name="output-tensor-1") + array_ops.placeholder(dtypes.string, 1, name="output-tensor-1") } problem_type = constants.ProblemType.CLASSIFICATION actual_signature_def = ( @@ -94,14 +117,14 @@ class SavedModelExportUtilsTest(test.TestCase): dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) dtype_float = types_pb2.DataType.Value("DT_FLOAT") dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[ - signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) + expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( + meta_graph_pb2.TensorInfo( + name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) expected_signature_def.outputs[ signature_constants.CLASSIFY_OUTPUT_CLASSES].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-1:0", dtype=dtype_string, + name="output-tensor-1:0", + dtype=dtype_string, tensor_shape=shape)) expected_signature_def.method_name = ( @@ -112,8 +135,7 @@ class SavedModelExportUtilsTest(test.TestCase): """Tests multiple output tensors that include classes and probabilities.""" input_tensors = { "input-1": - array_ops.placeholder( - dtypes.float32, 1, name="input-tensor-1") + array_ops.placeholder(dtypes.float32, 1, name="input-tensor-1") } output_tensors = { "classes": @@ -136,19 +158,20 @@ class SavedModelExportUtilsTest(test.TestCase): dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) dtype_float = types_pb2.DataType.Value("DT_FLOAT") dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[ - signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) + expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( + meta_graph_pb2.TensorInfo( + name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) expected_signature_def.outputs[ signature_constants.CLASSIFY_OUTPUT_CLASSES].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-classes:0", dtype=dtype_string, + name="output-tensor-classes:0", + dtype=dtype_string, tensor_shape=shape)) expected_signature_def.outputs[ signature_constants.CLASSIFY_OUTPUT_SCORES].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-proba:0", dtype=dtype_float, + name="output-tensor-proba:0", + dtype=dtype_float, tensor_shape=shape)) expected_signature_def.method_name = ( @@ -159,8 +182,7 @@ class SavedModelExportUtilsTest(test.TestCase): """Tests multiple output tensors that include classes and scores.""" input_tensors = { "input-1": - array_ops.placeholder( - dtypes.float32, 1, name="input-tensor-1") + array_ops.placeholder(dtypes.float32, 1, name="input-tensor-1") } output_tensors = { "classes": @@ -182,19 +204,20 @@ class SavedModelExportUtilsTest(test.TestCase): dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) dtype_float = types_pb2.DataType.Value("DT_FLOAT") dtype_string = types_pb2.DataType.Value("DT_STRING") - expected_signature_def.inputs[ - signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) + expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( + meta_graph_pb2.TensorInfo( + name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) expected_signature_def.outputs[ signature_constants.CLASSIFY_OUTPUT_CLASSES].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-classes:0", dtype=dtype_string, + name="output-tensor-classes:0", + dtype=dtype_string, tensor_shape=shape)) expected_signature_def.outputs[ signature_constants.CLASSIFY_OUTPUT_SCORES].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-scores:0", dtype=dtype_float, + name="output-tensor-scores:0", + dtype=dtype_float, tensor_shape=shape)) expected_signature_def.method_name = ( @@ -205,8 +228,7 @@ class SavedModelExportUtilsTest(test.TestCase): """Tests classification without classes tensor.""" input_tensors = { "input-1": - array_ops.placeholder( - dtypes.float32, 1, name="input-tensor-1") + array_ops.placeholder(dtypes.float32, 1, name="input-tensor-1") } output_tensors = { "probabilities": @@ -224,14 +246,14 @@ class SavedModelExportUtilsTest(test.TestCase): shape = tensor_shape_pb2.TensorShapeProto( dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) dtype_float = types_pb2.DataType.Value("DT_FLOAT") - expected_signature_def.inputs[ - signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) + expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( + meta_graph_pb2.TensorInfo( + name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) expected_signature_def.outputs[ signature_constants.CLASSIFY_OUTPUT_SCORES].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-proba:0", dtype=dtype_float, + name="output-tensor-proba:0", + dtype=dtype_float, tensor_shape=shape)) expected_signature_def.method_name = ( @@ -246,8 +268,7 @@ class SavedModelExportUtilsTest(test.TestCase): """ input_tensors = { "input-1": - array_ops.placeholder( - dtypes.float32, 1, name="input-tensor-1") + array_ops.placeholder(dtypes.float32, 1, name="input-tensor-1") } output_tensors = { "classes": @@ -268,14 +289,14 @@ class SavedModelExportUtilsTest(test.TestCase): shape = tensor_shape_pb2.TensorShapeProto( dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) dtype_float = types_pb2.DataType.Value("DT_FLOAT") - expected_signature_def.inputs[ - signature_constants.CLASSIFY_INPUTS].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) + expected_signature_def.inputs[signature_constants.CLASSIFY_INPUTS].CopyFrom( + meta_graph_pb2.TensorInfo( + name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) expected_signature_def.outputs[ signature_constants.CLASSIFY_OUTPUT_SCORES].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-scores:0", dtype=dtype_float, + name="output-tensor-scores:0", + dtype=dtype_float, tensor_shape=shape)) expected_signature_def.method_name = ( @@ -290,8 +311,7 @@ class SavedModelExportUtilsTest(test.TestCase): """ input_tensors = { "input-1": - array_ops.placeholder( - dtypes.float32, 1, name="input-tensor-1") + array_ops.placeholder(dtypes.float32, 1, name="input-tensor-1") } output_tensors = { "classes": @@ -310,17 +330,18 @@ class SavedModelExportUtilsTest(test.TestCase): dim=[tensor_shape_pb2.TensorShapeProto.Dim(size=1)]) dtype_int64 = types_pb2.DataType.Value("DT_INT64") dtype_float = types_pb2.DataType.Value("DT_FLOAT") - expected_signature_def.inputs[ - "input-1"].CopyFrom( - meta_graph_pb2.TensorInfo( - name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) + expected_signature_def.inputs["input-1"].CopyFrom( + meta_graph_pb2.TensorInfo( + name="input-tensor-1:0", dtype=dtype_float, tensor_shape=shape)) expected_signature_def.outputs["classes"].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-classes:0", dtype=dtype_int64, + name="output-tensor-classes:0", + dtype=dtype_int64, tensor_shape=shape)) expected_signature_def.outputs["logits"].CopyFrom( meta_graph_pb2.TensorInfo( - name="output-tensor-logits:0", dtype=dtype_float, + name="output-tensor-logits:0", + dtype=dtype_float, tensor_shape=shape)) expected_signature_def.method_name = ( @@ -379,8 +400,9 @@ class SavedModelExportUtilsTest(test.TestCase): def test_get_output_alternatives_single_no_default(self): prediction_tensor = constant_op.constant(["bogus"]) provided_output_alternatives = { - "head-1": (constants.ProblemType.LINEAR_REGRESSION, - {"output": prediction_tensor}), + "head-1": (constants.ProblemType.LINEAR_REGRESSION, { + "output": prediction_tensor + }), } model_fn_ops = model_fn.ModelFnOps( model_fn.ModeKeys.INFER, @@ -390,10 +412,11 @@ class SavedModelExportUtilsTest(test.TestCase): output_alternatives, _ = saved_model_export_utils.get_output_alternatives( model_fn_ops) - self.assertEqual({"head-1": - (constants.ProblemType.LINEAR_REGRESSION, - {"output": prediction_tensor})}, - output_alternatives) + self.assertEqual({ + "head-1": (constants.ProblemType.LINEAR_REGRESSION, { + "output": prediction_tensor + }) + }, output_alternatives) def test_get_output_alternatives_multi_no_default(self): provided_output_alternatives = { @@ -424,10 +447,11 @@ class SavedModelExportUtilsTest(test.TestCase): output_alternatives, _ = saved_model_export_utils.get_output_alternatives( model_fn_ops) - self.assertEqual( - {"default_output_alternative": (constants.ProblemType.UNSPECIFIED, { - "some_output": prediction_tensor})}, - output_alternatives) + self.assertEqual({ + "default_output_alternative": (constants.ProblemType.UNSPECIFIED, { + "some_output": prediction_tensor + }) + }, output_alternatives) def test_get_output_alternatives_empty_provided_with_default(self): prediction_tensor = constant_op.constant(["bogus"]) @@ -452,10 +476,11 @@ class SavedModelExportUtilsTest(test.TestCase): output_alternatives, _ = saved_model_export_utils.get_output_alternatives( model_fn_ops) - self.assertEqual( - {"default_output_alternative": (constants.ProblemType.UNSPECIFIED, { - "some_output": prediction_tensor})}, - output_alternatives) + self.assertEqual({ + "default_output_alternative": (constants.ProblemType.UNSPECIFIED, { + "some_output": prediction_tensor + }) + }, output_alternatives) def test_get_output_alternatives_implicit_single(self): prediction_tensor = constant_op.constant(["bogus"]) @@ -506,14 +531,14 @@ class SavedModelExportUtilsTest(test.TestCase): expected_signature_defs = { "serving_default": - signature_def_utils.regression_signature_def(input_example, - output_1), + signature_def_utils.regression_signature_def( + input_example, output_1), "default_input_alternative:head-1": - signature_def_utils.regression_signature_def(input_example, - output_1), + signature_def_utils.regression_signature_def( + input_example, output_1), "default_input_alternative:head-2": - signature_def_utils.classification_signature_def(input_example, - output_2, None), + signature_def_utils.classification_signature_def( + input_example, output_2, None), "default_input_alternative:head-3": signature_def_utils.predict_signature_def({ "default input": input_example @@ -624,17 +649,20 @@ class SavedModelExportUtilsTest(test.TestCase): (most_recent_export_dir, most_recent_export_version) = ( saved_model_export_utils.get_most_recent_export(export_dir_base)) - self.assertEqual(compat.as_bytes(export_dir_4), - compat.as_bytes(most_recent_export_dir)) - self.assertEqual(compat.as_bytes(export_dir_4), - os.path.join(compat.as_bytes(export_dir_base), - compat.as_bytes( - str(most_recent_export_version)))) + self.assertEqual( + compat.as_bytes(export_dir_4), compat.as_bytes(most_recent_export_dir)) + self.assertEqual( + compat.as_bytes(export_dir_4), + os.path.join( + compat.as_bytes(export_dir_base), + compat.as_bytes(str(most_recent_export_version)))) def test_make_export_strategy(self): """Only tests that an ExportStrategy instance is created.""" + def _serving_input_fn(): return array_ops.constant([1]), None + export_strategy = saved_model_export_utils.make_export_strategy( serving_input_fn=_serving_input_fn, default_output_alternative_key="default", @@ -655,14 +683,61 @@ class SavedModelExportUtilsTest(test.TestCase): real_valued_col1 = fc.real_valued_column("real_valued_column1") bucketized_col1 = fc.bucketized_column( fc.real_valued_column("real_valued_column_for_bucketization1"), [0, 4]) - feature_columns = [sparse_col, embedding_col, real_valued_col1, - bucketized_col1] + feature_columns = [ + sparse_col, embedding_col, real_valued_col1, bucketized_col1 + ] export_strategy = saved_model_export_utils.make_parsing_export_strategy( feature_columns=feature_columns) self.assertTrue( isinstance(export_strategy, export_strategy_lib.ExportStrategy)) + def test_make_best_model_export_strategy(self): + export_dir_base = tempfile.mkdtemp() + "export/" + gfile.MkDir(export_dir_base) + + test_estimator = TestEstimator() + export_strategy = saved_model_export_utils.make_best_model_export_strategy( + serving_input_fn=None, exports_to_keep=3, compare_fn=None) + + self.assertNotEqual("", + export_strategy.export(test_estimator, export_dir_base, + "fake_ckpt_0", {"loss": 100})) + self.assertNotEqual("", test_estimator.last_exported_dir) + self.assertNotEqual("", test_estimator.last_exported_checkpoint) + + self.assertEqual("", + export_strategy.export(test_estimator, export_dir_base, + "fake_ckpt_1", {"loss": 101})) + self.assertEqual(test_estimator.last_exported_dir, + os.path.join(export_dir_base, "fake_ckpt_0")) + + self.assertNotEqual("", + export_strategy.export(test_estimator, export_dir_base, + "fake_ckpt_2", {"loss": 10})) + self.assertEqual(test_estimator.last_exported_dir, + os.path.join(export_dir_base, "fake_ckpt_2")) + + self.assertEqual("", + export_strategy.export(test_estimator, export_dir_base, + "fake_ckpt_3", {"loss": 20})) + self.assertEqual(test_estimator.last_exported_dir, + os.path.join(export_dir_base, "fake_ckpt_2")) + + def test_make_best_model_export_strategy_exceptions(self): + export_dir_base = tempfile.mkdtemp() + "export/" + + test_estimator = TestEstimator() + export_strategy = saved_model_export_utils.make_best_model_export_strategy( + serving_input_fn=None, exports_to_keep=3, compare_fn=None) + + with self.assertRaises(ValueError): + export_strategy.export(test_estimator, export_dir_base, "", {"loss": 200}) + + with self.assertRaises(ValueError): + export_strategy.export(test_estimator, export_dir_base, "fake_ckpt_1", + None) + def _create_test_export_dir(export_dir_base): export_dir = saved_model_export_utils.get_timestamped_export_dir( From 6ee861444729063f7b0af8c2f8d37efcd854388e Mon Sep 17 00:00:00 2001 From: Jianwei Xie Date: Wed, 30 Aug 2017 15:49:43 -0700 Subject: [PATCH 08/67] Updates the docstring and model examples to use new style to specify metric_fn. PiperOrigin-RevId: 167062007 --- .../contrib/tpu/python/tpu/tpu_estimator.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 485c884bb30..6748a765623 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -980,18 +980,20 @@ class TPUEstimator(estimator_lib.Estimator): Example (MNIST): ``` + # The metric Fn which runs on CPU. + def metric_fn(labels, logits): + predictions = tf.argmax(logits, 1) + return { + 'accuracy': tf.metrics.precision( + labels=labels, predictions=predictions), + } + + # Your model Fn which runs on TPU. def model_fn(features, labels, mode, config, params): ... logits = ... if mode = tf.estimator.ModeKeys.EVAL: - def metric_fn(labels, logits): - predictions = tf.argmax(logits, 1) - return { - 'precision': tf.metrics.precision( - labels=labels, predictions=predictions), - } - return tpu_estimator.TPUEstimatorSpec( mode=mode, loss=loss, From 2e3d2f97de90508efd2729d90fbd74fb6d770961 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 16:01:58 -0700 Subject: [PATCH 09/67] [TF:XLA] Don't pass opcode separately in two HLO visitor functions. HandleElementwiseUnary and HandleElementwiseBinary never use this parameter and it is accessible from the HLO instruction anyway. No functional change. PiperOrigin-RevId: 167063592 --- .../compiler/xla/service/dfs_hlo_visitor.cc | 10 ++-- .../compiler/xla/service/dfs_hlo_visitor.h | 57 +++++++++---------- .../service/dfs_hlo_visitor_with_default.h | 6 +- .../compiler/xla/service/hlo_cost_analysis.cc | 6 +- .../compiler/xla/service/hlo_cost_analysis.h | 5 +- .../compiler/xla/service/hlo_verifier.cc | 6 +- 6 files changed, 40 insertions(+), 50 deletions(-) diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.cc b/tensorflow/compiler/xla/service/dfs_hlo_visitor.cc index 669ebb55bec..6efd0bcee58 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.cc +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.cc @@ -24,16 +24,14 @@ limitations under the License. namespace xla { -Status DfsHloVisitor::HandleElementwiseUnary(HloInstruction* hlo, - HloOpcode opcode) { +Status DfsHloVisitor::HandleElementwiseUnary(HloInstruction* hlo) { return Unimplemented("DfsHloVisitor::HandleElementwiseUnary: %s", - HloOpcodeString(opcode).c_str()); + HloOpcodeString(hlo->opcode()).c_str()); } -Status DfsHloVisitor::HandleElementwiseBinary(HloInstruction* hlo, - HloOpcode opcode) { +Status DfsHloVisitor::HandleElementwiseBinary(HloInstruction* hlo) { return Unimplemented("DfsHloVisitor::HandleElementwiseBinary: %s", - HloOpcodeString(opcode).c_str()); + HloOpcodeString(hlo->opcode()).c_str()); } DfsHloVisitor::VisitState DfsHloVisitor::GetVisitState( diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h index a1a3a882c7a..2f21043a1d3 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h @@ -63,37 +63,37 @@ class DfsHloVisitor { // These routines are self-descriptive, see class comment for usage // information. - virtual Status HandleElementwiseUnary(HloInstruction* hlo, HloOpcode opcode); - virtual Status HandleElementwiseBinary(HloInstruction* hlo, HloOpcode opcode); + virtual Status HandleElementwiseUnary(HloInstruction* hlo); + virtual Status HandleElementwiseBinary(HloInstruction* hlo); virtual Status HandleClamp(HloInstruction* clamp, HloInstruction* min, HloInstruction* arg, HloInstruction* max) = 0; virtual Status HandleSelect(HloInstruction* select, HloInstruction* pred, HloInstruction* on_true, HloInstruction* on_false) = 0; virtual Status HandleMaximum(HloInstruction* maximum) { - return HandleElementwiseBinary(maximum, HloOpcode::kMaximum); + return HandleElementwiseBinary(maximum); } virtual Status HandleMinimum(HloInstruction* minimum) { - return HandleElementwiseBinary(minimum, HloOpcode::kMinimum); + return HandleElementwiseBinary(minimum); } virtual Status HandleConcatenate( HloInstruction* concatenate, tensorflow::gtl::ArraySlice operands) = 0; virtual Status HandleConvert(HloInstruction* convert) { - return HandleElementwiseUnary(convert, HloOpcode::kConvert); + return HandleElementwiseUnary(convert); } virtual Status HandleCopy(HloInstruction* copy) { - return HandleElementwiseUnary(copy, HloOpcode::kCopy); + return HandleElementwiseUnary(copy); } virtual Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(multiply, HloOpcode::kMultiply); + return HandleElementwiseBinary(multiply); } virtual Status HandleDot(HloInstruction* dot, HloInstruction* lhs, HloInstruction* rhs) = 0; virtual Status HandlePower(HloInstruction* power, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(power, HloOpcode::kPower); + return HandleElementwiseBinary(power); } virtual Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, HloInstruction* rhs, @@ -101,73 +101,72 @@ class DfsHloVisitor { virtual Status HandleCrossReplicaSum(HloInstruction* crs) = 0; virtual Status HandleCompare(HloInstruction* compare, HloOpcode opcode, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(compare, opcode); + return HandleElementwiseBinary(compare); } virtual Status HandleAdd(HloInstruction* add, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(add, HloOpcode::kAdd); + return HandleElementwiseBinary(add); } virtual Status HandleDivide(HloInstruction* divide, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(divide, HloOpcode::kDivide); + return HandleElementwiseBinary(divide); } virtual Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(remainder, HloOpcode::kRemainder); + return HandleElementwiseBinary(remainder); } virtual Status HandleSubtract(HloInstruction* subtract, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(subtract, HloOpcode::kSubtract); + return HandleElementwiseBinary(subtract); } virtual Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { - return HandleElementwiseUnary(abs, HloOpcode::kAbs); + return HandleElementwiseUnary(abs); } virtual Status HandleSign(HloInstruction* sign, HloInstruction* operand) { - return HandleElementwiseUnary(sign, HloOpcode::kSign); + return HandleElementwiseUnary(sign); } virtual Status HandleNegate(HloInstruction* negate, HloInstruction* operand) { - return HandleElementwiseUnary(negate, HloOpcode::kNegate); + return HandleElementwiseUnary(negate); } virtual Status HandleExp(HloInstruction* exp, HloInstruction* operand) { - return HandleElementwiseUnary(exp, HloOpcode::kExp); + return HandleElementwiseUnary(exp); } virtual Status HandleFloor(HloInstruction* floor, HloInstruction* operand) { - return HandleElementwiseUnary(floor, HloOpcode::kFloor); + return HandleElementwiseUnary(floor); } virtual Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) { - return HandleElementwiseUnary(ceil, HloOpcode::kCeil); + return HandleElementwiseUnary(ceil); } virtual Status HandleLog(HloInstruction* log, HloInstruction* operand) { - return HandleElementwiseUnary(log, HloOpcode::kLog); + return HandleElementwiseUnary(log); } virtual Status HandleCos(HloInstruction* cos, HloInstruction* operand) { - return HandleElementwiseUnary(cos, HloOpcode::kCos); + return HandleElementwiseUnary(cos); } virtual Status HandleSin(HloInstruction* sin, HloInstruction* operand) { - return HandleElementwiseUnary(sin, HloOpcode::kSin); + return HandleElementwiseUnary(sin); } virtual Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) { - return HandleElementwiseUnary(tanh, HloOpcode::kTanh); + return HandleElementwiseUnary(tanh); } virtual Status HandleIsFinite(HloInstruction* is_finite, HloInstruction* operand) { - return HandleElementwiseUnary(is_finite, HloOpcode::kIsFinite); + return HandleElementwiseUnary(is_finite); } virtual Status HandleLogicalAnd(HloInstruction* logical_and, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(logical_and, HloOpcode::kLogicalAnd); + return HandleElementwiseBinary(logical_and); } virtual Status HandleLogicalNot(HloInstruction* logical_not, HloInstruction* operand) { - return HandleElementwiseUnary(logical_not, HloOpcode::kLogicalNot); + return HandleElementwiseUnary(logical_not); } virtual Status HandleLogicalOr(HloInstruction* logical_or, HloInstruction* lhs, HloInstruction* rhs) { - return HandleElementwiseBinary(logical_or, HloOpcode::kLogicalOr); + return HandleElementwiseBinary(logical_or); } virtual Status HandleReducePrecision(HloInstruction* reduce_precision) { - return HandleElementwiseUnary(reduce_precision, - HloOpcode::kReducePrecision); + return HandleElementwiseUnary(reduce_precision); } virtual Status HandleInfeed(HloInstruction* infeed) = 0; diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h index 10f8ae9b044..a5fe1205984 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h @@ -41,12 +41,10 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { // Default action performed on HloInstruction. virtual Status DefaultAction(HloInstruction* hlo_instruction) = 0; - Status HandleElementwiseUnary(HloInstruction* hlo, - HloOpcode opcode) override { + Status HandleElementwiseUnary(HloInstruction* hlo) override { return DefaultAction(hlo); } - Status HandleElementwiseBinary(HloInstruction* hlo, - HloOpcode opcode) override { + Status HandleElementwiseBinary(HloInstruction* hlo) override { return DefaultAction(hlo); } diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc index 9dbde0ec243..f6b764732b4 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc @@ -118,13 +118,11 @@ Status HloCostAnalysis::HandleElementwiseOp(HloInstruction* hlo_instruction) { } } -Status HloCostAnalysis::HandleElementwiseUnary(HloInstruction* hlo, - HloOpcode opcode) { +Status HloCostAnalysis::HandleElementwiseUnary(HloInstruction* hlo) { return HandleElementwiseOp(hlo); } -Status HloCostAnalysis::HandleElementwiseBinary(HloInstruction* hlo, - HloOpcode opcode) { +Status HloCostAnalysis::HandleElementwiseBinary(HloInstruction* hlo) { return HandleElementwiseOp(hlo); } diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.h b/tensorflow/compiler/xla/service/hlo_cost_analysis.h index 6d8fdfa64b5..eeb3d4edd1b 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.h +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.h @@ -49,9 +49,8 @@ class HloCostAnalysis : public DfsHloVisitor { using ShapeSizeFunction = std::function; explicit HloCostAnalysis(const ShapeSizeFunction& shape_size); - Status HandleElementwiseUnary(HloInstruction* hlo, HloOpcode opcode) override; - Status HandleElementwiseBinary(HloInstruction* hlo, - HloOpcode opcode) override; + Status HandleElementwiseUnary(HloInstruction* hlo) override; + Status HandleElementwiseBinary(HloInstruction* hlo) override; Status HandleConstant(HloInstruction* constant, const Literal& literal) override; Status HandleGetTupleElement(HloInstruction* get_tuple_element, diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 9ba2d54d024..eb6fe5d2e3b 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -32,13 +32,11 @@ class ShapeVerifier : public DfsHloVisitor { const std::function& shape_size_fn) : shape_size_fn_(shape_size_fn) {} - Status HandleElementwiseUnary(HloInstruction* hlo, - HloOpcode opcode) override { + Status HandleElementwiseUnary(HloInstruction* hlo) override { return CheckUnaryShape(hlo); } - Status HandleElementwiseBinary(HloInstruction* hlo, - HloOpcode opcode) override { + Status HandleElementwiseBinary(HloInstruction* hlo) override { return CheckBinaryShape(hlo); } From a4faf517b42f4bef8c9661f9142ba640aaeb25a2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 16:40:26 -0700 Subject: [PATCH 10/67] GroupByDynamicWindowDataset: A new Dataset where the window size is a function of the key PiperOrigin-RevId: 167068616 --- tensorflow/contrib/data/__init__.py | 2 + .../python/kernel_tests/bucketing_test.py | 115 ++++++-- .../contrib/data/python/ops/dataset_ops.py | 273 ++++++++++++------ .../kernels/group_by_window_dataset_op.cc | 76 ++++- .../core/ops/compat/ops_history.v1.pbtxt | 13 +- tensorflow/core/ops/dataset_ops.cc | 5 +- tensorflow/core/ops/ops.pbtxt | 13 +- 7 files changed, 358 insertions(+), 139 deletions(-) diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py index 5308ab64ace..1c0a5288f7e 100644 --- a/tensorflow/contrib/data/__init__.py +++ b/tensorflow/contrib/data/__init__.py @@ -22,6 +22,7 @@ @@read_batch_features @@rejection_resample +@@group_by_window """ from __future__ import absolute_import @@ -31,6 +32,7 @@ from __future__ import print_function # pylint: disable=unused-import from tensorflow.contrib.data.python.ops.dataset_ops import Dataset from tensorflow.contrib.data.python.ops.dataset_ops import FixedLengthRecordDataset +from tensorflow.contrib.data.python.ops.dataset_ops import group_by_window from tensorflow.contrib.data.python.ops.dataset_ops import Iterator from tensorflow.contrib.data.python.ops.dataset_ops import read_batch_features from tensorflow.contrib.data.python.ops.dataset_ops import rejection_resample diff --git a/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py b/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py index 71df1ee0a50..0111aae1035 100644 --- a/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py @@ -37,7 +37,9 @@ class GroupByWindowTest(test.TestCase): components = np.random.randint(100, size=(200,)).astype(np.int64) iterator = dataset_ops.Iterator.from_dataset( dataset_ops.Dataset.from_tensor_slices(components).map(lambda x: x * x) - .group_by_window(lambda x: x % 2, lambda _, xs: xs.batch(4), 4)) + .apply( + dataset_ops.group_by_window, + args=(lambda x: x % 2, lambda _, xs: xs.batch(4), 4))) init_op = iterator.initializer get_next = iterator.get_next() @@ -61,8 +63,9 @@ class GroupByWindowTest(test.TestCase): components = np.array( [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 0, 0, 2, 2, 0, 0], dtype=np.int64) iterator = dataset_ops.Iterator.from_dataset( - dataset_ops.Dataset.from_tensor_slices(components).repeat(-1) - .group_by_window(lambda x: x % 3, lambda _, xs: xs.batch(4), 4)) + dataset_ops.Dataset.from_tensor_slices(components).repeat(-1).apply( + dataset_ops.group_by_window, + args=(lambda x: x % 3, lambda _, xs: xs.batch(4), 4))) init_op = iterator.initializer get_next = iterator.get_next() @@ -81,8 +84,9 @@ class GroupByWindowTest(test.TestCase): def testSmallGroups(self): components = np.array([0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0], dtype=np.int64) iterator = dataset_ops.Iterator.from_dataset( - dataset_ops.Dataset.from_tensor_slices(components) - .group_by_window(lambda x: x % 2, lambda _, xs: xs.batch(4), 4)) + dataset_ops.Dataset.from_tensor_slices(components).apply( + dataset_ops.group_by_window, + args=(lambda x: x % 2, lambda _, xs: xs.batch(4), 4))) init_op = iterator.initializer get_next = iterator.get_next() @@ -108,8 +112,9 @@ class GroupByWindowTest(test.TestCase): iterator = dataset_ops.Iterator.from_dataset( dataset_ops.Dataset.from_tensor_slices(components) - .map(lambda x: (x, ops.convert_to_tensor([x * x]))) - .group_by_window(lambda x, _: x % 2, reduce_func, 32)) + .map(lambda x: (x, ops.convert_to_tensor([x * x]))).apply( + dataset_ops.group_by_window, + args=(lambda x, _: x % 2, reduce_func, 32))) init_op = iterator.initializer get_next = iterator.get_next() @@ -124,17 +129,20 @@ class GroupByWindowTest(test.TestCase): def reduce_func(key, window): # Apply two different kinds of padding to the input: tight # padding, and quantized (to a multiple of 10) padding. - return dataset_ops.Dataset.zip((window.padded_batch( - 4, - padded_shapes=tensor_shape.TensorShape([None])), window.padded_batch( + return dataset_ops.Dataset.zip(( + window.padded_batch( + 4, padded_shapes=tensor_shape.TensorShape([None])), + window.padded_batch( 4, padded_shapes=ops.convert_to_tensor([(key + 1) * 10])),)) iterator = dataset_ops.Iterator.from_dataset( dataset_ops.Dataset.from_tensor_slices(components) .map(lambda x: array_ops.fill([math_ops.cast(x, dtypes.int32)], x)) - .group_by_window( - lambda x: math_ops.cast(array_ops.shape(x)[0] // 10, dtypes.int64), - reduce_func, 4)) + .apply( + dataset_ops.group_by_window, + args= + (lambda x: math_ops.cast(array_ops.shape(x)[0] // 10, dtypes.int64), + reduce_func, 4))) init_op = iterator.initializer get_next = iterator.get_next() @@ -151,10 +159,9 @@ class GroupByWindowTest(test.TestCase): self.assertEqual(len(components), sum(counts)) -# NOTE(mrry): These tests are based on the tests in -# bucket_ops_test.py. Currently, different batch sizes for each key -# are not supported, although this would be possible to add to -# `Dataset.group_by_window()`. +# NOTE(mrry): These tests are based on the tests in bucket_ops_test.py. +# Currently, they use a constant batch size, though should be made to use a +# different batch size per key. class BucketTest(test.TestCase): def _dynamicPad(self, bucket, window, window_size): @@ -168,6 +175,7 @@ class BucketTest(test.TestCase): tensor_shape.TensorShape([3]))))) def testSingleBucket(self): + def _map_fn(v): return (v, array_ops.fill([v], v), array_ops.fill([3], string_ops.as_string(v))) @@ -175,9 +183,10 @@ class BucketTest(test.TestCase): input_dataset = ( dataset_ops.Dataset.from_tensor_slices(math_ops.range(32)).map(_map_fn)) - bucketed_dataset = input_dataset.group_by_window( - lambda x, y, z: 0, lambda k, bucket: self._dynamicPad(k, bucket, 32), - 32) + bucketed_dataset = input_dataset.apply( + dataset_ops.group_by_window, + args=(lambda x, y, z: 0, + lambda k, bucket: self._dynamicPad(k, bucket, 32), 32)) iterator = dataset_ops.Iterator.from_dataset(bucketed_dataset) init_op = iterator.initializer @@ -201,6 +210,7 @@ class BucketTest(test.TestCase): self.assertAllEqual(expected_vec3_str, bucketed_values[2]) def testEvenOddBuckets(self): + def _map_fn(v): return (v, array_ops.fill([v], v), array_ops.fill([3], string_ops.as_string(v))) @@ -208,9 +218,10 @@ class BucketTest(test.TestCase): input_dataset = ( dataset_ops.Dataset.from_tensor_slices(math_ops.range(64)).map(_map_fn)) - bucketed_dataset = input_dataset.group_by_window( - lambda x, y, z: math_ops.cast(x % 2, dtypes.int64), - lambda k, bucket: self._dynamicPad(k, bucket, 32), 32) + bucketed_dataset = input_dataset.apply( + dataset_ops.group_by_window, + args=(lambda x, y, z: math_ops.cast(x % 2, dtypes.int64), + lambda k, bucket: self._dynamicPad(k, bucket, 32), 32)) iterator = dataset_ops.Iterator.from_dataset(bucketed_dataset) init_op = iterator.initializer @@ -256,25 +267,31 @@ class BucketTest(test.TestCase): self.assertAllEqual(expected_vec3_str, bucketed_values_odd[2]) def testEvenOddBucketsFilterOutAllOdd(self): + def _map_fn(v): - return {"x": v, - "y": array_ops.fill([v], v), - "z": array_ops.fill([3], string_ops.as_string(v))} + return { + "x": v, + "y": array_ops.fill([v], v), + "z": array_ops.fill([3], string_ops.as_string(v)) + } def _dynamic_pad_fn(bucket, window, _): return dataset_ops.Dataset.zip( (dataset_ops.Dataset.from_tensors(bucket), window.padded_batch( - 32, {"x": tensor_shape.TensorShape([]), - "y": tensor_shape.TensorShape([None]), - "z": tensor_shape.TensorShape([3])}))) + 32, { + "x": tensor_shape.TensorShape([]), + "y": tensor_shape.TensorShape([None]), + "z": tensor_shape.TensorShape([3]) + }))) input_dataset = ( dataset_ops.Dataset.from_tensor_slices(math_ops.range(128)).map(_map_fn) .filter(lambda d: math_ops.equal(d["x"] % 2, 0))) - bucketed_dataset = input_dataset.group_by_window( - lambda d: math_ops.cast(d["x"] % 2, dtypes.int64), - lambda k, bucket: _dynamic_pad_fn(k, bucket, 32), 32) + bucketed_dataset = input_dataset.apply( + dataset_ops.group_by_window, + args=(lambda d: math_ops.cast(d["x"] % 2, dtypes.int64), + lambda k, bucket: _dynamic_pad_fn(k, bucket, 32), 32)) iterator = dataset_ops.Iterator.from_dataset(bucketed_dataset) init_op = iterator.initializer @@ -295,6 +312,40 @@ class BucketTest(test.TestCase): self.assertAllEqual( np.arange(64, 128, 2, dtype=np.int64), bucketed_values_even1["x"]) + def testDynamicWindowSize(self): + components = np.arange(100).astype(np.int64) + + # Key fn: even/odd + # Reduce fn: batches of 5 + # Window size fn: even=5, odd=10 + + def window_size_func(key): + window_sizes = constant_op.constant([5, 10], dtype=dtypes.int64) + return window_sizes[key] + + dataset = dataset_ops.Dataset.from_tensor_slices(components).apply( + dataset_ops.group_by_window, + args=(lambda x: x % 2, lambda _, xs: xs.batch(20), None, + window_size_func)) + iterator = dataset_ops.Iterator.from_dataset(dataset) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + with self.assertRaises(errors.OutOfRangeError): + batches = 0 + while True: + result = sess.run(get_next) + is_even = all(x % 2 == 0 for x in result) + is_odd = all(x % 2 == 1 for x in result) + self.assertTrue(is_even or is_odd) + expected_batch_size = 5 if is_even else 10 + self.assertEqual(expected_batch_size, result.shape[0]) + batches += 1 + + self.assertEqual(batches, 15) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/ops/dataset_ops.py b/tensorflow/contrib/data/python/ops/dataset_ops.py index 46af2b19494..abf7bcb384a 100644 --- a/tensorflow/contrib/data/python/ops/dataset_ops.py +++ b/tensorflow/contrib/data/python/ops/dataset_ops.py @@ -1199,28 +1199,9 @@ class Dataset(object): return DenseToSparseBatchDataset(self, batch_size, row_shape) def group_by_window(self, key_func, reduce_func, window_size): - """Performs a windowed "group-by" operation on this dataset. - - This method maps each consecutive element in this dataset to a key - using `key_func` and groups the elements by key. It then applies - `reduce_func` to at most `window_size` elements matching the same - key. All execpt the final window for each key will contain - `window_size` elements; the final window may be smaller. - - Args: - key_func: A function mapping a nested structure of tensors - (having shapes and types defined by `self.output_shapes` and - `self.output_types`) to a scalar `tf.int64` tensor. - reduce_func: A function mapping a key and a dataset of up to `batch_size` - consecutive elements matching that key to another dataset. - window_size: A `tf.int64` scalar `tf.Tensor`, representing the number of - consecutive elements matching the same key to combine in a single - batch, which will be passed to `reduce_func`. - - Returns: - A `Dataset`. - """ - return GroupByWindowDataset(self, key_func, reduce_func, window_size) + """See group_by_window().""" + return self.apply( + group_by_window, args=(key_func, reduce_func, window_size)) def map(self, map_func, @@ -1370,6 +1351,43 @@ class Dataset(object): """ return FilterDataset(self, predicate) + def apply(self, fn, args=(), kwargs={}): # pylint: disable=dangerous-default-value + """Apply a function to this dataset. + + `apply` enables chaining of custom `Dataset` transformations. + + For example: + + ``` + dataset.map( + lambda x: x**2 + ).apply( + group_by_window, args=(key_func, reduce_func, window_size) + ).map( + lambda x: x**3 + ) + ``` + + Args: + fn: A function that takes a `Dataset`, `args`, and `kwargs`, and + returns a `Dataset`. + args: A `tuple` or `list` of arguments to be passed to `fn`. + kwargs: A `dict` of keyword arguments to be passed to `fn`. + + Returns: + The `Dataset` returned by `fn`. + """ + if not (isinstance(args, tuple) or isinstance(args, list)): + raise TypeError("args must be a tuple or list.") + if not isinstance(kwargs, dict): + raise TypeError("kwargs must be a dict.") + + dataset = fn(self, *args, **kwargs) + + if not isinstance(dataset, Dataset): + raise TypeError("fn must return a Dataset.") + return dataset + class TensorDataset(Dataset): """A `Dataset` with a single element, viz. a nested structure of tensors.""" @@ -1927,71 +1945,6 @@ class _ResourceDataset(Dataset): return self._output_types -class GroupByWindowDataset(Dataset): - """A `Dataset` that groups its input and performs a windowed reduction.""" - - def __init__(self, input_dataset, key_func, reduce_func, window_size): - """See `Dataset.group_by_window()` for details.""" - super(GroupByWindowDataset, self).__init__() - self._input_dataset = input_dataset - self._window_size = window_size - - @function.Defun(*nest.flatten(input_dataset.output_types)) - def tf_key_func(*args): - """A wrapper for Defun that facilitates shape inference.""" - # Pass in shape information from the input_dataset. - for arg, shape in zip(args, nest.flatten(input_dataset.output_shapes)): - arg.set_shape(shape) - nested_args = nest.pack_sequence_as(input_dataset.output_types, args) - if _should_unpack_args(nested_args): - ret = key_func(*nested_args) - else: - ret = key_func(nested_args) - ret = ops.convert_to_tensor(ret, dtype=dtypes.int64) - if ret.dtype != dtypes.int64: - raise ValueError("`key_func` must return a single tf.int64 tensor.") - return ret - - self._key_func = tf_key_func - self._key_func.add_to_graph(ops.get_default_graph()) - - @function.Defun(dtypes.int64, dtypes.resource) - def tf_reduce_func(key, window_dataset_resource): - """A wrapper for Defun that facilitates shape inference.""" - key.set_shape([]) - window_dataset = _ResourceDataset(window_dataset_resource, - input_dataset.output_types, - input_dataset.output_shapes) - output_dataset = reduce_func(key, window_dataset) - if not isinstance(output_dataset, Dataset): - raise TypeError("`reduce_func` must return a `Dataset` object.") - self._output_types = output_dataset.output_types - self._output_shapes = output_dataset.output_shapes - return output_dataset.make_dataset_resource() - - self._reduce_func = tf_reduce_func - self._reduce_func.add_to_graph(ops.get_default_graph()) - - def make_dataset_resource(self): - return gen_dataset_ops.group_by_window_dataset( - self._input_dataset.make_dataset_resource(), - self._key_func.captured_inputs, - self._reduce_func.captured_inputs, - self._window_size, - key_func=self._key_func, - reduce_func=self._reduce_func, - output_types=nest.flatten(self.output_types), - output_shapes=nest.flatten(self.output_shapes)) - - @property - def output_shapes(self): - return self._output_shapes - - @property - def output_types(self): - return self._output_types - - class MapDataset(Dataset): """A `Dataset` that maps a function over elements in its input.""" @@ -2660,3 +2613,149 @@ def _get_file_names(file_pattern, randomize_input): if not randomize_input: file_names = sorted(file_names) return file_names + + +class GroupByWindowDataset(Dataset): + """A `Dataset` that groups its input and performs a windowed reduction.""" + + def __init__(self, input_dataset, key_func, reduce_func, window_size_func): + """See `group_by_window()` for details.""" + super(GroupByWindowDataset, self).__init__() + + self._input_dataset = input_dataset + + self._make_key_func(key_func, input_dataset) + self._make_reduce_func(reduce_func, input_dataset) + self._make_window_size_func(window_size_func) + + def _make_window_size_func(self, window_size_func): + """Make wrapping Defun for window_size_func.""" + + @function.Defun(dtypes.int64) + def tf_window_size_func(key): + key.set_shape([]) + window_size = ops.convert_to_tensor( + window_size_func(key), dtype=dtypes.int64) + if window_size.dtype != dtypes.int64: + raise ValueError( + "`window_size_func` must return a single tf.int64 tensor.") + return window_size + + self._window_size_func = tf_window_size_func + self._window_size_func.add_to_graph(ops.get_default_graph()) + + def _make_key_func(self, key_func, input_dataset): + """Make wrapping Defun for key_func.""" + + @function.Defun(*nest.flatten(input_dataset.output_types)) + def tf_key_func(*args): + """A wrapper for Defun that facilitates shape inference.""" + # Pass in shape information from the input_dataset. + for arg, shape in zip(args, nest.flatten(input_dataset.output_shapes)): + arg.set_shape(shape) + nested_args = nest.pack_sequence_as(input_dataset.output_types, args) + if _should_unpack_args(nested_args): + ret = key_func(*nested_args) + else: + ret = key_func(nested_args) + ret = ops.convert_to_tensor(ret, dtype=dtypes.int64) + if ret.dtype != dtypes.int64: + raise ValueError("`key_func` must return a single tf.int64 tensor.") + return ret + + self._key_func = tf_key_func + self._key_func.add_to_graph(ops.get_default_graph()) + + def _make_reduce_func(self, reduce_func, input_dataset): + """Make wrapping Defun for reduce_func.""" + + @function.Defun(dtypes.int64, dtypes.resource) + def tf_reduce_func(key, window_dataset_resource): + """A wrapper for Defun that facilitates shape inference.""" + key.set_shape([]) + window_dataset = _ResourceDataset(window_dataset_resource, + input_dataset.output_types, + input_dataset.output_shapes) + output_dataset = reduce_func(key, window_dataset) + if not isinstance(output_dataset, Dataset): + raise TypeError("`reduce_func` must return a `Dataset` object.") + self._output_types = output_dataset.output_types + self._output_shapes = output_dataset.output_shapes + return output_dataset.make_dataset_resource() + + self._reduce_func = tf_reduce_func + self._reduce_func.add_to_graph(ops.get_default_graph()) + + @property + def output_shapes(self): + return self._output_shapes + + @property + def output_types(self): + return self._output_types + + def make_dataset_resource(self): + return gen_dataset_ops.group_by_window_dataset( + self._input_dataset.make_dataset_resource(), + self._key_func.captured_inputs, + self._reduce_func.captured_inputs, + self._window_size_func.captured_inputs, + key_func=self._key_func, + reduce_func=self._reduce_func, + window_size_func=self._window_size_func, + output_types=nest.flatten(self.output_types), + output_shapes=nest.flatten(self.output_shapes)) + + +def group_by_window(dataset, + key_func, + reduce_func, + window_size=None, + window_size_func=None): + """Performs a windowed "group-by" operation on this dataset. + + This method maps each consecutive element in this dataset to a key + using `key_func` and groups the elements by key. It then applies + `reduce_func` to at most `window_size_func(key)` elements matching the same + key. All execpt the final window for each key will contain + `window_size_func(key)` elements; the final window may be smaller. + + You may provide either a constant `window_size` or a window size determined by + the key through `window_size_func`. + + Args: + dataset: A `Dataset`. + key_func: A function mapping a nested structure of tensors + (having shapes and types defined by `self.output_shapes` and + `self.output_types`) to a scalar `tf.int64` tensor. + reduce_func: A function mapping a key and a dataset of up to `batch_size` + consecutive elements matching that key to another dataset. + window_size: A `tf.int64` scalar `tf.Tensor`, representing the number of + consecutive elements matching the same key to combine in a single + batch, which will be passed to `reduce_func`. Mutually exclusive with + `window_size_func`. + window_size_func: A function mapping a key to a `tf.int64` scalar + `tf.Tensor`, representing the number of consecutive elements matching + the same key to combine in a single batch, which will be passed to + `reduce_func`. Mutually exclusive with `window_size`. + + Returns: + A `Dataset`. + + Raises: + ValueError: if neither or both of {`window_size`, `window_size_func`} are + passed. + """ + if (window_size is not None and window_size_func or + not (window_size is not None or window_size_func)): + raise ValueError("Must pass either window_size or window_size_func.") + + if window_size is not None: + + def constant_window_func(unused_key): + return ops.convert_to_tensor(window_size, dtype=dtypes.int64) + + window_size_func = constant_window_func + + assert window_size_func is not None + return GroupByWindowDataset(dataset, key_func, reduce_func, window_size_func) diff --git a/tensorflow/core/kernels/group_by_window_dataset_op.cc b/tensorflow/core/kernels/group_by_window_dataset_op.cc index a53e9456ad2..a4f9608b1fa 100644 --- a/tensorflow/core/kernels/group_by_window_dataset_op.cc +++ b/tensorflow/core/kernels/group_by_window_dataset_op.cc @@ -36,20 +36,14 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { graph_def_version_(ctx->graph_def_version()) { OP_REQUIRES_OK(ctx, ctx->GetAttr("key_func", &key_func_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("reduce_func", &reduce_func_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("window_size_func", &window_size_func_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("output_types", &output_types_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("output_shapes", &output_shapes_)); } void MakeDataset(OpKernelContext* ctx, DatasetBase* input, DatasetBase** output) override { - int64 window_size = 0; - OP_REQUIRES_OK( - ctx, ParseScalarArgument(ctx, "window_size", &window_size)); - OP_REQUIRES( - ctx, window_size > 0, - errors::InvalidArgument("Window size must be greater than zero.")); - - // Get captured inputs for the key and reduce functions. + // Get captured inputs for the key, reduce, and window_size functions. OpInputList key_func_other_argument_inputs; OP_REQUIRES_OK(ctx, ctx->input_list("key_func_other_arguments", &key_func_other_argument_inputs)); @@ -67,6 +61,16 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { for (const Tensor& t : reduce_func_other_argument_inputs) { reduce_func_other_arguments.push_back(t); } + OpInputList window_size_func_other_argument_inputs; + OP_REQUIRES_OK(ctx, + ctx->input_list("window_size_func_other_arguments", + &window_size_func_other_argument_inputs)); + std::vector window_size_func_other_arguments; + window_size_func_other_arguments.reserve( + window_size_func_other_argument_inputs.size()); + for (const Tensor& t : window_size_func_other_argument_inputs) { + window_size_func_other_arguments.push_back(t); + } // TODO(mrry): Refactor CapturedFunction to share the runtime // state between multiple functions? std::unique_ptr captured_key_func; @@ -79,24 +83,30 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { ctx, CapturedFunction::Create(ctx, reduce_func_, graph_def_version_, std::move(reduce_func_other_arguments), &captured_reduce_func)); + std::unique_ptr captured_window_size_func; + OP_REQUIRES_OK(ctx, CapturedFunction::Create( + ctx, window_size_func_, graph_def_version_, + std::move(window_size_func_other_arguments), + &captured_window_size_func)); - *output = new Dataset(input, window_size, std::move(captured_key_func), - std::move(captured_reduce_func), output_types_, - output_shapes_); + *output = new Dataset( + input, std::move(captured_key_func), std::move(captured_reduce_func), + std::move(captured_window_size_func), output_types_, output_shapes_); } private: class Dataset : public DatasetBase { public: - Dataset(const DatasetBase* input, int64 window_size, + Dataset(const DatasetBase* input, std::unique_ptr captured_key_func, std::unique_ptr captured_reduce_func, + std::unique_ptr captured_window_size_func, const DataTypeVector& output_types, const std::vector& output_shapes) : input_(input), - window_size_(window_size), captured_key_func_(std::move(captured_key_func)), captured_reduce_func_(std::move(captured_reduce_func)), + captured_window_size_func_(std::move(captured_window_size_func)), output_types_(output_types), output_shapes_(output_shapes) { input_->Ref(); @@ -182,10 +192,44 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { } const int64 key = key_func_output[0].scalar()(); + if (window_sizes_.find(key) == window_sizes_.end()) { + // Run window_size function + FunctionLibraryRuntime::Options opts2; + opts2.step_id = CapturedFunction::generate_step_id(); + opts2.runner = ctx->runner(); + ScopedStepContainer step_container2( + opts2.step_id, [this, ctx](const string& name) { + dataset() + ->captured_window_size_func_->resource_manager() + ->Cleanup(name) + .IgnoreError(); + }); + opts2.step_container = &step_container2; + + // Run the window size function on the key to identify its + // window size. + std::vector window_size_func_output; + TF_RETURN_IF_ERROR(dataset()->captured_window_size_func_->Run( + opts2, key_func_output, &window_size_func_output)); + + if (window_size_func_output.size() != 1 || + window_size_func_output[0].dtype() != DT_INT64 || + window_size_func_output[0].NumElements() != 1) { + // TODO(mrry): Support non-int64 window sizes. + return errors::InvalidArgument( + "`window_size_func` must return a scalar int64."); + } + const int64 window_size = + window_size_func_output[0].scalar()(); + window_sizes_[key] = window_size; + } + + const int64 window_size = window_sizes_[key]; + std::vector>& group = groups_[key]; group.push_back(std::move(next_input_element)); - if (group.size() == dataset()->window_size_) { + if (group.size() == window_size) { TF_RETURN_IF_ERROR(StartFlushingGroup(ctx, key)); break; } @@ -297,6 +341,7 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { bool end_of_input_ GUARDED_BY(mu_) = false; std::map>> groups_ GUARDED_BY(mu_); std::unique_ptr current_group_iterator_ GUARDED_BY(mu_); + std::map window_sizes_ GUARDED_BY(mu_); }; // A resource name for the temporary window dataset that is @@ -304,9 +349,9 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { static constexpr const char* kWindowResourceName = "__window_dataset"; const DatasetBase* const input_; - const int64 window_size_; const std::unique_ptr captured_key_func_; const std::unique_ptr captured_reduce_func_; + const std::unique_ptr captured_window_size_func_; const DataTypeVector output_types_; const std::vector output_shapes_; }; @@ -316,6 +361,7 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel { std::vector output_shapes_; const NameAttrList* key_func_; const NameAttrList* reduce_func_; + const NameAttrList* window_size_func_; }; REGISTER_KERNEL_BUILDER(Name("GroupByWindowDataset").Device(DEVICE_CPU), diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index ad290d123e5..22d4a0056f8 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -10467,8 +10467,8 @@ op { type_list_attr: "Treduce_func_other_arguments" } input_arg { - name: "window_size" - type: DT_INT64 + name: "window_size_func_other_arguments" + type_list_attr: "Twindow_size_func_other_arguments" } output_arg { name: "handle" @@ -10482,6 +10482,10 @@ op { name: "reduce_func" type: "func" } + attr { + name: "window_size_func" + type: "func" + } attr { name: "Tkey_func_other_arguments" type: "list(type)" @@ -10492,6 +10496,11 @@ op { type: "list(type)" has_minimum: true } + attr { + name: "Twindow_size_func_other_arguments" + type: "list(type)" + has_minimum: true + } attr { name: "output_types" type: "list(type)" diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index f6bd5768d7c..37d9a737e29 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -237,12 +237,15 @@ REGISTER_OP("GroupByWindowDataset") .Input("input_dataset: resource") .Input("key_func_other_arguments: Tkey_func_other_arguments") .Input("reduce_func_other_arguments: Treduce_func_other_arguments") - .Input("window_size: int64") + .Input( + "window_size_func_other_arguments: Twindow_size_func_other_arguments") .Output("handle: resource") .Attr("key_func: func") .Attr("reduce_func: func") + .Attr("window_size_func: func") .Attr("Tkey_func_other_arguments: list(type) >= 0") .Attr("Treduce_func_other_arguments: list(type) >= 0") + .Attr("Twindow_size_func_other_arguments: list(type) >= 0") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") .SetShapeFn(shape_inference::ScalarShape) diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 13356e1d8a6..63b7532b334 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -9611,8 +9611,8 @@ op { type_list_attr: "Treduce_func_other_arguments" } input_arg { - name: "window_size" - type: DT_INT64 + name: "window_size_func_other_arguments" + type_list_attr: "Twindow_size_func_other_arguments" } output_arg { name: "handle" @@ -9627,6 +9627,10 @@ op { name: "reduce_func" type: "func" } + attr { + name: "window_size_func" + type: "func" + } attr { name: "Tkey_func_other_arguments" type: "list(type)" @@ -9637,6 +9641,11 @@ op { type: "list(type)" has_minimum: true } + attr { + name: "Twindow_size_func_other_arguments" + type: "list(type)" + has_minimum: true + } attr { name: "output_types" type: "list(type)" From 66ed3d877ff6f2639339e7661e7fa8b2664190a5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 16:41:50 -0700 Subject: [PATCH 11/67] Move most of checkpoint_ops and tests from contrib to core (private in core with public contrib alias). Modified unit tests to use ephemeral checkpoints / vocabularies instead of checked-in testdata. PiperOrigin-RevId: 167068777 --- .../framework/python/ops/checkpoint_ops.py | 436 +---------------- .../python/ops/checkpoint_ops_test.py | 245 ---------- tensorflow/python/BUILD | 23 + tensorflow/python/training/checkpoint_ops.py | 453 ++++++++++++++++++ .../python/training/checkpoint_ops_test.py | 305 ++++++++++++ 5 files changed, 787 insertions(+), 675 deletions(-) create mode 100644 tensorflow/python/training/checkpoint_ops.py create mode 100644 tensorflow/python/training/checkpoint_ops_test.py diff --git a/tensorflow/contrib/framework/python/ops/checkpoint_ops.py b/tensorflow/contrib/framework/python/ops/checkpoint_ops.py index 848e26ab966..26146790b65 100644 --- a/tensorflow/contrib/framework/python/ops/checkpoint_ops.py +++ b/tensorflow/contrib/framework/python/ops/checkpoint_ops.py @@ -17,440 +17,16 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import math - from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_checkpoint_ops from tensorflow.python.ops import init_ops -from tensorflow.python.ops import math_ops - -ops.NotDifferentiable("GenerateVocabRemapping") -ops.NotDifferentiable("LoadAndRemapMatrix") +from tensorflow.python.training import checkpoint_ops -def _load_and_remap_matrix(ckpt_path, - old_tensor_name, - new_row_vocab_offset, - num_rows_to_load, - new_col_vocab_size, - initializer, - old_row_vocab_file=None, - new_row_vocab_file=None, - old_col_vocab_file=None, - new_col_vocab_file=None, - num_row_oov_buckets=0, - num_col_oov_buckets=0, - max_rows_in_memory=-1): - """Loads a 2-D (matrix) `Tensor` from checkpoint. - - Generates 1D-remappings for rows and columns using the - `GenerateVocabRemapping` op, and initializes any anticipated values with the - provided initializer. Then, uses the `LoadAndRemapMatrix` op to create a - matrix that loads existing values from the checkpoint, while filling out - "missing" values with the newly initialized values. See - contrib/framework/ops/checkpoint_ops.cc for more information on the wrapped - functionality (LoadAndRemapMatrix). This wrapper can be used to perform only - row remapping or only col remapping. If only row remapping is desired, - {new,old}_col_vocab_file should be `None`, and vice versa for column - remapping. - - NOTE: This only supports div-partitioning the vocabulary on the 1st dimension - (row axis) via `new_row_vocab_offset`. - - Args: - ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) - from which the old matrix `Tensor` will be loaded. - old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint. - new_row_vocab_offset: A 0-indexed integer representing what line to - start reading at in the new row vocabulary. Used for partitioned - variables. - num_rows_to_load: Number of rows to load for the new vocabulary (note: to - support variable partitioning and partial loading, this does not need to - be the same as the number of entries in `new_row_vocab_file`). - new_col_vocab_size: Number of columns to load - should be the same as the - number of entries in `new_col_vocab_file`, since we don't support - partitioning along the column axis. - initializer: Callable initializer function that accepts a 1-D tensor as the - arg to specify the shape of the returned tensor. Used to initialize - missing values. - old_row_vocab_file: A scalar `Tensor` of type `string` containing the - path to the old row vocabulary file. Can be None, which represents no - remapping on the row axis. - new_row_vocab_file: A scalar `Tensor` of type `string` containing the path - to the new row vocabulary file. Can be None, which represents no remapping - on the row axis - in which case, `new_row_vocab_offset` and - `num_rows_to_load` work under the assumption that the new row vocab is the - same as the old row vocab. - old_col_vocab_file: A scalar `Tensor` of type `string` containing the - path to the old column vocabulary file. Can be None, which represents no - remapping on the column axis. - new_col_vocab_file: A scalar `Tensor` of type `string` containing the path - to the new column vocabulary file. Can be None, which represents no - remapping on the column axis - in which case, `new_col_vocab_size` works - under the assumption that the new col vocab is the same as the old col - vocab. - num_row_oov_buckets: `int` specifying the number of out-of-vocabulary rows - to append. Must be >= 0. - num_col_oov_buckets: `int` specifying the number of out-of-vocabulary - columns to append. Must be >= 0. - max_rows_in_memory: `int` specifying the maximum number of rows to load from - the checkpoint at once. If less than or equal to 0, the entire matrix will - be loaded into memory. Setting this arg trades increased disk reads for - lower memory usage. - - Returns: - A Tensor of shape `[num_rows_to_load + num_row_oov_buckets, - new_col_vocab_size + num_col_oov_buckets]`, with values loaded from the - specified tensor in the checkpoint, and any missing or OOV values - initialized with the given `initializer`. - - Raises: - ValueError: If `num_row_oov_buckets` or `num_col_oov_buckets` < 0. - ValueError: If either `old_row_vocab_file` or `new_row_vocab_file` is - provided, while the other is not. Same for `old_col_vocab_file` and - `new_col_vocab_file`. - ValueError: If neither row vocabs or col vocabs are provided. - """ - if num_row_oov_buckets < 0: - raise ValueError("num_row_oov_buckets must be >= 0, but received %d" % - num_row_oov_buckets) - if num_col_oov_buckets < 0: - raise ValueError("num_col_oov_buckets must be >= 0, but received %d" % - num_col_oov_buckets) - - if bool(old_row_vocab_file) != bool(new_row_vocab_file): - raise ValueError( - "old_row_vocab_file and new_row_vocab_file must both be specified or " - "left unspecified. old_row_vocab_file='{}', new_row_vocab_file='{}'". - format(old_row_vocab_file, new_row_vocab_file)) - if bool(old_col_vocab_file) != bool(new_col_vocab_file): - raise ValueError( - "old_col_vocab_file and new_col_vocab_file must both be specified or " - "left unspecified. old_col_vocab_file='{}', new_col_vocab_file='{}'". - format(old_col_vocab_file, new_col_vocab_file)) - - remap_rows = new_row_vocab_file and old_row_vocab_file - remap_cols = new_col_vocab_file and old_col_vocab_file - if not (remap_rows or remap_cols): - raise ValueError( - "Must provide either row or column vocab files. If no remapping is " - "necessary, consider using `tf.contrib.framework.init_from_checkpoint` " - "instead.") - - num_rows_present = num_rows_to_load - if remap_rows: - row_remapping, num_rows_present = ( - gen_checkpoint_ops._generate_vocab_remapping( # pylint: disable=protected-access - new_vocab_file=new_row_vocab_file, - old_vocab_file=old_row_vocab_file, - new_vocab_offset=new_row_vocab_offset, - num_new_vocab=num_rows_to_load)) - else: - # Even when the rows are not being reordered, we still need to generate a - # remapping to account for initializing partitioned Variables (when - # new_row_vocab_offset is non-zero). - row_remapping = math_ops.range( - new_row_vocab_offset, - new_row_vocab_offset + num_rows_to_load, - dtype=dtypes.int64) - - col_remapping = [] - num_cols_present = new_col_vocab_size - if remap_cols: - col_remapping, num_cols_present = ( - gen_checkpoint_ops._generate_vocab_remapping( # pylint: disable=protected-access - new_vocab_file=new_col_vocab_file, - old_vocab_file=old_col_vocab_file, - new_vocab_offset=0, # Offset is unused for cols (no partitioning). - num_new_vocab=new_col_vocab_size)) - - init_vals = initializer([ - num_rows_to_load * new_col_vocab_size - - num_rows_present * num_cols_present, 1 - ]) - return_tensor = gen_checkpoint_ops._load_and_remap_matrix( # pylint: disable=protected-access - ckpt_path=ckpt_path, - old_tensor_name=old_tensor_name, - row_remapping=row_remapping, - col_remapping=col_remapping, - initializing_values=init_vals, - num_rows=num_rows_to_load, - num_cols=new_col_vocab_size, - max_rows_in_memory=max_rows_in_memory) - - # Add OOV row(s) and column(s). - if num_row_oov_buckets > 0: - init_row_oov_val = initializer([num_row_oov_buckets, new_col_vocab_size]) - init_row_oov_val = ops.convert_to_tensor(init_row_oov_val) - return_tensor = array_ops.concat([return_tensor, init_row_oov_val], 0) - if num_col_oov_buckets > 0: - # We need to add any row OOV to the new column shape. - init_col_oov_val = initializer( - [num_rows_to_load + num_row_oov_buckets, num_col_oov_buckets]) - init_col_oov_val = ops.convert_to_tensor(init_col_oov_val) - return_tensor = array_ops.concat([return_tensor, init_col_oov_val], 1) - - return return_tensor - - -def load_and_remap_matrix_initializer(ckpt_path, - old_tensor_name, - new_row_vocab_size, - new_col_vocab_size, - old_row_vocab_file=None, - new_row_vocab_file=None, - old_col_vocab_file=None, - new_col_vocab_file=None, - num_row_oov_buckets=0, - num_col_oov_buckets=0, - initializer=None, - max_rows_in_memory=-1): - r"""Returns a var initializer for loading and remapping a 2-D (matrix) tensor. - - The returned initializer loads a 2-D (matrix) `Tensor` with name - `old_tensor_name` from the checkpoint at `ckpt_path`. It will reorder the - rows/columns according to the specified vocab files and append additional - out-of-vocabulary rows/columns according to the number of OOV buckets. - - The format of the file at the `{old,new}_{row,col}_vocab_file` path should be - a text file, with each line containing a single entity within the vocabulary. - Let the function `line_of(f, "x")` return the 0-indexed line number of the - entity "x" in file f, and the function `entity_at(f, i)` return the entity at - line i of file f. Then, row i of the new output matrix will be taken from row - `line_of(old_row_vocab_file, entity_at(new_row_vocab_file, i))` of the old - matrix. If any entity in `new_row_vocab_file` is not found in - `old_row_vocab_file`, that row is considered a "missing" row, and its values - will be initialized using the `initializer` arg. The same logic also applies - for the columns. - - For example, assuming that: - - * `old_row_vocab_file` contains "mercury\nvenus\nmars" - * `new_row_vocab_file` contains "venus\njupiter\nmercury" - * `old_col_vocab_file` contains "good\nbetter\nbest" - * `new_col_vocab_file` contains "good\nbest\nfantastic" - * `initializer` returns the natural numbers `[1, 2, 3, 4, ...]` - * `w(i, j)` represents the value from row i, column j of the old matrix - - Then the new output matrix will look like: - - `[[w(1, 0), w(1, 2), 1], - [2, 3, 4], - [w(0, 0), w(0, 2), 5]]` - - If we further specify that: - - * `num_row_oov_buckets` == 2 - * `num_col_oov_buckets` == 1 - - Then the new output matrix will look like: - - `[[w(1, 0), w(1, 2), 1, 12], - [2, 3, 4, 13], - [w(0, 0), w(0, 2), 5, 14], - [6, 7, 8, 15], - [9, 10, 11, 16]]` - - If `{old,new}_row_vocab_file` are None, we assume that the old and new row - vocab files are the same, and no row remapping is done. If - `{old,new}_col_vocab_file` are None, we assume that the old and new column - vocab files are the same, and no column remapping is done. - - The returned initializer only supports div-partitioning along the row axis. It - does not support partitioning along the column axis or mod-partitioning. - - NOTE: When this is used to warm-start variables, client code should use - `tf.lookup.index_table_from_tensor()` like - contrib/layers/python/layers/feature_column.py does, as opposed to - `tf.feature_to_id()` - in order to ensure the underlying lookup tables are the - same. - - Args: - ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) - from which the old matrix `Tensor` will be loaded. - old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint. - new_row_vocab_size: `int` specifying the number of entries in - `new_row_vocab_file`. If no row remapping is needed (no row vocab - provided), this should be equal to the number of rows to load from the old - matrix (which can theoretically be smaller than the number of rows in the - old matrix). - new_col_vocab_size: `int` specifying the number of entries in - `new_col_vocab_file`. If no column remapping is needed (no column vocab - provided), this should be equal to the number of columns in the old - matrix. - old_row_vocab_file: A scalar `Tensor` of type `string` containing the - path to the old row vocabulary file. Can be None, which represents no - remapping on the row axis. - new_row_vocab_file: A scalar `Tensor` of type `string` containing the path - to the new row vocabulary file. Can be None, which represents no remapping - on the row axis. - old_col_vocab_file: A scalar `Tensor` of type `string` containing the - path to the old column vocabulary file. Can be None, which represents no - remapping on the column axis. - new_col_vocab_file: A scalar `Tensor` of type `string` containing the path - to the new column vocabulary file. Can be None, which represents no - remapping on the column axis. - num_row_oov_buckets: `int` specifying the number of out-of-vocabulary rows - to append. Must be >= 0. - num_col_oov_buckets: `int` specifying the number of out-of-vocabulary - columns to append. Must be >= 0. - initializer: Initializer function to initialize missing values. Accepts a - 1-D tensor as the arg to specify the shape of the returned tensor. If - `None`, defaults to using `zeros_initializer()`. - max_rows_in_memory: `int` specifying the maximum number of rows to load from - the checkpoint at once. If less than or equal to 0, the entire matrix will - be loaded into memory. Setting this arg trades increased disk reads for - lower memory usage. - - Returns: - A variable initializer function that should be used to initialize a - (potentially partitioned) `Variable` whose complete shape is - `[new_row_vocab_size + num_row_oov_buckets, new_col_vocab_size + - num_col_oov_buckets]`. - - Raises: - TypeError: If `initializer` is specified but not callable. - """ - if initializer is None: - # TODO(b/25671353): Consider using sqrt(6/(fan_in + fan_out)) instead, from - # Glorot and Bengio, 2010. - initializer = init_ops.zeros_initializer() - - if not callable(initializer): - raise TypeError( - "initializer must be callable, instead of being {} of type {}.".format( - initializer, type(initializer))) - - def _initializer(shape, dtype=dtypes.float32, partition_info=None): - """Variable initializer. - - Args: - shape: Shape of `Tensor` to return. Should include OOV on both axes. - dtype: Must be float32. - partition_info: variable_scope._PartitionInfo. - - Returns: - `Tensor` of shape `shape`. - - Raises: - TypeError: If `dtype` is anything other than float32. - ValueError: For shape mismatch upon invocation. - """ - # Sanity checks. - if dtype != dtypes.float32: - raise TypeError( - "Currently, only float32 is supported. Received dtype: {}".format( - dtype)) - if len(shape) != 2: - raise ValueError("Expected 2-dim shape, but received: {}".format(shape)) - if shape[0] <= 0: - raise ValueError( - "Expected 1st dim of shape to be > 0, but received shape: {}".format( - shape)) - if shape[1] != (new_col_vocab_size + num_col_oov_buckets): - raise ValueError( - "Expected 2nd dim of shape to be new_col_vocab_size ({}) + " - "num_col_oov_buckets ({}) = {}, but received shape: {}".format( - new_col_vocab_size, num_col_oov_buckets, - new_col_vocab_size + num_col_oov_buckets, shape)) - - offset = 0 - if partition_info is not None: - offset = partition_info.single_offset(shape) - - if offset + shape[0] > new_row_vocab_size + num_row_oov_buckets: - raise ValueError( - "Trying to initialize {} additional rows after {} rows have already " - "been initialized, which would exceed expected total row count of " - "new_row_vocab_size ({}) + num_row_oov_buckets ({}) = {}.".format( - shape[0], offset, new_row_vocab_size, num_row_oov_buckets, - new_row_vocab_size + num_row_oov_buckets)) - - row_oov_buckets_to_use = min(shape[0], - max(0, offset + shape[0] - new_row_vocab_size)) - num_rows_to_load = shape[0] - row_oov_buckets_to_use - - return _load_and_remap_matrix( - ckpt_path=ckpt_path, - old_tensor_name=old_tensor_name, - new_row_vocab_offset=offset, - num_rows_to_load=num_rows_to_load, - new_col_vocab_size=new_col_vocab_size, - initializer=initializer, - old_row_vocab_file=old_row_vocab_file, - new_row_vocab_file=new_row_vocab_file, - old_col_vocab_file=old_col_vocab_file, - new_col_vocab_file=new_col_vocab_file, - num_row_oov_buckets=row_oov_buckets_to_use, - num_col_oov_buckets=num_col_oov_buckets, - max_rows_in_memory=max_rows_in_memory) - - return _initializer - - -def load_embedding_initializer(ckpt_path, - embedding_tensor_name, - new_vocab_size, - embedding_dim, - old_vocab_file, - new_vocab_file, - num_oov_buckets=0, - initializer=None, - max_rows_in_memory=-1): - """Returns a variable initializer for loading pre-trained embeddings. - - Wrapper around `load_and_remap_matrix_initializer()` specialized for loading - embedding weights and remapping according to the provided vocab files. See - docs for `load_and_remap_matrix_initializer()` for more details. - - NOTE: Only for use with div-partitioned variables / vocabularies. - - Args: - ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) - from which the old matrix `Tensor` will be loaded. - embedding_tensor_name: Name of the 2-D `Tensor` to load from checkpoint. - new_vocab_size: Number of entries in the new vocab. - embedding_dim: `int` specifying the dimension of the embedding vectors from - the checkpoint. Must match the number of columns in the old embedding - matrix. - old_vocab_file: A scalar `Tensor` of type `string` containing the - path to the old vocabulary file. - new_vocab_file: A scalar `Tensor` of type `string` containing the - path to the new vocabulary file. - num_oov_buckets: `int` specifying the number of out-of-vocabulary - buckets to use. Must be >= 0. - initializer: Initializer function that accepts a 1-D tensor as the arg to - specify the shape of the returned tensor. If `None`, defaults to using - `truncated_normal_initializer()`. - max_rows_in_memory: `int` specifying the maximum number of rows to load from - the checkpoint at once. If less than or equal to 0, the entire matrix will - be loaded into memory. Setting this arg trades increased disk reads for - lower memory usage. - - Returns: - A variable initializer function. - """ - if initializer is None: - # TODO(b/25671353): This should be kept in sync with the stddev used by - # feature_column.py's _EmbeddingColumn. - initializer = init_ops.truncated_normal_initializer( - stddev=1.0 / math.sqrt(embedding_dim)) - - return load_and_remap_matrix_initializer( - ckpt_path=ckpt_path, - old_tensor_name=embedding_tensor_name, - new_row_vocab_size=new_vocab_size, - new_col_vocab_size=embedding_dim, - old_row_vocab_file=old_vocab_file, - new_row_vocab_file=new_vocab_file, - old_col_vocab_file=None, - new_col_vocab_file=None, - num_row_oov_buckets=num_oov_buckets, - num_col_oov_buckets=0, - initializer=initializer, - max_rows_in_memory=max_rows_in_memory) +# pylint: disable=protected-access,line-too-long +load_and_remap_matrix_initializer = checkpoint_ops._load_and_remap_matrix_initializer +# pylint: enable=line-too-long +load_embedding_initializer = checkpoint_ops._load_embedding_initializer +# pylint: enable=protected-access def load_linear_multiclass_bias_initializer(ckpt_path, diff --git a/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py b/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py index a11d373244d..b7b9f5c59e1 100644 --- a/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py +++ b/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py @@ -21,7 +21,6 @@ import os import numpy as np from tensorflow.contrib import framework as contrib_framework -from tensorflow.contrib.framework.python.ops import checkpoint_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -38,250 +37,6 @@ FLAGS = flags.FLAGS _TESTDATA_PATH = 'contrib/framework/testdata' -class LoadAndRemapWrappersTest(test.TestCase): - """Tests for the functionality of the Python wrappers.""" - - def setUp(self): - self.bundle_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), 'bundle_checkpoint') - self.new_feature_vocab_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), 'bundle_checkpoint_vocab.txt') - self.old_feature_vocab_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), - 'bundle_checkpoint_vocab_with_oov.txt') - self.new_class_vocab_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), 'keyword_new.txt') - self.old_class_vocab_file = os.path.join( - test.test_src_dir_path(_TESTDATA_PATH), 'keyword.txt') - self.init_val = 42 - - def _init_val_initializer(shape, dtype=None, partition_info=None): - del dtype, partition_info # Unused by this unit-testing initializer. - return array_ops.tile( - constant_op.constant([[self.init_val]], dtype=dtypes.float32), shape) - - self.initializer = _init_val_initializer - - def test_load_and_remap_matrix(self): - """Tests the end-to-end loading / remapping of weights.""" - # _load_and_remap_matrix() is the generalized wrapper that takes in row and - # column vocabulary files, calls the relevant remappings, and returns the - # weight matrix. Take this example to be linear multi-class by providing - # both row and column vocabularies. - remapped_matrix = checkpoint_ops._load_and_remap_matrix( - new_row_vocab_file=self.new_feature_vocab_file, - old_row_vocab_file=self.old_feature_vocab_file, - num_rows_to_load=4, - new_col_vocab_file=self.new_class_vocab_file, - old_col_vocab_file=self.old_class_vocab_file, - new_col_vocab_size=4, - old_tensor_name='some_scope/embeddings', - ckpt_path=[self.bundle_file], - new_row_vocab_offset=1, - initializer=self.initializer, - num_row_oov_buckets=1, - num_col_oov_buckets=1) - - # [4 in vocab + 1 oov features, 4 in vocab + 1 oov classes]. The offset - # means we read - expected_remapped_matrix = np.concatenate( - [ - np.reshape([18, 34, 50, self.init_val, self.init_val], [5, 1]), - np.reshape([16, 32, 48, self.init_val, self.init_val], [5, 1]), - np.reshape([self.init_val] * 5, [5, 1]), - np.reshape([17, 33, 49, self.init_val, self.init_val], [5, 1]), - np.reshape([self.init_val] * 5, [5, 1]) - ], - axis=1) - - with self.test_session(): - self.assertAllClose(expected_remapped_matrix, remapped_matrix.eval()) - - def test_load_and_remap_output_layer_weight_initializer_linear(self): - """Tests for the output layer initializer in the linear multi-class case.""" - loading_initializer = (contrib_framework.load_and_remap_matrix_initializer( - new_row_vocab_size=5, - new_col_vocab_file=self.new_class_vocab_file, - old_col_vocab_file=self.old_class_vocab_file, - new_col_vocab_size=4, - old_tensor_name='some_scope/embeddings', - ckpt_path=[self.bundle_file], - new_row_vocab_file=self.new_feature_vocab_file, - old_row_vocab_file=self.old_feature_vocab_file, - num_row_oov_buckets=1, - num_col_oov_buckets=1, - initializer=self.initializer)) - - expected_remapped_matrix = np.concatenate( - [ - np.reshape([2, 18, 34, 50, self.init_val, self.init_val], [6, 1]), - np.reshape([0, 16, 32, 48, self.init_val, self.init_val], [6, 1]), - np.reshape([self.init_val] * 6, [6, 1]), - np.reshape([1, 17, 33, 49, self.init_val, self.init_val], [6, 1]), - np.reshape([self.init_val] * 6, [6, 1]) - ], - axis=1) - - # The new weight matrix is of size - # [5 feature vocab + 1 feature OOV, 4 class vocab + 1 class OOV]. Use a - # partitioned variable to confirm that the offset logic works. - remapped_matrix = variable_scope.get_variable( - name='linear/obtained_weight_matrix', - shape=[6, 5], - initializer=loading_initializer, - partitioner=partitioned_variables.fixed_size_partitioner(2)) - - with self.test_session(): - variables.global_variables_initializer().run() - self.assertAllClose(expected_remapped_matrix, - remapped_matrix.as_tensor().eval()) - - def test_load_and_remap_output_layer_weight_initializer_dnn_output(self): - """Tests for the output layer initializer in the DNN output case.""" - loading_initializer = (contrib_framework.load_and_remap_matrix_initializer( - new_row_vocab_size=5, - new_col_vocab_file=self.new_class_vocab_file, - old_col_vocab_file=self.old_class_vocab_file, - new_col_vocab_size=4, - old_tensor_name='some_scope/embeddings', - ckpt_path=[self.bundle_file], - num_col_oov_buckets=1, - initializer=self.initializer)) - - expected_remapped_matrix = np.concatenate( - [ - np.reshape([2, 18, 34, 50, 66], [5, 1]), - np.reshape([0, 16, 32, 48, 64], [5, 1]), - np.reshape([self.init_val] * 5, [5, 1]), - np.reshape([1, 17, 33, 49, 65], [5, 1]), - np.reshape([self.init_val] * 5, [5, 1]) - ], - axis=1) - - # The new weight matrix is of size - # [5-sized input layer, 4 class vocab + 1 class OOV]. - remapped_matrix = variable_scope.get_variable( - name='dnn_output/obtained_weight_matrix', - shape=[5, 5], - initializer=loading_initializer, - partitioner=partitioned_variables.fixed_size_partitioner(2)) - - with self.test_session(): - variables.global_variables_initializer().run() - self.assertAllClose(expected_remapped_matrix, - remapped_matrix.as_tensor().eval()) - - def test_initializer_with_oov_only_partition(self): - """Tests for the output layer initializer where one partition is all OOV.""" - loading_initializer = (contrib_framework.load_and_remap_matrix_initializer( - new_row_vocab_size=5, - new_col_vocab_file=self.new_class_vocab_file, - old_col_vocab_file=self.old_class_vocab_file, - new_col_vocab_size=4, - old_tensor_name='some_scope/embeddings', - ckpt_path=[self.bundle_file], - new_row_vocab_file=self.new_feature_vocab_file, - old_row_vocab_file=self.old_feature_vocab_file, - num_row_oov_buckets=5, - num_col_oov_buckets=1, - initializer=self.initializer)) - - expected_remapped_matrix = np.concatenate( - [ - np.reshape([2, 18, 34, 50] + [self.init_val] * 6, [10, 1]), - np.reshape([0, 16, 32, 48] + [self.init_val] * 6, [10, 1]), - np.reshape([self.init_val] * 10, [10, 1]), - np.reshape([1, 17, 33, 49] + [self.init_val] * 6, [10, 1]), - np.reshape([self.init_val] * 10, [10, 1]), - ], - axis=1) - - # The new weight matrix is of size - # [5 feature vocab + 5 feature OOV, 4 class vocab + 1 class OOV]. The - # second partition has only OOV. - remapped_matrix = variable_scope.get_variable( - name='linear_all_oov/obtained_weight_matrix', - shape=[10, 5], - initializer=loading_initializer, - partitioner=partitioned_variables.fixed_size_partitioner(2)) - - with self.test_session(): - variables.global_variables_initializer().run() - self.assertAllClose(expected_remapped_matrix, - remapped_matrix.as_tensor().eval()) - - def test_load_and_remap_linear_multiclass_initializer_default_init(self): - """Tests where the zeros_initializer default is used for linear.""" - loading_initializer = (contrib_framework.load_and_remap_matrix_initializer( - new_row_vocab_size=5, - new_col_vocab_file=self.new_class_vocab_file, - old_col_vocab_file=self.old_class_vocab_file, - new_col_vocab_size=4, - old_tensor_name='some_scope/embeddings', - ckpt_path=[self.bundle_file], - new_row_vocab_file=self.new_feature_vocab_file, - old_row_vocab_file=self.old_feature_vocab_file, - num_row_oov_buckets=1, - num_col_oov_buckets=1)) - - expected_remapped_matrix = np.concatenate( - [ - np.reshape([2, 18, 34, 50, 0, 0], [6, 1]), - np.reshape([0, 16, 32, 48, 0, 0], [6, 1]), - np.reshape([0] * 6, [6, 1]), - np.reshape([1, 17, 33, 49, 0, 0], [6, 1]), - np.reshape([0] * 6, [6, 1]) - ], - axis=1) - - remapped_matrix = variable_scope.get_variable( - name='linear_init_fallback/obtained_weight_matrix', - shape=[6, 5], - initializer=loading_initializer, - partitioner=partitioned_variables.fixed_size_partitioner(2)) - - with self.test_session(): - variables.global_variables_initializer().run() - self.assertAllClose(expected_remapped_matrix, - remapped_matrix.as_tensor().eval()) - - def test_load_embedding_initializer(self): - """Tests for the load_embedding_initializer wrapper.""" - embedding_loading_initializer = ( - contrib_framework.load_embedding_initializer( - new_vocab_file=self.new_feature_vocab_file, - old_vocab_file=self.old_feature_vocab_file, - new_vocab_size=5, - embedding_dim=16, - embedding_tensor_name='some_scope/embeddings', - ckpt_path=[self.bundle_file], - num_oov_buckets=1, - initializer=self.initializer)) - - expected_remapped_embeddings = np.concatenate( - [ - np.reshape(range(64), [4, 16]), - np.reshape([self.init_val] * 32, [2, 16]), - ], - axis=0) - - # The new weight matrix is of size - # [5 feature vocab + 1 feature OOV, 16 (embedding dimension)], where the - # last vocab row (2nd last row) is newly initialized (wasn't found in - # previous vocab) and the actual last row is OOV and also newly initialized. - # Use a partitioned variable to confirm that the offset logic works. - remapped_embeddings = variable_scope.get_variable( - name='embedding/obtained_embedding_matrix', - shape=[6, 16], - initializer=embedding_loading_initializer, - partitioner=partitioned_variables.fixed_size_partitioner(2)) - - with self.test_session(): - variables.global_variables_initializer().run() - self.assertAllClose(expected_remapped_embeddings, - remapped_embeddings.as_tensor().eval()) - - class LoadMulticlassBiasTest(test.TestCase): """Tests for the load_linear_multiclass_bias_initializer functionality.""" diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 6597889fbcb..b75f79fbf47 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2550,6 +2550,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":array_ops", + ":checkpoint_ops_gen", ":client", ":control_flow_ops", ":data_flow_ops", @@ -3573,6 +3574,28 @@ py_test( ], ) +py_test( + name = "checkpoint_ops_test", + size = "small", + srcs = ["training/checkpoint_ops_test.py"], + srcs_version = "PY2AND3", + tags = ["no_windows"], + deps = [ + ":checkpoint_ops_gen", + ":client", + ":client_testlib", + ":framework_for_generated_wrappers", + ":io_ops", + ":partitioned_variables", + ":platform", + ":pywrap_tensorflow", + ":state_ops", + ":training", + ":variable_scope", + ":variables", + ], +) + py_test( name = "monitored_session_test", size = "small", diff --git a/tensorflow/python/training/checkpoint_ops.py b/tensorflow/python/training/checkpoint_ops.py new file mode 100644 index 00000000000..70460ceb480 --- /dev/null +++ b/tensorflow/python/training/checkpoint_ops.py @@ -0,0 +1,453 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Operations for generating and loading vocab remappings.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_checkpoint_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops + +ops.NotDifferentiable("GenerateVocabRemapping") +ops.NotDifferentiable("LoadAndRemapMatrix") + + +def _load_and_remap_matrix(ckpt_path, + old_tensor_name, + new_row_vocab_offset, + num_rows_to_load, + new_col_vocab_size, + initializer, + old_row_vocab_file=None, + new_row_vocab_file=None, + old_col_vocab_file=None, + new_col_vocab_file=None, + num_row_oov_buckets=0, + num_col_oov_buckets=0, + max_rows_in_memory=-1): + """Loads a 2-D (matrix) `Tensor` from checkpoint. + + Generates 1D-remappings for rows and columns using the + `GenerateVocabRemapping` op, and initializes any anticipated values with the + provided initializer. Then, uses the `LoadAndRemapMatrix` op to create a + matrix that loads existing values from the checkpoint, while filling out + "missing" values with the newly initialized values. See + contrib/framework/ops/checkpoint_ops.cc for more information on the wrapped + functionality (LoadAndRemapMatrix). This wrapper can be used to perform only + row remapping or only col remapping. If only row remapping is desired, + {new,old}_col_vocab_file should be `None`, and vice versa for column + remapping. + + NOTE: This only supports div-partitioning the vocabulary on the 1st dimension + (row axis) via `new_row_vocab_offset`. + + Args: + ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) + from which the old matrix `Tensor` will be loaded. + old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint. + new_row_vocab_offset: A 0-indexed integer representing what line to + start reading at in the new row vocabulary. Used for partitioned + variables. + num_rows_to_load: Number of rows to load for the new vocabulary (note: to + support variable partitioning and partial loading, this does not need to + be the same as the number of entries in `new_row_vocab_file`). + new_col_vocab_size: Number of columns to load - should be the same as the + number of entries in `new_col_vocab_file`, since we don't support + partitioning along the column axis. + initializer: Callable initializer function that accepts a 1-D tensor as the + arg to specify the shape of the returned tensor. Used to initialize + missing values. + old_row_vocab_file: A scalar `Tensor` of type `string` containing the + path to the old row vocabulary file. Can be None, which represents no + remapping on the row axis. + new_row_vocab_file: A scalar `Tensor` of type `string` containing the path + to the new row vocabulary file. Can be None, which represents no remapping + on the row axis - in which case, `new_row_vocab_offset` and + `num_rows_to_load` work under the assumption that the new row vocab is the + same as the old row vocab. + old_col_vocab_file: A scalar `Tensor` of type `string` containing the + path to the old column vocabulary file. Can be None, which represents no + remapping on the column axis. + new_col_vocab_file: A scalar `Tensor` of type `string` containing the path + to the new column vocabulary file. Can be None, which represents no + remapping on the column axis - in which case, `new_col_vocab_size` works + under the assumption that the new col vocab is the same as the old col + vocab. + num_row_oov_buckets: `int` specifying the number of out-of-vocabulary rows + to append. Must be >= 0. + num_col_oov_buckets: `int` specifying the number of out-of-vocabulary + columns to append. Must be >= 0. + max_rows_in_memory: `int` specifying the maximum number of rows to load from + the checkpoint at once. If less than or equal to 0, the entire matrix will + be loaded into memory. Setting this arg trades increased disk reads for + lower memory usage. + + Returns: + A Tensor of shape `[num_rows_to_load + num_row_oov_buckets, + new_col_vocab_size + num_col_oov_buckets]`, with values loaded from the + specified tensor in the checkpoint, and any missing or OOV values + initialized with the given `initializer`. + + Raises: + ValueError: If `num_row_oov_buckets` or `num_col_oov_buckets` < 0. + ValueError: If either `old_row_vocab_file` or `new_row_vocab_file` is + provided, while the other is not. Same for `old_col_vocab_file` and + `new_col_vocab_file`. + ValueError: If neither row vocabs or col vocabs are provided. + """ + if num_row_oov_buckets < 0: + raise ValueError("num_row_oov_buckets must be >= 0, but received %d" % + num_row_oov_buckets) + if num_col_oov_buckets < 0: + raise ValueError("num_col_oov_buckets must be >= 0, but received %d" % + num_col_oov_buckets) + + if bool(old_row_vocab_file) != bool(new_row_vocab_file): + raise ValueError( + "old_row_vocab_file and new_row_vocab_file must both be specified or " + "left unspecified. old_row_vocab_file='{}', new_row_vocab_file='{}'". + format(old_row_vocab_file, new_row_vocab_file)) + if bool(old_col_vocab_file) != bool(new_col_vocab_file): + raise ValueError( + "old_col_vocab_file and new_col_vocab_file must both be specified or " + "left unspecified. old_col_vocab_file='{}', new_col_vocab_file='{}'". + format(old_col_vocab_file, new_col_vocab_file)) + + remap_rows = new_row_vocab_file and old_row_vocab_file + remap_cols = new_col_vocab_file and old_col_vocab_file + if not (remap_rows or remap_cols): + raise ValueError( + "Must provide either row or column vocab files. If no remapping is " + "necessary, consider using `tf.contrib.framework.init_from_checkpoint` " + "instead.") + + num_rows_present = num_rows_to_load + if remap_rows: + row_remapping, num_rows_present = ( + gen_checkpoint_ops._generate_vocab_remapping( # pylint: disable=protected-access + new_vocab_file=new_row_vocab_file, + old_vocab_file=old_row_vocab_file, + new_vocab_offset=new_row_vocab_offset, + num_new_vocab=num_rows_to_load)) + else: + # Even when the rows are not being reordered, we still need to generate a + # remapping to account for initializing partitioned Variables (when + # new_row_vocab_offset is non-zero). + row_remapping = math_ops.range( + new_row_vocab_offset, + new_row_vocab_offset + num_rows_to_load, + dtype=dtypes.int64) + + col_remapping = [] + num_cols_present = new_col_vocab_size + if remap_cols: + col_remapping, num_cols_present = ( + gen_checkpoint_ops._generate_vocab_remapping( # pylint: disable=protected-access + new_vocab_file=new_col_vocab_file, + old_vocab_file=old_col_vocab_file, + new_vocab_offset=0, # Offset is unused for cols (no partitioning). + num_new_vocab=new_col_vocab_size)) + + init_vals = initializer([ + num_rows_to_load * new_col_vocab_size - + num_rows_present * num_cols_present, 1 + ]) + return_tensor = gen_checkpoint_ops._load_and_remap_matrix( # pylint: disable=protected-access + ckpt_path=ckpt_path, + old_tensor_name=old_tensor_name, + row_remapping=row_remapping, + col_remapping=col_remapping, + initializing_values=init_vals, + num_rows=num_rows_to_load, + num_cols=new_col_vocab_size, + max_rows_in_memory=max_rows_in_memory) + + # Add OOV row(s) and column(s). + if num_row_oov_buckets > 0: + init_row_oov_val = initializer([num_row_oov_buckets, new_col_vocab_size]) + init_row_oov_val = ops.convert_to_tensor(init_row_oov_val) + return_tensor = array_ops.concat([return_tensor, init_row_oov_val], 0) + if num_col_oov_buckets > 0: + # We need to add any row OOV to the new column shape. + init_col_oov_val = initializer( + [num_rows_to_load + num_row_oov_buckets, num_col_oov_buckets]) + init_col_oov_val = ops.convert_to_tensor(init_col_oov_val) + return_tensor = array_ops.concat([return_tensor, init_col_oov_val], 1) + + return return_tensor + + +def _load_and_remap_matrix_initializer(ckpt_path, + old_tensor_name, + new_row_vocab_size, + new_col_vocab_size, + old_row_vocab_file=None, + new_row_vocab_file=None, + old_col_vocab_file=None, + new_col_vocab_file=None, + num_row_oov_buckets=0, + num_col_oov_buckets=0, + initializer=None, + max_rows_in_memory=-1): + r"""Returns a var initializer for loading and remapping a 2-D (matrix) tensor. + + The returned initializer loads a 2-D (matrix) `Tensor` with name + `old_tensor_name` from the checkpoint at `ckpt_path`. It will reorder the + rows/columns according to the specified vocab files and append additional + out-of-vocabulary rows/columns according to the number of OOV buckets. + + The format of the file at the `{old,new}_{row,col}_vocab_file` path should be + a text file, with each line containing a single entity within the vocabulary. + Let the function `line_of(f, "x")` return the 0-indexed line number of the + entity "x" in file f, and the function `entity_at(f, i)` return the entity at + line i of file f. Then, row i of the new output matrix will be taken from row + `line_of(old_row_vocab_file, entity_at(new_row_vocab_file, i))` of the old + matrix. If any entity in `new_row_vocab_file` is not found in + `old_row_vocab_file`, that row is considered a "missing" row, and its values + will be initialized using the `initializer` arg. The same logic also applies + for the columns. + + For example, assuming that: + + * `old_row_vocab_file` contains "mercury\nvenus\nmars" + * `new_row_vocab_file` contains "venus\njupiter\nmercury" + * `old_col_vocab_file` contains "good\nbetter\nbest" + * `new_col_vocab_file` contains "good\nbest\nfantastic" + * `initializer` returns the natural numbers `[1, 2, 3, 4, ...]` + * `w(i, j)` represents the value from row i, column j of the old matrix + + Then the new output matrix will look like: + + `[[w(1, 0), w(1, 2), 1], + [2, 3, 4], + [w(0, 0), w(0, 2), 5]]` + + If we further specify that: + + * `num_row_oov_buckets` == 2 + * `num_col_oov_buckets` == 1 + + Then the new output matrix will look like: + + `[[w(1, 0), w(1, 2), 1, 12], + [2, 3, 4, 13], + [w(0, 0), w(0, 2), 5, 14], + [6, 7, 8, 15], + [9, 10, 11, 16]]` + + If `{old,new}_row_vocab_file` are None, we assume that the old and new row + vocab files are the same, and no row remapping is done. If + `{old,new}_col_vocab_file` are None, we assume that the old and new column + vocab files are the same, and no column remapping is done. + + The returned initializer only supports div-partitioning along the row axis. It + does not support partitioning along the column axis or mod-partitioning. + + NOTE: When this is used to warm-start variables, client code should use + `tf.lookup.index_table_from_tensor()` like + contrib/layers/python/layers/feature_column.py does, as opposed to + `tf.feature_to_id()` - in order to ensure the underlying lookup tables are the + same. + + Args: + ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) + from which the old matrix `Tensor` will be loaded. + old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint. + new_row_vocab_size: `int` specifying the number of entries in + `new_row_vocab_file`. If no row remapping is needed (no row vocab + provided), this should be equal to the number of rows to load from the old + matrix (which can theoretically be smaller than the number of rows in the + old matrix). + new_col_vocab_size: `int` specifying the number of entries in + `new_col_vocab_file`. If no column remapping is needed (no column vocab + provided), this should be equal to the number of columns in the old + matrix. + old_row_vocab_file: A scalar `Tensor` of type `string` containing the + path to the old row vocabulary file. Can be None, which represents no + remapping on the row axis. + new_row_vocab_file: A scalar `Tensor` of type `string` containing the path + to the new row vocabulary file. Can be None, which represents no remapping + on the row axis. + old_col_vocab_file: A scalar `Tensor` of type `string` containing the + path to the old column vocabulary file. Can be None, which represents no + remapping on the column axis. + new_col_vocab_file: A scalar `Tensor` of type `string` containing the path + to the new column vocabulary file. Can be None, which represents no + remapping on the column axis. + num_row_oov_buckets: `int` specifying the number of out-of-vocabulary rows + to append. Must be >= 0. + num_col_oov_buckets: `int` specifying the number of out-of-vocabulary + columns to append. Must be >= 0. + initializer: Initializer function to initialize missing values. Accepts a + 1-D tensor as the arg to specify the shape of the returned tensor. If + `None`, defaults to using `zeros_initializer()`. + max_rows_in_memory: `int` specifying the maximum number of rows to load from + the checkpoint at once. If less than or equal to 0, the entire matrix will + be loaded into memory. Setting this arg trades increased disk reads for + lower memory usage. + + Returns: + A variable initializer function that should be used to initialize a + (potentially partitioned) `Variable` whose complete shape is + `[new_row_vocab_size + num_row_oov_buckets, new_col_vocab_size + + num_col_oov_buckets]`. + + Raises: + TypeError: If `initializer` is specified but not callable. + """ + if initializer is None: + # TODO(b/25671353): Consider using sqrt(6/(fan_in + fan_out)) instead, from + # Glorot and Bengio, 2010. + initializer = init_ops.zeros_initializer() + + if not callable(initializer): + raise TypeError( + "initializer must be callable, instead of being {} of type {}.".format( + initializer, type(initializer))) + + def _initializer(shape, dtype=dtypes.float32, partition_info=None): + """Variable initializer. + + Args: + shape: Shape of `Tensor` to return. Should include OOV on both axes. + dtype: Must be float32. + partition_info: variable_scope._PartitionInfo. + + Returns: + `Tensor` of shape `shape`. + + Raises: + TypeError: If `dtype` is anything other than float32. + ValueError: For shape mismatch upon invocation. + """ + # Sanity checks. + if dtype != dtypes.float32: + raise TypeError( + "Currently, only float32 is supported. Received dtype: {}".format( + dtype)) + if len(shape) != 2: + raise ValueError("Expected 2-dim shape, but received: {}".format(shape)) + if shape[0] <= 0: + raise ValueError( + "Expected 1st dim of shape to be > 0, but received shape: {}".format( + shape)) + if shape[1] != (new_col_vocab_size + num_col_oov_buckets): + raise ValueError( + "Expected 2nd dim of shape to be new_col_vocab_size ({}) + " + "num_col_oov_buckets ({}) = {}, but received shape: {}".format( + new_col_vocab_size, num_col_oov_buckets, + new_col_vocab_size + num_col_oov_buckets, shape)) + + offset = 0 + if partition_info is not None: + offset = partition_info.single_offset(shape) + + if offset + shape[0] > new_row_vocab_size + num_row_oov_buckets: + raise ValueError( + "Trying to initialize {} additional rows after {} rows have already " + "been initialized, which would exceed expected total row count of " + "new_row_vocab_size ({}) + num_row_oov_buckets ({}) = {}.".format( + shape[0], offset, new_row_vocab_size, num_row_oov_buckets, + new_row_vocab_size + num_row_oov_buckets)) + + row_oov_buckets_to_use = min(shape[0], + max(0, offset + shape[0] - new_row_vocab_size)) + num_rows_to_load = shape[0] - row_oov_buckets_to_use + + return _load_and_remap_matrix( + ckpt_path=ckpt_path, + old_tensor_name=old_tensor_name, + new_row_vocab_offset=offset, + num_rows_to_load=num_rows_to_load, + new_col_vocab_size=new_col_vocab_size, + initializer=initializer, + old_row_vocab_file=old_row_vocab_file, + new_row_vocab_file=new_row_vocab_file, + old_col_vocab_file=old_col_vocab_file, + new_col_vocab_file=new_col_vocab_file, + num_row_oov_buckets=row_oov_buckets_to_use, + num_col_oov_buckets=num_col_oov_buckets, + max_rows_in_memory=max_rows_in_memory) + + return _initializer + + +def _load_embedding_initializer(ckpt_path, + embedding_tensor_name, + new_vocab_size, + embedding_dim, + old_vocab_file, + new_vocab_file, + num_oov_buckets=0, + initializer=None, + max_rows_in_memory=-1): + """Returns a variable initializer for loading pre-trained embeddings. + + Wrapper around `load_and_remap_matrix_initializer()` specialized for loading + embedding weights and remapping according to the provided vocab files. See + docs for `load_and_remap_matrix_initializer()` for more details. + + NOTE: Only for use with div-partitioned variables / vocabularies. + + Args: + ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) + from which the old matrix `Tensor` will be loaded. + embedding_tensor_name: Name of the 2-D `Tensor` to load from checkpoint. + new_vocab_size: Number of entries in the new vocab. + embedding_dim: `int` specifying the dimension of the embedding vectors from + the checkpoint. Must match the number of columns in the old embedding + matrix. + old_vocab_file: A scalar `Tensor` of type `string` containing the + path to the old vocabulary file. + new_vocab_file: A scalar `Tensor` of type `string` containing the + path to the new vocabulary file. + num_oov_buckets: `int` specifying the number of out-of-vocabulary + buckets to use. Must be >= 0. + initializer: Initializer function that accepts a 1-D tensor as the arg to + specify the shape of the returned tensor. If `None`, defaults to using + `truncated_normal_initializer()`. + max_rows_in_memory: `int` specifying the maximum number of rows to load from + the checkpoint at once. If less than or equal to 0, the entire matrix will + be loaded into memory. Setting this arg trades increased disk reads for + lower memory usage. + + Returns: + A variable initializer function. + """ + if initializer is None: + # TODO(b/25671353): This should be kept in sync with the stddev used by + # feature_column.py's _EmbeddingColumn. + initializer = init_ops.truncated_normal_initializer( + stddev=1.0 / math.sqrt(embedding_dim)) + + return _load_and_remap_matrix_initializer( + ckpt_path=ckpt_path, + old_tensor_name=embedding_tensor_name, + new_row_vocab_size=new_vocab_size, + new_col_vocab_size=embedding_dim, + old_row_vocab_file=old_vocab_file, + new_row_vocab_file=new_vocab_file, + old_col_vocab_file=None, + new_col_vocab_file=None, + num_row_oov_buckets=num_oov_buckets, + num_col_oov_buckets=0, + initializer=initializer, + max_rows_in_memory=max_rows_in_memory) diff --git a/tensorflow/python/training/checkpoint_ops_test.py b/tensorflow/python/training/checkpoint_ops_test.py new file mode 100644 index 00000000000..39c4d2911f2 --- /dev/null +++ b/tensorflow/python/training/checkpoint_ops_test.py @@ -0,0 +1,305 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functional tests for Python wrappers around warm-starting.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import partitioned_variables +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables +from tensorflow.python.platform import test +from tensorflow.python.training import checkpoint_ops +from tensorflow.python.training import saver as saver_lib + + +class LoadAndRemapWrappersTest(test.TestCase): + """Tests for the functionality of the Python wrappers.""" + + def setUp(self): + ops.reset_default_graph() + # Create the checkpoint file in a temporary directory. + checkpoint_prefix = os.path.join(self.get_temp_dir(), 'model') + # 0., 1., ..., 79. reshaped into [5, 16]. + initializer = init_ops.constant_initializer( + np.reshape(np.linspace(0.0, 79, 5 * 16), (5, 16))) + with self.test_session() as sess: + with variable_scope.variable_scope('some_scope'): + variable_scope.get_variable(name='embeddings', shape=[5, 16], + initializer=initializer) + sess.run(variables.global_variables_initializer()) + saver = saver_lib.Saver() + saver.save(sess, checkpoint_prefix, global_step=5) + self.checkpoint_file = '{}-5'.format(checkpoint_prefix) + + # Create the vocabulary files. + self.new_feature_vocab_file = os.path.join( + self.get_temp_dir(), 'new_feature_vocab.txt') + with open(self.new_feature_vocab_file, 'w') as f: + f.write('\n'.join(['zero', 'one', 'two', 'three', 'four']) + '\n') + + self.old_feature_vocab_file = os.path.join( + self.get_temp_dir(), 'old_feature_vocab.txt') + with open(self.old_feature_vocab_file, 'w') as f: + f.write('\n'.join(['zero', 'one', 'two', 'three']) + '\n') + + self.new_class_vocab_file = os.path.join( + self.get_temp_dir(), 'new_class_vocab.txt') + with open(self.new_class_vocab_file, 'w') as f: + f.write('\n'.join(['MISSING', 'knitting', 'flask', 'eminem']) + '\n') + + self.old_class_vocab_file = os.path.join( + self.get_temp_dir(), 'old_class_vocab.txt') + with open(self.old_class_vocab_file, 'w') as f: + f.write('\n'.join(['knitting', 'eminem', 'MISSING']) + '\n') + + self.init_val = 42 + + def _init_val_initializer(shape, dtype=None, partition_info=None): + del dtype, partition_info # Unused by this unit-testing initializer. + return array_ops.tile( + constant_op.constant([[self.init_val]], dtype=dtypes.float32), shape) + + self.initializer = _init_val_initializer + + def test_load_and_remap_matrix(self): + """Tests the end-to-end loading / remapping of weights.""" + # _load_and_remap_matrix() is the generalized wrapper that takes in row and + # column vocabulary files, calls the relevant remappings, and returns the + # weight matrix. Take this example to be linear multi-class by providing + # both row and column vocabularies. + remapped_matrix = checkpoint_ops._load_and_remap_matrix( + new_row_vocab_file=self.new_feature_vocab_file, + old_row_vocab_file=self.old_feature_vocab_file, + num_rows_to_load=4, + new_col_vocab_file=self.new_class_vocab_file, + old_col_vocab_file=self.old_class_vocab_file, + new_col_vocab_size=4, + old_tensor_name='some_scope/embeddings', + ckpt_path=[self.checkpoint_file], + new_row_vocab_offset=1, + initializer=self.initializer, + num_row_oov_buckets=1, + num_col_oov_buckets=1) + + # [4 in vocab + 1 oov features, 4 in vocab + 1 oov classes]. The offset + # means we read + expected_remapped_matrix = np.concatenate( + [ + np.reshape([18, 34, 50, self.init_val, self.init_val], [5, 1]), + np.reshape([16, 32, 48, self.init_val, self.init_val], [5, 1]), + np.reshape([self.init_val] * 5, [5, 1]), + np.reshape([17, 33, 49, self.init_val, self.init_val], [5, 1]), + np.reshape([self.init_val] * 5, [5, 1]) + ], + axis=1) + + with self.test_session(): + self.assertAllClose(expected_remapped_matrix, remapped_matrix.eval()) + + def test_load_and_remap_output_layer_weight_initializer_linear(self): + """Tests for the output layer initializer in the linear multi-class case.""" + loading_initializer = (checkpoint_ops._load_and_remap_matrix_initializer( + new_row_vocab_size=5, + new_col_vocab_file=self.new_class_vocab_file, + old_col_vocab_file=self.old_class_vocab_file, + new_col_vocab_size=4, + old_tensor_name='some_scope/embeddings', + ckpt_path=[self.checkpoint_file], + new_row_vocab_file=self.new_feature_vocab_file, + old_row_vocab_file=self.old_feature_vocab_file, + num_row_oov_buckets=1, + num_col_oov_buckets=1, + initializer=self.initializer)) + + expected_remapped_matrix = np.concatenate( + [ + np.reshape([2, 18, 34, 50, self.init_val, self.init_val], [6, 1]), + np.reshape([0, 16, 32, 48, self.init_val, self.init_val], [6, 1]), + np.reshape([self.init_val] * 6, [6, 1]), + np.reshape([1, 17, 33, 49, self.init_val, self.init_val], [6, 1]), + np.reshape([self.init_val] * 6, [6, 1]) + ], + axis=1) + + # The new weight matrix is of size + # [5 feature vocab + 1 feature OOV, 4 class vocab + 1 class OOV]. Use a + # partitioned variable to confirm that the offset logic works. + remapped_matrix = variable_scope.get_variable( + name='linear/obtained_weight_matrix', + shape=[6, 5], + initializer=loading_initializer, + partitioner=partitioned_variables.fixed_size_partitioner(2)) + + with self.test_session(): + variables.global_variables_initializer().run() + self.assertAllClose(expected_remapped_matrix, + remapped_matrix.as_tensor().eval()) + + def test_load_and_remap_output_layer_weight_initializer_dnn_output(self): + """Tests for the output layer initializer in the DNN output case.""" + loading_initializer = (checkpoint_ops._load_and_remap_matrix_initializer( + new_row_vocab_size=5, + new_col_vocab_file=self.new_class_vocab_file, + old_col_vocab_file=self.old_class_vocab_file, + new_col_vocab_size=4, + old_tensor_name='some_scope/embeddings', + ckpt_path=[self.checkpoint_file], + num_col_oov_buckets=1, + initializer=self.initializer)) + + expected_remapped_matrix = np.concatenate( + [ + np.reshape([2, 18, 34, 50, 66], [5, 1]), + np.reshape([0, 16, 32, 48, 64], [5, 1]), + np.reshape([self.init_val] * 5, [5, 1]), + np.reshape([1, 17, 33, 49, 65], [5, 1]), + np.reshape([self.init_val] * 5, [5, 1]) + ], + axis=1) + + # The new weight matrix is of size + # [5-sized input layer, 4 class vocab + 1 class OOV]. + remapped_matrix = variable_scope.get_variable( + name='dnn_output/obtained_weight_matrix', + shape=[5, 5], + initializer=loading_initializer, + partitioner=partitioned_variables.fixed_size_partitioner(2)) + + with self.test_session(): + variables.global_variables_initializer().run() + self.assertAllClose(expected_remapped_matrix, + remapped_matrix.as_tensor().eval()) + + def test_initializer_with_oov_only_partition(self): + """Tests for the output layer initializer where one partition is all OOV.""" + loading_initializer = (checkpoint_ops._load_and_remap_matrix_initializer( + new_row_vocab_size=5, + new_col_vocab_file=self.new_class_vocab_file, + old_col_vocab_file=self.old_class_vocab_file, + new_col_vocab_size=4, + old_tensor_name='some_scope/embeddings', + ckpt_path=[self.checkpoint_file], + new_row_vocab_file=self.new_feature_vocab_file, + old_row_vocab_file=self.old_feature_vocab_file, + num_row_oov_buckets=5, + num_col_oov_buckets=1, + initializer=self.initializer)) + + expected_remapped_matrix = np.concatenate( + [ + np.reshape([2, 18, 34, 50] + [self.init_val] * 6, [10, 1]), + np.reshape([0, 16, 32, 48] + [self.init_val] * 6, [10, 1]), + np.reshape([self.init_val] * 10, [10, 1]), + np.reshape([1, 17, 33, 49] + [self.init_val] * 6, [10, 1]), + np.reshape([self.init_val] * 10, [10, 1]), + ], + axis=1) + + # The new weight matrix is of size + # [5 feature vocab + 5 feature OOV, 4 class vocab + 1 class OOV]. The + # second partition has only OOV. + remapped_matrix = variable_scope.get_variable( + name='linear_all_oov/obtained_weight_matrix', + shape=[10, 5], + initializer=loading_initializer, + partitioner=partitioned_variables.fixed_size_partitioner(2)) + + with self.test_session(): + variables.global_variables_initializer().run() + self.assertAllClose(expected_remapped_matrix, + remapped_matrix.as_tensor().eval()) + + def test_load_and_remap_linear_multiclass_initializer_default_init(self): + """Tests where the zeros_initializer default is used for linear.""" + loading_initializer = (checkpoint_ops._load_and_remap_matrix_initializer( + new_row_vocab_size=5, + new_col_vocab_file=self.new_class_vocab_file, + old_col_vocab_file=self.old_class_vocab_file, + new_col_vocab_size=4, + old_tensor_name='some_scope/embeddings', + ckpt_path=[self.checkpoint_file], + new_row_vocab_file=self.new_feature_vocab_file, + old_row_vocab_file=self.old_feature_vocab_file, + num_row_oov_buckets=1, + num_col_oov_buckets=1)) + + expected_remapped_matrix = np.concatenate( + [ + np.reshape([2, 18, 34, 50, 0, 0], [6, 1]), + np.reshape([0, 16, 32, 48, 0, 0], [6, 1]), + np.reshape([0] * 6, [6, 1]), + np.reshape([1, 17, 33, 49, 0, 0], [6, 1]), + np.reshape([0] * 6, [6, 1]) + ], + axis=1) + + remapped_matrix = variable_scope.get_variable( + name='linear_init_fallback/obtained_weight_matrix', + shape=[6, 5], + initializer=loading_initializer, + partitioner=partitioned_variables.fixed_size_partitioner(2)) + + with self.test_session(): + variables.global_variables_initializer().run() + self.assertAllClose(expected_remapped_matrix, + remapped_matrix.as_tensor().eval()) + + def test_load_embedding_initializer(self): + """Tests for the load_embedding_initializer wrapper.""" + embedding_loading_initializer = (checkpoint_ops._load_embedding_initializer( + new_vocab_file=self.new_feature_vocab_file, + old_vocab_file=self.old_feature_vocab_file, + new_vocab_size=5, + embedding_dim=16, + embedding_tensor_name='some_scope/embeddings', + ckpt_path=[self.checkpoint_file], + num_oov_buckets=1, + initializer=self.initializer)) + + expected_remapped_embeddings = np.concatenate( + [ + np.reshape(range(64), [4, 16]), + np.reshape([self.init_val] * 32, [2, 16]), + ], + axis=0) + + # The new weight matrix is of size + # [5 feature vocab + 1 feature OOV, 16 (embedding dimension)], where the + # last vocab row (2nd last row) is newly initialized (wasn't found in + # previous vocab) and the actual last row is OOV and also newly initialized. + # Use a partitioned variable to confirm that the offset logic works. + remapped_embeddings = variable_scope.get_variable( + name='embedding/obtained_embedding_matrix', + shape=[6, 16], + initializer=embedding_loading_initializer, + partitioner=partitioned_variables.fixed_size_partitioner(2)) + + with self.test_session(): + variables.global_variables_initializer().run() + self.assertAllClose(expected_remapped_embeddings, + remapped_embeddings.as_tensor().eval()) + + +if __name__ == '__main__': + test.main() From 4aa83760c636085349da57e462acabedc8725937 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 16:44:47 -0700 Subject: [PATCH 12/67] Don't try to evaluate a Tensor as a boolean. PiperOrigin-RevId: 167069142 --- tensorflow/python/layers/convolutional.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index e46c0b29542..41c67743b6d 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -172,7 +172,7 @@ class _Conv(base.Layer): padding=self.padding.upper(), data_format=utils.convert_data_format(self.data_format, self.rank + 2)) - if self.bias is not None: + if self.use_bias: if self.data_format == 'channels_first': if self.rank == 1: # nn.bias_add does not accept a 1D input tensor. @@ -989,7 +989,7 @@ class SeparableConv2D(Conv2D): rate=self.dilation_rate, data_format=utils.convert_data_format(self.data_format, ndim=4)) - if self.bias is not None: + if self.use_bias: outputs = nn.bias_add( outputs, self.bias, @@ -1308,7 +1308,7 @@ class Conv2DTranspose(Conv2D): stride_w) outputs.set_shape(out_shape) - if self.bias: + if self.use_bias: outputs = nn.bias_add( outputs, self.bias, @@ -1611,7 +1611,7 @@ class Conv3DTranspose(Conv3D): stride_w) outputs.set_shape(out_shape) - if self.bias: + if self.use_bias: outputs_shape = outputs.shape.as_list() if self.data_format == 'channels_first': outputs_4d = array_ops.reshape(outputs, [ From d22ed610a0c7395a0e3ea8349b0da0e8afa62fe1 Mon Sep 17 00:00:00 2001 From: Ali Yahya Date: Wed, 30 Aug 2017 17:29:54 -0700 Subject: [PATCH 13/67] Adds tape.watch_variable(v) where v is any ResourceVariable. PiperOrigin-RevId: 167074863 --- tensorflow/python/eager/backprop_test.py | 2 +- tensorflow/python/eager/tape.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 010124ed56a..429eeabb421 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -85,7 +85,7 @@ class BackpropTest(test.TestCase): initial_value=tensor.Tensor(1.0), name='x') def fn(): - tape.watch(x.handle) + tape.watch_variable(x) b = tensor.Tensor(2.0) c = math_ops.add(x.value(), b) return math_ops.add(c, tensor.Tensor(3.0)) diff --git a/tensorflow/python/eager/tape.py b/tensorflow/python/eager/tape.py index 4d09db73c97..9cd29f630df 100644 --- a/tensorflow/python/eager/tape.py +++ b/tensorflow/python/eager/tape.py @@ -151,6 +151,15 @@ def watch(tensor): return tensor +def watch_variable(resource_variable): + """Marks this ResourceVariable to be watched by all tapes in the stack. + + Args: + resource_variable: A ResourceVariable to be watched. + """ + watch(resource_variable.handle) # py-lint: disable=protected-access + + def pop_tape(): """Pops the top tape in the stack, if any.""" if _tape_stack.stack: From 4807158b3df7fdfd1a597c581e7ba9f894b1b256 Mon Sep 17 00:00:00 2001 From: James Qin Date: Wed, 30 Aug 2017 17:36:00 -0700 Subject: [PATCH 14/67] Add functional cudnn_rnn_ops Will deprecate class-style cudnn_rnn_ops soon. As a transition plan, have the former depend on the latter for now. PiperOrigin-RevId: 167075637 --- .../cudnn_rnn/python/ops/cudnn_rnn_ops.py | 555 ++++++++++++++++-- 1 file changed, 509 insertions(+), 46 deletions(-) diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py index 694bd507d97..bc4fd10cac6 100644 --- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py +++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py @@ -716,6 +716,482 @@ _cudnn_rnn_common_doc_string = """ """ +def _check_direction(direction): + if direction not in (CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION): + raise ValueError("Invalid direction: %s, expect %s or %s" % + (direction, CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION)) + + +def _check_rnn_mode(rnn_mode): + if rnn_mode not in (CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_TANH, CUDNN_RNN_RELU): + raise ValueError("Invalid rnn_mode: %s, expect one of (%s, %s, %s, %s)" % + (rnn_mode, CUDNN_LSTM, CUDNN_GRU, CUDNN_RNN_TANH, + CUDNN_RNN_RELU)) + + +def _get_seed(seed): + seed, seed2 = random_seed.get_seed(seed) + if seed is None and seed2 is None: + seed, seed2 = 0, 0 + return seed, seed2 + + +def _get_num_params(rnn_mode, num_layers, direction): + """Return num params for given Cudnn config.""" + if rnn_mode == CUDNN_LSTM: + num_params_per_layer = 8 + elif rnn_mode == CUDNN_GRU: + num_params_per_layer = 6 + elif rnn_mode in (CUDNN_RNN_RELU, CUDNN_RNN_TANH): + num_params_per_layer = 2 + else: + raise ValueError("Invalid \'rnn_mode\': %s", rnn_mode) + num_params = num_layers * num_params_per_layer + if direction != CUDNN_RNN_UNIDIRECTION: + num_params *= 2 + return num_params + + +def _cudnn_rnn(inputs, + input_h, + input_c, + params, + is_training, + rnn_mode, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dropout=0., + seed=0, + name=None): + """Cudnn RNN. + + Args: + inputs: the input sequence to the RNN model. A Tensor of shape [?, + batch_size, input_size]. + input_h: the initial hidden state for h. A Tensor of shape [num_layers, + batch_size, num_units]. + input_c: the initial hidden state for c. This is only relevant for LSTM. + A Tensor of the same shape as input_h. + params: the parameter buffer created for this model. + is_training: whether this operation will be used in training or inference + rnn_mode: one of ('lstm', 'gru', 'rnn_relu', 'rnn_tanh'). + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + outputs, output_h, output_c + """ + _check_rnn_mode(rnn_mode) + _check_direction(direction) + seed, seed2 = random_seed.get_seed(seed) + outputs, output_h, output_c, _ = gen_cudnn_rnn_ops.cudnn_rnn( + input=inputs, + input_h=input_h, + input_c=input_c, + params=params, + is_training=is_training, + rnn_mode=rnn_mode, + input_mode=input_mode, + direction=direction, + dropout=dropout, + seed=seed, + seed2=seed2, + name=name) + return (outputs, output_h, output_c) + + +def cudnn_lstm(inputs, + input_h, + input_c, + params, + is_training, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dropout=0., + seed=0, + name=None): + """Cudnn LSTM. + + Args: + inputs: the input sequence to the RNN model. A Tensor of shape [?, + batch_size, input_size]. + input_h: the initial hidden state for h. A Tensor of shape [num_layers, + batch_size, num_units]. + input_c: the initial hidden state for c. This is only relevant for LSTM. + A Tensor of the same shape as input_h. + params: the parameter buffer created for this model. + is_training: whether this operation will be used in training or inference + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + outputs, output_h, output_c + """ + return _cudnn_rnn(inputs, input_h, input_c, params, is_training, CUDNN_LSTM, + input_mode, direction, dropout, seed, name) + + +def _cudnn_rnn_no_input_c(inputs, + input_h, + params, + is_training, + rnn_mode, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dropout=0., + seed=0, + name=None): + """Cudnn RNN w/o input_c. + + Args: + inputs: the input sequence to the RNN model. A Tensor of shape [?, + batch_size, input_size]. + input_h: the initial hidden state for h. A Tensor of shape [num_layers, + batch_size, num_units]. + params: the parameter buffer created for this model. + is_training: whether this operation will be used in training or inference + rnn_mode: one of ('lstm', 'gru', 'rnn_relu', 'rnn_tanh'). + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + outputs, output_h + """ + input_c = array_ops.constant([], dtype=input_h.dtype) + outputs, output_h, _ = _cudnn_rnn(inputs, input_h, input_c, params, + is_training, rnn_mode, input_mode, + direction, dropout, seed, name) + return outputs, output_h + + +def cudnn_gru(inputs, + input_h, + params, + is_training, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dropout=0., + seed=0, + name=None): + """Cudnn GRU. + + Args: + inputs: the input sequence to the RNN model. A Tensor of shape [?, + batch_size, input_size]. + input_h: the initial hidden state for h. A Tensor of shape [num_layers, + batch_size, num_units]. + params: the parameter buffer created for this model. + is_training: whether this operation will be used in training or inference + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + outputs, output_h + """ + return _cudnn_rnn_no_input_c(inputs, input_h, params, is_training, CUDNN_GRU, + input_mode, direction, dropout, seed, name) + + +def cudnn_rnn_relu(inputs, + input_h, + params, + is_training, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dropout=0., + seed=0, + name=None): + """Cudnn RNN Relu. + + Args: + inputs: the input sequence to the RNN model. A Tensor of shape [?, + batch_size, input_size]. + input_h: the initial hidden state for h. A Tensor of shape [num_layers, + batch_size, num_units]. + params: the parameter buffer created for this model. + is_training: whether this operation will be used in training or inference + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + outputs, output_h + """ + return _cudnn_rnn_no_input_c(inputs, input_h, params, is_training, + CUDNN_RNN_RELU, input_mode, direction, dropout, + seed, name) + + +def cudnn_rnn_tanh(inputs, + input_h, + params, + is_training, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dropout=0., + seed=0, + name=None): + """Cudnn RNN Tanh. + + Args: + inputs: the input sequence to the RNN model. A Tensor of shape [?, + batch_size, input_size]. + input_h: the initial hidden state for h. A Tensor of shape [num_layers, + batch_size, num_units]. + params: the parameter buffer created for this model. + is_training: whether this operation will be used in training or inference + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + outputs, output_h + """ + return _cudnn_rnn_no_input_c(inputs, input_h, params, is_training, + CUDNN_RNN_TANH, input_mode, direction, dropout, + seed, name) + + +def cudnn_rnn_params_to_canonical(rnn_mode, + num_layers, + num_units, + input_size, + params, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dropout=0, + seed=0, + name=None): + """Convert cudnn opaque params to canonical. + + Args: + rnn_mode: a string specifies the mode, under which this RNN model runs. + Could be either 'lstm', 'gru', 'rnn_tanh' or 'rnn_relu'. + num_layers: the number of layers for the RNN model. + num_units: the number of units within the RNN model. + input_size: the size of the input, it could be different from the + num_units. + params: opaque cudnn params var. + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + weights list and bias list + Raises: + ValueError: if rnn_mode or direction is invalid. + """ + + _check_rnn_mode(rnn_mode) + _check_direction(direction) + num_params = _get_num_params(rnn_mode, num_layers, direction) + seed, seed2 = random_seed.get_seed(seed) + weights, biases = gen_cudnn_rnn_ops.cudnn_rnn_params_to_canonical( + rnn_mode=rnn_mode, + num_layers=num_layers, + num_units=num_units, + input_size=input_size, + params=params, + input_mode=input_mode, + direction=direction, + dropout=dropout, + seed=seed, + seed2=seed2, + num_params=num_params, + name=name) + return weights, biases + + +def cudnn_rnn_canonical_to_params(rnn_mode, + num_layers, + num_units, + input_size, + weights, + biases, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dropout=0, + seed=0, + name=None): + """Converts params from the canonical format to a specific format of cuDNN. + + Args: + rnn_mode: a string specifies the mode, under which this RNN model runs. + Could be either 'lstm', 'gru', 'rnn_tanh' or 'rnn_relu'. + num_layers: the number of layers for the RNN model. + num_units: the number of units within the RNN model. + input_size: the size of the input, it could be different from the + num_units. + weights: a Tensor for weight parameters. + biases: a Tensor for bias parameters. + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + an opaque Cudnn param. + Raises: + ValueError: if rnn_mode or direction is invalid. + """ + _check_rnn_mode(rnn_mode) + _check_direction(direction) + seed, seed2 = random_seed.get_seed(seed) + return gen_cudnn_rnn_ops.cudnn_rnn_canonical_to_params( + rnn_mode=rnn_mode, + num_layers=num_layers, + num_units=num_units, + input_size=input_size, + weights=weights, + biases=biases, + input_mode=input_mode, + direction=direction, + dropout=dropout, + seed=seed, + seed2=seed2, + name=name) + + +def cudnn_opaque_params_size(rnn_mode, + num_layers, + num_units, + input_size, + input_mode=CUDNN_INPUT_LINEAR_MODE, + direction=CUDNN_RNN_UNIDIRECTION, + dtype=dtypes.float32, + dropout=0, + seed=0, + name=None): + """Returns opaque params size for specific Cudnn config. + + Args: + rnn_mode: a string specifies the mode, under which this RNN model runs. + Could be either 'lstm', 'gru', 'rnn_tanh' or 'rnn_relu'. + num_layers: the number of layers for the RNN model. + num_units: the number of units within the RNN model. + input_size: the size of the input, it could be different from the + num_units. + input_mode: indicate whether there is a linear projection between the + input and the actual computation before the first layer. It could be + 'linear_input', 'skip_input' or 'auto_select'. + 'linear_input' (default) always applies a linear projection of input + onto RNN hidden state. (standard RNN behavior). + 'skip_input' is only allowed when input_size == num_units; + 'auto_select' implies 'skip_input' when input_size == num_units; + otherwise, it implies 'linear_input'. + direction: the direction model that the model operates. Could be either + 'unidirectional' or 'bidirectional' + dtype: one of tf.float32 or tf.float64. + dropout: whether to enable dropout. With it is 0, dropout is disabled. + seed: the op seed used for initializing dropout. See @{tf.set_random_seed} + for behavior. + name: name of the operation. + Returns: + a int, size of Cudnn opaque params. + Raises: + ValueError: if rnn_mode or direction is invalid. + """ + _check_rnn_mode(rnn_mode) + _check_direction(direction) + seed, seed2 = random_seed.get_seed(seed) + return gen_cudnn_rnn_ops.cudnn_rnn_params_size( + rnn_mode=rnn_mode, + num_layers=num_layers, + num_units=num_units, + input_size=input_size, + T=dtype, + S=dtypes.int32, + dropout=dropout, + seed=seed, + seed2=seed2, + input_mode=input_mode, + direction=direction, + name=name)[0] + + class _CudnnRNN(object): """Creates an RNN model using the underlying Cudnn implementation. @@ -761,9 +1237,6 @@ class _CudnnRNN(object): Raises: ValueError: if direction is invalid. """ - if direction not in (CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION): - raise ValueError("Invalid direction: %s, expect %s or %s", - direction, CUDNN_RNN_UNIDIRECTION, CUDNN_RNN_BIDIRECTION) self._num_layers = num_layers self._num_units = num_units self._input_size = input_size @@ -772,10 +1245,7 @@ class _CudnnRNN(object): self._direction = direction self._dtype = dtype self._dropout = dropout - # get graph and op seed. - self._seed, self._seed2 = random_seed.get_seed(seed) - if self._seed is None and self._seed2 is None: - self._seed, self._seed2 = 0, 0 + self._seed = seed @property def input_mode(self): @@ -807,18 +1277,16 @@ class _CudnnRNN(object): Returns: The calculated parameter buffer size. """ - return gen_cudnn_rnn_ops.cudnn_rnn_params_size( + return cudnn_opaque_params_size( + rnn_mode=self._rnn_mode, num_layers=self._num_layers, num_units=self._num_units, input_size=self._input_size, - T=self._dtype, - S=dtypes.int32, + dtype=self._dtype, dropout=self._dropout, seed=self._seed, - seed2=self._seed2, - rnn_mode=self._rnn_mode, input_mode=self._input_mode, - direction=self._direction)[0] + direction=self._direction) def __call__(self, input_data, input_h, input_c, params, is_training=True): """Runs the forward step for the RNN model. @@ -837,22 +1305,17 @@ class _CudnnRNN(object): output_h: the final state for h. output_c: the final state for c. This is only relevant for LSTM. """ - if self._rnn_mode != CUDNN_LSTM: - # For model that doesn't take input_c, replace with a dummy tensor. - input_c = array_ops.constant([], dtype=self._dtype) - output, output_h, output_c, _ = gen_cudnn_rnn_ops.cudnn_rnn( - input=input_data, - input_h=input_h, - input_c=input_c, - params=params, - rnn_mode=self._rnn_mode, + return _cudnn_rnn( + input_data, + input_h, + input_c, + params, + is_training, + self._rnn_mode, input_mode=self._input_mode, direction=self._direction, dropout=self._dropout, - seed=self._seed, - seed2=self._seed2, - is_training=is_training) - return (output, output_h, output_c) + seed=self._seed) def params_to_canonical(self, params): """Converts params from a specific format of cuDNN to the canonical format. @@ -863,22 +1326,16 @@ class _CudnnRNN(object): Returns: A function for the specific-to-canonical conversion. """ - num_params = self._num_layers * self._NUM_PARAMS_PER_LAYER - if self._direction != CUDNN_RNN_UNIDIRECTION: - num_params *= 2 - weights, biases = gen_cudnn_rnn_ops.cudnn_rnn_params_to_canonical( + return cudnn_rnn_params_to_canonical( + rnn_mode=self._rnn_mode, num_layers=self._num_layers, num_units=self._num_units, input_size=self._input_size, params=params, - dropout=self._dropout, - seed=self._seed, - seed2=self._seed2, - num_params=num_params, - rnn_mode=self._rnn_mode, input_mode=self._input_mode, - direction=self._direction) - return weights, biases + direction=self._direction, + dropout=self._dropout, + seed=self._seed) def canonical_to_params(self, weights, biases): """Converts params from the canonical format to a specific format of cuDNN. @@ -890,18 +1347,17 @@ class _CudnnRNN(object): Returns: A function for the canonical-to-params-to-specific conversion.. """ - return gen_cudnn_rnn_ops.cudnn_rnn_canonical_to_params( + return cudnn_rnn_canonical_to_params( + rnn_mode=self._rnn_mode, num_layers=self._num_layers, num_units=self._num_units, input_size=self._input_size, weights=weights, biases=biases, - dropout=self._dropout, - seed=self._seed, - seed2=self._seed2, - rnn_mode=self._rnn_mode, input_mode=self._input_mode, - direction=self._direction) + direction=self._direction, + dropout=self._dropout, + seed=self._seed) class CudnnLSTM(_CudnnRNN): @@ -1036,9 +1492,16 @@ class _CudnnRNNNoInputC(_CudnnRNN): output: the output sequuence. output_h: the final state for h. """ - output, output_h, _ = super(_CudnnRNNNoInputC, self).__call__( - input_data, input_h, None, params, is_training=is_training) - return (output, output_h) + return _cudnn_rnn_no_input_c( + input_data, + input_h, + params, + is_training, + self._rnn_mode, + input_mode=self._input_mode, + direction=self._direction, + dropout=self._dropout, + seed=self._seed) class CudnnGRU(_CudnnRNNNoInputC): From 62742c14779062dc105b098142fff2c88c3a75a3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 17:55:02 -0700 Subject: [PATCH 15/67] Add code to extract feature importance. PiperOrigin-RevId: 167077754 --- .../estimator_batch/custom_export_strategy.py | 67 +++++++++++++++++-- .../custom_export_strategy_test.py | 18 ++++- 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py index c377c50e9fe..a8b60460c8f 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py @@ -18,6 +18,9 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections +import os + from tensorflow.contrib.boosted_trees.proto import tree_config_pb2 from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch from tensorflow.contrib.decision_trees.proto import generic_tree_model_extensions_pb2 @@ -26,18 +29,21 @@ from tensorflow.contrib.learn.python.learn import export_strategy from tensorflow.contrib.learn.python.learn.utils import saved_model_export_utils from tensorflow.python.client import session as tf_session from tensorflow.python.framework import ops +from tensorflow.python.platform import gfile from tensorflow.python.saved_model import loader as saved_model_loader from tensorflow.python.saved_model import tag_constants -def make_custom_export_strategy(name, convert_fn, feature_columns, +def make_custom_export_strategy(name, + convert_fn, + feature_columns, export_input_fn): """Makes custom exporter of GTFlow tree format. Args: name: A string, for the name of the export strategy. convert_fn: A function that converts the tree proto to desired format and - saves it to the desired location. + saves it to the desired location. Can be None to skip conversion. feature_columns: A list of feature columns. export_input_fn: A function that takes no arguments and returns an `InputFnOps`. @@ -68,9 +74,22 @@ def make_custom_export_strategy(name, convert_fn, feature_columns, dtec = tree_config_pb2.DecisionTreeEnsembleConfig() dtec.ParseFromString(dfec_str) # Export the result in the same folder as the saved model. - convert_fn(dtec, sorted_feature_names, len(dense_floats), - len(sparse_float_indices), len(sparse_int_indices), - result_dir, eval_result) + if convert_fn: + convert_fn(dtec, sorted_feature_names, + len(dense_floats), + len(sparse_float_indices), + len(sparse_int_indices), result_dir, eval_result) + feature_importances = _get_feature_importances( + dtec, sorted_feature_names, + len(dense_floats), + len(sparse_float_indices), len(sparse_int_indices)) + sorted_by_importance = sorted( + feature_importances.items(), key=lambda x: -x[1]) + assets_dir = os.path.join(result_dir, "assets.extra") + gfile.MakeDirs(assets_dir) + with gfile.GFile(os.path.join(assets_dir, "feature_importances"), + "w") as f: + f.write("\n".join("%s, %f" % (k, v) for k, v in sorted_by_importance)) return result_dir return export_strategy.ExportStrategy(name, export_fn) @@ -157,3 +176,41 @@ def convert_to_universal_format(dtec, sorted_feature_names, node.left_child_id.value = split.left_id node.right_child_id.value = split.right_id return model_and_features + + +def _get_feature_importances(dtec, feature_names, num_dense_floats, + num_sparse_float, num_sparse_int): + """Export the feature importance per feature column.""" + del num_sparse_int # Unused. + sums = collections.defaultdict(lambda: 0) + for tree_idx in range(len(dtec.trees)): + tree = dtec.trees[tree_idx] + for tree_node in tree.nodes: + node_type = tree_node.WhichOneof("node") + if node_type == "dense_float_binary_split": + split = tree_node.dense_float_binary_split + split_column = feature_names[split.feature_column] + elif node_type == "sparse_float_binary_split_default_left": + split = tree_node.sparse_float_binary_split_default_left.split + split_column = feature_names[split.feature_column + num_dense_floats] + elif node_type == "sparse_float_binary_split_default_right": + split = tree_node.sparse_float_binary_split_default_right.split + split_column = feature_names[split.feature_column + num_dense_floats] + elif node_type == "categorical_id_binary_split": + split = tree_node.categorical_id_binary_split + split_column = feature_names[split.feature_column + num_dense_floats + + num_sparse_float] + elif node_type == "categorical_id_set_membership_binary_split": + split = tree_node.categorical_id_set_membership_binary_split + split_column = feature_names[split.feature_column + num_dense_floats + + num_sparse_float] + elif node_type == "leaf": + assert tree_node.node_metadata.gain == 0 + continue + else: + raise ValueError("Unexpected split type %s", node_type) + # Apply shrinkage factor. It is important since it is not always uniform + # across different trees. + sums[split_column] += ( + tree_node.node_metadata.gain * dtec.tree_weights[tree_idx]) + return dict(sums) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy_test.py index 8d801fa1f38..4ed18b2d34c 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy_test.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy_test.py @@ -27,7 +27,7 @@ from tensorflow.python.platform import googletest class ConvertModelTest(test_util.TensorFlowTestCase): - def testConvertModel(self): + def _make_trees(self): dtec_str = """ trees { nodes { @@ -108,8 +108,12 @@ class ConvertModelTest(test_util.TensorFlowTestCase): """ dtec = tree_config_pb2.DecisionTreeEnsembleConfig() text_format.Merge(dtec_str, dtec) - # The feature columns in the order they were added. feature_columns = ["feature_b", "feature_a", "feature_d"] + return dtec, feature_columns + + def testConvertModel(self): + dtec, feature_columns = self._make_trees() + # The feature columns in the order they were added. out = custom_export_strategy.convert_to_universal_format( dtec, feature_columns, 1, 1, 1) @@ -273,6 +277,16 @@ class ConvertModelTest(test_util.TensorFlowTestCase): }""" self.assertProtoEquals(expected_tree, out) + def testFeatureImportance(self): + dtec, feature_columns = self._make_trees() + feature_importances = custom_export_strategy._get_feature_importances( + dtec, feature_columns, 1, 1, 1) + self.assertItemsEqual(["feature_b", "feature_a", "feature_d"], + feature_importances.keys()) + self.assertAlmostEqual(50.0, feature_importances["feature_b"], places=4) + self.assertAlmostEqual(50.0, feature_importances["feature_a"], places=4) + self.assertAlmostEqual(50.0, feature_importances["feature_d"], places=4) + if __name__ == "__main__": googletest.main() From 87c7e2c515d8ed032801483459cdc4528aea540d Mon Sep 17 00:00:00 2001 From: Shanqing Cai Date: Wed, 30 Aug 2017 18:15:01 -0700 Subject: [PATCH 16/67] TFE: Simplify tfe.Tensor.__str__() output PiperOrigin-RevId: 167079964 --- tensorflow/python/eager/tensor_test.py | 14 +++++++------- tensorflow/python/framework/ops.py | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py index 8d0f639ddcb..bd8e653b976 100644 --- a/tensorflow/python/eager/tensor_test.py +++ b/tensorflow/python/eager/tensor_test.py @@ -77,8 +77,8 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testMultiLineTensorStr(self): t = tensor.Tensor(np.eye(3)) tensor_str = str(t) - self.assertIn("shape=%s, dtype=%s, " % (t.shape, t.dtype.name), tensor_str) - self.assertIn("numpy=\n%s" % t.numpy(), tensor_str) + self.assertIn("shape=%s, dtype=%s" % (t.shape, t.dtype.name), tensor_str) + self.assertIn(str(t.numpy()), tensor_str) def testMultiLineTensorRepr(self): t = tensor.Tensor(np.eye(3)) @@ -95,7 +95,7 @@ class TFETensorTest(test_util.TensorFlowTestCase): np.set_printoptions(threshold=2, edgeitems=1) t = tensor.Tensor(np.arange(10, dtype=np.int32)) - self.assertIn("numpy=[0 ..., 9]", str(t)) + self.assertIn("[0 ..., 9]", str(t)) self.assertIn("[0, ..., 9]", repr(t)) # Clean up: reset to previous printoptions. @@ -103,7 +103,7 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testZeroDimTensorStr(self): t = tensor.Tensor(42) - self.assertIn("shape=(), dtype=int32, numpy=42", str(t)) + self.assertIn("42, shape=(), dtype=int32", str(t)) def testZeroDimTensorRepr(self): t = tensor.Tensor(42) @@ -113,7 +113,7 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testZeroSizeTensorStr(self): t = tensor.Tensor(np.zeros(0, dtype=np.float32)) - self.assertIn("shape=(0,), dtype=float32, numpy=[]", str(t)) + self.assertIn("[], shape=(0,), dtype=float32", str(t)) def testZeroSizeTensorRepr(self): t = tensor.Tensor(np.zeros(0, dtype=np.float32)) @@ -127,8 +127,8 @@ class TFETensorTest(test_util.TensorFlowTestCase): t = tensor.Tensor(42) # Force change dtype to a numpy-unprintable type. t._dtype = dtypes.resource - self.assertIn("numpy=", str(t)) - self.assertIn("numpy=", repr(t)) + self.assertIn("", str(t)) + self.assertIn("", repr(t)) def testStringTensor(self): t_np_orig = np.array([[b"a", b"ab"], [b"abc", b"abcd"]]) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index da3757190c7..ccaa2141b52 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -721,12 +721,12 @@ class EagerTensor(Tensor): return numpy_text def __str__(self): - return "tfe.Tensor(shape=%s, dtype=%s, numpy=%s)" % (self.shape, - self.dtype.name, - self._numpy_text()) + return "tf.Tensor(%s, shape=%s, dtype=%s)" % (self._numpy_text(), + self.shape, + self.dtype.name) def __repr__(self): - return "" % ( + return "" % ( self._id, self.shape, self.dtype.name, self._numpy_text(is_repr=True)) @staticmethod From 5c3977c297f07d1cc14591844e5df202b1994c85 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Wed, 30 Aug 2017 18:23:50 -0700 Subject: [PATCH 17/67] Don't use std::pair on GPU since it's not supported by nvcc PiperOrigin-RevId: 167080772 --- tensorflow/core/kernels/conv_2d.h | 10 +++++----- tensorflow/core/kernels/pad_op.cc | 6 +++--- tensorflow/core/kernels/pad_op.h | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tensorflow/core/kernels/conv_2d.h b/tensorflow/core/kernels/conv_2d.h index 4bb0b7f3b41..8de8f1b2650 100644 --- a/tensorflow/core/kernels/conv_2d.h +++ b/tensorflow/core/kernels/conv_2d.h @@ -225,13 +225,13 @@ struct PadInput { const std::array& padding_right, typename TTypes::Tensor out, TensorFormat format) { - Eigen::array, NDIMS> padding; - padding[GetTensorDimIndex(format, 'N')] = std::make_pair(0, 0); + Eigen::array, NDIMS> padding; + padding[GetTensorDimIndex(format, 'N')] = {0, 0}; for (int i = 0; i < NDIMS - 2; ++i) { - padding[GetTensorDimIndex(format, '0' + i)] = - std::make_pair(padding_left[i], padding_right[i]); + padding[GetTensorDimIndex(format, '0' + i)] = { + padding_left[i], padding_right[i]}; } - padding[GetTensorDimIndex(format, 'C')] = std::make_pair(0, 0); + padding[GetTensorDimIndex(format, 'C')] = {0, 0}; out.device(d) = in.pad(padding); } }; diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc index 6e8b09d0500..6196c5ed93e 100644 --- a/tensorflow/core/kernels/pad_op.cc +++ b/tensorflow/core/kernels/pad_op.cc @@ -146,9 +146,9 @@ class PadOp : public OpKernel { Tensor* output) { CHECK_EQ(Dims, paddings.dimension(0)); CHECK_EQ(2, paddings.dimension(1)); - Eigen::array, Dims> paddings_array; + Eigen::array, Dims> paddings_array; for (int i = 0; i < Dims; ++i) { - paddings_array[i] = std::make_pair(paddings(i, 0), paddings(i, 1)); + paddings_array[i] = {paddings(i, 0), paddings(i, 1)}; } functor::Pad functor; functor(context->eigen_device(), output->tensor(), input, @@ -180,7 +180,7 @@ namespace functor { void Pad::operator()( \ const GPUDevice& d, typename TTypes::Tensor output, \ typename TTypes::ConstTensor input, \ - Eigen::array, Dims> paddings, T pad_value); \ + Eigen::array, Dims> paddings, T pad_value); \ extern template struct Pad; #define DECLARE_GPU_SPECS(T) \ diff --git a/tensorflow/core/kernels/pad_op.h b/tensorflow/core/kernels/pad_op.h index 6a973833e2d..95a7c9a3ae5 100644 --- a/tensorflow/core/kernels/pad_op.h +++ b/tensorflow/core/kernels/pad_op.h @@ -31,7 +31,7 @@ struct Pad { // See pad_op.cc for details. void operator()(const Device& d, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - Eigen::array, Dims> paddings, + Eigen::array, Dims> paddings, T pad_value) { if (Eigen::internal::is_same::value && (output.size() <= std::numeric_limits::max())) { @@ -47,7 +47,7 @@ struct Pad { // In the scalar case we simply copy the input. void operator()(const Device& d, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - Eigen::array, 0>, T) { + Eigen::array, 0>, T) { output.device(d) = input; } }; From 114e129d6c31216e5b62ef1e78a77c3ae7afe82e Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Wed, 30 Aug 2017 18:31:41 -0700 Subject: [PATCH 18/67] Fix TSAN flakes in Keras io_utils test. PiperOrigin-RevId: 167081436 --- tensorflow/contrib/keras/BUILD | 1 + .../keras/python/keras/utils/io_utils_test.py | 63 ++++++++++--------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/tensorflow/contrib/keras/BUILD b/tensorflow/contrib/keras/BUILD index a09045d7fda..26f0e415180 100644 --- a/tensorflow/contrib/keras/BUILD +++ b/tensorflow/contrib/keras/BUILD @@ -551,6 +551,7 @@ py_test( size = "small", srcs = ["python/keras/utils/io_utils_test.py"], srcs_version = "PY2AND3", + tags = ["notsan"], deps = [ ":keras", "//tensorflow/python:client_testlib", diff --git a/tensorflow/contrib/keras/python/keras/utils/io_utils_test.py b/tensorflow/contrib/keras/python/keras/utils/io_utils_test.py index baa9781e71f..f6820ee0394 100644 --- a/tensorflow/contrib/keras/python/keras/utils/io_utils_test.py +++ b/tensorflow/contrib/keras/python/keras/utils/io_utils_test.py @@ -57,43 +57,44 @@ class TestIOUtils(test.TestCase): h5_path = os.path.join(temp_dir, 'test.h5') create_dataset(h5_path) - # Instantiating HDF5Matrix for the training set, - # which is a slice of the first 150 elements - x_train = keras.utils.io_utils.HDF5Matrix( - h5_path, 'my_data', start=0, end=150) - y_train = keras.utils.io_utils.HDF5Matrix( - h5_path, 'my_labels', start=0, end=150) + with self.test_session(): + # Instantiating HDF5Matrix for the training set, + # which is a slice of the first 150 elements + x_train = keras.utils.io_utils.HDF5Matrix( + h5_path, 'my_data', start=0, end=150) + y_train = keras.utils.io_utils.HDF5Matrix( + h5_path, 'my_labels', start=0, end=150) - # Likewise for the test set - x_test = keras.utils.io_utils.HDF5Matrix( - h5_path, 'my_data', start=150, end=200) - y_test = keras.utils.io_utils.HDF5Matrix( - h5_path, 'my_labels', start=150, end=200) + # Likewise for the test set + x_test = keras.utils.io_utils.HDF5Matrix( + h5_path, 'my_data', start=150, end=200) + y_test = keras.utils.io_utils.HDF5Matrix( + h5_path, 'my_labels', start=150, end=200) - # HDF5Matrix behave more or less like Numpy matrices - # with regard to indexing - self.assertEqual(y_train.shape, (150, 1)) - # But they do not support negative indices, so don't try print(x_train[-1]) + # HDF5Matrix behave more or less like Numpy matrices + # with regard to indexing + self.assertEqual(y_train.shape, (150, 1)) + # But they don't support negative indices, so don't try print(x_train[-1]) - self.assertEqual(y_train.dtype, np.dtype('i')) - self.assertEqual(y_train.ndim, 2) - self.assertEqual(y_train.size, 150) + self.assertEqual(y_train.dtype, np.dtype('i')) + self.assertEqual(y_train.ndim, 2) + self.assertEqual(y_train.size, 150) - model = keras.models.Sequential() - model.add(keras.layers.Dense(64, input_shape=(10,), activation='relu')) - model.add(keras.layers.Dense(1, activation='sigmoid')) - model.compile(loss='binary_crossentropy', optimizer='sgd') + model = keras.models.Sequential() + model.add(keras.layers.Dense(64, input_shape=(10,), activation='relu')) + model.add(keras.layers.Dense(1, activation='sigmoid')) + model.compile(loss='binary_crossentropy', optimizer='sgd') - # Note: you have to use shuffle='batch' or False with HDF5Matrix - model.fit(x_train, y_train, batch_size=32, shuffle='batch', verbose=False) - # test that evalutation and prediction - # don't crash and return reasonable results - out_pred = model.predict(x_test, batch_size=32, verbose=False) - out_eval = model.evaluate(x_test, y_test, batch_size=32, verbose=False) + # Note: you have to use shuffle='batch' or False with HDF5Matrix + model.fit(x_train, y_train, batch_size=32, shuffle='batch', verbose=False) + # test that evalutation and prediction + # don't crash and return reasonable results + out_pred = model.predict(x_test, batch_size=32, verbose=False) + out_eval = model.evaluate(x_test, y_test, batch_size=32, verbose=False) - self.assertEqual(out_pred.shape, (50, 1)) - self.assertEqual(out_eval.shape, ()) - self.assertGreater(out_eval, 0) + self.assertEqual(out_pred.shape, (50, 1)) + self.assertEqual(out_eval.shape, ()) + self.assertGreater(out_eval, 0) if __name__ == '__main__': From 9acea81c16c222f145515354f6176852dfdb03d7 Mon Sep 17 00:00:00 2001 From: Benoit Steiner Date: Wed, 30 Aug 2017 18:37:32 -0700 Subject: [PATCH 19/67] Annotate the graph properties with input values if they're known statically. PiperOrigin-RevId: 167081910 --- tensorflow/core/grappler/costs/graph_properties.cc | 12 ++++++++++++ .../core/grappler/costs/graph_properties_test.cc | 9 +++++++++ 2 files changed, 21 insertions(+) diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index 0ab6aff250b..1b1c88f2df4 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -396,6 +396,18 @@ Status GraphProperties::InferStatically() { } input_properties.push_back(properties); } + for (const auto& edge : node->in_edges()) { + if (!edge->src()->IsConstant()) { + continue; + } + const int input_id = edge->dst_input(); + if (input_id >= input_properties.size()) { + continue; + } + const NodeDef& node = edge->src()->def(); + const TensorProto& raw_val = node.attr().at("value").tensor(); + *input_properties[input_id].mutable_value() = raw_val; + } input_properties_[node->name()] = input_properties; // TODO(bsteiner): share this code with the input processing above. diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc index 954c5ead8fc..461e58cf736 100644 --- a/tensorflow/core/grappler/costs/graph_properties_test.cc +++ b/tensorflow/core/grappler/costs/graph_properties_test.cc @@ -345,6 +345,15 @@ TEST_F(GraphPropertiesTest, MergeWithoutLoops) { EXPECT_EQ(DT_FLOAT, prop.dtype()); EXPECT_EQ(expected_outputs[i], PropToString(prop)); } + + // The "Less" node should be fed by 2 int32 scalar constant values. + const auto props = properties.GetInputProperties("Less"); + EXPECT_EQ(2, props.size()); + for (int i = 0; i < props.size(); ++i) { + EXPECT_EQ(DT_INT32, props[i].dtype()); + EXPECT_TRUE(props[i].has_value()); + EXPECT_EQ("int32: []", PropToString(props[i])); + } } TEST_F(GraphPropertiesTest, WhileLoop) { From 424aa9aa9559f6fa29d8ccf3d74ff25528b39209 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Wed, 30 Aug 2017 19:51:38 -0700 Subject: [PATCH 20/67] Eager-graph mode should work with gradient computation. PiperOrigin-RevId: 167086826 --- tensorflow/python/BUILD | 4 +++- tensorflow/python/eager/backprop.py | 6 +----- tensorflow/python/eager/function_test.py | 14 ++++++++++++++ tensorflow/python/framework/op_def_library.py | 1 + tensorflow/python/ops/resource_variable_ops.py | 12 +++++++++++- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index b75f79fbf47..98dce82ee31 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1766,6 +1766,8 @@ py_library( srcs_version = "PY2AND3", deps = [ ":array_ops", + ":array_ops_gen", + ":dtypes", ":framework_ops", ":resource_variable_ops_gen", ":tensor_shape", @@ -1775,7 +1777,7 @@ py_library( "//tensorflow/python/eager:context", "//tensorflow/python/eager:custom_gradient", "//tensorflow/python/eager:tape", - "//tensorflow/python/eager:tensor", + "//tensorflow/python/eager:tensor_node", ], ) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index ca3ad1a2c33..326f56ebf9b 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -169,10 +169,6 @@ def _record_gradient(op_name, inputs, attrs, results, name): execute.record_gradient = _record_gradient -def _ones(shape, dtype): - return array_ops.fill(shape, tensor.Tensor(1, dtype=dtype)) - - def _aggregate_grads(gradients): """Aggregate gradients of the same tensor.""" grad_lists = dict() @@ -225,7 +221,7 @@ def implicit_val_and_grad(f): (end_node.progenitors, repr(start_node))) output_gradients = kwds.get("output_gradients", None) if output_gradients is None: - output_gradients = _ones(end_node.shape, end_node.dtype) + output_gradients = array_ops.ones_like(end_node.value) grad = ag_core.backward_pass(output_gradients, end_node, start_node) return end_node.value, _aggregate_grads(grad.gradients) diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 18b722e7923..c15dde9e487 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import function as tf_function from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops class FunctionTest(test.TestCase): @@ -52,6 +53,19 @@ class FunctionTest(test.TestCase): out = sq(t) self.assertAllEqual(out.numpy(), math_ops.matmul(t, t).numpy()) + def testGraphModeWithGradients(self): + v = resource_variable_ops.ResourceVariable(1.0) + + @function.defun + def step(): + def inner(): + tape.watch(v.handle) + return v * v + + return backprop.implicit_grad(inner)()[0][1] + + self.assertAllEqual(step().numpy(), 2.0) + def testTensorConversionWithDefun(self): @function.defun diff --git a/tensorflow/python/framework/op_def_library.py b/tensorflow/python/framework/op_def_library.py index aa373600669..76424ef579b 100644 --- a/tensorflow/python/framework/op_def_library.py +++ b/tensorflow/python/framework/op_def_library.py @@ -784,6 +784,7 @@ class OpDefLibrary(object): if arg.is_ref] with _MaybeColocateWith(must_colocate_inputs): # Add Op to graph + inputs = [ag_core.getval(x) for x in inputs] op = g.create_op(op_type_name, inputs, output_types, name=scope, input_types=input_types, attrs=attr_protos, op_def=op_def) diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 1d747f84008..1471b5909eb 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -19,11 +19,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from autograd import core as ag_core + from tensorflow.core.framework import attr_value_pb2 from tensorflow.core.framework import variable_pb2 from tensorflow.python.eager import context from tensorflow.python.eager import custom_gradient from tensorflow.python.eager import tape +from tensorflow.python.eager import tensor_node from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -574,7 +577,14 @@ class ResourceVariable(variables.Variable): def _run_op(a, *args): # pylint: disable=protected-access - return getattr(ops.Tensor, operator)(a._AsTensor(), *args) + value = a._AsTensor() + if ag_core.isnode(value): + # This avoids autograd trying to wrap a ResourceVariable. + value = ops.convert_to_tensor(value) + args = [ops.convert_to_tensor(x) for x in args] + return getattr(tensor_node.TensorNode, operator)(value, *args) + else: + return getattr(ops.Tensor, operator)(value, *args) # Propagate __doc__ to wrapper try: From 9624d165f1f2c717eda96464fee8bf7229cc14f5 Mon Sep 17 00:00:00 2001 From: Igor Ganichev Date: Wed, 30 Aug 2017 21:05:14 -0700 Subject: [PATCH 21/67] Add function support to Tensorflow C API This change adds minimal functionality. Support for FunctionOptions, attributes, output name rewriting, function name generation, etc is comming next. PiperOrigin-RevId: 167091238 --- tensorflow/c/BUILD | 24 +- tensorflow/c/c_api.cc | 37 +- tensorflow/c/c_api.h | 116 ++ tensorflow/c/c_api_function.cc | 496 ++++++++ tensorflow/c/c_api_function_test.cc | 1039 +++++++++++++++++ tensorflow/c/c_api_internal.h | 8 + tensorflow/c/c_api_test.cc | 2 +- tensorflow/c/c_test_util.cc | 137 ++- tensorflow/c/c_test_util.h | 20 + tensorflow/contrib/cmake/tf_c.cmake | 1 + tensorflow/core/graph/graph.cc | 13 +- tensorflow/core/graph/graph.h | 4 + tensorflow/python/client/tf_session.i | 27 + tensorflow/python/client/tf_session_helper.cc | 34 + tensorflow/python/client/tf_session_helper.h | 10 + tensorflow/python/framework/function.py | 19 + tensorflow/python/framework/function_test.py | 112 +- tensorflow/python/framework/ops.py | 8 + 18 files changed, 2075 insertions(+), 32 deletions(-) create mode 100644 tensorflow/c/c_api_function.cc create mode 100644 tensorflow/c/c_api_function_test.cc diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index 604dfab148b..1822e235eba 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -45,8 +45,13 @@ tf_cuda_library( tf_cuda_library( name = "c_api", - srcs = ["c_api.cc"], - hdrs = ["c_api.h"], + srcs = [ + "c_api.cc", + "c_api_function.cc", + ], + hdrs = [ + "c_api.h", + ], copts = tf_copts(), visibility = ["//visibility:public"], deps = select({ @@ -157,6 +162,21 @@ tf_cc_test( ], ) +tf_cc_test( + name = "c_api_function_test", + size = "small", + srcs = ["c_api_function_test.cc"], + deps = [ + ":c_api", + ":c_test_util", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + tf_cc_test( name = "while_loop_test", size = "small", diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index 07c8277a6f2..c454c94249b 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -165,22 +165,6 @@ void deallocate_buffer(void* data, size_t len, void* arg) { tensorflow::cpu_allocator()->DeallocateRaw(data); } -Status MessageToBuffer(const tensorflow::protobuf::Message& in, - TF_Buffer* out) { - if (out->data != nullptr) { - return InvalidArgument("Passing non-empty TF_Buffer is invalid."); - } - const auto proto_size = in.ByteSizeLong(); - void* buf = tensorflow::port::Malloc(proto_size); - in.SerializeToArray(buf, proto_size); - out->data = buf; - out->length = proto_size; - out->data_deallocator = [](void* data, size_t length) { - tensorflow::port::Free(data); - }; - return Status::OK(); -} - } // namespace TF_Tensor::~TF_Tensor() { buffer->Unref(); } @@ -559,6 +543,27 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, dimvec.size(), base, size, DeleteArray, base); } +Status MessageToBuffer(const tensorflow::protobuf::Message& in, + TF_Buffer* out) { + if (out->data != nullptr) { + return InvalidArgument("Passing non-empty TF_Buffer is invalid."); + } + const size_t proto_size = in.ByteSizeLong(); + void* buf = tensorflow::port::Malloc(proto_size); + if (buf == nullptr) { + return tensorflow::errors::ResourceExhausted( + "Failed to allocate memory to serialize message of type '", + in.GetTypeName(), "' and size ", proto_size); + } + in.SerializeToArray(buf, proto_size); + out->data = buf; + out->length = proto_size; + out->data_deallocator = [](void* data, size_t length) { + tensorflow::port::Free(data); + }; + return Status::OK(); +} + // Helpers for loading a TensorFlow plugin (a .so file). Status LoadLibrary(const char* library_filename, void** result, const void** buf, size_t* len); diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h index 43b50780137..ee110d88cea 100644 --- a/tensorflow/c/c_api.h +++ b/tensorflow/c/c_api.h @@ -357,6 +357,14 @@ typedef struct TF_Output { int index; // The index of the output within oper. } TF_Output; +// TF_Function is a grouping of operations with defined inputs and outputs. +// Once created and added to graphs, functions can be invoked by creating an +// operation whose operation type matches the function name. +typedef struct TF_Function TF_Function; + +// Function definition options. TODO(iga): Define and implement +typedef struct TF_FunctionOptions TF_FunctionOptions; + // Sets the shape of the Tensor referenced by `output` in `graph` to // the shape described by `dims` and `num_dims`. // @@ -914,6 +922,15 @@ TF_CAPI_EXPORT extern void TF_GraphImportGraphDef( TF_Graph* graph, const TF_Buffer* graph_def, const TF_ImportGraphDefOptions* options, TF_Status* status); +// Add `function` to graph `g`. Once `function` is added to `g`, +// it can be called by creating an operation using the function's name. +// +// If successful, status is set to OK and function is added to g +// Otherwise, status is set to the encountered error and g is unmodified +TF_CAPI_EXPORT extern void TF_GraphAddFunction(TF_Graph* g, + const TF_Function* function, + TF_Status* status); + // Note: The following function may fail on very large protos in the future. TF_CAPI_EXPORT extern void TF_OperationToNodeDef(TF_Operation* oper, @@ -1001,6 +1018,105 @@ TF_CAPI_EXPORT void TF_AddGradients(TF_Graph* g, TF_Output* y, int ny, TF_Output* x, int nx, TF_Output* dx, TF_Status* status, TF_Output* dy); +// Create a TF_Function from a TF_Graph +// +// Params: +// fn_body - the graph whose operations (or subset of whose operations) will be +// converted to TF_Function. +// fn_name - the name of the new TF_Function. Should match the operation +// name (OpDef.name) regexp [A-Z][A-Za-z0-9_.\\-/]* and be distinct +// from other operation names (at least those registered in graphs +// where this function will be used). +// TODO(iga): Allow null in here and have C API come up with +// a unique name with high probability (similarly to +// _create_hash_str in function.py) +// num_opers - `num_opers` contains the number of elements in the `opers` array +// or a special value of -1 meaning that no array is given. +// The distinction between an empty array of operations and no +// array of operations is necessary to distinguish the case of +// creating a function with no body (e.g. identity or permutation) +// and the case of creating a function whose body contains all +// the nodes in the graph (except for the automatic skipping, see +// below). +// opers - Array of operations to become the body of the function or null. +// - If no array is given (`num_opers` = -1), all the +// operations in `fn_body` will become part of the function +// except operations referenced in `inputs`. These operations +// must have a single output (these operations are typically +// placeholders created for the sole purpose of representing +// an input. We can relax this constraint if there are +// compelling use cases). +// - If an array is given (`num_opers` >= 0), all operations +// in it will become part of the function. In particular, no +// automatic skipping of dummy input operations is performed. +// ninputs - number of elements in `inputs` array +// inputs - array of TF_Outputs that specify the inputs to the function. +// If `ninputs` is zero (the function takes no inputs), `inputs` +// can be null. The names used for function inputs are normalized +// names of the operations (usually placeholders) pointed to by +// `inputs`. These operation names should start with a letter. +// Normalization will convert all letters to lowercase and +// non-alphanumeric characters to '_' to make resulting names match +// the "[a-z][a-z0-9_]*" pattern for operation argument names. +// `inputs` cannot contain the same tensor twice. +// noutputs - number of elements in `outputs` array +// outputs - array of TF_Outputs that specify the outputs of the function. +// If `noutputs` is zero (the function returns no outputs), `outputs` +// can be null. `outputs` can contain the same tensor more than once. +// output_names - The names of the function's outputs. `output_names` array +// must either have the same length as `outputs` +// (i.e. `noutputs`) or be null. In the former case, +// the names should match the regular expression for ArgDef +// names - "[a-z][a-z0-9_]*". In the latter case, +// names for outputs will be generated automatically. +// opts - various options for the function, e.g. XLA's inlining control. +// status - Set to OK on success and an appropriate error on failure. +// +// Note that when the same TF_Output is listed as both an input and an output, +// the corresponding function's output will equal to this input, +// instead of the original node's output. +// +// Callers must also satisfy the following constraints: +// - `inputs` cannot refer to TF_Outputs within a control flow context. For +// example, one cannot use the output of "switch" node as input. +// - No TF_Output of a function (inside any of `inputs`, `outputs`, `fn_body`) +// is allowed to have a reference type. Reference types are not exposed +// through C API and are being deprecated. +// - Every node in the function's body must have all of its inputs (including +// control inputs). In other words, for every node in the body, each input +// must be either listed in `inputs` or must come from another node in +// the body. In particular, it is an error to have a control edge going from +// a node outside of the body into a node in the body. This applies to control +// edges going from nodes referenced in `inputs` to nodes in the body when +// the former nodes are not in the body (automatically skipped or not +// included in explicitly specified body). +// +// Returns: +// On successful, a newly created TF_Function instance. It must be deleted by +// calling TF_DeleteFunction. +// +// On failure, null. +// +// TODO(iga): Add input_names argument and get output_names working (they are +// currently ignored) +TF_CAPI_EXPORT extern TF_Function* TF_GraphToFunction( + const TF_Graph* fn_body, const char* fn_name, int num_opers, + const TF_Operation* const* opers, int ninputs, const TF_Output* inputs, + int noutputs, const TF_Output* outputs, const char* const* output_names, + const TF_FunctionOptions* opts, TF_Status* status); + +// Write out a serialized representation of `func` (as a FunctionDef protocol +// message) to `output_func_def` (allocated by TF_NewBuffer()). +// `output_func_def`'s underlying buffer will be freed when TF_DeleteBuffer() +// is called. +// +// May fail on very large graphs in the future. +TF_CAPI_EXPORT extern void TF_FunctionToFunctionDef(TF_Function* func, + TF_Buffer* output_func_def, + TF_Status* status); + +TF_CAPI_EXPORT extern void TF_DeleteFunction(TF_Function*); + // TODO(josh11b): Register OpDef, available to all operations added // to this graph. diff --git a/tensorflow/c/c_api_function.cc b/tensorflow/c/c_api_function.cc new file mode 100644 index 00000000000..b4c6397d0b4 --- /dev/null +++ b/tensorflow/c/c_api_function.cc @@ -0,0 +1,496 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/c_api_internal.h" + +#include +#include +#include + +#include "tensorflow/core/framework/attr_value_util.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_util.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/lib/strings/strcat.h" + +namespace tensorflow { +namespace { + +// Class that maintains a one-to-one original node name -> new node name +// mapping. We normalize the names used as input and output arguments to match +// regexp "[a-z][a-z0-9_]*" specified in definition of ArgDef.name. +// Once we rename them, we risk creating a name collision with the other +// node names, so if necessary we add a suffix to make +// names unique. If we have an input named "A" and a node in the function +// body named "a", they will be renamed to "a" and "a_0". +class NodeNameMapping { + public: + NodeNameMapping() = default; + + // Normalize the input/output name and make it unique. + string GetIOName(const string& name); + + // Make the node name unique. + string Uniquify(const string& name); + + // Look up how a node name was previously normalized/uniquified. + // Returns empty if name was never seen. + string Lookup(const string& name) const; + + private: + string UniquifyHelper(const string& name) const; + static string Normalize(string name); + + // The normalized/uniquified names already used as + // input names (in signature), output names (in signature), and node names + // (in node_def). + // This is a superset of values in name_mapping_. + std::unordered_set used_names_; + // Mapping from original node name from the graph to the normalized + // and uniqified version of it. + std::unordered_map name_mapping_; +}; + +string NodeNameMapping::Normalize(string name) { + // Convert letters to lowercase and non-alphanumeric characters to '_'. + if (name.empty()) return "unknown"; + const int n = name.size(); + for (int i = 0; i < n; ++i) { + char c = name[i]; + if (isalnum(c)) { + if (isupper(c)) { + name[i] = tolower(c); + } + } else { + name[i] = '_'; + } + } + + // Find the first letter and start with it. + int i = 0; + for (; i < n; ++i) { + if (isalpha(name[i])) break; + } + + // Return "unknown" if none of the name's chars were letters. + return i == n ? "unknown" : name.substr(i); +} + +string NodeNameMapping::UniquifyHelper(const string& name) const { + // If the name hasn't been used yet, use it as-is. + if (used_names_.find(name) == used_names_.end()) return name; + // Add a suffix to name to make it unique. + for (int i = 0;; ++i) { + const string candidate = strings::StrCat(name, "_", i); + if (used_names_.find(candidate) == used_names_.end()) return candidate; + } +} + +string NodeNameMapping::GetIOName(const string& name) { + const string& input_name = UniquifyHelper(Normalize(name)); + // Record that we used this name, but don't add it to name_mapping_ + // since this name is not for a node. + used_names_.insert(input_name); + return input_name; +} + +string NodeNameMapping::Uniquify(const string& name) { + const string uniqued = UniquifyHelper(name); + name_mapping_[name] = uniqued; + used_names_.insert(uniqued); + return uniqued; +} + +string NodeNameMapping::Lookup(const string& name) const { + const auto iter = name_mapping_.find(name); + if (iter == name_mapping_.end()) return string(); + return iter->second; +} + +Status ValidateNoRefOutputs(const Node* node) { + for (int i = 0; i < node->num_outputs(); ++i) { + const DataType& dt = node->output_type(i); + if (IsRefType(dt)) { + return errors::InvalidArgument("Output ", i, " of node '", node->name(), + "' has a reference " + "type ", + DataTypeString(dt)); + } + } + return Status::OK(); +} + +Status FillFunctionBody( + const string& fn_name, const NodeNameMapping& node_names, + const std::vector& body_nodes, + const std::unordered_map& tensor_renaming, + FunctionDef* fdef) { + std::vector in_edges; + std::vector control_edges; + for (const Node* node : body_nodes) { + NodeDef* node_def = fdef->add_node_def(); + // First, copy the node_def as is. We will patch it next. + *node_def = node->def(); + if (!node->assigned_device_name().empty()) { + node_def->set_device(node->assigned_device_name()); + } + node_def->set_name(node_names.Lookup(node->name())); + + // Input names must be set based on nested names in tensor_renaming. + // Clear the flat input names we got from the original node_def + // from the graph. + node_def->clear_input(); + + // Collect regular and control inputs. Regular inputs are indexed + // by the index at which they come into the `node`. Control inputs + // don't follow any order. + in_edges.clear(); + in_edges.resize(node->num_inputs(), nullptr); + control_edges.clear(); + for (const Edge* edge : node->in_edges()) { + if (edge->src()->IsSource()) continue; + if (edge->IsControlEdge()) { + control_edges.push_back(edge); + } else { + in_edges[edge->dst_input()] = edge; + } + } + + // Add regular inputs. + for (size_t i = 0; i < in_edges.size(); ++i) { + const Edge* edge = in_edges[i]; + string original_input_name; + if (edge == nullptr) { + // A backedge might not appear as a regular Edge, but be only present + // in the node_def. Such edges are referred to as requested_inputs(). + if (i >= node->requested_inputs().size()) { + return errors::InvalidArgument( + "Graph to be converted to function appears to be malformed. ", + "Node ", node->name(), " is missing input edge ", i); + } + original_input_name = + ParseTensorName(node->requested_inputs()[i]).ToString(); + } else { + original_input_name = + strings::StrCat(edge->src()->name(), ":", edge->src_output()); + } + + const auto iter = tensor_renaming.find(original_input_name); + if (iter == tensor_renaming.end()) { + return errors::InvalidArgument( + "Input ", i, ", '", original_input_name, "', of node '", + node->name(), "' in function '", fn_name, + "' is not available. You might need to include it in inputs " + "or include its source node in the body"); + } + node_def->add_input(iter->second); + } + + // Add control inputs. + for (const Edge* edge : control_edges) { + // Add this control input only if the src node is in the body. + const string normalized = node_names.Lookup(edge->src()->name()); + // If we did not find a name for the source of control edge, this + // source must be outside of the body. Raise an error. + if (normalized.empty()) { + return errors::InvalidArgument( + "The source of control edge ", edge->DebugString(), + " is not in the body. Encountered while creating function '", + fn_name, "'"); + } + node_def->add_input(strings::StrCat("^", normalized)); + } + } + return Status::OK(); +} + +// Graph to FunctionDef conversion. This code is closely modeled on the Python +// code in third_party/tensorflow/python/framework/function.py. +Status GraphToFunctionDef(const Graph& fn_body, const string& fn_name, + const std::vector& body_nodes, + const std::vector& inputs, + const std::vector& outputs, + const std::vector& output_names, + FunctionDef* fdef) { + fdef->mutable_signature()->set_name(fn_name); + + // Keep track of names we used and how we normalized them. + NodeNameMapping node_names; + + // Mapping from original names of tensors (i.e. ":") to the + // name we used in the function: + // - For input tensors: + // {flat_tensor_name -> normalized_name_of_src_node} + // e.g. {In:3 -> in} + // - For tensors produced by nodes in function's body: + // {flat_tensor_name -> nested_tensor_name} + // e.g. {Add:3 -> add_0:z:1} + std::unordered_map tensor_renaming; + + // Fill inputs in function's signature. + for (size_t i = 0; i < inputs.size(); ++i) { + const Node* node = inputs[i].node; + int idx = inputs[i].index; + OpDef::ArgDef* argdef = fdef->mutable_signature()->add_input_arg(); + argdef->set_type(node->output_type(idx)); + const string& input_name = node_names.GetIOName(node->name()); + argdef->set_name(input_name); + tensor_renaming[strings::StrCat(node->name(), ":", idx)] = input_name; + } + + // Fill outputs in function's signature. + for (size_t i = 0; i < outputs.size(); ++i) { + const Node* node = outputs[i].node; + int idx = outputs[i].index; + OpDef::ArgDef* argdef = fdef->mutable_signature()->add_output_arg(); + argdef->set_type(node->output_type(idx)); + argdef->set_name(node_names.GetIOName(node->name())); + } + + // Populate tensor_renaming and node_names. + // Generate the new output names for every node in the function. + // The NodeDefs in FunctionDefs use a different naming scheme for + // their inputs than the NodeDefs in a graph (see the comment for + // FunctionDef.node_def in function.proto). We do the + // graph tensor name -> function tensor name conversion for every + // possible input (i.e. every node's outputs) and store the result + // in tensor_renaming. + for (const Node* node : body_nodes) { + // Make sure node_name does not collide with an input or output name. + const string& node_name = node_names.Uniquify(node->name()); + // For each output_arg in the op_def, the output_ranges + // map will have [start, end] range of indices that this arg produces + // among all the output tensors of this op. + NameRangeMap output_ranges; + TF_RETURN_IF_ERROR( + NameRangesForNode(*node, node->op_def(), nullptr, &output_ranges)); + for (const auto& output : output_ranges) { + const string& output_name = output.first; + int index_start = output.second.first; + int index_end = output.second.second; + for (int i = index_start; i < index_end; ++i) { + const string& original_name = strings::StrCat(node->name(), ":", i); + const string& new_name = + strings::StrCat(node_name, ":", output_name, ":", i - index_start); + // Record the mapping if this tensor is not already mapped. + // Tensor can be already mapped if it is used as an input. + if (tensor_renaming.find(original_name) == tensor_renaming.end()) { + tensor_renaming[original_name] = new_name; + } + } + } + } + + TF_RETURN_IF_ERROR( + FillFunctionBody(fn_name, node_names, body_nodes, tensor_renaming, fdef)); + + // Remap return values. + for (int r = 0; r < fdef->signature().output_arg_size(); ++r) { + const string& ret_name = fdef->signature().output_arg(r).name(); + + // We convert this flat tensor name to the nested value + // (e.g. `add:z:1`) that we stored in tensor_renaming. + const string& return_value = + strings::StrCat(outputs[r].node->name(), ":", outputs[r].index); + const auto iter = tensor_renaming.find(return_value); + if (iter == tensor_renaming.end()) { + return errors::InvalidArgument( + "TF_Output ", return_value, " is neither in the function body ", + "nor among function inputs. Encountered while creating function '", + fn_name, "'"); + } + (*fdef->mutable_ret())[ret_name] = iter->second; + } + + return Status::OK(); +} + +// Converts `ninputs` and `inputs` into `inputs_tensors` and `input_nodes` and +// does various checks while doing so. `input_nodes` will contain the same +// information as input_tensors just in a different structure to make +// following processing easier. TODO(iga): Simplify this nested structure. +Status ProcessInputs( + const TF_Graph* fn_body, const char* fn_name, int ninputs, + const TF_Output* inputs, std::vector* input_tensors, + std::unordered_map>* input_nodes) + EXCLUSIVE_LOCKS_REQUIRED(fn_body->mu) { + input_tensors->reserve(ninputs); + for (int i = 0; i < ninputs; ++i) { + const Node& node = inputs[i].oper->node; + int idx = inputs[i].index; + + TF_RETURN_WITH_CONTEXT_IF_ERROR( + fn_body->graph.IsValidOutputTensor(&node, idx), + "Encountered while processing input ", i, " into function '", fn_name, + "'"); + TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNoRefOutputs(&node), + "Encountered while processing input ", i, + " into function '", fn_name, "'"); + + input_tensors->emplace_back(&node, idx); + + const auto& iter = input_nodes->find(&node); + if (iter == input_nodes->end()) { + input_nodes->insert({&node, {idx}}); + } else { + auto& indices = iter->second; + if (std::find(indices.begin(), indices.end(), idx) != indices.end()) { + return errors::InvalidArgument( + "TF_Output ", node.name(), ":", idx, + " appears more than once in the input list"); + } + indices.push_back(idx); + } + } + return Status::OK(); +} + +// Converts `noutputs` and `outputs` into `outputs_tensors` and does various +// checks while doing so. +Status ProcessOutputs(const TF_Graph* fn_body, const char* fn_name, + int noutputs, const TF_Output* outputs, + std::vector* output_tensors) + EXCLUSIVE_LOCKS_REQUIRED(fn_body->mu) { + output_tensors->reserve(noutputs); + for (int i = 0; i < noutputs; ++i) { + const Node& node = outputs[i].oper->node; + int idx = outputs[i].index; + TF_RETURN_WITH_CONTEXT_IF_ERROR( + fn_body->graph.IsValidOutputTensor(&node, idx), + "Encountered while processing output ", i, " from function '", fn_name, + "'"); + output_tensors->emplace_back(&node, idx); + } + return Status::OK(); +} + +// Populates `body_nodes` with the nodes that will become function's body. +// Performs various checks. +Status ComputeBodyNodes( + const TF_Graph* fn_body, const char* fn_name, int num_opers, + const TF_Operation* const* opers, + const std::unordered_map>& input_nodes, + std::vector* body_nodes) + EXCLUSIVE_LOCKS_REQUIRED(fn_body->mu) { + if (num_opers == -1) { + for (const Node* node : fn_body->graph.op_nodes()) { + const auto& iter = input_nodes.find(node); + if (iter == input_nodes.end()) { + // This node is not referenced in inputs. Add it to the body. + TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNoRefOutputs(node), + "Encountered while creating function '", + fn_name, "'"); + body_nodes->push_back(node); + } else { + // This node is referenced in inputs. Currently, we place an + // artificial restriction and require that when num_opers=-1, such + // nodes must have a single output. + if (node->num_outputs() != 1) { + return errors::InvalidArgument( + "When `num_opers` is set to -1, nodes referenced in `inputs` " + "must have a single output. Node ", + node->name(), " has ", node->num_outputs(), + " outputs. Encountered while creating function '", fn_name, "'"); + } + } + } + } else { + body_nodes->reserve(num_opers); + for (int i = 0; i < num_opers; ++i) { + const Node* node = &opers[i]->node; + TF_RETURN_WITH_CONTEXT_IF_ERROR(ValidateNoRefOutputs(node), + "Encountered while creating function '", + fn_name, "'"); + body_nodes->push_back(node); + } + } + return Status::OK(); +} + +} // anonymous namespace +} // namespace tensorflow + +using tensorflow::Node; +using tensorflow::string; + +TF_Function* TF_GraphToFunction(const TF_Graph* fn_body, const char* fn_name, + int num_opers, const TF_Operation* const* opers, + int ninputs, const TF_Output* inputs, + int noutputs, const TF_Output* outputs, + const char* const* output_names, + const TF_FunctionOptions* opts, + TF_Status* status) { + tensorflow::mutex_lock l(*const_cast(&fn_body->mu)); + + // Process inputs. + std::vector input_tensors; + std::unordered_map> input_nodes; + status->status = tensorflow::ProcessInputs(fn_body, fn_name, ninputs, inputs, + &input_tensors, &input_nodes); + if (!status->status.ok()) return nullptr; + + // Process outputs. + std::vector output_tensors; + status->status = tensorflow::ProcessOutputs(fn_body, fn_name, noutputs, + outputs, &output_tensors); + if (!status->status.ok()) return nullptr; + + // Process output names. + std::vector output_names_vec; + if (output_names) { + output_names_vec.reserve(noutputs); + for (int i = 0; i < noutputs; ++i) { + output_names_vec.push_back(string(output_names[i])); + } + } + + // Compute body nodes. + std::vector body_nodes; + status->status = tensorflow::ComputeBodyNodes( + fn_body, fn_name, num_opers, opers, input_nodes, &body_nodes); + if (!status->status.ok()) return nullptr; + + // Do the actual function creation. + TF_Function* tf_function = new TF_Function(); + status->status = tensorflow::GraphToFunctionDef( + fn_body->graph, fn_name, body_nodes, input_tensors, output_tensors, + output_names_vec, tf_function->fdef_lib.add_function()); + if (!status->status.ok()) { + TF_DeleteFunction(tf_function); + return nullptr; + } + return tf_function; +} + +void TF_GraphAddFunction(TF_Graph* g, const TF_Function* function, + TF_Status* status) { + tensorflow::mutex_lock l(g->mu); + + // At the moment, we have only one function and no gradients in fdef_lib. + // This makes the following operation atomic. + // TODO(iga): Add an atomic version of AddFunctionLibrary when we support + // gradients + status->status = g->graph.AddFunctionLibrary(function->fdef_lib); +} + +void TF_FunctionToFunctionDef(TF_Function* func, TF_Buffer* output_func_def, + TF_Status* status) { + DCHECK_EQ(1, func->fdef_lib.function_size()); + status->status = MessageToBuffer(func->fdef_lib.function(0), output_func_def); +} + +void TF_DeleteFunction(TF_Function* function) { delete function; } diff --git a/tensorflow/c/c_api_function_test.cc b/tensorflow/c/c_api_function_test.cc new file mode 100644 index 00000000000..c9dd38ea15f --- /dev/null +++ b/tensorflow/c/c_api_function_test.cc @@ -0,0 +1,1039 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/c_api.h" + +#include "tensorflow/c/c_test_util.h" +#include "tensorflow/core/framework/function.pb.h" +#include "tensorflow/core/framework/op_def.pb.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +// Specification for expected input/output and its type. +// DataType value of DT_INVALID signifies that we don't want to +// check the data type. +typedef std::pair IOSpec; + +std::vector M(const std::initializer_list& names) { + std::vector v; + for (const string& name : names) { + v.push_back(IOSpec(name, DT_INVALID)); + } + return v; +} + +// Specification for an expected edge. +// src is either: +// - input name (as it appears in FunctionDef) +// - name of output tensor (in nested "add:z:0" format) +// dst is either: +// - output name (as it appears in FunctionDef) +// - : (this looks the same as +// output tensor naming, but it the index is actually an input index) +struct EdgeSpec : public std::pair { + typedef std::pair Base; + + // Inherit the set of constructors + using Base::pair; + + string ToString() const { return strings::StrCat(first, "->", second); } +}; + +class CApiFunctionTest : public ::testing::Test { + protected: + CApiFunctionTest() + : s_(TF_NewStatus()), + func_graph_(TF_NewGraph()), + host_graph_(TF_NewGraph()), + func_(nullptr) {} + + void SetUp() override {} + + ~CApiFunctionTest() override { + TF_DeleteFunction(func_); + TF_DeleteGraph(host_graph_); + TF_DeleteGraph(func_graph_); + TF_DeleteStatus(s_); + } + + void Run(const std::vector>& inputs, + TF_Operation* output, int32_t expected_result) { + Run(inputs, {{output, 0}}, {expected_result}); + } + + // Run the host graph, which now contains a function and check that + // outputs are as expected. + // 'T' stands for 'tensor' since the outputs are tensors, not scalars. + void RunT(const std::vector>& inputs, + std::initializer_list outputs, + const std::vector>& expected_results) { + // Create a session for this graph + CSession csession(host_graph_, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + // Run + csession.SetInputs(inputs); + csession.SetOutputs(outputs); + csession.Run(s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + // Check results + for (int i = 0; i < expected_results.size(); ++i) { + TF_Tensor* out = csession.output_tensor(i); + ASSERT_TRUE(out != nullptr); + EXPECT_EQ(TF_INT32, TF_TensorType(out)); + EXPECT_EQ(1, TF_NumDims(out)); + CompareInt32Tensor(expected_results[i], out); + } + } + + // Run the host graph, which now contains a function and check that + // outputs are as expected. + void Run(const std::vector>& inputs, + std::initializer_list outputs, + const std::vector& expected_results) { + // Create a session for this graph. + CSession csession(host_graph_, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + csession.SetInputs(inputs); + csession.SetOutputs(outputs); + csession.Run(s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + for (int i = 0; i < expected_results.size(); ++i) { + TF_Tensor* out = csession.output_tensor(i); + ASSERT_TRUE(out != nullptr); + EXPECT_EQ(TF_INT32, TF_TensorType(out)); + EXPECT_EQ(0, TF_NumDims(out)); // scalar + ASSERT_EQ(sizeof(int32_t), TF_TensorByteSize(out)); + int32_t* output_contents = static_cast(TF_TensorData(out)); + EXPECT_EQ(expected_results[i], *output_contents); + } + } + + void CompareInt32Tensor(const std::vector& expected, TF_Tensor* t) { + int32_t* data = static_cast(TF_TensorData(t)); + size_t size = TF_TensorByteSize(t); + ASSERT_EQ(expected.size() * sizeof(int32_t), size); + for (int i = 0; i < expected.size(); ++i) { + ASSERT_EQ(expected[i], data[i]) << "Different data at index " << i; + } + } + + std::vector ToOutput(const std::vector ops) { + std::vector out; + for (auto op : ops) { + out.push_back({op, 0}); + } + return out; + } + + void Define(int num_opers, const std::vector& opers, + const std::vector& inputs, + const std::vector& outputs, + const char** output_names, bool expect_failure = false) { + DefineT(num_opers, opers, ToOutput(inputs), ToOutput(outputs), output_names, + expect_failure); + } + + // An explicit `num_opers` is needed so that we can distinguish between the + // case of no operations specified (-1) and the case of an empty set of + // operations specified (0). + void DefineT(int num_opers, const std::vector& opers, + const std::vector& inputs, + const std::vector& outputs, const char** output_names, + bool expect_failure = false) { + ASSERT_EQ(func_, nullptr); + func_ = TF_GraphToFunction(func_graph_, func_name_, num_opers, + num_opers == -1 ? nullptr : opers.data(), + inputs.size(), inputs.data(), outputs.size(), + outputs.data(), output_names, + /*opts=*/nullptr, s_); + if (expect_failure) { + ASSERT_EQ(func_, nullptr); + return; + } + + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + ASSERT_NE(func_, nullptr); + TF_GraphAddFunction(host_graph_, func_, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + } + + TF_Operation* Use(const std::vector& inputs) { + return UseT(ToOutput(inputs)); + } + + TF_Operation* UseT(const std::vector& inputs) { + TF_Operation* op; + UseHelper(inputs, &op); + return op; + } + + // All the *Helper methods are used as a workaround for the restrictions that + // one cannot call ASSERT_* methods in non-void-returning functions (when + // exceptions are disabled during compilation) + void UseHelper(const std::vector& inputs, TF_Operation** op) { + TF_OperationDescription* desc = + TF_NewOperation(host_graph_, func_name_, func_node_name_); + for (auto input : inputs) { + TF_AddInput(desc, input); + } + // Set device to CPU because some ops inside the function might not be + // available on GPU. + TF_SetDevice(desc, "/cpu:0"); + *op = TF_FinishOperation(desc, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + ASSERT_NE(*op, nullptr); + } + + FunctionDef fdef() { + tensorflow::FunctionDef fdef; + EXPECT_TRUE(GetFunctionDef(func_, &fdef)); + return fdef; + } + + // logging utility + template + string ToString(const Container& v) { + std::stringstream ss; + ss << "{"; + size_t i = 0; + for (const auto& e : v) { + if (i != 0) { + ss << ", "; + } + ss << e.ToString(); + ++i; + } + ss << "}"; + return ss.str(); + } + + void VerifyFDefNodes(const tensorflow::FunctionDef& fdef, + const std::unordered_set& nodes) { + ASSERT_EQ(nodes.size(), fdef.node_def_size()) + << "Got unexpected number of nodes. Expected: [" + << str_util::Join(nodes, ", ") + << "] Actual nodes in fdef: " << fdef.DebugString(); + for (const NodeDef& node_def : fdef.node_def()) { + ASSERT_TRUE(nodes.find(node_def.name()) != nodes.end()) + << "Got unexpected node: " << node_def.name() + << " in fdef: " << fdef.DebugString(); + } + } + + void VerifyFDefInputs(const tensorflow::FunctionDef& fdef, + const std::vector& inputs) { + const OpDef& signature = fdef.signature(); + ASSERT_EQ(inputs.size(), signature.input_arg_size()); + for (int i = 0; i < inputs.size(); ++i) { + const OpDef::ArgDef& arg = signature.input_arg(i); + const IOSpec& in = inputs[i]; + if (in.second != DT_INVALID) { + ASSERT_EQ(arg.type(), in.second) + << "Got unexpected type for input " << i + << ". fdef: " << fdef.DebugString(); + } + ASSERT_EQ(arg.name(), in.first) << "Got unexpected name for input " << i + << ". fdef: " << fdef.DebugString(); + } + } + + void VerifyFDefOutputs(const tensorflow::FunctionDef& fdef, + const std::vector& outputs) { + const OpDef& signature = fdef.signature(); + ASSERT_EQ(outputs.size(), signature.output_arg_size()); + for (int i = 0; i < outputs.size(); ++i) { + const OpDef::ArgDef& arg = signature.output_arg(i); + const IOSpec& out = outputs[i]; + if (out.second != DT_INVALID) { + ASSERT_EQ(arg.type(), out.second) + << "Got unexpected type for output " << i + << ". fdef: " << fdef.DebugString(); + } + ASSERT_EQ(arg.name(), out.first) << "Got unexpected name for output " << i + << ". fdef: " << fdef.DebugString(); + } + } + + void VerifyFDefEdges( + const tensorflow::FunctionDef& fdef, + const std::vector& e_edges, // expected edges + const std::vector& c_edges, // expected ctrl edges + bool is_exact_edges = true) { + // Build a set of edges from fdef + std::set a_edges; // actual edges + // Get edges from inputs to body nodes and between body nodes + for (const NodeDef& node_def : fdef.node_def()) { + for (int i = 0; i < node_def.input_size(); ++i) { + const string& in = node_def.input(i); + const auto& v = + a_edges.insert({in, strings::StrCat(node_def.name(), ":", i)}); + ASSERT_TRUE(v.second) << "Duplicate edge " << in << " -> " + << strings::StrCat(node_def.name(), ":", i) + << ". fdef: " << fdef.DebugString(); + } + } + // Get edges from body nodes to outputs and from inputs to outputs + for (const OpDef::ArgDef& arg : fdef.signature().output_arg()) { + const auto& iter = fdef.ret().find(arg.name()); + if (iter != fdef.ret().end()) { + const auto& v = a_edges.insert({iter->second, arg.name()}); + ASSERT_TRUE(v.second) << "Duplicate edge " << iter->second << " -> " + << arg.name() << ". fdef: " << fdef.DebugString(); + } else { + const auto& v = a_edges.insert({arg.name(), arg.name()}); + ASSERT_TRUE(v.second) << "Duplicate edge " << arg.name() << " -> " + << arg.name() << ". fdef: " << fdef.DebugString(); + } + } + + // Verify edges + for (const EdgeSpec& e : e_edges) { + ASSERT_TRUE(a_edges.find(e) != a_edges.end()) + << "Failed to find expected edge " << e.ToString() + << " in fdef: " << fdef.DebugString(); + } + + // If caller specified all edges, check that we have seen all + if (is_exact_edges) { + ASSERT_EQ(e_edges.size() + c_edges.size(), a_edges.size()) + << "Expected edges: " << ToString(e_edges) + << " Expected Control edges: " << ToString(c_edges) + << " Actual edges: " << ToString(a_edges) + << " in fdef: " << fdef.DebugString(); + } + } + + void VerifyFDef(const std::unordered_set& nodes, + const std::vector& inputs, + const std::vector& outputs, + const std::vector& e_edges, // expected edges + const std::vector& c_edges, // expected ctrl edges + bool is_exact_edges = true) { + tensorflow::FunctionDef fdef; + ASSERT_TRUE(GetFunctionDef(func_, &fdef)); + VerifyFDefNodes(fdef, nodes); + VerifyFDefInputs(fdef, inputs); + VerifyFDefOutputs(fdef, outputs); + VerifyFDefEdges(fdef, e_edges, c_edges, is_exact_edges); + } + + const char* func_name_ = "MyFunc"; + const char* func_node_name_ = "MyFunc_0"; + TF_Status* s_; + TF_Graph* func_graph_; + TF_Graph* host_graph_; + TF_Function* func_; + + // Workaround for not being able to initialize empty map using {} + std::unordered_set empty_; +}; + +TEST_F(CApiFunctionTest, OneOp_ZeroInputs_OneOutput) { + /* + * constant + * | + * v + */ + // Define + TF_Operation* c = ScalarConst(10, func_graph_, s_, "scalar10"); + Define(-1, {}, {}, {c}, nullptr); + + // Use, run, and verify + TF_Operation* func_op = Use({}); + Run({}, func_op, 10); + VerifyFDef({"scalar10_0"}, {}, {{"scalar10", DT_INT32}}, + {{"scalar10_0:output:0", "scalar10"}}, {}); +} + +TEST_F(CApiFunctionTest, OneOp_OneInput_OneOutput) { + /* + * | + * v + * negate + * | + * v + */ + // Define + TF_Operation* feed = Placeholder(func_graph_, s_); + TF_Operation* neg = Neg(feed, func_graph_, s_); + Define(-1, {}, {feed}, {neg}, nullptr); + + // Use, run, and verify + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({func_feed}); + Run({{func_feed, Int32Tensor(3)}}, func_op, -3); + VerifyFDef({"neg_0"}, {{"feed", DT_INT32}}, {{"neg", DT_INT32}}, + {{"feed", "neg_0:0"}, {"neg_0:y:0", "neg"}}, {}); +} + +TEST_F(CApiFunctionTest, ZeroOps_Identity) { + /* + * | + * | + * | + * v + */ + // Define + TF_Operation* feed = Placeholder(func_graph_, s_); + Define(-1, {}, {feed}, {feed}, nullptr); + + // Use, run, and verify + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({func_feed}); + Run({{func_feed, Int32Tensor(3)}}, func_op, 3); + VerifyFDef(empty_, {{"feed", DT_INT32}}, {{"feed_0", DT_INT32}}, + {{"feed", "feed_0"}}, {}); +} + +TEST_F(CApiFunctionTest, ZeroOps_Permutation) { + /* + * | | + * \ / + * \/ + * x + * /\ + * / \ + * | | + * v v + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + Define(-1, {}, {feed1, feed2}, {feed2, feed1}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({two, func_feed}); + Run({{func_feed, Int32Tensor(3)}}, {{func_op, 0}, {func_op, 1}}, {3, 2}); + VerifyFDef(empty_, M({{"feed1"}, {"feed2"}}), M({{"feed2_0"}, {"feed1_0"}}), + {{"feed1", "feed1_0"}, {"feed2", "feed2_0"}}, {}); +} + +TEST_F(CApiFunctionTest, OneOp_TwoInputs_OneOutput) { + /* + * | | + * v v + * add + * | + * v + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* add = Add(feed1, feed2, func_graph_, s_); + Define(-1, {}, {feed1, feed2}, {add}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({two, func_feed}); + Run({{func_feed, Int32Tensor(3)}}, func_op, 2 + 3); + VerifyFDef( + {"add_0"}, M({{"feed1"}, {"feed2"}}), M({{"add"}}), + {{"feed1", "add_0:0"}, {"feed2", "add_0:1"}, {"add_0:sum:0", "add"}}, {}); +} + +TEST_F(CApiFunctionTest, OneOp_TwoInputs_ZeroOutputs) { + /* + * | | + * v v + * add + * + * (output ignored) + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + Add(feed1, feed2, func_graph_, s_); + Define(-1, {}, {feed1, feed2}, {}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + Use({two, func_feed}); + VerifyFDef({"add"}, M({{"feed1"}, {"feed2"}}), {}, + {{"feed1", "add:0"}, {"feed2", "add:1"}}, {}); +} + +TEST_F(CApiFunctionTest, TwoOps_ThreeInputs_OneOutput) { + /* + * | | | + * v v / + * add1 / + * | | + * v v + * add2 + * | + * v + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* feed3 = Placeholder(func_graph_, s_, "feed3"); + TF_Operation* add1 = Add(feed1, feed2, func_graph_, s_, "add1"); + TF_Operation* add2 = Add(add1, feed3, func_graph_, s_, "add2"); + Define(-1, {}, {feed1, feed2, feed3}, {add2}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_, "two"); + TF_Operation* ten = ScalarConst(10, host_graph_, s_, "ten"); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({two, ten, func_feed}); + Run({{func_feed, Int32Tensor(3)}}, func_op, 2 + 10 + 3); + VerifyFDef({"add1", "add2_0"}, M({{"feed1"}, {"feed2"}, {"feed3"}}), + M({{"add2"}}), + {{"feed1", "add1:0"}, + {"feed2", "add1:1"}, + {"add1:sum:0", "add2_0:0"}, + {"feed3", "add2_0:1"}, + {"add2_0:sum:0", "add2"}}, + {}); +} + +TEST_F(CApiFunctionTest, OneOp_TwoInputs_TwoDuplicateOutputs) { + /* + * | | + * v v + * add + * | + * +-+-+ + * | | + * v v + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* add = Add(feed1, feed2, func_graph_, s_); + Define(-1, {}, {feed1, feed2}, {add, add}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({two, func_feed}); + Run({{func_feed, Int32Tensor(3)}}, {{func_op, 0}, {func_op, 1}}, {5, 5}); + VerifyFDef({"add_1"}, M({{"feed1"}, {"feed2"}}), M({{"add"}, {"add_0"}}), + {{"feed1", "add_1:0"}, + {"feed2", "add_1:1"}, + {"add_1:sum:0", "add"}, + {"add_1:sum:0", "add_0"}}, + {}); +} + +TEST_F(CApiFunctionTest, TwoOps_ThreeInputs_TwoOutputs) { + /* + * | | | + * v v / + * add / + * | | + * +-+ | + * | | | + * | v v + * | add + * | | + * v v + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* feed3 = Placeholder(func_graph_, s_, "feed3"); + TF_Operation* add1 = Add(feed1, feed2, func_graph_, s_, "add1"); + TF_Operation* add2 = Add(add1, feed3, func_graph_, s_, "add2"); + Define(-1, {}, {feed1, feed2, feed3}, {add1, add2}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_, "two"); + TF_Operation* ten = ScalarConst(10, host_graph_, s_, "ten"); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({two, ten, func_feed}); + Run({{func_feed, Int32Tensor(3)}}, {{func_op, 0}, {func_op, 1}}, {12, 15}); + VerifyFDef({"add1_0", "add2_0"}, M({{"feed1"}, {"feed2"}, {"feed3"}}), + M({{"add1"}, {"add2"}}), + {{"feed1", "add1_0:0"}, + {"feed2", "add1_0:1"}, + {"add1_0:sum:0", "add2_0:0"}, + {"feed3", "add2_0:1"}, + {"add1_0:sum:0", "add1"}, + {"add2_0:sum:0", "add2"}}, + {}); +} + +TEST_F(CApiFunctionTest, FromSubsetOfOps) { + /* + * | | | + * v v / + * add / + * | | + * +---+--+---+ + * Ops used | | | | + * for func | v v | + * | | add | + * +-------> | | | + * | v | + * | | + * +----------+ + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* feed3 = Placeholder(func_graph_, s_, "feed3"); + TF_Operation* add1 = Add(feed1, feed2, func_graph_, s_, "add1"); + TF_Operation* add2 = Add(add1, feed3, func_graph_, s_, "add2"); + Define(1, {add2}, {add1, feed3}, {add2}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_, "two"); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({two, func_feed}); + Run({{func_feed, Int32Tensor(3)}}, func_op, 2 + 3); + VerifyFDef( + {"add2_0"}, M({{"add1"}, {"feed3"}}), M({{"add2"}}), + {{"add1", "add2_0:0"}, {"feed3", "add2_0:1"}, {"add2_0:sum:0", "add2"}}, + {}); +} + +TEST_F(CApiFunctionTest, UsingOneOutputOfSplit) { + /* + * feed + * | + * +---------+---+ + * | const0 | | + * | | | | + * | v / | + * | split | + * | | | | | + * | v | v | + * | | | + * +------+------+ + * | + * v + * + * Only the second output from split is used as function output + */ + // Define + TF_Operation* feed = Placeholder(func_graph_, s_); + TF_Operation* split = Split3(feed, func_graph_, s_); + DefineT(-1, {}, {{feed, 0}}, {{split, 1}}, nullptr); + + // Use, run, and verify + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({func_feed}); + RunT({{func_feed, Int32Tensor({1, 2, 3, 4, 5, 6})}}, {{func_op, 0}}, + {{3, 4}}); + VerifyFDef({"split3_const0", "split3_0"}, M({{"feed"}}), M({{"split3"}}), + {{"split3_const0:output:0", "split3_0:0"}, + {"feed", "split3_0:1"}, + {"split3_0:output:1", "split3"}}, + {}); +} + +TEST_F(CApiFunctionTest, UsingTwoOutputsOfSplit) { + /* + * feed + * | + * +---------+---+ + * | const0 | | + * | | | | + * | v / | + * | split | + * | | | | | + * | | v | | + * | | | | + * +---+-----+---+ + * | | + * v v + * + * Second output from split is not used as function output + */ + // Define + TF_Operation* feed = Placeholder(func_graph_, s_); + TF_Operation* split = Split3(feed, func_graph_, s_); + DefineT(-1, {}, {{feed, 0}}, {{split, 0}, {split, 2}}, nullptr); + + // Use, run, and verify + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({func_feed}); + RunT({{func_feed, Int32Tensor({1, 2, 3, 4, 5, 6})}}, + {{func_op, 0}, {func_op, 1}}, {{1, 2}, {5, 6}}); + VerifyFDef({"split3_const0", "split3_1"}, M({{"feed"}}), + M({{"split3"}, {"split3_0"}}), + {{"split3_const0:output:0", "split3_1:0"}, + {"feed", "split3_1:1"}, + {"split3_1:output:0", "split3"}, + {"split3_1:output:2", "split3_0"}}, + {}); +} + +TEST_F(CApiFunctionTest, UsingTwoOutputsOfSplitAsInputs) { + /* + * | + * v + * split + * | | | + * | v | + * | | + * +---+-----+---+ + * | | | | + * | v v | + * | add | + * | | | + * | | | + * +------+------+ + * | + * v + */ + // Define + TF_Operation* feed = Placeholder(func_graph_, s_); + TF_Operation* split = Split3(feed, func_graph_, s_); + TF_Operation* add = Add({split, 0}, {split, 2}, func_graph_, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + DefineT(1, {add}, {{split, 0}, {split, 2}}, {{add, 0}}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_, "two"); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({two, func_feed}); + Run({{func_feed, Int32Tensor(3)}}, func_op, 2 + 3); + VerifyFDef( + {"add_0"}, M({{"split3"}, {"split3_0"}}), M({{"add"}}), + {{"split3", "add_0:0"}, {"split3_0", "add_0:1"}, {"add_0:sum:0", "add"}}, + {}); +} + +TEST_F(CApiFunctionTest, NodesUsedInInputsMustHaveSingleOutput) { + /* + * | + * v + * split + * | | | + * | v | + * | | + * input --->| |<--- input + * | | + * v v + * add + * | + * | + * v + */ + // Define + TF_Tensor* tensor_123 = Int32Tensor({1, 2, 3}); + TF_Operation* c = Const(tensor_123, func_graph_, s_, "const_array"); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + TF_Operation* split = Split3(c, func_graph_, s_); + TF_Operation* add = Add({split, 0}, {split, 2}, func_graph_, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + DefineT(-1, {}, {{split, 0}, {split, 2}}, {{add, 0}}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("When `num_opers` is set to -1, nodes referenced in " + "`inputs` must have a single output. Node split3 has " + "3 outputs. Encountered while creating function 'MyFunc'"), + string(TF_Message(s_))); + + TF_DeleteTensor(tensor_123); +} + +TEST_F(CApiFunctionTest, FunctionWithWhileLoop) { + // Inputs to the while loop and the function as a whole + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + + // Outputs of the while loop corresponding to the two inputs above + // The first one will the function's output + std::vector outputs; + + // Add while loop to func_graph_ + { + // The inputs to the while loop + std::vector inputs = {{feed1, 0}, {feed2, 0}}; + std::unique_ptr params(new TF_WhileParams( + TF_NewWhile(func_graph_, &inputs[0], inputs.size(), s_))); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + params->name = "test_loop"; + + // Initialize outputs so we can easily detect errors/bugs + outputs.resize(2, {nullptr, -1}); + + // Create loop: while (input1 < input2) input1 += input2 + 1 + TF_Operation* less_than = LessThan( + params->cond_inputs[0], params->cond_inputs[1], params->cond_graph, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + params->cond_output = {less_than, 0}; + + TF_Operation* add1 = Add(params->body_inputs[0], params->body_inputs[1], + params->body_graph, s_, "add1"); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + TF_Operation* one = ScalarConst(1, params->body_graph, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + TF_Operation* add2 = Add(add1, one, params->body_graph, s_, "add2"); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + params->body_outputs[0] = {add2, 0}; + params->body_outputs[1] = params->body_inputs[1]; + + // Finalize while loop + TF_FinishWhile(params.get(), s_, &outputs[0]); + EXPECT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + } + + // Define function, use it in graph, and run + DefineT(-1, {}, {{feed1, 0}, {feed2, 0}}, {outputs[0]}, nullptr); + TF_Operation* five = ScalarConst(5, host_graph_, s_, "five"); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({func_feed, five}); + Run({{func_feed, Int32Tensor(2)}}, func_op, 2 /*+=*/ + 5 + 1); + + // Verify input, output, and subset of edges in fdef. + // The subset of edges we verify is a chain between feed1 and output to + // make sure that the correct output is picked. + tensorflow::FunctionDef fdef; + ASSERT_TRUE(GetFunctionDef(func_, &fdef)); + VerifyFDefInputs(fdef, M({{"feed1"}, {"feed2"}})); + VerifyFDefOutputs(fdef, M({{"test_loop_exit"}})); + VerifyFDefEdges(fdef, + {{"feed1", "test_loop/Enter:0"}, + {"test_loop/Enter:output:0", "test_loop/Merge:0"}, + {"test_loop/Merge:output:0", "test_loop/Switch:0"}, + {"test_loop/Switch:output_false:0", "test_loop/Exit:0"}, + {"test_loop/Exit:output:0", "test_loop_exit"}}, + {}, false); +} + +TEST_F(CApiFunctionTest, ControlDependency) { + /* + * | | scalar + * | | . + * v v . <---- control dependency + * add < - + * | + * v + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* five = ScalarConst(5, func_graph_, s_); + TF_Operation* add = + AddWithCtrlDependency(feed1, feed2, func_graph_, five, s_); + EXPECT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + Define(-1, {}, {feed1, feed2}, {add}, nullptr); + + // Use, run, and verify + TF_Operation* two = ScalarConst(2, host_graph_, s_); + TF_Operation* func_feed = Placeholder(host_graph_, s_); + TF_Operation* func_op = Use({two, func_feed}); + Run({{func_feed, Int32Tensor(3)}}, func_op, 2 + 3); + VerifyFDef( + {"add_0", "scalar"}, M({{"feed1"}, {"feed2"}}), M({{"add"}}), + {{"feed1", "add_0:0"}, {"feed2", "add_0:1"}, {"add_0:sum:0", "add"}}, + {{"scalar", "add_0"}}); +} + +TEST_F(CApiFunctionTest, ControlDependencyOutsideOfBody) { + /* + * | | scalar + * | | . + * v v . <---- control dependency + * add < - + * | + * v + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* five = ScalarConst(5, func_graph_, s_); + TF_Operation* add = + AddWithCtrlDependency(feed1, feed2, func_graph_, five, s_); + EXPECT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + Define(1, {add}, {feed1, feed2}, {add}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("The source of control edge [id=3 scalar:-1 -> add:-1] " + "is not in the body. Encountered while creating " + "function 'MyFunc'"), + string(TF_Message(s_))); +} + +TEST_F(CApiFunctionTest, ControlDependencyOutsideOfBody_FromInputNode) { + /* + * | |. + * | | . + * | | . + * v v . <---- control dependency + * add < - + * | + * v + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* add = + AddWithCtrlDependency(feed1, feed2, func_graph_, feed1, s_); + EXPECT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + Define(-1, {}, {feed1, feed2}, {add}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("The source of control edge [id=3 feed1:-1 -> add:-1] " + "is not in the body. Encountered while creating " + "function 'MyFunc'"), + string(TF_Message(s_))); +} + +TEST_F(CApiFunctionTest, DuplicateInputsAreNotAllowed) { + /* + * feed + * | + * +++ + * | | + * +---+-+---+ + * | | | | + * | v v | + * | add | + * | | | + * | | | + * +----+----+ + * | + * v + */ + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* add = Add(feed1, feed1, func_graph_, s_); + Define(-1, {}, {feed1, feed1}, {add}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ( + string("TF_Output feed1:0 appears more than once in the input list"), + string(TF_Message(s_))); +} + +TEST_F(CApiFunctionTest, InvalidInputTensor_HighIndex) { + /* + * | | + * v v + * add + * | + * v + */ + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* add = Add(feed1, feed2, func_graph_, s_); + DefineT(-1, {}, {{feed1, 0}, {feed2, 2}}, {{add, 0}}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("Node 'feed2' (type: 'Placeholder', num of outputs: 1) does " + "not have output 2\n\tEncountered while processing " + "input 1 into function 'MyFunc'"), + string(TF_Message(s_))); +} + +TEST_F(CApiFunctionTest, InvalidInputTensor_BadNodePtr) { + /* + * | | + * v v + * add + * | + * v + */ + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* add = Add(feed1, feed2, func_graph_, s_); + DefineT(-1, {}, {{feed1, 0}, {nullptr, 0}}, {{add, 0}}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("Node is null\n\tEncountered while processing input 1 " + "into function 'MyFunc'"), + string(TF_Message(s_))); +} + +TEST_F(CApiFunctionTest, InvalidOutputTensor_HighIndex) { + /* + * | | + * v v + * add + * | + * v + */ + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* add = Add(feed1, feed2, func_graph_, s_); + DefineT(-1, {}, {{feed1, 0}, {feed2, 0}}, {{add, 3}}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("Node 'add' (type: 'AddN', num of outputs: 1) does " + "not have output 3\n\tEncountered while processing " + "output 0 from function 'MyFunc'"), + string(TF_Message(s_))); +} + +TEST_F(CApiFunctionTest, InvalidOutputTensor_BadNodePtr) { + /* + * | | + * v v + * add + * | + * v + */ + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + Add(feed1, feed2, func_graph_, s_); + DefineT(-1, {}, {{feed1, 0}, {feed2, 0}}, {{nullptr, 3}}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("Node is null\n\tEncountered while processing output 0 " + "from function 'MyFunc'"), + string(TF_Message(s_))); +} + +TEST_F(CApiFunctionTest, NodeMissingInput) { + /* + * input---> | | <----missing input + * v v + * body----> add + * | + * v + */ + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* add = Add(feed1, feed2, func_graph_, s_); + DefineT(1, {add}, {{feed1, 0}}, {{add, 0}}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("Input 1, 'feed2:0', of node 'add' in function 'MyFunc' " + "is not available. You might need to include it in inputs " + "or include its source node in the body"), + string(TF_Message(s_))); +} + +TEST_F(CApiFunctionTest, OutputOpNotInBody) { + /* + * | | + * v v + * add scalar (scalar not included in body) + * | | + * v v (function has two outputs) + */ + // Define + TF_Operation* feed1 = Placeholder(func_graph_, s_, "feed1"); + TF_Operation* feed2 = Placeholder(func_graph_, s_, "feed2"); + TF_Operation* scalar = ScalarConst(2, func_graph_, s_); + TF_Operation* add = Add(feed1, feed2, func_graph_, s_); + Define(1, {add}, {feed1, feed2}, {add, scalar}, nullptr, true); + EXPECT_EQ(TF_INVALID_ARGUMENT, TF_GetCode(s_)); + EXPECT_EQ(string("TF_Output scalar:0 is neither in the function body nor " + "among function inputs. Encountered while creating " + "function 'MyFunc'"), + string(TF_Message(s_))); +} + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/c/c_api_internal.h b/tensorflow/c/c_api_internal.h index f7d25dce8f5..6e44a72e2b9 100644 --- a/tensorflow/c/c_api_internal.h +++ b/tensorflow/c/c_api_internal.h @@ -130,6 +130,11 @@ struct TF_DeviceList { std::vector response; }; +struct TF_Function { + // Currently contains a single function and no gradients + tensorflow::FunctionDefLibrary fdef_lib; +}; + namespace tensorflow { class TensorCApi { @@ -142,6 +147,9 @@ class TensorCApi { }; TF_Tensor* TF_TensorFromTensor(const Tensor& src, TF_Status* status); + +Status MessageToBuffer(const tensorflow::protobuf::Message& in, TF_Buffer* out); + } // end namespace tensorflow #endif // TENSORFLOW_C_C_API_INTERNAL_H_ diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc index 0aa60fb45dd..c4420290099 100644 --- a/tensorflow/c/c_api_test.cc +++ b/tensorflow/c/c_api_test.cc @@ -829,7 +829,7 @@ TEST(CAPI, ShapeInferenceError) { TF_Operation* vec3 = Const(vec3_tensor.get(), graph, status, "vec3"); ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_Operation* add = Add(vec2, vec3, graph, status); + TF_Operation* add = AddNoCheck(vec2, vec3, graph, status); ASSERT_NE(TF_OK, TF_GetCode(status)); ASSERT_TRUE(add == nullptr); diff --git a/tensorflow/c/c_test_util.cc b/tensorflow/c/c_test_util.cc index 21603c1a07c..9cd978c97ea 100644 --- a/tensorflow/c/c_test_util.cc +++ b/tensorflow/c/c_test_util.cc @@ -15,7 +15,9 @@ limitations under the License. #include "tensorflow/c/c_test_util.h" +#include "tensorflow/core/framework/function.pb.h" #include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/logging.h" using tensorflow::GraphDef; @@ -36,6 +38,23 @@ TF_Tensor* Int8Tensor(const int64_t* dims, int num_dims, const char* values) { return t; } +TF_Tensor* Int32Tensor(const int64_t* dims, int num_dims, + const int32_t* values) { + int64_t num_values = 1; + for (int i = 0; i < num_dims; ++i) { + num_values *= dims[i]; + } + TF_Tensor* t = + TF_AllocateTensor(TF_INT32, dims, num_dims, sizeof(int32_t) * num_values); + memcpy(TF_TensorData(t), values, sizeof(int32_t) * num_values); + return t; +} + +TF_Tensor* Int32Tensor(const std::vector& values) { + int64_t dims = values.size(); + return Int32Tensor(&dims, 1, values.data()); +} + TF_Tensor* Int32Tensor(int32_t v) { const int num_bytes = sizeof(int32_t); int32_t* values = new int32_t[1]; @@ -44,19 +63,40 @@ TF_Tensor* Int32Tensor(int32_t v) { &Int32Deallocator, nullptr); } -TF_Operation* Placeholder(TF_Graph* graph, TF_Status* s, const char* name) { +// All the *Helper methods are used as a workaround for the restrictions that +// one cannot call ASSERT_* methods in non-void-returning functions (when +// exceptions are disabled during compilation) +void PlaceholderHelper(TF_Graph* graph, TF_Status* s, const char* name, + TF_Operation** op) { TF_OperationDescription* desc = TF_NewOperation(graph, "Placeholder", name); TF_SetAttrType(desc, "dtype", TF_INT32); - return TF_FinishOperation(desc, s); + *op = TF_FinishOperation(desc, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_NE(*op, nullptr); +} + +TF_Operation* Placeholder(TF_Graph* graph, TF_Status* s, const char* name) { + TF_Operation* op; + PlaceholderHelper(graph, s, name, &op); + return op; +} + +void ConstHelper(TF_Tensor* t, TF_Graph* graph, TF_Status* s, const char* name, + TF_Operation** op) { + TF_OperationDescription* desc = TF_NewOperation(graph, "Const", name); + TF_SetAttrTensor(desc, "value", t, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + TF_SetAttrType(desc, "dtype", TF_TensorType(t)); + *op = TF_FinishOperation(desc, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_NE(*op, nullptr); } TF_Operation* Const(TF_Tensor* t, TF_Graph* graph, TF_Status* s, const char* name) { - TF_OperationDescription* desc = TF_NewOperation(graph, "Const", name); - TF_SetAttrTensor(desc, "value", t, s); - if (TF_GetCode(s) != TF_OK) return nullptr; - TF_SetAttrType(desc, "dtype", TF_TensorType(t)); - return TF_FinishOperation(desc, s); + TF_Operation* op; + ConstHelper(t, graph, s, name, &op); + return op; } TF_Operation* ScalarConst(int32_t v, TF_Graph* graph, TF_Status* s, @@ -65,11 +105,39 @@ TF_Operation* ScalarConst(int32_t v, TF_Graph* graph, TF_Status* s, return Const(tensor.get(), graph, s, name); } -TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph, - TF_Status* s, const char* name) { +void AddHelper(TF_Operation* l, TF_Operation* r, TF_Graph* graph, TF_Status* s, + const char* name, TF_Operation** op, bool check) { TF_OperationDescription* desc = TF_NewOperation(graph, "AddN", name); TF_Output add_inputs[2] = {{l, 0}, {r, 0}}; TF_AddInputList(desc, add_inputs, 2); + *op = TF_FinishOperation(desc, s); + if (check) { + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_NE(*op, nullptr); + } +} + +TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph, + TF_Status* s, const char* name) { + TF_Operation* op; + AddHelper(l, r, graph, s, name, &op, true); + return op; +} + +TF_Operation* AddNoCheck(TF_Operation* l, TF_Operation* r, TF_Graph* graph, + TF_Status* s, const char* name) { + TF_Operation* op; + AddHelper(l, r, graph, s, name, &op, false); + return op; +} + +TF_Operation* AddWithCtrlDependency(TF_Operation* l, TF_Operation* r, + TF_Graph* graph, TF_Operation* ctrl_op, + TF_Status* s, const char* name) { + TF_OperationDescription* desc = TF_NewOperation(graph, "AddN", name); + TF_Output add_inputs[2] = {{l, 0}, {r, 0}}; + TF_AddInputList(desc, add_inputs, 2); + TF_AddControlInput(desc, ctrl_op); return TF_FinishOperation(desc, s); } @@ -81,11 +149,20 @@ TF_Operation* Add(TF_Output l, TF_Output r, TF_Graph* graph, TF_Status* s, return TF_FinishOperation(desc, s); } -TF_Operation* Neg(TF_Operation* n, TF_Graph* graph, TF_Status* s) { +void NegHelper(TF_Operation* n, TF_Graph* graph, TF_Status* s, + TF_Operation** op) { TF_OperationDescription* desc = TF_NewOperation(graph, "Neg", "neg"); TF_Output neg_input = {n, 0}; TF_AddInput(desc, neg_input); - return TF_FinishOperation(desc, s); + *op = TF_FinishOperation(desc, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_NE(*op, nullptr); +} + +TF_Operation* Neg(TF_Operation* n, TF_Graph* graph, TF_Status* s) { + TF_Operation* op; + NegHelper(n, graph, s, &op); + return op; } TF_Operation* LessThan(TF_Output l, TF_Output r, TF_Graph* graph, @@ -96,6 +173,32 @@ TF_Operation* LessThan(TF_Output l, TF_Output r, TF_Graph* graph, return TF_FinishOperation(desc, s); } +void Split3Helper(TF_Operation* input, TF_Graph* graph, TF_Status* s, + const char* name, TF_Operation** op) { + TF_Operation* zero = ScalarConst( + 0, graph, s, ::tensorflow::strings::StrCat(name, "_const0").c_str()); + TF_OperationDescription* desc = TF_NewOperation(graph, "Split", name); + TF_AddInput(desc, {zero, 0}); + TF_AddInput(desc, {input, 0}); + TF_SetAttrInt(desc, "num_split", 3); + TF_SetAttrType(desc, "T", TF_INT32); + // Set device to CPU since there is no version of split for int32 on GPU + // TODO(iga): Convert all these helpers and tests to use floats because + // they are usually available on GPUs. After doing this, remove TF_SetDevice + // call in c_api_function_test.cc + TF_SetDevice(desc, "/cpu:0"); + *op = TF_FinishOperation(desc, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_NE(*op, nullptr); +} + +TF_Operation* Split3(TF_Operation* input, TF_Graph* graph, TF_Status* s, + const char* name) { + TF_Operation* op; + Split3Helper(input, graph, s, name, &op); + return op; +} + bool IsPlaceholder(const tensorflow::NodeDef& node_def) { if (node_def.op() != "Placeholder" || node_def.name() != "feed") { return false; @@ -196,6 +299,18 @@ bool GetNodeDef(TF_Operation* oper, tensorflow::NodeDef* node_def) { return ret; } +bool GetFunctionDef(TF_Function* func, tensorflow::FunctionDef* func_def) { + TF_Status* s = TF_NewStatus(); + TF_Buffer* buffer = TF_NewBuffer(); + TF_FunctionToFunctionDef(func, buffer, s); + bool ret = TF_GetCode(s) == TF_OK; + EXPECT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + if (ret) ret = func_def->ParseFromArray(buffer->data, buffer->length); + TF_DeleteBuffer(buffer); + TF_DeleteStatus(s); + return ret; +} + bool GetAttrValue(TF_Operation* oper, const char* attr_name, tensorflow::AttrValue* attr_value, TF_Status* s) { TF_Buffer* buffer = TF_NewBuffer(); diff --git a/tensorflow/c/c_test_util.h b/tensorflow/c/c_test_util.h index 0c0ba667bd0..a927739d462 100644 --- a/tensorflow/c/c_test_util.h +++ b/tensorflow/c/c_test_util.h @@ -33,6 +33,13 @@ typedef std::unique_ptr // Create a tensor with values of type TF_INT8 provided by `values`. TF_Tensor* Int8Tensor(const int64_t* dims, int num_dims, const char* values); +// Create a tensor with values of type TF_INT32 provided by `values`. +TF_Tensor* Int32Tensor(const int64_t* dims, int num_dims, + const int32_t* values); + +// Create 1 dimensional tensor with values from `values` +TF_Tensor* Int32Tensor(const std::vector& values); + TF_Tensor* Int32Tensor(int32_t v); TF_Operation* Placeholder(TF_Graph* graph, TF_Status* s, @@ -47,6 +54,13 @@ TF_Operation* ScalarConst(int32_t v, TF_Graph* graph, TF_Status* s, TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph, TF_Status* s, const char* name = "add"); +TF_Operation* AddNoCheck(TF_Operation* l, TF_Operation* r, TF_Graph* graph, + TF_Status* s, const char* name = "add"); + +TF_Operation* AddWithCtrlDependency(TF_Operation* l, TF_Operation* r, + TF_Graph* graph, TF_Operation* ctrl_op, + TF_Status* s, const char* name = "add"); + TF_Operation* Add(TF_Output l, TF_Output r, TF_Graph* graph, TF_Status* s, const char* name = "add"); @@ -54,6 +68,10 @@ TF_Operation* Neg(TF_Operation* n, TF_Graph* graph, TF_Status* s); TF_Operation* LessThan(TF_Output l, TF_Output r, TF_Graph* graph, TF_Status* s); +// Split `input` along the first dimention into 3 tensors +TF_Operation* Split3(TF_Operation* input, TF_Graph* graph, TF_Status* s, + const char* name = "split3"); + bool IsPlaceholder(const tensorflow::NodeDef& node_def); bool IsScalarConst(const tensorflow::NodeDef& node_def, int v); @@ -66,6 +84,8 @@ bool GetGraphDef(TF_Graph* graph, tensorflow::GraphDef* graph_def); bool GetNodeDef(TF_Operation* oper, tensorflow::NodeDef* node_def); +bool GetFunctionDef(TF_Function* func, tensorflow::FunctionDef* func_def); + bool GetAttrValue(TF_Operation* oper, const char* attr_name, tensorflow::AttrValue* attr_value, TF_Status* s); diff --git a/tensorflow/contrib/cmake/tf_c.cmake b/tensorflow/contrib/cmake/tf_c.cmake index 87d946c3462..c5a10181271 100644 --- a/tensorflow/contrib/cmake/tf_c.cmake +++ b/tensorflow/contrib/cmake/tf_c.cmake @@ -18,6 +18,7 @@ set(tf_c_srcs "${tensorflow_source_dir}/tensorflow/c/c_api.cc" "${tensorflow_source_dir}/tensorflow/c/c_api.h" + "${tensorflow_source_dir}/tensorflow/c/c_api_function.cc" "${tensorflow_source_dir}/tensorflow/c/eager/c_api.cc" "${tensorflow_source_dir}/tensorflow/c/eager/c_api.h" "${tensorflow_source_dir}/tensorflow/c/eager/runtime.cc" diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc index 7d938365c5a..a274c799704 100644 --- a/tensorflow/core/graph/graph.cc +++ b/tensorflow/core/graph/graph.cc @@ -523,6 +523,17 @@ Status Graph::IsValidNode(const Node* node) const { return Status::OK(); } +Status Graph::IsValidOutputTensor(const Node* node, int idx) const { + TF_RETURN_IF_ERROR(IsValidNode(node)); + if (idx >= node->num_outputs()) { + return errors::InvalidArgument("Node '", node->name(), "' (type: '", + node->op_def().name(), + "', num of outputs: ", node->num_outputs(), + ") does not have ", "output ", idx); + } + return Status::OK(); +} + Node* Graph::AllocateNode(std::shared_ptr props, const Node* cost_node) { Node* node = nullptr; @@ -572,7 +583,7 @@ int Graph::InternDeviceName(const string& device_name) { } string Edge::DebugString() const { - return strings::Printf("Edge %d %s:%d -> %s:%d", id_, src_->name().c_str(), + return strings::Printf("[id=%d %s:%d -> %s:%d]", id_, src_->name().c_str(), src_output_, dst_->name().c_str(), dst_input_); } diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h index 51ede642d27..25875185e47 100644 --- a/tensorflow/core/graph/graph.h +++ b/tensorflow/core/graph/graph.h @@ -519,6 +519,10 @@ class Graph { // Returns OK if `node` is non-null and belongs to this graph Status IsValidNode(const Node* node) const; + // Returns OK if IsValidNode(`node`) and `idx` is less than + // node->num_outputs() + Status IsValidOutputTensor(const Node* node, int idx) const; + // TODO(josh11b): uint64 hash() const; private: diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i index 08dd3922dbe..fa49e66e87b 100644 --- a/tensorflow/python/client/tf_session.i +++ b/tensorflow/python/client/tf_session.i @@ -373,6 +373,33 @@ def TF_Reset(target, containers=None, config=None): TF_DeleteSessionOptions(opts) %} +// We use TF_GraphToFunction_wrapper instead of TF_GraphToFunction +%ignore TF_GraphToFunction; +// TF_GraphToFunction_wrapper does not use any Python methods and +// does not require GIL to be held. +%unignore TF_GraphToFunction_wrapper; + +// $input is a Python list of wrapped TF_Operations +%typemap(in) (const std::vector* opers) + (std::vector opers) { + if ($input != Py_None) { + if (!PyList_Check($input)) { + SWIG_exception_fail(SWIG_TypeError, "$symname: expected list"); + } + size_t size = PyList_Size($input); + for (int i = 0; i < size; ++i) { + PyObject* item = PyList_GetItem($input, i); + TF_Operation* oper_ptr; + SWIG_ConvertPtr(item, reinterpret_cast(&oper_ptr), + $descriptor(TF_Operation*), 0); + opers.push_back(oper_ptr); + } + $1 = &opers; + } else { + $1 = nullptr; + } +} + %include "tensorflow/python/client/tf_session_helper.h" %unignoreall diff --git a/tensorflow/python/client/tf_session_helper.cc b/tensorflow/python/client/tf_session_helper.cc index 60a589fa8bb..72f560fa878 100644 --- a/tensorflow/python/client/tf_session_helper.cc +++ b/tensorflow/python/client/tf_session_helper.cc @@ -337,4 +337,38 @@ std::vector TF_OperationGetControlInputs_wrapper( return control_inputs; } +TF_Function* TF_GraphToFunction_wrapper(const TF_Graph* fn_body, + const char* fn_name, + const std::vector* opers, + const std::vector& inputs, + const std::vector& outputs, + const NameVector& output_names, + const TF_FunctionOptions* opts, + TF_Status* out_status) { + if (!output_names.empty() && output_names.size() != outputs.size()) { + Set_TF_Status_from_Status( + out_status, + errors::InvalidArgument( + "output names must be either empty or equal in size to outputs. ", + "output names size = ", output_names.size(), + " outputs size = ", outputs.size())); + return nullptr; + } + + int nopers = -1; + const TF_Operation* const* opers_array = nullptr; + if (opers != nullptr) { + nopers = opers->size(); + opers_array = opers->data(); + } + + const char** output_names_ptr = + output_names.empty() ? nullptr + : const_cast(output_names.data()); + + return TF_GraphToFunction(fn_body, fn_name, nopers, opers_array, + inputs.size(), inputs.data(), outputs.size(), + outputs.data(), output_names_ptr, opts, out_status); +} + } // namespace tensorflow diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h index 3bc63f822fe..8fae6206c07 100644 --- a/tensorflow/python/client/tf_session_helper.h +++ b/tensorflow/python/client/tf_session_helper.h @@ -148,6 +148,16 @@ void TF_SessionPRun_wrapper(TF_Session* session, const char* handle, std::vector TF_OperationGetControlInputs_wrapper( TF_Operation* oper); +// `opers` equaling NULL are converted to `nopers = -1`. +// `output_names` must be empty or have the same length as `outputs`. +TF_Function* TF_GraphToFunction_wrapper(const TF_Graph* fn_body, + const char* fn_name, + const std::vector* opers, + const std::vector& inputs, + const std::vector& outputs, + const NameVector& output_names, + const TF_FunctionOptions* opts, + TF_Status* out_status); } // namespace tensorflow #endif // TENSORFLOW_PYTHON_CLIENT_TF_SESSION_HELPER_H_ diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py index 2f35f0e04b6..7a866ee6e8a 100644 --- a/tensorflow/python/framework/function.py +++ b/tensorflow/python/framework/function.py @@ -26,7 +26,9 @@ import hashlib from tensorflow.core.framework import attr_value_pb2 from tensorflow.core.framework import op_def_pb2 +from tensorflow.python import pywrap_tensorflow as c_api from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.framework import graph_to_function_def from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -290,6 +292,7 @@ class _DefinedFunction(object): self._shape_func = shape_func self._extra_kwargs = kwargs self._definition = None # Constructed lazily. + self._c_func = None # Constructed with definition. self._sub_functions = dict() # Constructed with definition. self._args = [] @@ -396,6 +399,22 @@ class _DefinedFunction(object): if self._func.__doc__: self._definition.signature.description = self._func.__doc__ + # pylint: disable=protected-access + if temp_graph._c_graph: + with errors.raise_exception_on_not_ok_status() as status: + output_names = ([compat.as_bytes(x) for x in self._out_names] + if self._out_names else []) + self._c_func = c_api.TF_GraphToFunction_wrapper( + temp_graph._c_graph, + self._func_name, + None, # opers + [t._as_tf_output() for t in inputs], + [t._as_tf_output() for t in outputs], + output_names, + None, # opts + status) + # pylint: enable=protected-access + def _create_hash_str(self, input_arg, output_arg, node_def): """Creates an 8-character string unique to this input. diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py index 589db9ef4dc..40205ddf053 100644 --- a/tensorflow/python/framework/function_test.py +++ b/tensorflow/python/framework/function_test.py @@ -33,6 +33,7 @@ from tensorflow.python.framework import function from tensorflow.python.framework import graph_to_function_def from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops @@ -63,7 +64,51 @@ def _OptimizerOptions(): do_constant_folding=cfold))) -class FunctionTest(test.TestCase): +class FunctionTestMethods(object): + """Test methods for verifying Function support. + + These test methods are used as mix-ins in two test cases: with + and without C API support. + """ + + def testIdentity(self): + + @function.Defun(dtypes.float32, func_name="MyIdentity") + def MyIdentityFunc(a): + return a + + with ops.Graph().as_default(): + call = MyIdentityFunc([18.0]) + self.assertEqual("MyIdentity", call.op.name) + with session.Session() as sess: + self.assertAllEqual([18.0], sess.run(call)) + + def testIdentityOutputName(self): + + @function.Defun( + dtypes.float32, func_name="MyIdentity", out_names=["my_result_name"]) + def MyIdentityFunc(a): + return a + + with ops.Graph().as_default(): + call = MyIdentityFunc([18.0]) + self.assertEqual("MyIdentity", call.op.name) + with session.Session() as sess: + self.assertAllEqual([18.0], sess.run(call)) + + def testTooManyOutputNames(self): + + @function.Defun( + dtypes.float32, func_name="MyIdentity", + out_names=["my_result1", "my_result2"]) + def MyIdentityFunc(a): + return a + + with ops.Graph().as_default(): + with self.assertRaisesRegexp( + ValueError, (r"Length of out_names \(2\) does not match number of " + r"outputs \(1\): my_result1, my_result2")): + MyIdentityFunc([18.0]) def testDefineFunction2Args(self): @@ -77,6 +122,35 @@ class FunctionTest(test.TestCase): with session.Session() as sess: self.assertAllEqual([5.0], sess.run(call)) + def testValueErrorOnFunctionWithNoOutput(self): + # TODO(iga): Remove this restriction and this test + + @function.Defun(dtypes.float32, dtypes.float32) + def APlus2B(a, b): + print(a + b * 2) # Create some ops to have nodes in the body + # Using 'print' to make lint happy + + with ops.Graph().as_default(): + with self.assertRaisesRegexp(ValueError, + "Function can not return None"): + APlus2B([1.0], [2.0]) + + def testDefineFunction2ArgsOutputName(self): + + @function.Defun( + dtypes.float32, + dtypes.float32, + func_name="APlus2B", + out_names=["my_result_name"]) + def APlus2B(a, b): + return a + b * 2 + + with ops.Graph().as_default(): + call = APlus2B([1.0], [2.0]) + self.assertEqual("APlus2B", call.op.name) + with session.Session() as sess: + self.assertAllEqual([5.0], sess.run(call)) + def testDefineFunctionDuplicateOutputs(self): @function.Defun(dtypes.float32, func_name="Duplicate") @@ -137,6 +211,7 @@ class FunctionTest(test.TestCase): out, = sess.run(dx, feed) self.assertAllClose(1 - np.square(np.tanh(inp)), out) + @test_util.disable_c_api # Function gradients don't work with C API def testCustomGradient(self): dtype = dtypes.float32 @@ -169,6 +244,7 @@ class FunctionTest(test.TestCase): out, = sess.run(dlogits, {logits: x, labels: y}) self.assertAllClose(out, np.exp(prob - y)) + @test_util.disable_c_api # Function gradients don't work with C API def testCustomGradientError(self): dtype = dtypes.float32 @@ -194,6 +270,7 @@ class FunctionTest(test.TestCase): "SymGrad expects to return 1.*but get 2.*instead"): _ = sess.run(dinp, {inp: x}) + @test_util.disable_c_api # Function gradients don't work with C API def testSymGradShape(self): g = ops.Graph() with g.as_default(): @@ -209,6 +286,7 @@ class FunctionTest(test.TestCase): self.assertEqual(x.get_shape(), dx.get_shape()) self.assertEqual(y.get_shape(), dy.get_shape()) + @test_util.disable_c_api # Function gradients don't work with C API def testSymGradAttr(self): @function.Defun(noinline=True) @@ -312,6 +390,7 @@ class FunctionTest(test.TestCase): "assertion failed.*-3"): self.assertAllEqual(Foo(constant_op.constant(-3.0)).eval(), 6.0) + @test_util.disable_c_api # Op._add_control_inputs doesn't work with C API def testAssertWrapper(self): @function.Defun(dtypes.float32) @@ -326,6 +405,7 @@ class FunctionTest(test.TestCase): "assertion"): _ = MyFn(100.0).eval() + @test_util.disable_c_api # Op._add_control_inputs doesn't work with C API def testWhileLoopCallsFunc(self): with self.test_session(use_gpu=True) as sess: @@ -345,6 +425,7 @@ class FunctionTest(test.TestCase): ans = sess.run(loop) self.assertAllClose(ans, 131072.) + @test_util.disable_c_api # Op._add_control_inputs doesn't work with C API def testControlFlowStrictness(self): """Inlined functions must not execute in a untaken control flow branch.""" @@ -607,6 +688,7 @@ class FunctionTest(test.TestCase): self.assertAllClose(vals[0], vals[1]) self.assertAllClose(vals[2], vals[3]) + @test_util.disable_c_api # Function Declaration doesn't work with C API def testDeclare(self): foo = function.Declare("Foo", [("x", dtypes.float32)], [("y", dtypes.float32)]) @@ -626,6 +708,7 @@ class FunctionTest(test.TestCase): expected = rand * rand + 1.0 self.assertAllClose(expected, y.eval(feed_dict={x: rand})) + @test_util.disable_c_api # Function Declaration doesn't work with C API def testDeclareUsedInDefun(self): foo = function.Declare("Foo", [("x", dtypes.float32)], [("y", dtypes.float32)]) @@ -649,6 +732,7 @@ class FunctionTest(test.TestCase): expected = rand * rand + 1.0 self.assertAllClose(expected, y.eval(feed_dict={x: rand})) + @test_util.disable_c_api # Function Declaration doesn't work with C API def testDeclareTypeMistake(self): foo = function.Declare("Foo", [("x", dtypes.float32)], [("y", dtypes.float32)]) @@ -861,6 +945,32 @@ class FunctionTest(test.TestCase): self.assertEqual(len(f.signature.input_arg), 3) +class FunctionTest(FunctionTestMethods, test.TestCase): + """Test case that invokes test methods with _USE_C_API=False.""" + + def setUp(self): + self.prev_use_c_api = ops._USE_C_API + ops._USE_C_API = False + super(FunctionTest, self).setUp() + + def tearDown(self): + ops._USE_C_API = self.prev_use_c_api + super(FunctionTest, self).tearDown() + + +class FunctionWithCApiTest(FunctionTestMethods, test.TestCase): + """Test case that invokes test methods with _USE_C_API=True.""" + + def setUp(self): + self.prev_use_c_api = ops._USE_C_API + ops._USE_C_API = True + super(FunctionWithCApiTest, self).setUp() + + def tearDown(self): + ops._USE_C_API = self.prev_use_c_api + super(FunctionWithCApiTest, self).tearDown() + + class FunctionsFromProtos(test.TestCase): def expectFunctionsEqual(self, func, grad_func=None, new_func=None): diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index ccaa2141b52..659bc394b92 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -2948,6 +2948,14 @@ class Graph(object): if self._graph_def_versions.min_consumer < 12: self._graph_def_versions.min_consumer = 12 self._functions[name] = function + if self._c_graph: + # pylint: disable=protected-access + assert function._c_func, ( + "Cannot add function created without C API support to graph " + "created with C API support") + with errors.raise_exception_on_not_ok_status() as status: + c_api.TF_GraphAddFunction(self._c_graph, function._c_func, status) + # pylint: enable=protected-access @property def building_function(self): From 6523d8303c4df74cca1d914d4d5d4c126292b019 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 21:16:58 -0700 Subject: [PATCH 22/67] Make SummaryEntry a msan-resistant plain-old-data (something that can be safely memcpy'd). Without initializing padded bytes from memory alignment, msan complains when it's used as plain-old-data. plain-old-data = http://en.cppreference.com/w/cpp/concept/PODType PiperOrigin-RevId: 167091849 --- .../quantiles/weighted_quantiles_summary.h | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h index 314c44fddc5..dad3b4e10de 100644 --- a/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h +++ b/tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_summary.h @@ -15,6 +15,7 @@ #ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_ #define THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_QUANTILES_WEIGHTED_QUANTILES_SUMMARY_H_ +#include #include #include "tensorflow/contrib/boosted_trees/lib/quantiles/weighted_quantiles_buffer.h" @@ -34,10 +35,27 @@ class WeightedQuantilesSummary { struct SummaryEntry { SummaryEntry(const ValueType& v, const WeightType& w, const WeightType& min, - const WeightType& max) - : value(v), weight(w), min_rank(min), max_rank(max) {} + const WeightType& max) { + // Explicitely initialize all of memory (including padding from memory + // alignment) to allow the struct to be msan-resistant "plain old data". + // + // POD = http://en.cppreference.com/w/cpp/concept/PODType + memset(this, 0, sizeof(*this)); - SummaryEntry() : value(0), weight(0), min_rank(0), max_rank(0) {} + value = v; + weight = w; + min_rank = min; + max_rank = max; + } + + SummaryEntry() { + memset(this, 0, sizeof(*this)); + + value = 0; + weight = 0; + min_rank = 0; + max_rank = 0; + } bool operator==(const SummaryEntry& other) const { return value == other.value && weight == other.weight && From 9514a703d3c670fa48ffec6c856f1459813eb02c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 30 Aug 2017 21:31:18 -0700 Subject: [PATCH 23/67] Update profiler doc. PiperOrigin-RevId: 167092704 --- tensorflow/core/profiler/README.md | 11 ++++++++++- tensorflow/core/profiler/g3doc/profiler_ui.jpg | Bin 0 -> 220483 bytes 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tensorflow/core/profiler/g3doc/profiler_ui.jpg diff --git a/tensorflow/core/profiler/README.md b/tensorflow/core/profiler/README.md index 5c50a86c88f..f0d4dafd3ea 100644 --- a/tensorflow/core/profiler/README.md +++ b/tensorflow/core/profiler/README.md @@ -56,7 +56,7 @@ with tf.contrib.tfprof.ProfileContext() as pctx: ```shell # Profiling from Python API is not interactive. -# Dump the profiles to files and profile with interactive command line. +# Dump the profiles to files and profile with interactive command line or web UI. with tf.contrib.tfprof.ProfileContext() as pctx: pctx.add_auto_profile_dump('/tmp/profiles', [100]) train_loop() @@ -66,7 +66,15 @@ bazel-bin/tensorflow/core/profiler/profiler \ --run_meta_path=/tmp/profiles/run_meta \ --op_log_path=/tmp/profiles/tfprof_log \ tfprof> op -select micros,bytes,occurrence -order_by micros + + +# To be open sourced... +bazel-bin/third_party/tensorflow/python/profiler/profiler_ui \ + --graph_path=/tmp/profiles/graph.pbtxt \ + --run_meta_path=/tmp/profiles/run_meta \ + --op_log_path=/tmp/profiles/tfprof_log \ ``` +![ProfilerUI](g3doc/profiler_ui.jpg) Detail Tutorials @@ -239,5 +247,6 @@ bug fix. `OpLogProto` is a good plus if it is used. #### Teams * Xin Pan (xpan@google.com, github: panyx0718) +* Chris Antaki * Yao Zhang * Jon Shlens diff --git a/tensorflow/core/profiler/g3doc/profiler_ui.jpg b/tensorflow/core/profiler/g3doc/profiler_ui.jpg new file mode 100644 index 0000000000000000000000000000000000000000..36aa94502a8c3de7915fb0e388c861cd706c3af8 GIT binary patch literal 220483 zcmeFZ2T+sU*Do4DrB~@y=}MC>H6kxfgh-JVib@R_I)oBHP>NImftM~#M4FV)2^|y! zq)Vs?y(iQF0dD@^Irn_$-uum&^PRafcjnypNp|L$d7kV%Yh~@V*Z!^F+Sh-smjLun zwRE)rH*Np`H;6C5^(;UGKuSXLuQ&0yNqpZTzjf>8&0CaYWVgwwDXFQcD5GbV+`L6`>lOtQ4HXU3fBLxY05Fh~ z6p?--y}<_{VYorcaO1iMz)O72TQ~mg0Q}eE1_|*QZ2IM14bJQ*&2$Pj6rUkDmi$;}erp(|=}AXw34;>e~9o z<`(YY@aXvD6n}R9FS%|2NdJpi#LxdC*#Au~2BKUfH*b>OB>R_KH%MT_M#^yW);+1) zjOvDDuRQPYNr#a$JxR*1?xf(CF~Tyx_8O&R5s*a-;{GMtze)B#Cs_FZl4So)uz!~e z1)wIqLHzSb82~_lNI@Qxq)IH8^O!Q%EyHBj^JXJM10z43t2=R4nQ<>>*);>{qSTm% zR9t;4^iHVUkk5gwul6^=>F2`l5JDVABIA{Y+NmD5sV;T5@za>IhS+R{)MdNy_p4yr zK$B_NSBLFw>w}@~8Yl3)E(4+AhsJ$c$t4c{eK4c+fsyp9g@*ak6%#XcO#5vcf1?*N z()!SQJ>FXWKtt5`{7AhI$s$R`X_v!(5G4X7~Qn2K{?p3ucs^;PpL<27vlGE5%&VgTSh9rC7) z=d4jm^?mf9`zG(x{SwO@Tl7z$ws&$R4}vPl{Lx~w>mg%|NX0Xfm|eIi*0wKKek`=^ z2ukVNLh`hHSIm=)zjA(u3dZG>U+G9gYTVq?THi3f=jJ9hxv=(#4?_JQigu`uOJom3 z(p2~%U-2o!*Q|uCZII2POpUqNvwM`u#e(bviQ%y$zv-3d1u>#U%akGoZ~wPyO+{NN z8>gf5;2(CWN)LBUHmYP~2ERJ~dC*9$fA&C_70R8lV}Di*PkC6F@haK7SWxg+Wo1ic z|K|n#@v)+v^_uA!Ova;dn&dOCJxXG?F!4o#d2zi1;5|IaD&g;N-7cQE#}#?>)<26o zP)6K1g^{MjyMbo7FLxp!t8m}T4{1g(AVr@h^_0gw-(?}wlG~P{Ipv`~%Z*>m7ipbw zJo2sIk5^WfHm!rUyyN(%=f}p7@iuEAB~^o^=3rE#^6SpIeu0nVS9SYj*MJ|h1)xS3sQ7YFX%(YzYns#RGOkx4>Wv99*|0BLxg^-iYBW_I zlfS{CX>$M5#u!yIgNYtnNP@x}v7~02qC0)$OQYCY%Q0U%GJI91WqmPijBg@ekm{3l zzMHv10s`-YCtvmL{izTQD~5~WUtz2{FG8P9Bla0&83F=Tt^v-GDDuSs$B)!*0?VWX zS7h-sO{Kle?50nj3z{FP**MO(_rik3kRh6s=9YNp$<#+)jV9i(+PG;20V8DAlYx<} zjK#rXJTjN16`R)$rg|}j2`GPcHA1IiW96yxS@F}YwI?zIOHN$Ia8a}}h!k#!FU~g~ zR5?it;cC-aX?i&(ZzQiA4dBt;wHir83@IIqqaQX*h&#z!zG0XrUo$;mdrD!}d8Wgi zAX?stGl*H_2zjtbW79EXvtjmp)++YX!MZoo;|KB{Js&-8gKoDL8&s$$)R;l~3Tvlr zA)Fzsq4Faz)bEOf_yDg`EpwmDn$gje^ks)t`$XVx0vS4owNnZ9p%oJFO-!k-M!RJ5 zT&YjUCl0;pnDVx1iQ4`eAb!ca$XV08=1$JSQ`ic+z3oUE4PrqdDq^v-^EPP;V zu$JwD)$W%E4P1Pl2GTxmmlKt%yv{|J#i%L0^d!rly8%FUTpbTHq9vS;YUJCzfO_)U zvXS}kr%Tw;ZwKE3&>vJqvr2xcr~X)RlO4Eld4A$y2yItrIveIIC^I!SG_+0|JS6&2 zgkl%+fK8r&88PN>OZ-%b04~{oo}fL_}2j0 z{Rdx6iqmwN&p+i6seH(;^;;3}QOQqkdD&x9O77+7ohFAW$GJCCW!5_*p9iZyefN3r z)MY=D!&3DulG%7XrMft)&z9m}6u|%6J3I(EGg3=fMqO_RTSP8-FkSPrk0`(IHa(wPcEy z8t?j~-ffAFlpQw@Sk*u^e?p+aGp?tNy(f^`$b2UZ zp_q@`qta4l7Hu|^nzzPgT^MGpECoy7^FJNtm#PLTB}W9_-|Xcm=~mWrMq~xi(QB5v zT8cr%kCSI*Aqa8{c&YB$$(`XRDl(ZWePp9NIM8^MHvnWbe81vPKY7Zx^8yV{JTpam22X!CeKu;_a& z>scF6Nmf5Jy-oM1Z_KT${`a4ycVXF1q(fz+uki6-`j;;ytsYtjCr|5u$sWh9vENF1 zch{|NzF*ZlVFuH>)#4Fqeho<9ZR0!Eg9TME&VpJGYw5l@V}FW1)zg~CGzHjH?86$M z;S~^5nr~Zvemb2zYxWW5N4Nf_zCxH5H8duVVeX^7l6yKwqn>xHa2z=VsacQk(LT@} zw41qF{2bafehqj}dc}|+6lnH=6oN2x7M6yex^FIWoq?Uh{@7+)w=M zF6d^J&XA6<-NnP5+3EN{r3=g5(|MF|eqfCKq)xE0(I>Ucq!u6{DBYCSjQvM9iRZh6 zY}|pT?}n0#%BaBrSbn8Y9@TasifrfK7y8~<{NiVqo61-ME+#frj%jybDjub93a@;b z_x>92XPRRhpJHmwZ!lJ622zkeSzE0XdTH?&q{*x0_JJ|&9gBZ;81PVl9noON@SiP2(?sp!m#Cg}+;Z`f%2v5^lE!(4@2<7AUhNj;|x1Hrn z()fHGpKm=E#!)kkz|DY_a2>>CT`22J>v)k=1Z@mloJ2*#(7jVRFd0`4#D#`b zbM-uRkv_-AEk}MAmJR^TQuc0Hrf_d46>8quvUrDM{UkE0IMgAQ(=Tx+1|AtZZ1r8B zez(^{?X|)3@55$b!ckB#S?9<3UU=EA||1Gn>bKYj1bkhE>b?Yt;IdbQ$G{5$JJjqkyx zn6B}i`rD>C%kB%(J#07QVUk_}9b`;Lcy;D?_@eF1BTFdqo#0o<=X%#LZ|)8|&OTDf z7nF}5p_}-bE=IWUuDyO{9HG&ClhEDIlWqtqsvK4R0F|a0yUfEwmwM>eCs9B*`p zh^2p&KJJsalU&)>0c6{Ok$#g}{@V}cKOemtobGygkJZQaxBKlvgZb^73SA*pO~2;% zon|$14F^)!o!d}TX70(*L)YSxfkIbn*wox&0$cz|Hwpb5dFCrBI;c}s;nt(YFJMF+ z{ye2FoSa{{#B$xW1UK?w7ye8|^~8KF@j_XrVSM%Q6U~^=PHHDm@|RRLugjX~T(?p} zTyR70#98_4f@;PSUNX?Zw6#iC8>|D#JW4Rc=IXI8MRJE+n5S{eU#4mVl-_*vUB(x6 z_VTs|+q;;92b)nbuM*L_^&jYSRiAKMAK#E(x@*jMw{sXqQgn2*)`Xk*X8PVvcjizy zWcG@6-dCvxb>wW4U8X5FFqsFQ)to_{1S`lNgJ@=T(UK(U(w#-~)>gI}FqoLY!`TNn z+MUIE0{~t@Cg+KrKgAt>^N6-g)J)2W9CZuje^D)}Zb^zVpGi3G6rk_kTf!iG>Eiu3 z(#n={Org?-!Nr_W6Yqlz9nA1g#Txv-AL*fYy)$nc zeL4{RaDYjwb2Ba_J3DEtHF|`Xse&c16*XGnAc3?5w@qc`sXquXS(Sj?7gZPj7Wjl) ze)3Rwrlj)gmumpk5vY1=S{JkGUk91u)>aA%&(BDD#t++3hXZ)M3AK{GVAzv{etkwAr49oIyK_Q+V88eRwdsXKm04`tiK$>GJbjskK3VFz zx=pXT=eP%R!r~|Yd_dE1oKx+3Z9mWI&ZJJ?huD$MW@UvxrZ#yy*d;0=p1pD;^d9dG zabPJ9D#M3ULHVY3EHnb_C|WK|B4Ag$8j=I_;kx{sZ#z|9TAY*izzyMQf?!L9WE?dbcWTlp4-HqTmu^Z$i<2sbUh`X z;nEpkd-y`}=F5cJk4laD1oyoq&V?PmjQCwFT>6b2i#Y6=C5kQ#HEHdG4+VDF7G1$# z&S$#OR(=f}o9$D{G(i2@h6%pb@cCla=g&46QoYCp9xa`+3D82yun4!R^5WVl{;bE1 z-_wntM&^`kuRp{LaNN~h;G|@_lRWrtgj+rp1C{LMuuTwbY-p+bUEveN$;Ka%y2mK8 zchAHwTU0*e^`l{(AKNBARk4Agk-jwYBie~ZQ5uMGr*YJZQg7h1F$x}s&Id)(Us#1B zI#qDPWy>o?g)qV9nudH#oBsE0zlo7RkjXXRGj#nD zABru!V%Rml3TwzaMH7RacCBkbb$RIe81KcB;bpwtH9+m-HDHmXo$DH4Qw=;^0|o)} zLosYi*MJQnTQ8s@5Y00Q+~5qj288euMEULhXpL)JPHVw4?Ac zu!ng1_vH@opML()&p-C_KNuHkv{=b58smOm=r!O&LOOAT*6dsZKJM&8*WKo?s#cV% z8}jrAx2??xGAzWUyhcg*H2~{!4IsE7`>*<{uK{^Fq5EOlCvu5|bB${NobqrBm+PR| zWV{-)>v|x&{cqJ+A@Oz28EAZ!WbHINGlPoP}5Hu|!{yKi(ON zmkYg8uqAqge;>&g9Q{|p_pSlRTXt8CYwbh4gqs`Jfasv4Yk*igR_K4dJKpUCSHkEu zphM&8-kvt0hIp2N>;HcUnTU53H@=Dj-MR)WOle_57X(&kcK13JsD0k~!$`fmj2oLhzU8t~5S8nEnp0{nNy0sos< z{68fHVt7_U2PFv%s(nRLLc&J8-}I z8W6#YO(gyvP3twFo$iYHT;V?wntAIQP-UKXcF5b7%S+JU;=Q~%xc9%(5mV@Y?I!A< zUWhZ{e`i(y=;c2$ynpQFKOT_(=;a^1{AV2aKlbvEz5HV@{}#af6M6h=4gUYD$U_q) zQqV39bpB6sO=JxDhAkeC+@$utItE{L@*ZAzUjq_Cak4jPorm-(>AxtbuX<=juP&_{TQ>84Le-hW~^oq3@}cK4~T)s@w0`SS3r{n4-2nm6-b zeI#GcM18HSxQlEE3i{iwF=;1Gvt)9Td9DV_y+`b45v5}&6EI>Xx~ zrOx6X?Ab1mw${><59D_Hw^-CSH6m*2FZ2gTeAR?~9sWujJi7sTej|M~?9tWc2zMdg zTQ_-_Q)_G|bBbk=>-Ib)f8e)ez~%i@OO~v_ED6~Q@cToR-I|VaVm2)(X?n2&;0?36 zH(80Eu%UP(W*5I&Z|-=jg!cPcRv0xYaULKiT|H~?jM->a6+7Ipgf~c+kGD0;{3xrF zdO+)$LfCTODU)oF+yAqB_2=hmR^okl7c#3lJIJIy?V~xk-yKTrht+zR^jFvm?a!oP zj~PcbnE;0W6KzQON+XsCt{;SM=z=HTm4yEA4BcJ;T~de!zNHi$xl;CDH^+Nphpz!v zzwg#GPA<1pBf?cRtva5adUWW$x!1^;{4g+Vx;S8n2s{xHI|xiEF@npPRZWsdvJTt( zrMzCIzvLU}bpsNfwh*LfpZH5Z{8dwU&Hx3#=CJ07U_@AX=KveqUrB)x=<>f-<;X53 zgec-zii`cFdsS4OmgT2?_4zK$8f$Z*%7ZmAiqERSAExxjDA1hL{jUxcgA3I>v9rsG z9|*$Ob@XY3Pf$=0)aiNMlw8m(4n41D8!p9DEby9bsir4S`A2Br4)c@5)Yu!kb;-fU zHZF717l{3_Y3^Jk_)HnK7&^ETT&(FGN~@wDybQeIA8Iuz8`uVCChcE|{4D_G zwA$lsdD#I~4#73Bl>D5&qO{HW>%K6L-G-9rdc5mNnfwtKk%Z5h_C&0 zw?hwUatv#5W+T6vqf}xx5uYi`r2__%tjykqx`alXWhO)gV4PLt)@@52=+dkIJp`S&v4)?xXzmA1_&VAhMb_KSprWuuCC)gjGp1sU(QC6@z+3}N zzBjq**^htmFZQj2+{=Q;(U*hxq;EreK6P%xHJ3p=?eVR^C2^MR?lc>|6wP#tiR0Ky zB@E{GH$5)CN8)K1g~8fny{CSFM_!QQhR{*@tChc-6=p+`i@8x%qqZ{m#gjzbm)PZN z0E2V0d^oDW-=h3w+xHqQvh1r~nZB6R(1T82z(dq6T{HJ=uIgbwYN_p31@x|{=4A!k z%)11V78u-Dh!*9{`U$NV6K+c1P1jq~F(KJ~dwcC?2NR|#+e|yOQ!zxIDNoF*1&o7) zu®Ijcpt@A6|tc>MSE59-+0u$9H-Hk(ft=%}`m246AJNDHnsrzFkk0}Gn z4y%nszhZ@Z+`0TS;I9ML_;&LQV?1V&PHi>`#t0twzyyPtjEtelE8kMOg&KLuljdzP{gKwSwiD`$~CnXwzcT zF%9$KFakH;hvdQs5Ops<{bYgwK?CVG&+lfkIW~;%M~u(Z=&oj1W7MqID9;ijpRIk9v!Mg|*dB#}CZVfnTV1v2HW{O0^* z_8#trM`I+(2HOimMy)Qxc#{Ju?KFj(3++K#(nf=V~?bf;XS#f=E3Z(uwvejsjZzh zU)xNgX1z|`Lt^&Fwa;X)0qBe)2j)YYvI`x24Jd|VA){bJ#&#o zfw`kPu-KzE=vGZJ7~sZM)2C}chgiVh=|jrkqn2p`4~C&+Ago(rh1EvIE7eb9TM9yS zX}a$*4hi?xoZdJJ54?rZ{J9x?l(|)PFm6l82);7tX`33~*gc#E5mGM~qRK{Xn3vk) z7X)x{UTVxiU6x)u^QitL-)NvBN4`uZ4NQTp)%2Qto5Xqt`y7%cWZE;g*BEtQlK<%Tm5gtv zm;JC0m%NLepN;Q8dLZ>T=rAAM*!@ec5!=^r6{n zK(cM_s2<(4To)qgHgCd7Lg+a@eb?%hq~rv(e+Mc^*u#l5ph0Qs?~_sr)cM z{Q;9yi9d=f)Aug4=lvH(Oz|!%!WdT?5~}t)>JX1~OP^Dr*T0856ce`SN8znK+Q0mA zy4@(ID1s}&p|ex9gy%k4p-!3CfRo+*GoD$^0J0wNB}^M1f%Owz`z;4np10mvUo8HW zPCp+nf1=?H6-GVnA&*ZPwEwu9gbX~`j|SG2^n_C3r+cN$d(klLNxa%;Q_fhbv}XS| zh2N7O+q2$){g5;#t)*BE*H{N6wn=ENT|He~3%A+QKW7C=IE=xQE_#puSmJ)u6!LRy|1M3kFMjSx}@ znS=)Ydz->MWdsyqoNxk(U|!9;-M-?B?o%`1&Q`WL;34~q2!CWbFuk3}aWP+L?(fhRMV(ZIjQ`8E1 z2GnDXGgb=~|sfa9_5sm9DcT*8tbhmP}ws7UO*CqAQ_o z|4K9WP}7r_tkp9JN7Z?_HI19qRB|~l4zKw3U4t8Obb`w@&*|keGVo0H04xPW_2ngA zJ;3Jm=xcW^+!?zXS4%4BE;r!qw#@yGdY_^)$XpJ!mq)Gd>}M3uNq3Xann!y*o!-7O zBgE*ooQ3AJ1My0{8iqEc%XEn$LcfuM{u-TA=YHu>*s`r3h)!(ZJY*8A$&FQ@~Pr9G}c5KC-kP-LHO^tep7N#q!2>#chwE=oAjPevK89fbx$LnyAmGrYQ^HX!VB-<2#rzRIQf zmyEMK4^{P){0QSH!%VM{0*MLCx1KTx)qHJ4zwRICN8AowU*%niuhpI0%QK$bEf)&rTMllm-eQ%C{M5Fb@tV^_Om8*y@tuIa zt)JLwqF?AD`h}^~QgcWqmks{}93&q`G7Hwbo!ubg$~eoJ7Co@XlrvTnkl`qBoBfAU zQG;JYnV;T%!{WAG+qo^aZIQ_LD6`!l&Rn8-@mIixW)Rg4bOX-1J^m#Yu~2%#8D=NI zxY>8?#@}!_SQsJlMMjjm^el48YywH`zi5nGd@tdKD5|rz1x+Go__p-OQICXzz@>K& z4Y_kZK8%we8;VOxi5u6;uqGzDZvZcwg0BkPQ;UchOk!x96bj3qAGP7eEiCtcCxGyV z?r2-4%}nlyt1?e_al~G_r;niT_m&jtGnmH*{w@jryYyRCjmuD1{59bDbg*S|R_#p>_||xdb`bqb_4qb&todj(#QiwX zh4GZD>cl$lTl*RCr)&OE&i_kPDz?ON-=NmP5$A%Tl?5+h99y?waiHw{k7W z%}3Seeg!_rCh?%X^ju^?awo>vkT?AX*3qVmT7hjed{@Lak~>wr``7;Jp`fpGVW(if z8JAFZW8H@L@>|`!ac79+&oSwT*raJO2!dou34O_YQtT4Yp_d-jowNpmzg-%%ZDE^kAKKhEBqoK z_OY!kJ}D{jU-6IaqbuD)ill>DSf?yqEG6>19bZ%|h@2u{*gVQ|WehQ*;+}(S0dEuO zY%KonrU)D;!C}HTWA9Cs?Di1Q01Spo2kHLxTbH>-S7=QdU_ZC+e?#l{Er3Q9JOGIg zUCN{1wt0lDBJek4zUW2N8TNlIuU8oH82I@2-ippA5x<^@N%(gvuDCnvSAv%Xi9MhQ zJ-$-H*OIo*p6_vI~(T>olw_4HM%L; zeP;X=6lC7gf z)%k_|l7-^l$mp~*ta!zQ0w;^4Tz0w?y+!e(`Huij9Wrs;os&*1vA9shwZeH!C`ml` zo#(B}ucwwrEDaYc=5twK`CFCDj4#As!uL{P%zVw4RG5>dDtvnXRQ(m*p{y;%+>rhB z=$5maWF;etGa^C6g4vAkWl`%}(uS0jdd~gu^WKv9;~~555b@Y2<8l{Y-4S{eb>dD(``pXRdROY-Np4zt|37p&$6#p^;$(p=>}u@p1y zDgU6~`w8Iw)6agFC)EAIU@JuIw@N^Xb3G4_Ez1UqVoJQdRz;7e(B(s#DdQj)*5r8X*fAo^pA_?2yMy?OVF>km8URHDt52Gktv;^cqdu(U-< z8cpmF*jc0CoR|4k{^6&F&UtXQK=Pr&I8D zOI$o`?O4^7z`1Fd8gJ{wG3U&lcz;Ieye*lG&eMbkD+^3&q4Vb4W5xLr0a2T>&_wh2xrT0vTThD1OH6Sxb{RL{*BP+agN`9w;wCR zcIjH%im}y`INaZFb-aZhH9^8$I}B;o!SSNJY6_hK%zrtTDgK4YkSANTOtJ%?&^N!*(}~!;i?pIiBb5IgiVf zj4vKj5iQ*&1-FjzeK+=U5~~vPoOeIFmpUzh1S<@qt<^CWAvaOB{=ohPp0-IxfB1{C zp|XnJ)+dT$_UNuiNLKSc1Sg3*w9OP*pdic#%fe z9coh37~k4Bl3C9Dl)2AZzLvCS@wVE-Eneo&uBAs*%G^p|ohorTZc&Rwqk50CU((VA zZ0j~uLTujMTpW%O*dR-#K17v&UZ0W6f?KVNZ$)>TdE@vyo)dJi`s-19?v6D9 zmh4OIVJ1J_J*i7&WdQzy_VJu#@8V!pLim2{cHb%Q?N%^LK1Q#KW&1pJIIW%tn3yHe z6lexfu}03qi|VPAYh_62J-#)5rU~{wiI`O3e6=jns*4tR&IW9jk*vv4wn zag4Bbe3)-)rcM1t?O~$;SHk&7f4eKby3Aj6M_H}2U}l@w?Xb3eBzH|x(s4olGE2FE zYrpTdw7?ct zEl?5s5pLwYieVKVW$p9~xA@_hMq==3LKoP(0M*ZR2XyZh-(`NJW;YkJoB9d~$=e_r z=7^X?mQ)BGs7SE@K1@SfMQ> zs`dgqn_@4K637xF=C9q8OExM;EPm}81KtKk@07!h zF?7_6XMDqhgWgds@81Cy?rM7s*qJHbpAd99Ek14BB2>9&!=vhv`jC_JP|5@6u|@rHgubb4Q={R5h`Q)ma;{XauIkT9m*Wt6 z{JTEnJ#5_h!aHBguF=Ve;lvTsRR8QkkVC}o{AUZ;m&r($j_2f7AJr9c?FqTUSWrYO zcRrj6=N{)nCCR*vx+&kh7^7nGwVNr)T*4HbDDOujHX@w0dRN$$ahvjvN@K5ONZ7Z?uZw{;KRJkvG|;3dNLi z@A}?o{P}QvYT4ckK5P};sr+}nmuuZ9@+O}1ah85sw+1~jTU@hL!MHIF+$;MWJFEfK z%ZO4OCS-pkKO@t{Da{dH=rgy1(L)yHA(B?}RpGK1aQ|@Wai!qW;=@Ij4Xz*EKjN5- zR2KTb;{$N7_f0SJ;UdlAm%mt=clWBHUpG0T;wlk(h=YQRWmXRF{*}#NJab^*L4jJ*<6|dhfyKV6s1(qbyffJ}WWD?Ptl5#xJ8XYvD4E@a@1>He#ukuw6se5UjtX_+<(S2n*iIZta)CEEG}p3y^fuE_s@(C-+Tt|dum-sB4}A9(zR z-b1}AL#~RGn0cVMQX5PkPJ0!3k{A#l3 z*VQ}uc48T*<$KYaz214uJ0X;K)5UIi9yOdz2AV31O(0RHc|yfnBIzre!RUjEAB_nC zhSB;u64{4W~JC=;fqgBzw(9yY6UpDQ_xtdQ6 zA3m9MN7qk?qZ8qd=!#+ngD|=Qr#Ut*3GbMi?v7j~eBbD@*}MJ_HiRzwM;9d*U6%vy z{J#S{7P_oCDW831APqF633ECg#SB)YD+C4;+1&QsllhLwdFv!#)JcOE*p$tP3LB` zp9Gki_CXqgFWpQMIka)n;yP`p_tYyG%G8s50v6*C6g0 zP=azzx3v$)*o%#+zRmif%WYM^%;q=;7Ra7fcfy`_PpR=v`8+!MF{?Y}*phoS<^(Z9 z?^$Fn`(gaOc2l{imjI$&Aw13@Y}lw>8JzsYcIJCxnPCuko>ymrU>LZFVnCz`_|&I) z)fO~24o@-ccA%K-QZ7?fz?eh^|7rCUQ(R<*T9ZVU2N7=~EIe$PaG%=Pxtk-cXP;%W3EUVLo|tq6zRe}C#)cH4&5s6~RFs4GbWpy1WdX9L zJBe*-zj%yKtP@B@-n}&XrbW)S9O?k1)rVo%HE|(j7&->m%#(cXv1jMG2wzp?NL`~Q z*zaRc@e5{+_lt3>3Hm`f3g4#Br>+6H$O1%e-c%Z)?&uK|xK#4aXM}vt-$h))!lx{2 z<5)QlVLJ!%^!7>af1oeklPp{F$iHxPZso3VQZG!_&;V2Y1HXs{MPA*T|1-aJa$3DO z!{2}FD)pN+q(PT+sWrs6E~!cDK~eteqOSs;Ci_?9{-Q4$CNQm|@zW=%#TB0p^kB+Q z@6{_;Oes?BMHj-+g%4nav-u&IiIL?S)nT5CrS|vx1S^$iQ_T_eI1YR6(nD4xD7e_L_yDPJ9PcjY2Jije}74YfTPUWd|bg{JJ$7bf> z9(k`D75W+-yi_5KcpK3+H_Z5xF{rOIB7L$|qf)PFKh;lLY#JuRni~H0v(C=L&vpPa z?h9%EhORvN<2-JH#+Zr*db?%*jm6oyM8wzG`TBGxN*!_J@Dd9q4iqIhb3(s;*sLJ>~Kqp12t1M0TDT&pi zBqP*#T_G65FaD+@y}A6amPhCx;NlZp`S`<&`Z8CRlRRCLR3>?FnzTE z0&CtZIMgfFl;3H9FREHE7S=bn+3Z(M*ofjO(2^Vr8|i_y_!w{wvx&K`JEp?uPuC!4 zpTOnhz}}~#gfFIp-(OR4q?_PrmY^>Bn)^Oljlq9ei;v6g_BpUpkbsf}mc39N(n@cB zbUKdw3S~v!^Dj8<{jlyIDH(NQ>~98Y#a82hac)P9=`~`JrpY1|tJY8KUs1NTmTLjJ zcx8M8^@-&?KAUPvTc*4W1lM46Y?*)E2wwAOR^M&;8Dhd>T&C5t@N=uT+KNp59GvX+sp5__z=tqGyuAJ*2t@0QY1wmrS|-Lso=F18m* zf-gpM`@Wop12I)JVqPeh3ilwI$!`+lYyC2!%vSdmY2s)R2N(W>-8RHrUgS6INcRZC zMqpmFq|YKgjfnjoN$X4L-{(siNRa>8lkwy+dYG2Ri(@2pS(O(fREpNun(Ym_(|}$0 zaJx|A!>iTD^*EPA8FN|ZAtX468}~!)gzfeVVd&Gkm(&5(oBVsb&mje4x^%{THF zTF*}gcFFYm77lrMJ(W(uP`HE@}KO!l|_+QZioD65*B z68@N_P$%(_DIh(;1{Z+iPK<69zIrE$Pkf4CFkb-OD&#CJ&>0M80oL~QTGWg(Owe;* zXyS7Rt;VkC@l)MBrhy9YISrG~;QC)J1L{p_)UrJ4&Aj{XJ*HQQxqa|ircy`FsMWZ^ zH0$i3_!>}jfqdU9T$R!=uzL;IGz}>!*-(n=289D@ea0b`8@U`<`$Ls_SOcm#7E;uh z^hHoOQ=_#-`TJRdcRw>(9Gn}wjp2FqQV@n*Kr8K)Q=(b9XnLB4Xs>~^xb*WcD_YohTM$Aa+6=F z3*2uckyld5n<`313nRZ0EnCg+A#!$$>SG`adVUx85+=&kvxnFnpTDFz z{Ecc*X{mYcIdyR$;+WPzW!B)@^;gAOZ$oSexpOZTZ$t?u+lbIX)&Clq>ftmw7~4 z=o8&fskmf>H^Vq4h8E#DIJcY1eG{1=n60jE!{-ggZz3GhJzh{Xy$E{)P$jMnzGZr* zXI`~GF*Zs}QOWB%H|v6GVEW2@(LfjjwGi-1&1I7MPM6-)jGI zOm7=>Lj_vwt~ykuSRVPRa#by!x92to%$>(bM�CBceX+y-t7r`)Jo}xOw7P;N@;x z>#dBUOy{@Rf~8`=cCcsm$+~hs@j!H9=M0j`8*))C*&B)@i+Lr3H7p}8|amhV< ze|R8L!%QrsVPUqiokTVRy9(8^Xe=)SFDr!O*5ztK4rWkick(z#8Z5%Ib&t7~gtNf@ zHqGz9DGvH(m?Gzi;bF9yEirRt?;dS5H?LjlvInd!uYBWH(1b`kdGQuiTwdMF^F&`3 z5>6!$y=7g$r&Qs$llSxBN>cN{5>jpHys@U?B!R~cw+1IzBcEvx>JsxLI(Zv2YdiZyORg`X~0BE4-@)Jc*Prk%T8DxIU=$*&J}quMi2+ z{R)%VCem^E(d|^T{^_PT(+|aXuT49Dx!8FvHmHypdtA}4PPT_DFAtPVcT;zfB~G{I zb9&57Q~GMURcLt>86sb#M%|=Wzv)|*prW`;M;F=+gx|owEB5d2yrYfQR}zDQ1)l@A6_M9y`ctRRW)FZ9!GUc zTShs?9BU~jRRg7qF9a+zPd@UR$Y!bcvsFd9=MPdeJ-HM7VsZPx8uWUa5R6Wwbjh^I zsK6)cjvkAffnQ6L!`P;Pnu zp|a%e_n5mq?q19ZlRrCDJZ(9zCcAftE7fAiR~Nq%h1`4C_?)z`k~s^C^{Evqc&w9S zZar1MCZ@#q*vKuPq*& zphaGv^Zo7ZbN66Yl!;kapj#D2 z)IFM_jE)UopmnvVaiZuuSWY=LqZ)@S+L)#uMU6R@AV+4)|S`n@P)rLVYjt!=)CZ341cV6 zGn|}r5|an0k{x1Qs;`HGJe?psIThPc)_F!gI(DzVTIn1-9XW^)88~y3{}^b}&|b2I zI3gzal%ZSc#4J2FP-d(`?H=Bgh`iBY25s*+pQ^E#ql_yxLho4IG<`6%9y#+?{pNZ& z`B#aM`*?}L&QbsR<=%5@9_B0_n-+-@%0Ebh+rr}CSK>SZr~3oR;t0x^_L+38^d~r> zq^}T5x*kPIM3Y!a!hyKbC?crzZewFSXSDjmp~II<`ZFkLnR21bu8XI*H&>$LvirAm zm-Q`!K8>d>TR;fMPT2-rdZLL~ONo()4MUB`q2wJVZ zilQirqGoC&u|jKB)m{}dQPc6(e8Wl+O@C-46@qDE5RijRGzurqd+j2^DVwV z{Q;GivKtH-=IbysBi{!e{yiqRIum99;%+&jf7(2-1`6SJo9U38)KI!aF@>TyM;9Gz zWI}owYqt0baF=T^AGEbKiF@_*Kudh$Gg-u1_u3 z*ILENeD^i>AA8TB@$O`+{tj-^_#!&C-QW{B#o{{W3ayIo%5XD)*D9FGJ(q&2q=ZN?6l}z_Nm2z_VFcZ!+71(9T)#FwT$X5%3J2o_z}=2FCzT*P?Y7`} zrwmDfYu1?TA5@~K467h?tfFZ~XTGeQp>ErqhxGx(IWTeL8oX~;Dhd(Zx1IoO2fm^d+pxmyEz{ZW_@bgw+UxH zTg3;dwfGbT-O0ZZS|@+(95(kc$czR#n=)&xY5q$rbB{l@u)2vt;uIpk-56gi>4piW(?xco%>2sxnbXo_zQC5`>`ca|{q^sYS%nQpt|0ZOEdKkwb`zm<9dWJ})m8gMha>9$yG#2& zpImMJ|N0jhFmCvkm3sYPR!BB~_g@K1j1!%|?{*&Sil4v9!dRLdrJP7EkAlwV1RYUd>NfZz;H*U^Sd^x6e^;!1Xb$@e?AtZ-j~b zx*NMwU1=o#`XAL78K!ciJKidB+F02*arqGrD_xTzELq)fwVN@trU(n}&f87U60qsV zEpSF%-LzCYPpcA8n|8aA?N*DtEdEXyEaJB_%lA~_vv~4M8mw)(BCXv+sfZ7iAO3L@ z-61o9oFq`#7YQsJcuie|;ujH4p)02T9jecSc}wuas-;HA;tsw|R7tu3L+^pv7= zR&p)~S4ivH+v>PI?)8x*(53fGb<%5GIVJ18eRe22H|;26QhE-=mI(Ss#kASU0yONU zk0DoJISx|@xKq{U-UjdLbcPaV$oQ%i?EL50Y$?3#aF<6k!4ny58O5mKgzJ1GctHYp zI2lyH)jLkkSm{Qo_|iW^<>7b0jd~)fZyS4Z9;ybJt-fV>w1iEA_MNBB93U4v9Qei} za=ZwVVRg`mL)*|p+Y`|v0v!&(E7&6ir<7{>Koj0gNt&kJw(V2JiF|9#d2TDQ(!ZDu z<9u)T2k+4FXf9Di)l?3R&nwRm&(3n6zOWlyYO2Wed?xZp1ddbYcseWWJ@MgryDT2v zRU3|Ht|jqV#&v6BX0&($-zz0m3AcLs8sk~%KJtvhCr zXR;tRgXLPQ$4F5AjnDe^^VlWb3mzG2-mA;u{)2^f1U9c>V^*ss$Na7JpX1=)$Z(JR zi6=(0D5Pz~d_AUpXR#1Gdg22jrYl zG1hn9rcW{kHEx7U`QhwlEPdfo!KcwlsWSgRfZC}=E@ z5Z-l8hv99X(Vh$8^%`mq36XISpU9(ymMsVm2upvvLf!S!L*|%0V|qMFaUmybsT#iClAs;LIvft-Hn4AFLyGGcrl$od7GP1proL57n{ckE^;gj(UyTJ(S@(kiDdCZ zXOawJ*}eS+{xaO6Z1ETQ_GuRin|08a7??bqUVKOT12pI+XNoiJvIMv4pq$d5^E(q7 zQ6$&pR0Iy#x>Q3r09RVe?m5s4Z!B#OdZ0TIk zD$3tvZp$msxqP7sI8wjQ?S~yhbf=}xrN}~$iOnQwL~^@!0B+(A`g|tN{1riCT!((0 zZnjN?(N$F`#DH`ZtD)ypZtw0WJg#(CNk3gmw6P=6qRs7_{N==YW^+kSh7V8I%iu>* z#ERYym8pXzR(qJ1@dYMGZwK!r?W>3&xBdQ0$@Te3mGDYOfgWZKhxauS^8{X?h_zeM zROvpKpKcOhlA)MYzzFSTxH%RiKo-C0W;8QsNByGw!V1BOGuLGF z$6kJl4hu%8lDJ|zu9NAOm!jM0ak~s#TG!7<6ZJ{n^|;fy%ZTftEct;qPAz`Y!1o4k zvBhfH?p)`bOCCvxx`N#JyWkCGrpOnbn#@EBM0JTf;f}p)gd;f36*v-~PaH?u(K}r# z4)uL108>(8FQwP>MnFTF@{d)VJ3=Foox-M$omaImd?!UfAwaN0^>LGOlgn zz)gvT-rbTHRqUOv-vz5HPSufq-@LEUsjL+7?T$@^UKos=gQF!S5Z{t4- zEORKrLXmJV#cTcU#t=M?q;Tj;j;kVr`?y*;V>4hG_z30c#aYBtFzT0bu8S8eKZ)Rw zX;<@7jz@Wghmn4V60t0l2;k_s6^19lC4d8D1@3r^uJ#m&gxBk)jL%HFLrleJOCr6;r)u=A35at%#aiBKK z&?>lc7W#NzMEyC73O0Dz?w$H8OQ)`#yTf#vI4{W_K`!#@O2Qg(b#?I--mPk=j6j-B#%AX^h*ZEW^w>d;C%}}oWDkI6H`vjvJHc?L&l&T?8 zvc9i|THo?MZ5hdCxFtR6&A)Fin*gN2gD-CgUIa=*#Jx6fJg0P}oja>8PI3^0&fW>l z;Knmgh&&82Y?^T7C9nxwcVJQkC#cuUk8jm z=zwL`;-!C7FFa6-Qq8C)sN`#2!_RwC2nKH>)_0$hFMGFKWX?On%f9)$L~nqv9=Y~s zx`R8N^TBk*da31N9EPIcCz6Wo&*g7e!VyPcWaQN)saMp zyxV$Tq2PF}cs*wHAOb+893fYA;VGVP=Zrk#E?%gqkpcNEQn4Nl>woyqkOKy<%3tmi zu!-(558h~9(Sp*=FZAA)kOO^1*&G&%2u&Lb+I_A$?XtoYEbAxAiJ!u}6F&bq zdH;m$K=Y=+FsYB;4I+VS(Qwr67|23UEd-R3{;ORzRco&qBv~}(t>39^<;GicqX$OKvZKzPrmZX=Q78JpW9ZV#> zDMEP+9&^p1qcNtUa0E2@tT|7S?ouvm0*!_N+3tCqcNkaB=(tl z`f(*$j&`&8eCyOz>=pLMcjr0Z1$!Mr(o~F3WH+*;x7&bVb_d|aaC`e))PbCdOZi8Y zNRg$BqdXuD_&8QGneIk#+r1uv6);sN8)f&q>&6YCu)#BK|7I%g$6 z>ug2A5HC`voY;Cr82W}?(ZDNG{*EpS(V^-jK2O6XX;5 zV_W*LJ6*FU-16#uXQvpk%34Sjga`*HG3TPz{KX;4)L)A9dU)Y}F%;{XsF>tnr9bJV z4xJZE;23gsNXW)1>&AYm>w3^!DJX*(qon5FTT- zz_aJGtFi^F-V3X{eZlJ=TzS@NpBP^Fyp@;l)~dOBV!hzu<8|$!)bQFXtITc~% z3}Om+a{jYnB2d>;EEJyLDuVn;>|+;U0`DK5~*B3~wTO0j=xQhj@d@ey&5l`I_b=#?H zm(Q_<6Uyzmt*>>gxs_x-3o-sNyMCQlG-4)aA=U4A^FqQ59^9|q~>JzGy| zN@45n`6Al)PAUIa&8JpI>MNC!5>hHu=FYEs;b$R4?fyG3Ssb#k?c-L5@c7^NL4JP4 zWTzf?*UDc?JzszI$vWa`L>|v&znXeM9dCvL00ayO7u8e5OqTt+Tl~j14{rE{*b^p| zSD2)gb(UPke+StSyA8kl`H}`et^}lpw@O6^pvQHwiC}B}G8>V4Z1q<(E zs;JNG=HiulVt4swibf0eM8S{ShP6x|{o)^$wV}nc?7v_}H?5mQ+1@nX<3+}1l3*?# z^J304t|xd#`IS@&$nC~n9h;3MWI<7CV^!qkaZy)*8B?g)K4rSl4joZY_FXBv{a`Ju6EafD!1Ow&e>X%GfsXu+<$;H3T`U@$zY9E?7iRr_5$s5o4hwu zb*JL{gth76#m~GU4ZA)QJ7G=d2IZ883pR9-C(G zBfJZ?%s$%i#+Ly5qdp%NSU29MyNG%Lj5EGZkDIJTYF{N=8BRZYZTjHqz-hG|PUK;O zfx-jOr+QBFr~IyTF%#OZBMDqZjEebon&3yo31Ih4O{K*tovEKXTH0m%dAb(zeYdDf z7kS5HvwEoT1Lmbe}AI!vmGVhWl_b~00iw+daIvwED1ZOWk`H}VI0E?_|2kt$Of{7Ve* zYt?@YPtK9gda`<^YgGVJoZgt8m?6Ska?-Y`qtsYI9mzOqVf`&r&ClJRDI4-f0 zRZ7))=ZwiY%CXX3ES=;%VL`61gwYz)*-jw&)M+XWK0E4N!8+a-wN7|bQ?48P>?r-- z`o)l}BrJ8OLIIkH zO@e?p&mfRz;X^{i-_x5*d7Zd`YJdyvd#?l*>c%lybpa8)F+u8sp*u|*N-;<|57ztN zMctCG{TM|5)J?e0#zM8cQbPiYtC6_#tJ82$%@4psVf+b=Hr0>$Wx$*5dF%HFW~wbm zX@Dl8+fu6_Tyj~l)?&X$+|K7~i@Lg{lCiOGzjdj-^Tti6yNEwWre7$$@g7xPO7LPMB-+P)FH&gOsNOT4P9L&gOwZ^P@84>t>DdV%AG zFDg5`gCNPpUH}5&S|M>1t=3v(OO6Ubz04XG(t|$90jv|BHd`5k6~MTy=9=?4#gL3+ z%Mn&k_RBcmZl>E=zviMz1)4r9KO@oC4(Te%mU$Qr@M+-~MFg|OA zwIg**#%{b_p{DN*)qbUQZPQk;U}YkLSZy~27ps6>g=0#-i4CPoj#l3zi@+$SEXRSt{6-{oW#H zwbX*=_4W%fnS*Ww21LP~W0OaI*_l2(bIHjw)~O9QTi^cZzMlHV>ZfuC1F5g*R0;Nv zs$1pc(#+v-C+qn3N}i;0f9mCL4!uB=-XV4Pqez}d51ncuqiFhjSi-X z;)yz?>H<59rcp0kOjq-KjiXaxHNT&caJWR3yV&?n#zBOZZ3NzUul!W`+#xdTtUpMP z-TWQ<6Fn*p%L!1!#Ez^uvN0MPKJZ{hUYD&Dd|gjxM`uuZ(qdr!3Y`uBu2WfYkfQJ@ zo!^#dAV39me_H93?_y|O@tAPTS|YoCNLKr-lS5Tiz`m;p6*ClxkU6W6QX+&bLy`l9 zm|!0NsCw0gTgvS>d~L14Q2Lbem`RYZy)oOHi^&Ah7H+fQn_iz6c+HH4Bs`%Mb8>Gn zT!;(}U-9x*A86WwT^w`=%Gn^_hdcDVoK&dMp4FGs))`b{ZA`L{(<~>XWs5DX>z)~< z+f#kBxyREV30>Pw(Y~T}lk%MG-5Luh6%{^rhyE*r~7@GkX)>~VV{s|O`a`}i*RNZI zo1D#lRYGR{ZXb6{L7s?*|0zVNPUJ}OZOq|$CRS18Jao2iZH=4} z-#~`g>60#c;k`F{LLnkFdFuzVFC9gS@Gh$*WN&w%M{}i1uF#uOp! ze_Hz*%QIw~wwUho>_{7`gY$XioY&LdGeg&Ws3S)f?~H2P!{yIX)CJJ9qGN)1ix#w0 z2}6O!4hu_WOzRuchvWA=-A)+tH5>~!Ouhfw#+X~{dfd5y-K>3l6QJptcSnoyRWw^Y zagwfX>bt;L7BjF=Bun*lIS$jAcbef&*>!HK?bQxyiwYrOW(IYawg#V)Ps`I{;&DP5nG(^tF1Tx(d{4WRc!d;*|4Twj(_^597F@=itAlDA3O

~^i0k(WoWafVHH?%Le*-Z4J7ArHG@G4(B`FmM+O z9SpG?^>t94TaiZtj1N7qLF_d3O*vV`J34dsXORN$GIgPG%C*GgVdt}8tc+I%OJbl0Z zeZ1-YDtVonC9MBkFK=BOE?kI_ENn%B11-JrRGpA<>z7wg6Q_413hW40cQSUWzTJ=I z5t4JH-vbs-n_eSeWr=UAwYMThDSr@SW;}6Cr%>RYqDaqCa`(VGt0Kj-qK7o{Juwvf3Y=wT+) z)DQg|?VS5(h0DFjL2CZa8~YpzAL>@u+NzERSLnV;yTcv4@PKng*L)I5?+A3u^S`9T za2lNrXXUCEoOVHHBzLNejYzQc%kKz4#(9&;4qx83{5^0WH!wne{AAsnStDI)_m*7* zCL|^)YAmN*-*Ah2DZ*!OX76-y?h|OCqloBrd`@Q(5l)-y;ujip`y14U44b%Qb-vRfdcfqYH*A#kW+h zY`1&icFCZ29)#O{pT+`{F8tt7UvOx8gEvLxMLWx3IN1QNN!QOGghP+* zFrJh+)>TAE+8$5Ly~zgOo|k?XZt=!dUTC3?UPLM16ub1P(}H7vt}0-hzCZlD{*1qa zaH1VTT2BFr7cKS<5^dc?MB?c6Jwl26kYeg)dM_`t&BE*x1OKj;19EXS`b`wdpj(^k ztbHNkgPX`k*WbE@MekiyD7AXPEl(1vg(cbXDgdzos6$?AekksB$yU0@=t6G^>T|Ns zjiyec$Mh>*#f7&TZu&(l(JCHkor36oU#HkUJ6GEv z)$l%V*_K*lpzVhd_^5+)v&^AVWsjmgREI(>JRIK$TOyrj4;-o2svH&*>uRATBVNy{ zTOmJ27ibuDHoVc%1KG4H-d3I$`#|dzN}gM?u4l@1~0(rrN7mkcj*-hvfaE|u!v># zPrNe7%fXXsq3_M#`g^*tc)92rgu+d%h~5cQ-LCV2KXIGWL$+#W#bAi-``#%ZPt_he zp|Ou#p++}XGHs=8!L6O8(4B@QE3cgwb4JK@vN)nBJMq-|L?-c8<*lBQ#ENQTr*tL*ZrGlzu zifF9dkH0SPf$~d)Sre+Y(svcefUwofTFPSjY3c9CBKw2w`{&0_k^ATM`oYVtqc2{n z{HZ!TiK$)6`EB2=lFOZ|(L&>${MRqFDk99{|KFgV7X|;QZuA0qBB9!UMQS&{{dX2! zhhMudDn1h~Lyicov$YW`t5oW(D|+TG`7YJ2IzOI(WB$r^4PIsa>H|$GRB*q)P?&6m zd%pt@mrhFziqL%=2Ojrq7Xs#DasBOx-5hTTSuKFjqi)5tA5k=E50AAwhG+KXNP7|s zfJ&pU8X!Z<@MHzsDGJE1#_`W#o?%my%83_6c~7%Tec(>2!m4D-#tlm=?&r06i^TPQ zz;H`Wb~CxzIq`JrZ~l<21Y*T-T(l`&xFz;JKi+!wm_{<@IXAvRL-JH>u-N0Af7+*A zUrFFaKV%YsJ4~{*9)+N}ED_$8iu-&64=jy8HpQ_nXBc^OlOWsW8X#iP1EOc-?HrTV zi6Hek)YQj@ORLECj1_`qo5sjH4Ha3zv;IpKq&}=@xTH7XPXThjA?&E(i=@MnSA=8T zqLs%Wa^XyfB)h_u61W?KT4L9@l?P~jW7BCrD{8a;`Wd8oZ&elKwkXl1D;%3*w)#E? z&cTsX4UZ(P#UaRZ_RZ)~QKD=N#pv0`1^4GaUN3y*EhekYsnW%Ns!QVhYDqZr@~zwu)4(PQjEN zrUIC7qWVdH{zO?W+uAA!VHc)}5x@jtrc%|rf#yW3^E2E@i%0D!`oJ=ZdjaN2RtKZ9 zl;-RVkMQRPr)JSFufOi z42Hsc6`@zR8CQl3k8WouBrY$X_(9wiFcn%Q-9$iA)pwOEL<2y?0M%?66n>El1uME~_ZmEe=QPN(7x zlyT)|t4q5v$JFm6-ocABVaFN5%}Y%eO&`We)s$LwXh%FKVfT0e#?J2x6H@tp1<0?D3iaEA;&(#7e`&6pz!XgjxhTf zXyxn>n@gxrjA@p#Gdvwl3gvv zls*RvvBCTgc0FZfwv7pmKGB9Esn!qD6{9;m!Z0?1Jeqy_y(Z3oO^0u$!vB5i4Up`G{%;{ZN zHV`d%m$>wX0xfe_)?#jrd$J!?#4_;NI6MS6Zx`iK_bxqflFWEzG{F6oR-)Qb`X7~i zXNAJru^n`4W-Ktnq0GlF9KDY{oCqi(aS=GmV7fggbum%JTREi63|5x4cn49?qzZu}hk z=qEV(lZN#X3H;R^ZXuLPAw=BB?=>!KYYGCp-L{Y1cH4}%y{gN!_|rIXcRRw&`9z_LCagp0)f>lYq#J(3++8ullF_ zZH5EtnK%6XQiz}x@%VGGt-?F?({QFra@dmL)Q;43Od&}4s=1+I;ysS}x)*0FY(F|2 z<@Q|@N5-^WYpwK2b*~~dAdH2)W5B^rPMk30HR56t-Ly{-&R7|_ktMjJZz}AlW!ARK z-1hY!6*DsGB5b5LWONG^;>b6HSh6_(nQ83OhHi!7^?SePLk&(#?sMy2Sc;-yFD5zK z-8~yw_(w&E)2h-0xP|b)s}z&-#9JRP{O79tR{G0!(<&@k*>ZG~%bh=j%de+e>PJ3G zvYF>;TRv5x3V)E2|9{m46`wXZd6|otVsl-5Dy+NH1(2UW>_!kwW!Mt6L>6 z725B&HvY*;DmHRqu5-GHHP%03VA=8n+w>udy?ii4TmaY4a?P%<_xJ4^|o6#@Az6DY&n#-F<1DNF6xy!+FE-ccsOUQrfY*Pf@cdwGnD<# zzXy$cMktaEi1~d&jOT0+m|7I?Mh-T*?E5$}Ux2zcCI9ERQHP;(Ywin4k(if1E>M}W z*jhD2%`BYVm{q@eJ_cc<7!&amUA4GbX;1tll1KW0>ETv)4t8_)D{B0^(Dt-{zqm|% zKKiNmK+Q<}5+~fOn8HgONrj8olC63yGX|e4F|1U~hjNXL_%VvsI@u+6pVy}f{GNNs zYy!oQxZ)_GemDZ>oDL~Ft740DOe!q4xXO(wRA9@K`jfR&OWcp=U%&9ZXZrQ6H*^<} zFt{>#N4>&ocSkXj`vyU=i>orb2bxZL&|~j4uKU{!A|ZFA?8+~u?=w}n^(h#K*B(qo zEJQ*=O2zSLW1)ksqm$BRpnzwHVj-W;eVcaVpKdp*!DM(?NtQ)qnGR2_x@FsF9OGqF zgemp(++M5t`T=mG)BA3byd;z2GOe`QCHE8KDP(p%0;@~^WqiHpS)yvv!v?vYdQLfg z+{Hxu&o$afLR3MW$;u}pywez57S!nEh&$=H+7KJmMBQ}vlbKp!G$*H(C;E2aT{ zxFve~1>0Ho@1Nf!N%~H-z+D3~6%ZmX>x-#e67Wr(Sn=K&K*z1&SBu?Lqiw^V$@F1Q zMhw0Xhb!nZ8s(^y0}Mqv;&CWrD!ojn=a`$a(di2tog5lUl_35?Hqxi@`P(|b?R939 z@sefV96ox3h4pwgSxr6D@b=@ zjq(hnbma%QH;`5ze)>Q`$4*DH`%%BA3FRZDm9|G|``WFWK)mSwf+hgeejO14uK?xg zlD#9j$cUs|>>f5fQJ|pHHpV*|V|p`hd^7Zg(VY;N#8n$#%ufh6B^*I^vLIT;QIy28 zoT|vBrrEtp#@>sdB)ca9O^qLV`Rf%^7cHkCa$%X7$=AL{T?y*E(z{}8_L%fYEiFak z@st%cQ;+@O2wl>U@G=40?oFczhsEk$+v9|rGkz5;#Jd;Y=%+S0~Ms#zC zb};VUq$k4pLHBphH)B-_D#Lev%RXYY&KT8Nu189`*ly<29xqLlQUdbLBlL9>z8r7-pAzO5T8GCw#rcr2mNoh+l5r# zXc)UxY_(qMDD#n?t}l@$l&#OL8r2nTj}5WB>wCbbti@q;pWEWN)8$NT38xZAx#8sZ zOnN7ssfno)%4_P-AH^>kAG^i@`}uZtQ0m@S)6YO(meb%95_GbF=r-1rt+qRs)E?ei zpFrfJ;+kyjYgcPjE;rnM_-e`}rPTNGFpu?xipi^&cTH#V-5rHUIWb7U0v0JW;S)Kx zd?smBDBxq-@#}~yQ^e1}v$Wt_g1q6-_pYZO(j7C+2@%RO`(GsI97ZI8!%V~eAC-`X z(ejX2lTDyupj%Li+_rRi>MF;MXp-~rUPDWd4(0QEam-VSDjB)LWUvxju`C_xiA)Oj zQiE;;38;7HoIJ8L6&WkhU2`zH=@7^K+lZbj)Ic~rB`GQ+w?PT~r_;i{i`2rDQx6w~Qqp zsCjybyCpZ6829v9_1vOeNK{o17qM@5l%}huW!AuFJYt)s|Tca|NISa3NB)4UkbK+`p1Nejr8+n&cd; zBKuEU-4xJ?Y69l;l;iiAJpZuMcD3s`>i2KEi~&b>LBtHjfD=$RA>WvHIlFZ5vgLGod^?Q3%Rxq`WtK;+LnE}%D}Bgfr8ADdKi zSJJcuLdsE-fRH4qtw42$*SMBueo(^biSxHCL)WJ^EnH-12UMusayzjI4SNr(?Tx z#n4QsY>irTZku3zK?A4@Th`lWA8^4L{>FYwtXiBvF$p|}gzy;Q-V?#}3Yb?ptZ8nE zc_>7x&+Vs~cY)7ynp=_f2784jq+Tn?*6ulSkGV*8Bg!^3LvN*s^Vj*999j3V7bk=P z@<+uMnKN5kZuhH@zP9A14X!-z&BL1uBzC5|4Y4 z0c84Q@Q4kKVl4I3A85e)OoXwV{KONri#e)k5Xqk8W7RTN;9vt zgrS0Ros)l6eUJ>=^@DV=e_6KU>x|wRddtj=lAMx*^#D~K2EhP9Cj|lm^%9)9J)l|uZy>gJozA`z6y7a4<{c~8^>`S>4nrc z^_|==`myV%RE7x4DA-Szoj!p@cXu!9t<^6NUD!7v19Y)Ri zW@3Nv*a@7FuhcxvMxN|9f@$}%Via>lkxm=ulLD<_V^@$`?K_m*;CU*V^C!v2N95)g zW~ygzjs07-w~NK!H0?4wkbQd8UK9X3#=LDRa!Ms`>Tio6{t_BC4vbpeOTTG2u9>lV z-yag-(&iXER}rz|(;@-E9GHA};EM1$0_}S%6TDPh%PAbBj#82^&L@rns z-WoC2W*AnvIU3JhYYyc5tMGViYX}q5tci481g@?^L6YAWOyi{9!D9J|v%mBY%{hDf zuWvc$uG|et@ij^TPzzOXauj2`9mxPZ*{PE@Y2^-<_me+X96J% zFg=lp5@hovMKUVrtW4y1iEUFNK%j})WKw6dCx>4}w|2QGRSmh+6A{quC^6Cdx{q)u z338C%BHYp%6EVu%PkN74GBWP*sPFG|;!mDhiWj2QFj~P0#jTNIQgXUG<%d?sO2!yG z7_$j{@i80CSdPN$MtclYhKy=QQ9maZAZ`s5{>QkX^dYtet{+NVqJ#GsP=3Y)u-<&>2IX@?_Ti@{_V>B{c}#Y|4gsb&kk<# zR$mC@ng)0o!8!O5uo2lt6n<2KFk_9O?9%EFwFVXaha~nd^zq;E%kq6FqaQ)jOCl$F zU2!HN`*NW*jn@++p3{q?Qvk`uSl^%F;B_js-z(|%;$PEZ3Z=*v13ip0qg?ydU%gH&2E zh3ZxKy{B*)APqEPr`S74xuGRO!8~zXNbYwm>)jyreO6k_E2M#aWl5_(2)~9N4%B(k zVd$AF?dxhg>%ou^rB6fr8Q_~W-y7rh2#=l=DTWreZy&qwGfr)dyk3-VC10zAYb5oY zUz-qXMP`Cx_l=2VJ(k(ivaBhkc21uactW3x$giZo)5wfmSw>EG63dtV>pkM$Ap(EUPCPGIE@IornAa;=fF9;~7rTZ!j~Xee_($X^&)udMcFr?|pv;x; zeMf*1f@#ZM+k)4KDlv0U8k@)droftu*_t# zxwtrq`Q$r0ry&R8AD7B3YSNX@>59(XGZeLcYn|>GqetbB6^eN=frP5hBywNS4Q$Xy z%g-wxJF+n!)7r(Sf(7*H%*+vRuAl(CUE3;hB4@Y*-J149CUWYEe!8T7sjL3puL#Ps z;d_6=J=OtoE1xA!i#H;`G!Y*T^uJ9K9#!K$&CRI`%slU&YE*$Wsr$eeFR;*TFu9zS z1rFXcP$URc4e_LmFWBU=SN?wXDU*k0bNinVig zD|^3u!GubKI?tHTs)Xy;`ox&C!?J2vwd3oDs5Vk;WoUJ;Gl4zBNMirRzD=+WIl)=SCiY75Oey?< zAg9kosKiQW;CBtN!|l9xc|R*TKE5Hs`~5i#gfsM+)o1d_*S-NIq?Q;=!E~%$9M9*& z??zhG_45mG!k-pc;)iimcN!d$a`G15$(T9SsrIRs304oOYy<`UZVEe!)D4#{z<8DJ zZ&bLf1x!crYlPQ8$RXXzue-B5gmZdrb7F%uA3}Ln8*gHVb}aFiFK({*vcBef7@X1V z_f-@dLQwgSIoKP#pcu`&DM+c%I>m z4Rk=AEWbRov8Y2IKV`P{;n1WVmgeX^qb!k1M)T+jb|VLoB4}q;K4=%ns9B)4QN~W# z$>zI5A_qEVj*n&g7{FJd+RW{8mGjWn!xR$0dOt;vHEA<#&TwWw+0=vgXMmoripEMW zVcd`EdcIP>r}<7UHJ_nqR#L1;^MbblI0UVsvJxTw2sr@|FguV(d01=3-t4*UM{8I4 z=YQ(nrepjXi^ufiXSyJ9Co!u>JrItex{Vw)t<0PDn|UhnNC(xDhWMu8D>lRVUsXN5 zM3#fT)_Kb^Q)mEbUmq{{>c%zry|y;=Wak&3b}Ok2&0o<<(JQCQpyUnwqBWM;PP-W331I8yz#_Bfs-42_4%ArVk$xKBWU z)6MP>M)_(?dkm*e#w}I-TYtXt^UD{$F@I-Q(n!%o{DPV3Od@MdqZ$BaqbVoHY}<7Y zp8{(0dJHJ!e5u91RGm~L1ADgHsfV`?;qvKExbWLe-sCN>3pFwKIYTcqG&j8Z;IFf# z|Bq@W;x)SH;H30tgbo%;tA2zX^Zf53K%A z%5lKCEI9@!no!d-f)XT|G+cFfuEu-OX<2-e5#+SW`6&-3PLX=_$)iN`LXiv=8)07_ z2WDNG(bw~4f2c|tg3>tL!KeTHiidve7Lx-qS)!BD~Lca?fg(J>>4E6Srl#*T4guH{=c|8({QNc_us2z z3CWh7vL?GMWlge9nyBndk$o%cV3;EN7D5P-ogvE@%b3X?lB|=RvF|g4!I;kX_y3;< z=h3;&d2pVbXD*lPn#=f(&;7aY_x*akOEsY$0pGmeov-;+_NU?5%*ezD>aeY5y5`MB zF)*3d5j@H#wHK4`5ms})%tBFMg56hqK=u^Bvk5k7&|@J{#RkNes*tSW`Yhq%;}er_ z50F8|y|H4%{B00npr-|%q?D6u-!ZMj=BE0%O-@CH+1H(^e^%W)>l2$$-4F0j>(_&! z;?-LedJ<^ij8_A0JA#&6Tz_e-I(e9DD82Nj9O=N?zfe1+mI}67pa_u|7KX^TYp%Fx zN=_{NhS54@98KK??kX)?Mvb3tm!lx6YM%z%FR-MA-21_JZ}oF1P7FDre4=g-#_m0M>j!GZ@A%(Q-P28)7>6WdlWPfH>dt`!`0|l9QytPeS zd&Etj#^-r&g=~3d0BywK;W2;nnb+^Qse+@GKpHYj;D}D1;=`Uo$0T~ERJA{}lZf}O zujwo=W&H7ZqL6*a4BYbi3Ev#-lIu_T^^A%(8}a?pm+re~2*|7X zRRo}_@Xl@pKbzXjT&X#khiAm+6EYLg&cOXpfb?M_&Be<5JxrsjFtJ4~zqhBq(5+{wAucNx zPX14*Tv=kSpb7GmK%*BNhG{g1s|0%GFp-TB*qT27O{i?dHWJR`0_WfJn~p{j9VrNS z;h!1$M_@PUoeW-GbG|Q(gL1+JSutA{Kn6g{s%O4?o{4M$(_e&Nvv$p}D&8#ze(_mK zUA5KXFo0_*s@g-Kav&L^KLUWNjn%n}n3etDT z-!_CUOGtL4>`t9raPW}uK;5Bj&yP43;3e)wF%Fbj+Y~(^m4Is4W@qMmd1SNL1690V z<5+Gyu#ia6oA6+i|0x@wOxb>V;A%z53=BQpasrZw|ERo`(P@FG+>ChlU|Z(%Z2TC+ zh^X<`DzCmADt7Rc6>0~*W(Nv_PS|seFCUahOX`Xg>L>rSpQgLld5d4LS6Z!ekZ|^F zf@~aGZLtx|gr54CT;PtIdRE1%V3gUAEe890r7Iz&$E~K5`3hNU3{77?P11oh6XB7E zjv=nQuZWJT&Ad#J0(aN4@lOS5sr3f7Ci0gM;)|K#7|sn0KQXDG7#h-k9r6meWpVob*ni;o%P4HhcGs&3LlZeb4_iEO2KPMkI>k z`G?wNCu=&xsBx4VeT&JwTiG1D<^EOfm>VxcJSw7`QK;so4}BtYuWGG(qhvUJuReKx zE{S)-77s2!_$=Mc0b$V2ZssfXY02r<_@G<_k65F;h&TJL*YlE~f&;Io4mXouh+*=G zSBn=Dy}zrbuOiA__o12x;FRh^TC~NE)$r2InO|I&AG{|oM$z7@V%U*f+>86$#`}Hk zXmG_#wioea{o)ib2$*9#^++;}CgMaRV6| zRTR(tt>c>Xaikja)h#(veCMfGrMD8AU^lJrlR9PaDt3HK?pD{ufg$R5DS!MBJPY}Y zqMyT*2zoV+L1MXINg!}bUEQ$sJBzGhi?8MZvj=obiDMG9St`y#ty5luvq2R2N}1&J z(=M{CqkjQq+)i?I)MjGOZ)=ay+ARu*ewk+Dr{)6DO%Gngek#&&yVY^wx3f|X0jt|Q z8m8)&LE$2*#47F4yJq_}=H1A#n-Cv+k!Gf#H!Rwm{z;G6BYI#aU7M_+B-wPWjV5|Q zE3yUKljsZ?f$6*{BR{AE%bGSIiMf9VJ=v}^@HIYI@8t{0>MuyRglCcT7a(NH*wKRuNs-fO7h}3*NYEY(=mox zZQ>(?W8=SOZQ#^32cDsWr@5$Zcq|X3LpC~lEgl)mA_Y`|9U|%W<^eBK3Ic9E^{uO3 zy}`H)_*M34BQOe^JZta5Zf*nzcGHm^D(3(ct-y} zWgAbm7`3tNGX~?|Xh}kvF7RmZWt`HazX|fKcFW8>uDWUX>ZBxV&Rj@o=`Qu{rMHri zpuKl?Eu{{7P24r%FSQJ47pi)~G4(_H`OBz#a+&Dq z?P~1KdSe?uu+UH`v6*T+i&;%OG;gap7#83VC?#HjFhgq|HH^WuU*&O4@NjwTDOJYD zw_eiBat)w9UbpqST$BplpXvYhgI}j_zdBS4AKev0h6C{1&hExqL{$0Wj9Z3@ppuvL zt|z+nlloZ3tx5i#pJkUVMR;w;-)Z39Gk3tRz|_N;1g(EB6Y^ucL=wHLkp;ooZC#V* zO#X`1I0elspIhY_k|H>mV>P)+3>^aW(Ylz7<43=}k(Kn~L0+Y;Ap`FeLEh{K#;aNx zgM9LfpQs9+-oA?np8QAUFGHRZI}b)o2khUp*586FRZ?1Ha=}j{e}3!PZUx;E4ZT~6 z69kTd=+fSnlMbzp&usf>U8NBitI0DG~0JFxYD3~)^& zj)TI@>gK_{5EA?oaED>MRrueNYa*~pDEXZtOGK7Nns;<;t}^iJfpWoMBoUBpWbZQ9 z8^z?iHPAd3b(!|oL47ulAkE`7){+`(=$NlqlTP<11nJ$(WfymtyI9-drL&!?)5YT} zOEwa%4^t&ZT~p0`ROpKPdPnkk8n9F|><)*U;L)N%1S)UBY5-DAr3+WNzs)kBf8k z>lfHJpCXdd2gvtHZwUI@!@y?|Z`~%@ek3=ooL0nz){%+xlAgbe(CJn;R+Vp;uYJ+A z9RAo{(`+{r-kT0Q@nGz;(b_^+#d+_}EEN)D;H$AfJfYPhtxJ$eUA)4$LbuvDsA$T@ zu)Li+Nv+Z4c|t|e#e!I=3sebzIrjIuA^cFxD4sc4-O$m$x+{;3D3Dn^DcxB8a5cMh3)JGB^X`xeDjpS?-G(zm)^JN<*G?*fZ@OBK^@ zK{xwR@*<57&3&aGq=CR;JHkx0CpA^D>rq6+OZ8=5c8xNlKaAnwvEA>7j~ z|3Oc((DX4@)e7RB9e7yhHf=qilPA%WDY=d9pUZJ?I~5QuC!xB5bG@zDxJ)GxyC{Ms zo{WmbIRpZ4Dmk0IM;FC%i4tS$us?GG$6igcUKwR3IcUtg) z@TiB(%O-$?7NiahZ1O*!w!G$33e8N+(e$>c8=dRlgSaFey3X$V?YrrjiS{NKQ;7BpAKF2nHz7o9RhnN&C|3PK|>NW@uDdNe9y0d6f zJf3bRah@#nkLr&~hKjO_=pTcYZ1cEN`E(i!4s(%LjPLCF;A&HUW{mlq%#)P;wZ~() zFE$HaCi4?eZ~fyFtqmKoX{OoCb?24O#`lysb1-0kMP)K5;l7mR?F?@gz|d72V~;xjrDBJY$c@PY@4C@mc)WrarbnD z>M(KzfpS%e5Iqu0G5PjyrnMM9X~Fh<5BEWFTQIv)R!HeQ#s$fd>lg&_d-vJM5+!Hv z<2q|+EX`llYTGmK6K@zaV5?)T>u-KCX#`Ztv-5?cI`t$Dd?lQVEHk|Kr4dsv2asWp zPSXz{VD_T}b=FE7d1++wHmRNTHjTi{zt_QF4ZotXnY8E zLDBR79tg$hZ$&)ZT`4)Hp!Y&HLR;n?X$sMZ^B_-8PxrsIYxyjrwgg}ijFg2asjY#j ziyK%6H6-)6@dQs^f64pP@5khe=ER~@e3M zNAwe5wBMSd@yrR1zm;}EtV#Zia+cOJ?x$6!nfc|tB88hES7#^qtxG(7%AG<@_mw{% zX$h)J<00H+dCUk*)K|I@DLwn~ZD%F8xV0nmjXE*D>_VzEQ7X2XCd{p$#Ih(GrODga z7QI*+vpTU|_1td*;Xq7WMR_{Z?{?K1``*6a$<+~gmutf+xTaI^0u;PxLlHm+soz_e z77v+};k0S4Rqif!%?3A04M;hJ)DC^3`lb5jxtb3mpdL3t2@bGrm^OVG<^)U~K%wVn zt!;nR!zl~y^YyY_I||$|%Z2%CP}wtWuL)T$2xO*sCLrG>p`>s>Xyn0w;$!yo-r3u7 zy4Iw01gY|VJ%Kh3f#f$^dxRR<_L3E9Y-%=+wB-r#XIQpXT<>7v`^9fK%lc!DaMIpj^v@26{|{DA zCUN1}#6s)u!%qeyh#IoK((K^&i)fm;ASc&+=l=d5YfhZ5Hdov>P`KH%k-)RP#+I9E zikaY$T~DQBYX}6JLP}YZ-Pxlis}Q?CMDbfqcejLA=z8oo4!`j~;w|GHOp>@jFO`$B zWLhIAENORW=UxsKZnxuWBhVzOADhWfyN%9nDEoj!`a;59=IObvd{8sI!KC=5HT)L? ztjoOEq|jJqhm7FxZa-hx^P6`MQvgDLK-=Tye=%?W7jq<6`U=2?#*|-AA-G#* ztn&iCQQZ$Mf?j75$rLMI*a^k*tibocs|EqQN~*exx?DaD%x9ifMl8A0v)mOv1up3n znR*m3=Lh#xj%5&5J>fo>;N<3gd%HXFg?{lyqaY2;TR8%sc**X!7ZC3K%2XPu*j)9u zQX;C@O=KT5c)R}Y{y@z1m4P3Z6GIWYR4`+rzH!qypj@30yauh;Av9u?)Pxs<4w9r-E)bod(BIksymlDPN$OCIH!DmJ3+$3gaI`(|F~B39BiJRPp5HWwQM-53{TV@)_M_n_Dc>Znw6 zb>p>kMsyMR&2;pDCu)InU}cBbY#=OzB9Ai}=O=|}oPYmJ`P#lUPywOH!N2Xj`*|*=gW-#QD zTvnmQrj*P@y3~=!ISy#!6Gd;Jh+cXAO!^P%a*P_H+GF)NFM_h_awY@iUo=Uk^tcaG zZo`ZkNG6N&)w3JHNhr+GC^!N<@?H!dwCijj^!%VGNj@{h$8zjK+B29mvA#3x2H~i+ zV$;+R*!bRNh;T(%LDTWbG~+b;Cbqi)@$Moq{hQnG+E28_ws@CxvFAn$bg zp>gO&%>k<5pa;2h1Z?OVn_GIv99cb1$iJp^YsA>O>sVE^vD778o4+dfqe!1umZT8Z zK9?TN6{IQgxU0?FY#75nJ7Wi4n7^=TC)m^+gw|b(AEZ@T#9NqpKECxT)l^t!y^fvj zqGg2VG~}r**@hqkD3i48T8O1UAY58s%YJNi`~}GU^F?r`L}<-h`V+%-cUnB{4kzJY z@pBj_02Mb&^Q4X0WtF7=_0=n-!K12Ux&FtO@5*mq*(9`_j;z!@+3Cl?&@Cs}*Z{Zl zKXyAqWbn%1;IVYRf2ljH5?E+gIo4uSH}oX(;;D~N@??dmC7n#&aVKOw&U;qIY-5}% z!H)I%AFf%t=O4f6&ZZFL=~1R*b(ctwlZEqfxdcmrjnn^*o4VdwU{o9V#DSn*Bg^=(=aohA;4 z*1$^%d3zMrT;4M=Ag@HheWU#Rc1L<;Z|Z4Zqg<*OtLE#RE8jd)P_Dn^0k8egR1mf! zkSvgW;#$k(YsGJhn&+apLLTq^Jdv~^)x5?wXvQddKG^ICg1MhZl)2*Of*bFG_eVa)CX-%iSPLf{Sy(fYCK?pMw@=dmy7QNKiRH)p-ZO1*1F zLCN*e#~uExQkAP5o;k|l1@{GpiplyT$e|cc4A-c?2HTVDo^G#dJaQbLdNey3)tx%7 z?f9UI%fMih=H8&yXqX5TWJxsMi}Fx6DV&F<_JM3&j;7u#*jc|z$b2vFe@VEEU-#kD zC!k7W2%>Gg)YE?66B3eSH8@a`pW{A0S-fu-Jz&Kg;|IA$84A;C8bf=QZ^Aho2~HIh z5yIgSN zHgHwYoloQ*O;c69tVHE6!Yj}FYO4Rk8qaA$>xkCGH-P3z`VK+<@9W2ea{nIQ2W5?O zeIjr)y-?*D1YjwD(5@x3ZYVo099pzc*A4aC4I>0{xzUWek*^f+c!%qX3P+a13p&x@ z-_6#oRw0P$44YqbhI3nFjo^DFJD&Wrt+P#GwrC^1c#zL!affpSJoad0}DP^ZAkjv zQv!*Fr1~CJ|MvlMGm&8SO;$BSWn`)Ky&wUnK!Il?WsC;4%VpGzK0x?$CXaSpR^1Ta z1_w8MB$@5$6MCALG>!vmG^!D>v}v`5vJqnuNhi5i%wpfeJd4c7^At$U%uxu|B|X9% zYH5tsM%Nd(R)0f&wrrr5s+0t&yYU+e4^TJX{w|~IM9MjeBY>kZ*C$ESXCaTQJ(Y`qksHFCu&pTBD*2C!+gANxy4he^vhc+7;`BEdqEn&SLr(o| zsJ-CSLC0-%XQsLQO6@Wz$mrWRlEE)dnXTKO(dU9kcW>6pNp0mWefYqpA!IVGW~Nt74mAC=0=E>`-CZ{uJzJ#WK( zdIm%sfs7BOsozAjSHav#&y{QOgGR@A1`^87)}a%n`Z=}RfyWwB++_WrTl(Wep+EhV ztCbm|r^(2zLG;bDhhz=nP#2SLd$xX@UuDfT*QZiU?g9l~dp>jKpD0FBk@*AM<|ld+ zX0*j_q(FRi1fH2>w7ZB?*OG}|JVBQqs!6-?zK{0MUOjs}!{7rzZGS>-d%3@N!z4%k zQAu;oxJo~Fv1AX$re+>sp}jYODtvE@@GUj0XPcISB`(Xku()zXG*L{~q>5o+yY zkWei0LdXBd4L=-~1twlXGScF(1JtfaLRU?wund8MQcAVD zhb~UL1@QRIDH8r#MSc0u=j?J?b&oI~X4(#Ev);PZ> z$4_WHJ~{$+v#hstYfk!i1L%ZNTiT!Bh&Rakr<8C<>&3v2uhBjxwV!$N=JA}7USw;n z$QQr}RwZ^Jc`)-~5=E>nQSR#Mdu4*0b|3$SbO7(D@0Y^nP+ENSdv)88K!?RWb{CA$ z7{dMz=w-3n6Pw2sR~4Q^R=AhMFKKM0k^G4iTrY%>0V*SbYzfM_i?KzGV~4VAHm1D@x!F4TfL%1&K;2grEvuKBpW_C0=xLBaNHAZSYqcoCqNm_f0o*@;uY4-)tu)& zC*>GN*}F=kpiO-#`$!TJN1?+EBbn@`dsHv?q?JBuJVp`SJ}6!vslDKkdVks^){#n{ zIzs^IY)Fb-G=>Adtp)&6IhB+f=qH8P-I2v-a93-s#i`aZk_A8V5l7NeA!K-BY#d(&wSr^|KQiO*cU#3l-8pu_^0(FvVEJH0%_iIgjRe0q>RysIh z6G+#S+-WWxsq5Gh=vRZr zaX(MJQS*`AOc-Y)TNUAFbC1bb`pXYNw8%$=B{|lRgLt#hzv*WhGTE2KdBvxR#%k3- zBhwZf>x4sUMFO{g`!EomTNG1A$jGE8@P$dX4l6}9&7%gx>7ZjG(~Wn$W!QuCQ<~P` zc$EZon=lPhY6txKMi>VGm$JR5F7H`M>e+h)ioe6hB%WEK??H4Ec$N5u_l}yv2de#} z@UZ?ulrQ>ZK7euz#}=Qdc{mU3?w^PrONP$b9?l(~;5z1nhxaI|&}>`}MschmnQPj; zW4?2aM|Jt=wT&y~p`LhLpRGfyxtQs`g2D55+_n*dx+c0#4dKb&`$m+Sz}bw$v`qNs zT)g8lROKy_3Ep|i*?WChF@I->cZ!s8ik~+QG8OjQ^SopkKB7?elqJ^3g84&2g845q zKpEn_Sd%;@MY%#0`)q>NDswm89CT2}xz#YDRv#jpb9@H{^To#dcluMWUdNE5jp|Ti zUf%e6RNUzX>%HAnQP%$QR{Pcq84S(en2me+jrFVDJPX&4GdCRk%i9`^Z`If82Cd$3 zwx_vqU4zCz0AC!EVrK*#3m44Do)w$s-Qug$XTZ)IQ>MdO{^S4zncSttiME63JK$yM z|51JErwETA9-677wY&6ex}+*BL?&+&Yt1|w9{N)=KOn^u_1jrtN+ZbD`50vk?Ff?O zkZ(X#$InR@{5Gh{cTX%d?MsPeyK7HHTM8o7<2CZTlP_D5Qaiz8il3bFRwt{Fx`|#l zQ|agEIl8cv{eeAG=EdVKcNo-2!c%cxncw#b z%HU;v~g=2wi88Xx#hxl)N~>BrJ1s6j=!n81(9n0CO1D`)M2cW|+9a-A<|p zg+~1#rtch=ZhYpc)C$S0OkNzE)(@Gq49#m~o4~CXlcv{-k(gsqLQU78z&|QOf}#q} z*J4ZH)@b*5WK>~+SIOj+0hxsgPM^w)f2e}5j!&C2gF31&^FtKlh)vx$aUNDh~L^EXAGA0za zyUcoDzUwFd3GEjRiN>Pcj2%iy==aRq_3Kc(d?ItZ1MolAr{+ZF#41m}zFiNplzq5+e87QdbBhc0zBb=| zA2XGR+?j^R*UHP?D%&HrY2@YIR#-sjUpu7ekQmMkD}sSDk&(_CL^i-BPAVrv;F%(E zacqwWhs~|Om2xd)KPsMlj9I1r(drW!Js94$uN&kY39yoFnL9PFlHw7h*6s=TH4H!0 zYqj@GxI5L_st{shbg$7;_!%nYYi9q&=T|CMVO<_hH3rE?S--cdfW1*+!d?tn5dOtN zw|ua~?dUhMd$6H;xQh^)=V~%wP^^5Nk?ZYurZJDULQYmyY`Ld;Oy_!y4atJkn53KLsM*>*T}%_^Zn65`_yi8$wuI)^lm9lFpkLrGrULq)jp z@s;a8lnfE3)4Ch((i$9XGS!FI%e&il=MlSr3_c&x0N~fENv(zin3d{Qc7mCLTi}O{ zjHpKx%I5t5y(WHZS8}Bfy1rXLfe=+s1a+#?AimnoyIDDI2VPf=WjL}1ZFqVyx(%$- zILk>bt7QDZ&Y5)wf4IlZ7pGzj+?Q5Jd;7Jw`}+bQ)tG!lcVXOquIy3#ZzP0Q=SxX^ zwa&{oRW{HtT-Q?_y3t*KuIqXkv+@$3SeE-$8jRi-Tu9~ynfIe8ShLJZiS_2Z`B8FO zSZw`xS($9LNJCvyo}{a{W1O;IzbLA)-aF57RheE zy{X{2wI>sP1AEbPGQ-F=<<*fNu9?xsK{8HFafYdPm!pmh-dPm7av5wyN{)S_Z5hMd zEXu!9LPSMxLu}t}K&Fl|bbqqibU7Sc41cy`-*$l?x`bdV<`nyDO@V(9nfYk84fJ|E zWFU)KZDu?Y^%=`yWg)S4d+50-z^CW?388c zBy25~c7Pe7JttM<=M}CQ*)~reCGBY}n}xk9GoZeBU4mZf{z>o}$9J+TDSiQbC2Vw_ z7J&q_K|}N>%#0t-bOyqyIWIUi$hV{K&~#$^vo@jJN`FH+(5suzd_%3_K_7*;7rwMe z!XYc9d|9`7R!v@@Vk&Ha&`(0sI;J#m;fKxMqVH-v7sn2zZ3|NGcvGvrk9=k{DD~;& z&uzxV9|w?k*+Vfeh{HFVV_SsJY zc0K3)hb?t|x@B4*hi4OcvEI!-$Di~4M)v3CeFG0Mvzn0ebn+`y*u4pLE%8|MUff1$ z&Oi;@MwnE~xJQwTGt#ZpO)h`*dg~n`PR9P4 zD3)FYONPN;k2cI?hWw+l9s0QKH|?tBP!l=zxurV3U|>;Ykpl6O^wxF)dU+^(L;5=Z(_L$L)Dy1fk9*V}3(dZ8p_jxo3_l-V$P)84zBXp>_FPD`lDW zDNqE*i3l53HA^E%J|-b(qsY3IE}YpuZU|=wre<_wqg7b#>ByFIj^mS|<&#tTY)HnP1sW}i~r zn(E#dH)mV%_DgQoE0c!X^!FDSacNu z{C9UvIQkC`E`Lf>*Y8u)uBrFfH9Hz%cuy_zqwbd17nYRl0|)@RZlFmd%-di_oq-{< zi>sB7+v(t33blex(c~S~s)N0B?sxmeYW9|bEVa{d{u&#z>%f0Ic3f$|sWs^Gy^zeL zih+MrM#e!~g|^;^UDzHG*z4`~g<{)PF~x+TuKM*;v$n-Q>gt3!R|HRHHn)sQtk;Oi z>%GPc7`X}|aVV>jALh0#XhE=>SF)k)Ofw4s(wg*$T-ICaQ>m1kwpN?Aw=4#R*8k<{ zngl#uI0PyF=mQ{;MS}uM`6VsK6x&p-?<#>=EOi-NPdzfV85ty8&CF&*5@VB8p0P^U zf5AQ0R3K52-mdHPYVxZ7X}K=`h=N0%88nH0)LHSKU1d#BUb^?Rm!?`~(Df+iZ0sq7 z5M1icZFzU<)Tel*cnVHX`)ZHt;ZfR)J7i|khO=kyL^h~ont^hOI+ChAYY8tCylAyL z+y$3p@+&s6bg(y$1+lzV&A<0v)?WL3FZatEE=XvXeZ{(JZ)Yn!mLgO)f^54A7KBvE zR%{U6-t>i;vQG=Ft7Pat_j-5#BzChaZ}(d{v#W7$ixVMVR9AHP>oRuIXLrbsgc2xN z9JCs+{Oy6}@GhOz%yE+b`Ymne0>ba)twYvAf(^}Vl)#JrYQ5n3DZ$=g&Mv`8yzDh- z*e1V@ewytlxy=sGeYrZ1;3i)=YopiCpu?ELx)~64CCpJ!ZV%imeS4^5PKBs@UNu4C zCrlJ5f6kOkQ$&qCG`pwX^qosH$mQ9q-B&|YqkiDTt?CWT&1)n{VC0_p0D66|qh9pW z*p;7m*kd}Dr|aYx1A}HfUcc6pbRC-VL5_@^tqgAA#=S!}9qIOMVVL!ON?xwCC-XIu zE0LjNDlnWPG)IcZKTcj8t%BAWN@H$r#Cy_YhH6bp$ENmqy(fI3%LCB_1u7WmWN68% z0A^g!%QH=AU{5gb%=F~?&Yhva3Ih&*mxb}7yop@{Smb^|5Z|n~eu7?N|D(6kp%B6l z-YcJ|1se16l=A)?KPz);e8*(uP$|L2a;C6S(aSzC5o4}*ng3%H-(dxoHB4#>riva> zT>gES%+m$*{*^@4O!C9=@RKIdq%JBKrhNHKL zsSPeU2RpY59ND=O8R}oV<^a;qJROF6wpTqfdu2XRaec=Ny}rsxGk22M^?QK*jmQUN zg9>AHZOQr%5ytFOJ#>7RARk$RBe}DNt$T46+!bsdQ!QOMsx+;3mq+7X_2kXOBG*^? z$!|VN&~bm`V(IqFhIj7>o04LE7wL9nM>e@dP@KCd=NXnuaT2d0U#}Vu@C`RwUgTSl z2ac-MIEX72bgKko6}k~1Ug2FwUdC-5D+dAtP(y8?xO|6o3BA;dk!Tp+QEYSSnKuQM zD|*_U(q#;pTRYR=LrDtXoqDXFoswZww`-EU0}EQ>)rtmCpR{rQT-D;=hCRf`yo8X? zUG46Eb$;NDNo?)`P} zJYQpF-RFzq1BPZ{DzD`q8*k~=8F9C@+=iICUqX+wX=57BpBm@krLFavQ7BF;du{fo z#)?Ic9Q-UwBYRW#@6gkX+tYD42LNlgo$zd|S5fKDG+THL$?Kfit*}`(HcS&7Ex-b2 z8E`(DRh_`kK;$xCq8fvqi{ux~2%FEDCsgEx27mWaP&K7sYqs_o;sG1m9oDZ|{9|;> z`#Bl`t3`!xZ=dI_QNsRF?Q?zIKGr%o1{&GjR{!69TR4)RJyztvS|b2o^grWw|65p` zfccpr$=4QuVO8EGs}&C6`HgxZ!-qPhR)f?EgB*|kr;8aE`lkXH~$bZv0R_3G6X6e8Ufvg>MLtM;CP@f1lTs2sF%4&F^7-Ql1CM}`{@16LXU zosb!G2xoPh@Nv7;*#ZkF*`JtG2)7h-&VBg=AtZBMPwMeC4Zi;2)JY%Yz7c5URnB(n zO(0;p^)r94UzX@8v)8KrkLv%W5-pgvM{ug5)0kvivnfDVH-SVvgzAA=bSqpNKcKB1 znfBg5<}Hf(i!12Ow?0gHn;3t~Bzx%)3z&wi1iQ|y?+e@@*mh0u^duKgkrb?5Y1%4! z122Cac*7pPMRl_&1-ta8`cIODkM5{h*;kJ0dDtwDLcRTuN`qGn&U_~j5tt>eyW~zIQykYDqOJ}gbP2c?(Oi! zEsukEHOH|GlBh7yb*IEfY43xNwYb98DZ;4N9Wd;i0sW<9=xpQ8wG%fhvYG*zgeaF{z>0Dt{uc@(?_>As@8_T?{DB8Cy-2Op5 zz1{?{mgRN-uJY=%=Sk>JU#Zt|t2-e$rN0o5o*!5M+s$9W;xAN+Sw@R7Iia}K8mX(Imom2q!u{^TM6>W`!W#~ab{*YZ?lD}-`JnsF!U_o7X*(M z%$C{CZZgPt#sqPwRQ7PDz6)q}`u<}RXf3!hU3yFkErXUE=;1u`jVEsY_WpavlRY~U zv0bmrs%Ikd3Vuk`bO^X?!67}8DJ%KW?094D@6kz*fZefET-%sUMqVgFARsB)q_5ZXZc`e+&F^G-a9KqJ({eJ%Z$#gIiImhMr zEa6?#3ze}3vue+==K>q0D_`TuOC)TDNqC~{t41W21}jby?ng@(LnAu=QSqVQsL%1C zTYsw?O$3^qt{}TnDIUM4V)7)ux|+S{{@^LWTRx{?PV-HZ*8&=C2Jq}PU?9Sh%`i*@ z-~7^L*IyE$Y>{B-$luy4)E+1t-fhigDnFK#zWdS4SS&lZq{=-YcOv?SmH9T2$j~)U zj}h7s5TbX)3zt}tK-yW=g=sr2j%k^Oxo`b-{A?p=pT6bvN$EBtI=4eONNMe1H^!Kw z&>?WdoLe4$NTQbMxF3jjAl{mHNNKPr+wX%uTb0Lx!T#6JH+Tc$&UoUy_T9kdUr$Oy z$a6L0Qyr6vXv!+WJJo0|=0vSi@G3=w^z{DtbSH&>;xCi93DL?;($PP()Gbd8g;cf{ z&Dl}$HKtB&*!-k0C@H1;R{i#S0=NS#K+@?HpdUYEW}MwR#t0C@P*ui@8{SfvmkGAW z?@^0aKJCDk47;;La%RTnsKFi~O@i&(L3vo~w@eqqFe#X(>(eOZRfyO}R;H zoif{^H#4@qtX1{5z5Y}$I093)i_n=jW4{}WG-T%IL(Y%MoVlBOVJ%_T0ia{|iS>}8 zVDxdod;@bF**vIJUo>$(s-f*R&cr8SmpBAHW?54EXm3FOgKGIjZmGb_#l1)Yc_c9` zqHt)@*W2|gY0HH83ZrmlTN}4KwJ@vDRVH%hq{~J`hDYw5!`*-d^U*O=D?Zk^xa1gqf6PH-h1S=r2krhH`ccB4+z4<13wvpV$~Gh{zT8tzP`u zp?~n4T@e&MtCySn9vy>#1<}4}DeEz~opIE-K|R80nb?%RZ}3#+ zN>}rX$Fx;pzcE5-9S}O?puL8$@r2!v>A~eKmXnMR_IRzoS?1*yTDWU;-m|Z@+XW1; z4QnyjMGJSUzp6D6NJ&XS)@hg6r7f3bTdFC86vfOQ*@ippvCDGbtP@)^)y8N#6WUf! z%f$~?BeA^R#clj`PfQY96mfmJrKErfN;xO*VsE0qY&bvu*x!LO_ILIVtMEAhS34vkyPr^aD}3IenHUy~)=ObXoZ> z;%YM-L&*PgXo|6UpOYar!wBGMb4ThwjoOMdcAdF|EP!#OOO~c z8!=;EJJc88szmhUJNQgB)-T%Xm?Q^c=|#O=E>fd-r#1itdU(a387Ree3|}D6#KPuIp_xJpm` z7lmtc3-5_cOi`|5SHtbLTz4DJfLZ{v1HNAo-2HdTDLpXIvvg(?ckSVb{!6D}K#fB4 z(8ba3e(#sLP4xc#)oSyPFaWp|uX&TKjw9<7&$(`ehE_pz3VX}%TTP8jF=RB_;x%f- z78&}rJ$BQ)R-VY^KVLZq-qRoMAT2FmB}pFZOh7h*I|{kLZ>}>@wRfm?2I7Bk z+73)&ret>sAi;a^0v3dyIrFhXZ&+?W&X=x`JfXyD>5)_>uw@~lnSQfvY(>{!vw zgiG^xkVX@A{8x3kQzw9?6ljxwx=d1cd6^tsy}uns3!Q&LN+D`Wk*Y8@Pv}9B@jKNJbwXZzBmAwo>LFnk2S^7 zceYSG`B4oBdo|Y`fisinCymo;gRFOBca{w0#Rr>XSeZZe`4Y@~;mk0Wg&glIuBrf@ z=^Ai6Urm&;bq2MJTI-&%^OYwg80lz-C&9&u6-w@4>C2#P|V?nOnSJ>|P=&7n_E7!X{L!1gXiQ_zQ`-Q*2;~z0%<-F*$Ju+$ao9Cw)GX(Y%pZ8Jq(7c2cGy1&WL(Gu?W zyT0^e>ZMgU{ooDKVx z5@GjjplV$_Ymw^OMV2lRPciv=gu`FCN)2^`D^tBKptDF%yjqm7%MCs3Tv7V+NPwpj zoSXyL3awB5C0h0No@K*Jw;WOkfe}i3F=4F4zd(bG7n(QFZFi#`&XB<+b-+^5PKS8ghu0O6VWDs-4S|&yE{uBNp>O`Vs8=%B9I!l zlPRqvEtbE=%ss%A4Z}7eQcpC?UzC=#K#s_GNKN2@OSV{^3OXxSC@37@^RFdl3g4sc zni2l$Dth252|-Mm2fNq2@n|ka?oy0SPb}>QY!yd!ib;dOGXXOu`7WrHlO@Wu)m76Q zOYwqNHh6x@xfz6WYx%pwL}JYhR$XHKGKBgQE+s4wfu`=ffU8y=bDTPx<3b|HY{I|X zUouq1Ae_=&?crIPVR+MhI@XIt^n>%7e~@ifyXddAO4aBcd0-}CNED$N)g3z&q~B8E zF+)_k9&yS#yx#DpCg=%UewsYN1SOVa=qFu|MX_=@jAA6o;Cv&}y(;KZr#YL56`R=C zyLp2t-qJ7qLt;h;wVUmv7abqdw=?RwZa6J9RGqV#1pY;$juBx?=Bsfbz2IygNKP2| z@stl>Zwlj@hZzxe)A4^`N|hTJiMYF-MI&4r5hmfdK_NfGK(xZ0n_4Xyfo~)Fy{b$9 z&t+oe`ZqF^{8ub4YJS`dMB)PBBpO$-G+V?!Dr(8)Jw`Ggu`Rg=p=H@bI4W#;uPKck z_g2@}tYOR+ner%C(91Xyc;ixyZhXgBNNxXo;L&E6V3kWz+hq9>u+H(13f$Bj^8583 z#W)NbnicugooDAl3`Y-8z<4A zp!ox`5}5^$xkQqcjS1s$JF-)+w+?mh&MumoLpk+%1_%d!2r^r4G32FNOrk@{80_y? zU1jpkXF_W%X+>$$lcJSN7h5Z{uJwzs&O#hwrBcDzge z>Qa2GT(|S9*@@M_F6hBmv7+K}HS%8I@u7O6Kev`s%_K@Y3EvO7W=HZh9BnA7L%y!F zo1tvZ3P122F!;`RibIr(Y74sxr)=2dcR@&xI6-k>dbd1*;i4FHW*|nWK6_In)?8i0 zS7*MGlj_26+)#GrPEfVYx*c^Q8;DR{Q?;%@iZE$G57!))*`o|$#V($R|H0gQ1~v7! z|GFp$A|L_+(vhmtlq%gu7Z6ZdfT(~FVu(l!H3|aKe}I5erAa4B3q8^WM5IPSO=waA z2{n+AXRqhX{?C7(J!kfw=iPa+hFNdcViv6Oz3+U z1i;8~Ic@kDo%cuhYc%uyI-c$2mVtdkGw!g>|A*MZ594u}e)$XNk)f4l>m#Sa6EJCE zWi=$1_j~Ad3IWde_JzAH@TO>~aM{zDtOw5F~u8)=fQHzy28Wa+j9aFBbCi zUkqOmt1ThOgpM0Ai$-tT9;NX9_CRDYAA%A$H`q<;8;5wvN|w0YV>IAK${JA z3qL2DfgaNHRxiz!Wb|tNqjL2rM=-V%ECQxIq&&f+E_($BLjrNRpYzNRpR!a2&KeIf z1l&G@)gk(10T@yUI7tiDWqx5!N6Fp2d~@sfoKp!UAY!ba$ki=X!}VeNMY*d*pwc0=O89P_OGaxLEIj&geqF@Mv@}J#bTb6>YtHgE1w+-`6usJxWTU(o>E*+{ zXy(i0Yp5_Nw>Ky+&`nd#XLPHeTf5>@NL-NVsx*-OINCJ$yhWdh<}+DCm)y()z~6cDq8jjTj*jb@V94FWSc!-E zIZl0}6{g;jPm*-0SC(JoxXl?)%u$C!$4(XCsq;UCA48gyqF&R5keyU&ol2+ZC2~x3 zhx#ni)e|?LxCqTPvYJq{KWI%`q*XEto9pdpks@KVh_C?lKc+I)y7!!H#l+s)}~hR;eZ!m-i%V?EqKC$ z{xta69}?j$e|CWDms}174#?Dc>HPX!ZD8O{i-_s7)00i=f2b#dZQ!_JZo2x5H{E$< zH%ps$B=7r!sxyy=Zh5G!BN=sk2q=CrVF+CbChBa{-9m+!smH<;+PIAlK|!5zQnl)f z9^yG~0+P-JhF^Dv?6Y;^r>hhWQj506Hhgp%H(o0{bESWD)nOK1$}+#!sh4z%T9te1 zxG)jemPOz1pwdlIfNfDhAKXxTRx4+Uc7=PhljXIvqVC)szIG zf`%m}8^JkQ3UdjMK-thAH81XX*F2NA{8e6mDS+0iEElpA(>Zs)R~d)E-1`kL#qRX@ zcnO9=zOa3)?qRp(-}5=w72Bitd|2xHHDA7q)Iy%N7e^+$iy9r zF!Lue^ES30$MRNj|E*%VY|O9fvIPfqg_qMVw=-^@|0U(CIO3Vry6QY=RMx9-*>v1C z6$sLzj}`Rnq!pd{aN%Zq!L+T=0+RlsSgg+VrR1|l#7yU_c_ZJNf%1|b!J#YW z)dh|3OouP+3gnp&@c&}yKHWHSw%)TNYrJcNxz`|==0Q@Ke0(i;G3ZQcz|X#-aQg~_ zRd2uPt5;?Q7_j~Z*Fy%!ux7~4gLT~81&d#snEiHrF_l#%MuZ0dX<@6+j}5;{((n4@ z#}-0)nRb^FCR6e}e;vB2&^ygFE9Cj)T?%*XHKoJX7E{k0!-XiSbQOvf(jAZ%IE;l_ zD3wgEX7|dP2Q@f9?2T$xb~Jwl`>oyszF}N!kod%6bY%3gb@0Nl^t$(g8nU(aYdz$6 zfrMT;?f|kIH!T+}ub{U^fCnUt%U`<`uW*SbLGvv0bCoVkE%wjzHi%VwTz`nK=+Cqh zJhOl8`tC2^djH(G^M}C&*2B_K+qnaJJs7xWgIFD;aexNmUks(Qz?`j#q4Z@tH8T4= zvgg3nj+mA|WXY*6+$qA2GK94%} zzke1oY}q}PHfNDCIQA(qXBQMBArV}}n6MXN{&bzLGG%`k3h{1v^VZR*`NQWuJoVu> zNpA=bQ}Hu2!?EIRfn~-~Pn0G$^jTF+RE3%FVTwwBc`@^~?4R#u>fZG1bpEhty7I#2 z2ciDzlp2)tl1r?jcym4dcVh$Mr}ud!rl4E2Wr=OgnaG^Hkl!pYpi_M2|FSWv?i^oK zIt>~47ej$G-1Z^>68K-_TMSEjH)&b`5U-B3;6N&v(68+){HBDXC2J2&-!M7SQ1q=6 zF>>_i*@;m`Y~CM-?GN7*VY3-KFoU@aZkSot->bVf-IKzS&{CUW%~-xBPIf9KTIu3K zmB)!hH3cJ6YG7==HtV)(<$6^OJCBzwc`#m8zE);A!1~GCP91E9GwlVI&h908i{zaj zh$P&F6Wa9+M&Sn`A;VtKE8BhmqusQ&O0%hT_ucDs=b4d@Z*$J1+zQ1Vn1d#7gT@_{ z6hw%V?W((h`}L^J!tk(~V1d)TgEZhj#g25^*N(3aUb4g1G_)=eEH8w0bb)o_gYIF8 z#)e&nQ+ur6mqi;AtHS$nVe>JIo`vnd8*%}gxl`7i{JwBubFW{kEdIXqq!)=rZf<8T z4D6QQ2)vlSrFT2j2+J8!9}=c-KI)(7(W8OW!F_4PQ>n(@3X63iA6g;?AXG zE>_~-Gvth14mG62EMeF}PY$#y&qy>_flDxD$~slR9=@I`C2u!l(y( z5_WpiaHN2 zXNICd=hyUJ5O+&{hAJ`!Sued{C}sJ1T-1SDE~cM8AyVu=$>}-33aLwCErWyfT!&JB zssOI@s1M$4B&IOM|{%G!pvEe%^@e8rx z*Rj?9t(rQz1e52j9b>ZN4mGxTg`!3dksmoX3XEN)1!Xi@*j4&uGRrr9)5rO>TuMb_ z{4eM6Qk_dyzU0emd=+ngfDpCKoqC{@dF6rR+h5AB+LXwc9xKFAk0U%Ge4s$Oa6{z} zBCMY6M~!!N^~bF|5rYzSET{-a6!xsj{;uzr#-DIqj;5Y=tmJ<}liS98f@zw!LJ=s; zyh}b0mADKZ4QHgp7*inCp;~Z3M<;}`Gxz3$vFEVM@te1o?+d?x&_^46xJ5q8`|vS% zKMyJ)Be&Loc_Zf)pyEaP=QMM8Tvb;a{s#@Dl4N(?kV`)GN$A}P zJz*$a2F_Xkb!c%Dz@`5g;EZZ8yW&^A6y+8@56v!Eae>=OnOwT_&^+pDq+Nka5si_e z9k?W-?Jz}q25@=p?}+ti%?-~L9|iIlU9gaRA$Jy@qauHX-}5_Tq$AEMeSTve-Ihv& zqd?VD6=L#Z&LXFJ-AF3*ShwZ3$jKCKi^xwa90EtCs*RHhL^biq_8ReS63AKbCN z`(9vBaAL_eaRw_umHFwKNX5jklu;my$rQVAuK4>@_+j)=>kvwRaJO!4&P;EvHGBKG z)i*C-EnyKc@}6eS(gTms@ zxBufxrl*uIhC38YzNZv*(*M?5(eJw2sSqA@h>6`fxF+>sVMN}hYxIsQVoLI(jO%qf zfoo@D!)q5 zsS4<~4EY=?W?R0Brd{?)K@?WrV*cn>J-1{_+L=2u78>4*`=-JTys8Jm;n$TqHh1ha z@p@h^lSC+;pC*sZ%}p*SLtUWQ==aFwCoS+)A0YW4j>$1wN-rn<{_bbL^Rufpy35wE zRAn#4e>ww)^}PIb4#T)He+|G_32rS$cGn?_EL+~$&*RpCa&Vxey^(ZN%l%g?gWC_} zt3EwhRCtQBS`9-TBR^HxW!~6v#g(D5lrT`U={B)y3d(47v180H8OU-b_M$W8EKe&# z6n_u_a{BokR~TzQ)%bF&^nc&y*Ch#!Sm+BlWgt(6o7_g_7Ji?>&* zT_M~`X!&N#U3%fQoS7<)NQZ@I?tjx5`xnDhx&G8VdMkMDvX%AC!nkV;|8rxEr(j6a zdU6!OrS4VQdYVu+XeOkHA(tB(w3Mf$EO94QnOx>3<;JbdDLL$||H2vk$c*@WuaA#& z$t;-*hxnu<{yPq0;sb{r`rUW)7MOnMmLU_i()jgR{e{2(V(^0VZH9ADG?4!I)#l_U z0LZ7lX{FW=cs*qQfX(*pq#Mmhp7ZTW<@8z(<@n-4$^1^&AjwZT5;Nm;w2=Z?qaEU2 zHf+%uOc@GUjiawqZhG52#dUrye)=gH_aRun_cexY1oQj4@Ze-&8-m(fxHnn4)n5rQ zD~9rk-uwm@wBKxrZo{I3xiZZlaLL`(2eqIm#Kbl0VhelqT?h2nn(i+B^9RWcp^BrS z*YP|2LTod-55UeF@GnLZvqf_ay+2vcz`hp6Ah1jyClb6R^Cd}py-Hg%D6a*shPiOq z{Z8~rg*mJMfwn<6p*>gNH~P$)q$14vE<85~IwNl%D66+dy5H4;j_BZ~*co9D4R%|~HC=;mW!}+a1$8bRo(iL`==HpXh9+QLg^3CJF zb6LCJV#x7$yEwWWZbMHDEMUJb$}&kb$SW!3U>^=Ar#>hYDtc8edZ7LKce9JJ=6ZfP zO`P2JF9t!LzQ#P&-%?}M_@uFJqscZk-JdP~9sIhZ*%+iUHCjj}YFe$a3A1nCLY~sYe?o$_?CTIQkiW-Fh)9DkeiCx~$_S z1*nN%B4@M{SY^rC8{alhZcyCo$Nx<{+dp|+4H4=3$$xsewJlRT788~cKePnggdbq- zz+CbW9gbE~YbmVo%nHLi0MsbXJJ5ZPFa1=7<3Pe1o%R4L#zlG?K5)#|$naG00Z zX2SKO8+Eivf;;Av3E)u=KXB$8v9Bq3{F|nKn(L29O~)y1>l{(-GA_f8d06tj_SiUa zmW2BV{IlkmE+W3WvbrEzjxDNnSI9Zdlxt ztc)YOgZ0y3f8Cn4Gz*F)A(0Iq6HgPvFYwGDUJ&-I+M9kK6gOzVT{(XC1#+@_cKQW* z7<8Y`CFZO^r#Aca&f1F!hF-%Su+3Dq0(W}WJWBdtGG|rjW8AzMOt3h9@CrWSTBmSg zL_y`8-pC$!lJ0D3F%=HoMNVRVjFU9F!GZ`$eAq}-b(;y^c<1?DzE6Q3`1p_2)e|{C zJfaTxgXUsiD6L(U@6Go4^^D?A0=_jFC>)$`o{@j$sSRs&`Vl1o+Rd5wj)jKhLIW^` zpOtbEtg^4me+U^W2O%RJ6^oYO(={;lFdNV$3Ye-02?<&yORwb45;#m(t|pmLhAb)@ zxXd{E84K1ynA+jPcG;0=Un7ow531UHcol~SBo}& z#d!k?PO{lEIVmX}pw8)e6J7MqiOFC`!0iz1GP-fC2OiaejZ?#1q$GFe$wqc}$c+-c ztIi$Fe>$8AB5pH3Z^-ikk@C_3wax{nVudOE&x7eYjwUK?7 zXpf>7ep*uQ5LJ2a%Tq-OJ64cyIgw3FvC7j7w?4XRTj52(goCS(#gM=r|=Pu#yWrB@BS_qI78(X@qo+Hk)Ipr@ME(DK$ohl$-OS6yc7ujTpc=-2b)%TegWa zZg1#l&ahGF8OH@xwwXv8_v&;eE+7c#aOJJdlRA0uvwF9OL-8B$pT@rS;car#B5%h|{$ z<8Ejys(^}|me1;%Vw1RP_5uAyr`5%n@Y*C)HBFnGW0!yfO0K~>F*>d+F%9~B289%8 zj~!pu)2VF~@!&i@gbn(Bttsn{a|UzJ$NQmwlWK1aZaltlpcE&@EeXI#w+5Ct`EY%W zf~7G&6%G$VC!5gCjl0`V_pI%U+yaG^Kl8f%0}Hfe+Q`T40!Fj$B0HBX&`S7NMul8k z^=1H?I2k}(n9>Q>0Rzn~77)b*hOdV`SE-);#r4kdwSlBoMJj8c5BJ$-Ngvra(AqEe zR_$E_oJ&4%cjVZt!~ECi;FaUltWhgv-K4us@HVYI-Aij$uJ{uTM&sn z?c0UxEy~vFR*L6m`P(F#0lY=z z?-hHW<=t9giQgZqmTF8-*U=P$atA51GD%uNOpFC4%_MqQ1%$rKwA*yw^a_|49xIrt zk!+oktdCXlzUo{qk7r2v5OA*c&JpTExK8+(ppL$Xj%>kwvZOhwAwQaC^USpZ;^7!K zMmSwf%(GE7=oa!!DHM42Ezc&;mG}3I|B}$W1?Z$6wT@a^eI|565#YwmjB3Ea{g#qA zf6P08iGS6w%U(7U8=`egmdn*9@kN=(pFgO2lZULT|6zCS@=b}Z9pgoN2mDzE= zHAKE>$)PM_pDRBdAFFWggZh3v_R^wNT%uri0!Bzql=&qz0e*AENt`vi1^c7)Pk+@T-Y zyM@V{V?Aj-6zVwz!QjzNy()zs7!~WK^$f_hL^tyg4=w#EUvL$I| zfm}&Ep?rbjqKM#`bdg!sP-3JM8ix!Uz> ziSXC8k~gIXYj>FZew6>mAXb~iRUm8-S1T=s6=QaOo?AgYPR__-hf{)m(A}*nl*x28M^=dUM#dk<| zkUYAQ1vXEf$4vND^X;BYI|MSyztS!hIkxI`wKl!fLhiRV{@{Rkr zI~kT#PWUiVm1@jpJc264^G7axmPpwhdPG!OEV^Z3Y03G^!q5IyeZj2Xv_?ZG?G zCQHZW@McYUh~5c?0X|B`gV{)ZHK_H)n+YxpoQ0N|ShL`n!eD`WrmtavnEB>*trKv| z#G;NIRUJ{(ozJPmQ>34`AD_%ckvXoZS9qa+(M0&HdbF`y-0JN7t<2ZMvOiOMlheff zwioJ8b6#GhIEUbe;;A7EmjJ^?pvHUm@idu`$~50sg$mR5A_KPXO!ka+Gh|m$O4&7u0(hFj_nl zCQ?HyMj_3lH^YxaE9NG*+J(Sjv+Y{m$>IN@WecIylPrLQBV_TIB>8tYH>Tzl#);N) z9_fwo@u;tJcjjOE7@s$Bn@tu>M;r?{9#sT4D3SF$MzP!ISp~&*eTQ%~wJYvg`L+MT zuK(B1p0YJ`+{%*|vC>c6Uy0m3^I^qOx04}qc}ydlv!o|8T^pf#u^>1 zS^UQPTwK+g0_nR9U&TFc-|l4q37WsZ!WTd}U8B{4mo(GRWKm00aP&+!?n&x7^tk+@ zRgS#v-_LGO_eJ`}OpnKn$^EgaVt#*k66a}TP}Voo9TuJmO47$WjQJP^48gPN$i>=W76%?Sx;QpI7szSeO#@Po!R(F zf_y2&_N>DShJkS%xi1hT7oeOvRA2BfgJ_B5iv;Uiv>;G&2|19e2p@DzXcnM&ZnomnJe_pC;$8Ula zA^r}keN@MvsF#v?nv(u$!;f*iZ>1Pi?1SyPMxwAd(Qk_iT5@KXuP(jFa?ssMGnrBt zVo}GA$BET^(BjEm2Z!!-MwYY9JGi56zIingf3zxFm+g!V**4R28xzDhX!q(!;Kvc~ zBH7WICQ4@M5z^-aH_$OP=AT7C_Fu6czvX);hRcjypnp3v=4YAg-&!>h)aW-p>IYcD zgXe30L-%NsyX7)Du$s($m)%o+{rumes;61|7%neShQ`JWDwXZ-#P)$ue+%}2dBVB4 zKJ_x9hBmH8+HO0pjACK=)VZ%`xZk-g}z= zD+zv6WhEEvqb$<1tw{m^Ux5h?oDow*e39nMLEuiU zYW+|EE3g0TbS8S6=-!@BH~HDIDZMhbJSU6F`WM4MGkBqLZ+GJ41~`g-i7XPeHRjk@ z39s_J{(06}uRv!wu=Lqr!G|AaJZqYJzHQqBuD#Y1`W}E?;iRTP5fzAU@|bXM8As*e zOEYDl!tq9Zv==vx_n#f1?S3v?^82I54B1%>*%5`dZ~%-&4i?%~bN8~Lr~WV}lULGg zmml(N+!^xXn~W@TxP3ooa)BQ~b&l!F!XBdI)L@c?cY+;J{LyW)uyS*bc4AHnPN+~m zy{+|w;!&kkRn>E@1Z5a=jrs>7Kh_Aa)~Hs`&F?mD94*|F-mWz9!va93_`uzU^uw_k zsSt}uyk#_9mNHl4Wov^dLs0Jr(f)3=%ufchV>P2hL)rXAmm?L#lP47Q^UYilfw=wH6lz2wiuj;RkK?%j<5QW`3 zfSHb)wUAk&RX_T@N6FY{(e1tcc~oCTOaAoZ?lY}1zIgE_bRhg}=zTbw3r(JaWY4!(&1tmifoBzrmiMI=B30!^wq@(7}u;=$_@u!X&uaU)2|0 zl213yAKrmmrk%)jfthGm7elgRRpW(ze9mHN5De?94D@wnSG?l>%GKT6kYpsk88fsXSmR_y!w-Q-IdZa!fMQcovq z_n15*t5}eZ-Y>%hMT8z;pw+T{8sXQV=T|YQa>q+;O|dSTeA5mIo(`_p1dOHPtU`y4 zD$U5Pb-N8ad+TK@+w;yP@GOXP&>Si=FJ?TChXN%{IBkXtj1j$+;Pr`)Lr*5MPEHP2 zH;HB6hM-cJDy=5!llCOG+BauYlH@*P!`Ug?JstdH%NT!!D^7puvMta#$JyDRUkJ}7 zg}N)F6X>ftFXbXWCkSWl=1;#cXSRF~EIXrh?)wtLIQ(@rVGV4#rsl?9sZ2pnT2gYq zJen5z^e4$qUy)_h-`dgP0GtJ0?pT@T;hU-~T}SRALv;4~^cmXtmkRJGqL8k|WSa(0 zXK_b-4R{^ynGlyyF;&-RVZ|FHn~=07PuAa}E+z}C2Q&&!Mi(L31%1~QJt1i>l7&5r`+UrxVL;BBtTLF!KvFLus7(RA_slkDYhBoLXKNzN_V~Qq zYlU}(Tr!W{Biw5PJ!JJ_4YXVL`~f-GzZj&C6uWAzR)8!!Cmp`yA&+6F>5C;xlM`;b zfS}_&UuR8VWXeIF+Fva>&$0tlUo$yS)fE3X^|$Cet9or z$ciN)!pq`!MlK&+Oz3>_n7jEA!v4UYTnD@?3!1l)6i7| zf!j~SeaBkjt~tV%JIvg7|8hE?hLZ~ayT$VFRtuwSuM6*3C=9uf=`ecJ&YrcTcI;36NGQCnc zP}VT`YxwKSGbPl|3^(PloAH2|19g#N*`*Au_<3g#5>te`Fy_|Rzi;4v4))rs{w2#j z12G%O&9pzLHwU1XfOLM|UBMZc^~a;&L|cXIE3B?{qeVN5Xyz}1!_nv zK1?(58>63QxwAh^n)z{z_N(TM4_Bl-#4*3aTxxU156hXfwnRP|E16XLvnAP*F*oM9 z97~(nJB!ZfIr}(5`MeGdRX7AIq}bpO8}pVtvqWvT9ww=Pp*lC#mGeeEg_H7IwYJI^!5PE>LpkNl0-*ntq zsA;Z*?3C%Q6u&Z8NGkV*GZ(#NMJzP|TGZ{r<3CX!wD3vHtW^s1&Dw3JixflVj|Y7M3)URM*wq z=2+X^FY8B-92}vJA9nm*-StW(Ed8bIFgV3gL${%=Cb(Ke1jB88*&&diSSRKJ1=+hp z2!TS657O3xtic;cnJJFPRqPm#Xn)3n0zusI;-kV#6Pmwm7JK`%tnfw~Su)<}jNm6kjvjZU-jxG94vSA_IA`Z)U zlF{t%50}O56HT^NE~oNphIn2%t9LJ7ac;QvcB_F}>G9Am{V4zSTHDUJ-P#axcYI%O zv&P1R0xj{xfnx#O(ZTup)8Jjgp(kudbgp_C5Cs0nkVrLXJ~H`dCh|uQ*LP7~svKp& zo8}8l3|n%el2PN98ksJ#V>Ac^v|_D`x2^q`cIoCfy3q&2!ZU5FgL#vPI3bE z84O+u9$1r#GqRp6A@vf&&xbMsu(srW4YQKjE2U(y_uC0C>E^xN0&7YkkF!bEWmffPW2j#)zh;Bt85BBE~# z9JFHR9?17bl>1&mQ;c*vz~Q<5ODl%}$X7bW&RJyRhfsou45_giSV7%a`V6@m=jb(h zeL--r);TD!XPZSimd(vAYegRtONb%#cXI^VkaNP%l5!lqt5{-X(2aiRcAu9|A7QQ9 zZndRUedoK^``~?($1q~Pc`2W}GX%1P;-O@4o`s_IIfzoft5?Pp1~>T!)P_vlh3&YS zMZIC4s_`=_tdiXf>0Ft13u~ns1Y#*j_v2Re_Mt7YGLD*lsygQ#7Ec&v`HsB2lihU! zBxwrD*P&|zrS@JZcpyyp{iB^r#Huot3S_Yo*g2XG_W}cv1FLtW2+|;(s>aBK$@&|)!pSZrYF+{e)KU5Od5~a z*rr$0hL9lb6Dk_p*dj2ewmg-WX}dp#3mkW8t(6!Mk_($+67}f~;_L(*jpC6#C%{0| zn}!)BZc4;2v2V`6!<+R1Zl(8~G&+d!SDuY`)CZCHdhVj6bHV!wXO`Zn-l{SlJbx}{ zDfsQpsgg{OWBn0m5W3cVXt8x0qrXmTtq<`Ey+NwU#bdbEZ#j;*e}QJ!ZDggyST)5N zHl;@n-1sTQ$$x`$d-@(AjeAc@iYGaTiy+Erx`Yt6e2(FpVSzn-9g0~YQ5iRzR5rj6 z6-n_YjLH#3Z{K*u5V;8N!)3yUF_#xf=F^jK)t!2X#dR2=0mgyER7Lx7!?VbMVX?9tb_KI|Z-JEa)If`;;Rq(z}t0 zky4ze*3aOc#>2%(#AvTXKhMo)4U8?#SF{_?B>6f=J=^S1919iWOKMpy;!ViPUDk3h zNYlr({3DW5!(xj=-k2?Sl*z!-USiC`&xB036DiJg^o47MZ~x0NBaH6JkIv%kMX@;{ ze0#nvj1)O7Tf(1JINJ++D7Q}4ZA+;uV83f9Z7INVwhhw{d(?5Oa^vj1E@n z25(x_{5TZai3w(|^u>cJhafwOp(@w&P4hyg=Z%YR)=SUDuReDTU>XETZBu8bz{&pL z7_5r~sV{~w3^!`T+C7T58ugo*vx3#UXf_`b>g79t_%uh)-qidH!CA)9q!(dA-}S_j zK_-in_;~Z*FW9kXX+|&ItflWfUcEMblzOJ0B?Wv7(cj~Ud84O8Gw6u#A>j}e0PYQoHH5Su=C#9Wx^ z0%KQQ=gnm!fNQ9n>*Ay2e zPbits3&Z7Yz0;}W>ir_qB-1DPk)kWrLNan_=33gog$Ui&TBzLOi#oy;9oI#nY1%VF z;)D5I-Wr7ih1VM!^OgKm)zysyxEYsnW6<$YK@x9E%`NxK<4mxE= zPq&i2QdxvA5TPsoVweW^nFTK8x_0j((nG_-vx3@M$asS`e zOTFp1>Tyrr{oxaH>BO#?uV&GS?_P$e-`3C7L<@R>fZUP>dgM36r6_*>5 z8R3RQ06#m^b_Zig)M0b!pHTUhfb;`7IPZu%G}v8HcsNX?ORfLoFKXMs6;0~2xZN4o zFEb`okx<2nl}SJ6i(uA{%ra+SHr(h-#)=9GUfe^=gR47m@DbVq&U=!<_Qg$woReyy`}O~?F(bK%b}wZKe7|Hm_-6?>5( zVZUa}!;S(rnuBK*Pp1{p#}FC%rW|Gm(vbWO7S2b}b|PJjD&TB@h6~pZdVT*hHJm-N4iL?jV>q`OuzYl9EiGm>! zFa4I9AK1x_{N#K7kSr!eWd4v_+zzfjh96B!oy-)CGD#vDmOHkgVOUsA9@I7@KQUa4 zcD0?7Tw+8ByEmrKyD=`a9gr#pe{05v_Nw1-!hMnTv5M}L}=0V?G8UfLDzh& zse!4r+jWlxX{b*L7FEvdxci=)c=ba=@=8jA%~_z>X(6`^k0&fs;`>i*Toe-tmY4p; z@YD>y!?}P9YKi{LC+Cjl+s&B$GPR=UVqP&5$Orqpd?Fh+ZbdmqHvP4sy(9T>>PwE} zboH{KFKfGl3vRNwv8Q%Odfepjxxir&vFT9()PS$#$}uLbmKLnvabiDgO7h)wFU>p# z!CTAsFGi8}*Y$aHY?cHX9CoCqU{zZM-^8nIUws!|Wx9O!`x1tmWs ztya5N{Z)6lfF9q@H?QLx{bO?YW~wFh`>mfOLq$gKZqA$Tc9MFh{5?C2hwwnMnd;n4s79piJHE;t>M@8 zF}59+2DL*Z;ESwl;T@7gVzM4{$urm}V#iG*e+Jt!Hgs+)b@xTcbg`?7x*k=V`?Bv_ zo0_+zH3O1u6lsm<2>TL?^lHQzemME$P$ThBIXh4}sK##P>EGVBolL@l$0J=}?hY}a zf!(R){iT^~8NC`Y(w&**lj?kLE`#kb0<6fMas1+H^aQp4)%VpF&MTen1k}5EUP?Ni z!`A}nZmO7k<{6Hx9pJi)NnraZEXzMTsJN8R4{V|rfXCi z59)xo-=+s5)s8|GOBR#QAzmMS7gXJwv+svhS_T$R56GuU6OW^V1WGgR5T)Lzjzw&< z-B#G+a!ChkAhLUHC{#Ruw|3K7>Yh_8`Ae=yH8xkcYjI}bR#TS1^CYQMzQ$;L`jqON z0Mpwvu<@t|1YDd&lh~omEX!3^2AsN=;*T2tJwbE#f14j+mcIPh*CJy>&=6O=bbFW~ zC_VFg_AL^~juMEhF!P0#jg-wIJwZKJQ~8?Pv(^P3X9yKS?3i-6KYjnj?0K4Bqhk(P z;v=ucV(`)vG@~M!H&|Oi#Q50Q=)(kdpX1bU(!llCD_>1NUts7B9y!cK(+&W?nkB{R zv=j!A)R5$H`sFjs)@{ePXoE3Wc?IUy1UJA7sTzc|xv53?u*A@4^%InRs#pGM99xe@at^f4t3LMrM~Ksf^{vwDt| z>UZjbkKWB%s`achnRfW4^eOKunrXdJ_H_v7p)BEdF8$4uoKSPm?^MFKqP}g~=E!8=&5@ zyYky1X@Qt(60);nh7pXy{VDDp8>TGE9H6xtfbzu4m?qO^Yw}lR@Wf(Wa0e5dItjqW%+en?T~!2yv7Ix z0juo8D}JhE0ksNK}3JI zR%2<|<~+lgDqA{<-pmoT9baMJuh_35^uCT261{Xe0U6NkT8aHik;H$gE2RMk&m!go zCi1`CCvr77NISTg73T#uTUhK~bba2#FfE*VQEkQ4TVqrMmD!G_pUpgN_YeD^ejHRi ziX>)Y=;xF^gu^=;DW4XpBm6^nV;#FfOTxzw^;uHE2@1h)*i)Z!w6Xh+m-wQATUke4 z@0zD5V9-!d0+OyZ-Z0c+-9%B)eq-F+s_09(=C`oyS2I4?y!2%~DH3O|5;qMP`I?*0 zEl0YoyQm9h#oGedHX92z38}$X@Vv#MvVT-=q=FgVG+RbvSZQGVj!>vF{`RR%!wKMlS6k-#tXK-}BR(SiASw2Zru0Yl3kx++ zg6?*K>tf=w{c2`v4|*3=e+$y`GJ`yP-3NWOMmp{$bD(h3qOjYda7D9WPL9T3sEMJC zDM&bXjd-RQ^{$PxYCVg&6*Z3-1eWr-oR&$mE^UTqwCg^-Fn%P#)%3|H_3XCP3s#IL z@#5wKQtw*KzHUfwR_6PHybyC9Q*;zu5-7mF8V}m-6?S=?APF1e%)ti=?!jcg&tKC& zqsL!85)le5pzL){{a#Ozi7NN34B=B?QZW_`T4@Lgy2&i0%+7rNLZs6>{o0NKJdFQ4 zzu?YWeY|JCYq8g1aB@u4XjfpDR+Tf#u@BGJz}816V)F3s^g8(oqO z4yg@8{qD;4=6X%3rp%9QFO2-8%Ji+EURG}lvceqgr|vT?vPw#MKpBjG{C#CS5=;me z5B2=5CyYRK(uF5n+gm1etU~2slC`l;ck_>-x8&80TG>)kO%{c32`pSB>Og?;!0&*L zP@`-;k&t*-;p$iH!P_#4(|7$ZO+Fsys~)G=>SwMVABmnp-J+mLVuSz`S;>N|5z}V! z*^94^QffXS%0^^1Y#xUx5d5F)bu0)DaMume9eZjV12hTQW;2HR9-m}rHJQshVO4pb zvk%P=R$#%?Gh`6FF)J^vlIGV8{0coB{9PxR-T8bD{Z$*Vl0Q&Gb7eElY1dR6#ff&v zl)j?u%lbQSG#PU23_gdtlUU-_JXPCy0GjxezvfVttz$R}t}}M|(Jx-E0qw^TuWmE( z*eByTT)~u+P@{5(V?8kvJ?`bc9mJ|*H1IXGrUb_-S9|+e0w1HE+sDP#w)=RG@0?dC zU?-Bn+**ulJ^dHGi|Q8d>unmA*KHHM1hBTCvt4Z-+Plh(Tshzt)Twg=#qq??-7E-H zi1~%BJ(!vF*ZW9X1M=61{wEWD!+=6cvAG;lU3)TOK5?}vYqBoE^nFjk56M-=6lboh zccm6Dny4*;>#*;1o-K}WTb48?SN|2`l&>E@my}I_CaSN7K$EqvDoEG82tc#8>O-C% zPbVF0A2n8nx%<5AsTdYAG~PIKr7-Z(YEsQY66V!p^Q;S&-fkfVq)zsoLHOc%6?47u zZavZFkalzTb5GnG_e46TWq9u>m+e`+4i7icti+tBUx(xig#HiG-ZQGHFz)t51sfpJ zr6X0P2~wo1bR(cBH3HHJA%sqV09JYtkRV;UNR!?INbevuk^mv0NeLv>00D2_nR#c{ z+`I0~eb@bvbv`HOIXTa>|9k&7BN(40H1uM!tU_8E@~OV`dY5Mc?arm`MAlHz_#F1WQn!=%ti1JHd7Q6UYr5l0$y8RM1)oJ)>r`p`xj?l_MAX`{jo@4Kr6nhpI@9B`z*HKY zu$-2v%?g9$cpoCpO6Lw2jl$xB&{-&WWCw?706o-M#s81A9tqeBmlU zWCUJg$kkFO`P+`q<+bi{gjDkN&Cq`|1v&EtLFFD)vy|oRk!_hd)_DPAIXj!bA;U;< z{Y^1lvZN?GK|`9%94_Yz=}LK|Gha({=t{d_T&t7x_uKX{;0?-<^P_Jh66l5ZtFa}_ zZskvA6I(b=RaE<{-WpBwWJUd=C1ny`!(@6j-!&8d{v>h46vcsilwOFa{P@#h$E|NN z7nw5Qr@z*)H_i32*ML228ltZ5(T8UnBWVa+Y(q@U>wGJ!3Mm0Lo~pOVFTC<32rGLh z%jM|Dft-918=ELyjE!0QN=XtFu{%3^;+AB*1|2iXYW2)vYLh}!56eo?`)t4c%C9^p zE`Jv7++=?8-Y?BMKS@T~TvcD|s53j$Jb>EogM?LUQelkD2kPdy!=~K7D$D+NT}o}N z*Q=jko#M6a2N>s);#NA0lZzMrz)FCB+EUfSx~f#QU5+b}w}i-4NNs)Nta`0*)+{jp`eMOV%GkCtT`=kkIru60 z1F>b1Bfn%lo1MDXios_$1V~qg6zTh8s_3@M=b#C7I482Lm@SxF<5U|{K zINeUaaH?w4V|8c)CAM#ZfuXVEnsu7tdE=`sjpJ&MLDc6u;BrZUF_flxqUVR6Xm@hx z>5!9lI_KGBzlFZy!&5) zth0>uAeARF6W9s55aou3)_>R%SlFh8|1;Z2={pYt>I>(EmoJ=|a1zB%(X}6+UYph7NI?(WRHGUz) zO50fZM!ktttm17u=e?P0{Ca*M_UA$^%E=c1}jKyR{ zJI2_E{?s#cPS?5}1>-Jsa&$C}~lCf>WU-GjFXl2;$L{x7~| zZPh^SH8U-_a4RLNun(~+$uN8vb55ktE>>XHhcgMt*ye)1U{4T8Svl6DrrxKiZfP! zO!3INl&E~YLERd7HmEig+n}|ca09MKOpW!k)_eAwTJgJ?j!4JIlkC_JlL|_T$U6GsDJ^;11s+t8^RB~@Wa+}ySA9p{cD4w3J zo=8bSbSj%qTChX-_PJxo^k!(pbm#@rXevp(lbZLH_s`hk?6)+*Kj5uWsnUzrIp@?f zGvkX+|3xb9ISsoGtH5-bQ5PB9y28LsrnvTZ-rKYIKTSI6F4?Z1FFSeuC>zD_T=xWL zlC8hNT5vKQtsDbz#q^!Xii1Fp14c#0Cl3(=GSKC0f0lNZz-!8j-shI|Q|4Kr<2e@& zWH=~VB+f`FHF~NVIGm3Cgdg8;>LDPnJf5;o!sR11d?R1k!~JP)-777T>Y zTfZc$n(cLTANJGoqQ(SR&aY4&&!|C5tmAl0NH?lfTubv|?=j}#Sp8bX7KV`$%i13v zkzWjjk{On-JP%josu;pUJNO)o$)B9?0$0`r&^~TTh5?}??C*a`n_0v%`mz4F5_NMf z^niW)gL~JK0!X3&Yt}9i2K1fY(80ujEN39%3TA3c z%axry64UHly9Df76yo$m4F^?c|IA#Mc9X*Z{?XV(s1v)E;!{bgFEfl1v}Y*F-|4R^ zfWZ*&tNFo0TrpR-g?E=fLqt7I5Bm$^`kUBi$nD+U5uIp++9sXectI8>u8N%FA+?$x_Dv1V<_ufcX8;gQFU%N2}+?m<=3f zFWbioB}J@4=ZlngkN_&Nw+b$RobgSFNK%TQUEm30q>4D`p_PP(y*2XXA}<~?;~^PS znOptJ!fqux(zcsssS9O0MJI*TgE{gUtGkrsFrVgQtDS%>0fJPX@i#pm&Iq`UOr+yX zPFGi_J-t^+%t7SnIPb9FuaALkE?Fytj``$h|Kw9cm^tvgOA%q!()YdAYpw;;n=|X( zim@y+zc+Cid-w$w8ENpoAc>O!UipHo{$u0(GO>J}9D)zd7?H7RZAR_K0bl)Wn68^f z)*$F(zOU6I@8@r)D3+v4m2jD&FFK#KZ!MJph%jsGKY=zX3D2hvO06|!fIgzh3pPjs zulch}*G$A`yG!XCkGl$`vK9tW4SCJh?(J@|Tcv_M0muZ~eNBpPvrguK58GNTaAba6> zT3G-R`S=*{0J7Berp2Gx`MPC@*P-3ANX6&QSm9%aq%5R!+owFV>JfOw^}PJkflW>v z_HXLi6xH#r_1|j=vj1s=|F2yCzbjL$PYvKkQyR>0sf=zbr|pEoIYxj{HAKrcs-U_y z8ozr-B;C}~Ui5{CGga)*Us=XCjz#3uZrW0N<);*Ks>mJRNEcnX#fld*yrqx;;m)KF zuGPbXiplp>xrQAEe2_F=)V_EP>W;K?n7D#Sp@s~%ur+Qc{-tYy8Xh~<^F2N*OWsRi zDGmOx`Apt7$0+Jf*eQmI$GF1b#ZPOE9m{$WUwm@O-}Nvj@b>~^h8+>fMF891C0_?S zrlziK4>2Y4RcZ{6$!CAtll^SDMn<#}!1jl|df#r9e6xA#e%(YPj z`FVhTQ+-XKc3dEot^Vjjl2c#m{Wrh+<@ofyC8qtL0_!JAaa84(!H{vt77I#Eul3gb zVb2UyyC#%m!Ph+}Mnmg4D;!Hq3>_P%eyj9pHI^E3m{v2g39#Hu?T&x>tU4U8EQ}C8 z6-+s@%QeIl3|)xpinSsccZN~%`p=0A$ZVNONj3AL^#-)z%CZV?9LR@mJSwyAk~K%v z^tZ?okfT8GJ@5NQ8yc(sXewov3+lH{Hg4909_?2iV@|C2u2+Th6J1V(03$^TrxoNUoV)IihGH0O%yU+> zOWK`{<_M#5UfD&>K{LhoGM3RN%sjglkk05k-~Wfy#s6%%Nu-lw{O~*im)!z1I5VC# z%)E&lVoOJ|7<*jY#Uv@|I=uC~bR!rAu;5plP$&=Uv_fRz&W!r>rls>_^@2+*)c0je zaC_GcuQ@abh7#X=HuCzAVk&O*lxP1yp>VLM>2M=ZDVl0=oC_UL6kwBQ#+`=UlnKu# zv2St573^8vKNZgr@ydG^FZYM0CX->!t2B&NLlRbLL{f~vQ7e02!vgf08;JyLx zYK~QP+A(`;OF2rzn13|SHnsY~Xpwp(reAIR<9;E~4pe$}R{Q8wC7kQ9_DX>_T1x?Q zmhSjm!yoimjnH?=?PSygftp;+S>7Vd%*=i@X^vA+-`e>{lXUK%)DFPr&QtD?wlvw! zZ;{P=Ify(Kb@OuA7aS`Mfi{kI(JrxzNe{5>OPV*sT(QismrFW*pCGHmDZq{V!dXk0 zJwXKGV*UQ_xAg7l?hccON(P_plh3pej)7hP>QP)2b$EMp$f+G5B# zmFYXWutkSg_sQd^57cPtd!q^2@QO>l0yTy^QOK8g0|L?<>VO+#**FQU`|)R>AP|Me z|9*56XcIVi`-xOOF7I=R^s9<(X(Q>@B_fvCd?tgPbp8^UBB2?LO*lB-X_lt|mf8hi zGWg<8zNg1(3O`^0-OXj_khW9@uor(Z_~K)cb~jMEnE^Yz?na&n{i-x?=B1oP0W`Hy zT9d?)P|$^gvVnw6@5clap3<8(ob9t3Ul3D`fG&t4$(@FR?q!0`>EwT*%?|t3tCF4eZOpb^|#0IEakm*(;LrRZ#DK)DeA*25QfW0#D~>o zah}SooR7_}g~X$GZ?u6h?SzaL!Y*laD>mPcJxnGk_yiMj1aV7}wq+}y#f)r1wfAkJ z)OlR|h{gD`_8tr?_2kPtQ+syVm4e}Ocz!CA7@85i9cQ1lY?Ow%r~8L6W$x$Wi9k$@ z-R<_e?XU5cA$?c#e=CD}C+k>!EE*eMe|EWibf@kg&82=dfhEmbu++oANebvju?wVZ zqrMLqw<8}|cZ&cKb&&KdFE7=T9R6H==Q<|so!T0E$*CIU+4Q*tMKjfM#vOsi^XYE% zs(#&a`3d-o|C~1`Rl7Vy1$vbX;U5>|zMt99 z);5bC$%;V4aMQ$Y8C{b!C_M@+nwgnw+rWN04=&Ds={)sv;pA{LtapANjV`Vv1-u|- z$K|#a;~=)3Cwy6-kVipr*eHT{W312}C;Ambj*hpdSuv1bo!Q_XYG;#q4`~4=y?qiQ zj%#~Y3UlkTY->g#rM|n3W9G($uQUXIl@^TubMMpf>iN;bb(;)QDNZx`T)YQY8Y0ny zHcKrA)tr?+pY)5nZukUcn&SWUSt;Lq)75zTtvS;SkHshNUcmLqmekIoD2-bgT>&=g zhXDWZIZ!y z@kTq3_|bM*^j*@B>4e68Te43#hXg2J&S>Tuv}mq%>c@Wo+Kmbke+^cjK7Epr%M4#j zKm6u>;ozWB zZ={c!Y_}WsSkG^&8qz%<(&%e!nX+ zKJsQuOiK(<=!iAFp`7F<_lgy(uBW;{u+&(Ak9jdhXCa zJCQV35n(_|#WUp4Yfz&kJvS{Y^8M^+6lda3IY9G>hbdoOu3Mu-e7PzaplI7(y^d_f z=-x0USVDIYUvJVOx;G-vW^NV$3V=b1eC(5Jx2a9vmakG*cc)?<F45xWxoZ1vm3Vwm()new&Zs+9AunV|JYV(giMeB!7 zB`m{sN_^VGuThc(wbfwkffy1ka4NX(9xZ=8pvIBKVzr6f?xK38xwu-lct^on5fBn| zBG7P_G*Bk>rdo)t2>`n^RJR;2E|S`1@SvTm$L4fr8-oH=YM&dzcT^QHz!NYutA6ho zI4}Gez}>uCc0u5&pI4hYzszaahjNNYJrS)~#Wl0-anQ?Gf&T;5}>?AHzZ3Q&{@nNUdBiC~FYO+uiNvRQA?h4j2T^hVp z-J#NtG1!b)kz3wHn>cViPr0t~)_LdEa1&OiqyJVT(Y=!z%BjQf1FP0I+ zS)XKy)$_~c*?cb`h-3-LQ;4h2gs%(jtGbB>7xU*VA02O^em7!X{i7*2{~N%;c{kVr z;>Am{){_Jaja@luxUwv7V(*IugYlZW1o{PKWgydRXV53fgZ z4;45cJtI$98YnmKdD`t+7BL?KsiRnaETRM^aYzlWeAJ*4MU;f8EJCn~%%SbjSx(L9 zDPP?(AAGVQ45Ji)Sy{@OnmW|8tOfjX0&Z&WFQ@(BgW}nNvUL~JYDBMc z*!T>7Hd~6Y{L$>fF9x}56=3bN6l@1t?2~3VT+6Ni)(4ob@N=394{x6xMct$i6y%T{ z;_s&gu;i8HrVc!@Y=cY}u3?J9YM@PbZA;tGx52JQo75%}+Z_+ZzvZ(v89FTT%s9?JlFqHo!j}Zh?-r7texlFB;Yz~CUqb*-;y-S@j6YZj$_V4a@1gGi*mGWBf#B+(f9lsd8yQzqK8sbz-Osh zsCi)RYwISZ+a^8`ShUkLA2!A1zxpz#Tgn@JDBC~c3>!KGHjy~W@elWRIK*;rW2}4+ zHx5zFK3F%+zf?XoWGCCxLDc?PZPbIyl8fQ?;5>5LsfHzN^Yw%upWH@Y7Xc!lJhwGT zTPad0V&q}x=et`Gkz8w}^||^X+DF?x6FQF_$Lt;)i~6#(m^3&+P1~{cSk>$%3mY|( zBQ?RI83{M^V>O24gHYVDRkyoTdU*6-37BxQr{)UWG9*3dktw+ai8_YKc#@_jokBk$GxAgk%uji!go=5)l;#kSJL zN)~D=mz>;VfNwR!=zi!rf&W}@x4${tyE5=OusSeV)m?hYJ}ah;k}BbLrw!{ZUnN`Y3l*hJVgWC z7J%6`r$C{IDA%Si28tgFi`$9x@B^qnFeMox+BL&{Qzy*y-RoKkuQM;{xwwv7^ot&2 zU1nERBdu2Dnb5$VJ@NxwK$Jp=5}tkQ~GLx8caC)_U!dg zj+BPmcJ0{g0?Letc$ZHe-9_V}H(KJj!!Feq;y%_yFb^`tqSym|0px<{@{pyO`?c_r_V zbVba)Yr>A{`nL726d(6j8dviF7atc+gWeqhZg=PKOzyt+5(`=97}kCgq?@cb&Z% z-oy=GuUema|M-CR_sI#Pud-?wGuZq=_h-7k4eG z_Et5fEHzjPH8g}q&@BWCg`R0VdPJZqBn6WM2f@PHds@dI+~3$866Oyuy&(_3ixJOL zJ&DL>NI~TdcoB^4>#n!|ve%fJX?;|Egmjw-s<=x}EVJaEb^p8aWl>rdNXOaQNF(#u zt$l^izHU@%P*1fArprNN`f+W`gIn-UrSQ(5Yb#D%yy0u?zooO>zR}fxq~T@IeaPUw zL{iJKvUfdXwh={JU_>Xx5jAB``OI&jdANF2UsZ0NoQ5Arjq9-e=Y9E~UPn~VwW!$+ zo87R#2EgeL#bNzkVVeuaXV)Zx-l}OGHia=Ob(P};N1{ZPI@PByB0zdeY<*^kIv zUC|d)6ukVhi{U+W!_P=^=;Jnz3>Dd+3hd6r-jXtnIE{>4Sa|QAsr7v$k7>xaNy4%v zzjHLLzka`HL)E&iIV`73MKcPr6<>34>_$c)NAV^3@>|g`qvg*j!m7p(m9I1^3>XrZ zaBDr0W6|ZvEGNI0U>}aAyp2h*Jr7X}$M?a3x}ABTR9|WGF0i`Kt?D^yd z&ND@2%i2?!-dzeiYCmzs@bncpNT~yu5NVa2L4|VR&O(>;W(dMnjg#9@2Wn7ZrSAPo zR~NMiIhJMn;ld@yi=O9q!5a5Snz5lQ=nemD5!AKL<{AUfJ@3^Q~u&GZp2hy09OXe*b7#))mv$Y_1cx ziwi-i&3*)Ad5QH|wW=k*MUmwbot5$}m~PzB7igNC#015dn2+1(9WN$_pDd{RU)$=I zU06>qAUzIq9YN~cZAdX$y%;zvw!cgoDCO?^RY?n8kcsI=H9ye!o&+2o;v;mM)eKx# zixR6dpnaeQZyHAv>{LYuui zeF+DIJa|?!X+zy+ri-p%tbhf&c>kuUE`Zo?rKVp)yvgrIf^~ejbGkyVn3C!uVw~S^ z^Mwuy)QC=J=X$wUchldGhb<+V_%dug8LVKd61)g5F0s9Ci%K-M9JD4Ji~OSzq=v!& zlfC=@cfCZdw{cY!VCo{SnNsI(6$hta!;3TEX~Rq1oF~tc8*LGCGs)VHk{DCdnjx=& z$~)w!#hjZkrb8*lG$dIR>Xr z1Ka6M_iKpo69;BVTfpt4lcC=;ONE6obKTN3x48n}+_>7Y`o*`!dWjrbaT+KiTNi9w zLkZJ4srj?H21T>#lf*wEU2v7R{x+wqrWW!!7)F>)d#JsHOO@LNf5o|)8(gK*@wI5> zSjx)9c5ov>)H*_H&mu+Wr7_(!rR~<|IgpZ)Rm$*g?<&W~mi}Fi%&*IKX@E85%@41Q zqbSV84<_UWdn(HtBpz2G7GqLRJlKgUrdO-Furk_gmow;Z{WfAZiq0o9cGBuq3>8x) z5gN39q6nB_q0aHvD{;BYZERGI;1zOH2)>d1SUYJPK zexTibaZ-Qv(az@InpY_(3sc9*qo13n$+ZPEgChpsaNC_e>p4P4zvL1)RB6$bx_b$e6 zSG4GmITyp2jEfA5|K_+P8=lI{0tr41sfVsb(M-rSRg+b_QJ;2kk&A5d#cW5D+tsVb z4FO>qN02eg(RnCTyu9t{I&jD%LYOFnM@7NGct$OKQYJ!0CQ`UwDU3+{cYd>AF*o}y z)lO&wWFzsn_Z8P0)7Ocg<>-P%DLY^XFL|#tn}@U0SIh(*uP?(ni26tqLx<%r&5=_!ca%JK^OWQUR`gn~I0nD^{BY46tc{Xaw%;P0hY^*yCm`FEXJPt8 z%}*!J$))Fm{M^$>Jv`6Z^|2R|$=&HQ<;89$pxxS<)z!sXQi`w1?tN8DBO03cl z4)j@jSCh!hwx1=zLxHC>j$L8h#`=krY>wiP=Qf~*FceBmy*^kLaJLN5eu*MJ6MEfx z-KwopkQ`97u{Re|v1XR>GY7rGre9W#Uz&IfgJ+LH0;L=t+;r@#V z(tc`k>v6_N(#!BJ4h+k?TFBM8tT{Kdzk#Lie$DB@B+2=nw1jy8YGgh%HSGOgztoz# zy39QxF8=>>0lTp83T)w)G&#nWjKj^*g@;XkobkT&t$^n$MU{s`z!Gi6)QiCuNpPrldg4{|M6Q9phw;&rs|UFsX*iH{g!s-KRb18HMKM82eoxc#+}($wtm7>k*%SHx`vz@3Ej296`e1?ypc zHvpS^{F3vli)BfS6&t^@yMDcd^AN|nRzGwFQLSJeYIiGbhM=Nb>3FPINx&V9kos_S z%j>7V=x%3yeS77x2`m|x^A>q~@vARz9qK!B`PAxXQUGEjxB=P7s$EYqmS1oU z|MfVv(NSnvcn&di=l;_f;5wPAhA{0>$LjxgTAGeM29MH>YxQ|=XY&g1ULuPI$P;4 z^i}1Do<1kv!&*Wv@+kpHHYwD8BbE(BvsBFA1&OIGyZ85>HL#wg(|`QX6eXk*O>rd^wU=6L2G}qXZCVTeZXOD-85b8e8#0y#=Ge#iV_8LK9QWpEGM-$% zUAq3|x0*73-l6MUF7ok@xmh&eEaiKLx8p}G&zmj1?8y3s?_c4p4G#rn$F402T!-^d zolAl?@qAiOBl?u$pgtTi(#XS|@A6X3*BpPc-g{l^=)|?;-J*N-SJ(UHbmNxE?K9S` zv$cwk1A97kn3FSA!m7{7c9a+7^fB3==^8AfyP^3PG6c|l*r0+u;60JSM*8?z;C4^8?JQ3cKAvCgB8EN9 z4gu;-Qm#OJp<8errpSdUsi9P}ECrD)M>*v{_MDNq)2}SvI;y3kX5(8ewg{|u%*sK~ z?5CXO=28k_BrF4Jt250bP@@!iV9ZM4pOjXJnXOaZ1deDKo`!T>27V<}x-`U_3-3G- zc?t*+$vzl5pFELIngtquBPVyyXx==y+7t08v-1bEVeb+a7QB=&vqCFKh_h=M>LN(&zucI6q?BJ_a3W zS)UAU{lw{7U*9kk(#q0Mf%fpyWOJvYDE`sl%g4H9SUSqU{B+@lVzdzP-t8h^20brQZ{U6pR_QjmO@z1}$&bLh~vyW5gRzTO;EuQiv z)d68J{3|BPks)DCapiW0S?}B%2e)z!8w7Z@snP=z+ybnw50{@bpP|;$wSZ6NZTAAE zk2SlfSaWKbic7Z){b;se`caqFO5>z22Fqh(F|RH5v#v2|bjh?{&hSpuw&;bZ}s4x`n)#QFqC8cu!Oy45TnzB`s48ps`#*g zR;1cPFYYN49$Owf9a!;S3pxX|)2%VbR`;+UPCFLX+h!LyDD%M6wVnP+Hce-bCr7iEYC^YY@*`N-?$r<0R^`PL2Aulfpy7*h}f-c@cG_-+Y zl?k!bUU{RMv(GXLtfgUN5r@xg!k+!3Dcx=GT`TmWo;UpHLo~Yso`4(osdV7Y765o4 z$6ibRb9Jr^MKHM`05ZF7ZI&hZ8NG2rdw%z4hGpZpU0r?4sg>kSx@MNvO0-Yyid=vP zhN6sbck6ybly`fmM{hJ^Yv<<=)u|0S2UqTev)&)BVoNb_aGP(a(;K(ev$WZrh5kLL zaU$=KY;Ns{B)@8jSG~(2kqS|~Dd^vC>azkbAh=V@!%xgoB6?)v(4}*tsfRsFn4gvi zthSzPlH4VG{%f@S!Q7KjK36kDs7OtFTfXRKzqGVFcz&;1S4 zU?qjn+vG~^nktf-Y?w1FVPqzES2jTKnC`2svRVmSVq&~^6E&2wf#COfHFNn|i{-BL z555UZ+JO~tgukp5zdq^blr{yj*+d(t^Ac(haXRkV<@+^(ieWq&GH#syhxHApBPAu- zK%&--6jo%_uF{Xl>YDPq#@g0phB%h7A4D=V8F<*J52 z!zrXXv_R(v8iRce3+;?3`^cDCY|Ha>$Z?OionnnTUpPTa641# zbkvytFu9?pE$5i(^_>ymIG3+WnO{sbRYZ7@UXRMVQX2U*PFeyjOjDv>QrZeY;{0eVNZ?Z)(06o2B$CN9gCwtF%o00JcGVeF!Hn|TLGAC?A6#qI| z4BNUrQF2fIruxN^ciDv(+HKY_1?T%VXVsHCEI3h?huBuPw%V8bW)_Q~R?Ms&1>PDB zntG8Ro~7SebR*N_q*EQe>9Npe?DI@lnCEHi#>DWwk;0IBfa-vWvqOw=l+DmLveT&^ zd33E8t&XVrUQ}!s?88@^*oyQ@S?JKZ7_ChEVHCWjp!!9?in2xyp-!rI!rQmRA^&Ld z&DwQx|Ixh4d2GwtYoVsW$_m)npOWEoFKgIJn1HJd%fcvpVdbV@<{ZnGF+mkOFZN&P zgu+9A{-;skzrQ{(`bYDbx|K#Db52;y4&JO|cX@43j*|b;JbAkCjY;sV3~mtejY3bN z%T8c8vO0=Xt7HPz&y>Z;*UUG*4Ro4)C0E+AlJ;me=@D0Dwi-3}P8RQ^;*%@O{o2F% z*~Te8mu&kKC}s_2+;m$HUTZaPOaxdsfO}k@B@SM%W(TmbUb4Cm?45V;}dYmTGk+~etCwN4_~4%LgcLAvLr-`=xFhGGN7}0riC`? z?%p;5Wl8V*@r~1Rec(^oqd`(nNCHjk$t!?jBs0 zwQ1F1FEsEO8EO%_TaZjWiGBO_Ur}ItH9<^d9Y|^?cTVKFm>#it2hf(j`D|J=Schd0 zvivE(XF?6K`wT0s#6O2ZkV_9ECYH#z7cpUc3osWQ(ovkd^cDn4F{%O`Mmh5qq)wD= zUC-BBc`DU0M-ci>bJ^+SWuJyj8!_Y~CkH?448>Nm+Fh}ktByriHfhVl<YSzI00tr83Ow;lm}*y>XyaMZutKM*$A|l zD_t^Swn{Z)>AjxW!>WOQlHHeRt4Il~YMp4iO^)ddx>1e`F}5a!Mf*y5^C-3?A|KoT zR?o{q3~^ni12`v-fo|%S-$8fBo`v8v*+2_Q#QH=$pOAx3p<8{Tt<|&VC9r}=fA!U3 z-#=6shCd>Ye}g|H*?yoV68eJue!%j`7HB;umc|d7yEFC{?g{>!!XnAP9@?vjtPrLO zNj{M2jjYzdq%dieGCsB1P4&(hT0KN+l7wCm2jejUv~d3z1EP303~3sS7hr#_6iHza zs!!YRb_I?$&2wK{WUiInlVF4Vxg^59hkchm7Wsyh@D}PqiilQV4e~W>Y?!JWDlHAv zKoz?gdwnhVTrz}t^ox0y?bv@Cf;HLDcU;L8O-zN5rdC5@AsWs^;OVCLxa!>z-*27nNA_S9b zzZP#&bHdvq&A@H5qhR#>K@GvdVDm2aM_6!s>t^VD%N&X6f1hb(NA|&MendL_*idnP zN5$Q~o~oUh4RI;jxH)I&Hrp$zZU20kdGA7hMdasiNuF0k(qBIHow9|TpPAbggPNpI zLonG~L@TyPr~UTPx<1-I(gxrqZDu zxnXfc_)wV`5#>x0%c0t%$C!Y9rSr4Tq(3G1T;vi?w4WQ?xH{u33VR=$y6?g(*(X^IMIB%yok0u^VP?#QW# z4y|)wKSL@-m&%DmbD4J5+dU4a#^JZ{6VB)u-pYaaof`Ke_b#!ab- z7jI~=ki<$zHZtMdJt^g+36DacIcJ_}Oi*CJolfJ%93i{Wt&0+C(3rM|(-fUT!ym98 zA)LR`k0RSRFU*s0}nJK+mG=TlTS>b+w% zGXny;+dBuEhU;s3yvY%bJuj%a zsG4*8*o2}w9>b$Y>AC;^@c;|Mec}X zUKkKVw&Sn_`fn6H5@S&+sBlxoF(|75bt0Wx@|Xn(RvcF;NCvVce!U~Z)m_TN?%DBl z?hXWnJ;QO<=S6Jk`E0`dnn5oPg_#^*F!v3;q&sEw&ND1-U)TsuRrf zgO)6DUb;DY2Q1DE%=292(BF5f!56-UD0lumafA&1P!)<7wj~mX zGQHIMYJj)c@txP}RLKb+uAy{AnC=P|IQEaPR%$b1W`Qp6Qq*bYkZMBi0|vxqvq${g#|hk!>T7#{X$%x zAqKBBN9sUn!vCgS_~0vc5nffVbGyNf{9j~JCWI7)&MdIZsBuB?V2E({QPDWT?JnoX+n+fRaunw8-P7^>$IR8?0&ZZROjtyDJCW6k8; z0zR)J!k?X=Z#?qXZR7)jD5c(2bz@imo{LS%0|-wCpwhm;nvmj_7JIH?W6rc%UfEgU zca6Lr{T&_U>Dfm+WY7tgNL%h$->_7<4(%+-wD$n(GpCYo{v;ZQ04`!ip)y ztS&JBG#dDs+hkNWsbZW0R-dNI#o|v~&rda5*F&t!;?e(TdNg>6?5IyjRUaSz z48`)hlfJ!ypQ0Io?V`CqujS}k@sq7dd*Z~!_l+1Y4b zuk=k*v>EADkqFD7Fmg0QAxxPH84_noOJ)&^5PCBrHJ8QIZJ=iE=`OvT3aou^ngsaR z{8q32>Wnf2_`%bFlBkOpeJnLEDdgPjSQ4OF7+Ydua}(SDTkZud0jHXq?S@nuO=U01 z8r*)FTL12?@bcYV!W^b8uZ4^Q7s7VCx`G-$!42{;Zst2h7T_ZUkx^rN<;9$Cp)<(b zyZ6)*Z|d&ZC#=eSrPr1QR%7-7Fs|li#?X)S#)O$7|eoZfD{u_1g8P(L|uX|!eqzOu|(vhb0j*4^v73l<2KuUxN7zhxE zg3^nCfCd2-0Vx3~p+o3ZKx!nBgoNG`Y9N8%&Hvmrcb#?Cy=TtMoi(%OMP6jD=T z1Zu%Tid_U347u%n>n*?7PiU;FW@#|{cPR3!b5w2onL7p3AJuWEoaB_UAFFW21XV!L zqX}ov{k)tXh5>42Uc8wc@rLhBeldzQiQSDdYZpxSv6P*77*S8%2gKwSEi`W)`a?_p z-{4suCz?~HzIvmWHlcnzG0XKo(Q1amnMK~@-b3y0**ub>AI>d)YdPXf-(XP4{3F%g$Bmi6iOi*b-?&U{3RVGUVeEK2pxt?{$SnCX7Oor1h0Pv~{^$|LN z&VLv_*k?{n;Ytd~8~(~dVn^lYg{T^tORu+{OFUs)cx~glV~wJnZ#%); z7UjGJWTbDxOv3RCbpVlwRB6g8ofUl;t*=gFPqZQ1$G*y;?8`1^Rw8-D@;4Agvpp;+ zj?tlh;uq|6xitmhon$9p1BN!)nOT#F1t0k2M541=OWep)jWRi`Q%lyLEJu$fFX}D3 zh#yq{Ycfs;`9e?-k`h`eS%Dx}bF#8;a!=Pl@&q6PR`uy2kCW7&U*%|>viN?#)LP=n zykUn3&t9r9B-UnJ9-*}Py@pfQp2vqbVMlX0JX9hPSi?}=yBg2MJ1=@A$Vzd?W*3%K zHjO>BTzzWh#HIu^id4j>;CzHGR3@nc@my=d{86W}YFK7(kYahZ4l%|KoIJ|g#ru~H z_I7F*_^^6xFMR#P{9ddT^376Xy3yY<&Jsn_WpRcnOO;N3<5Z%M7ofJ<0U6;wz=IC| z1b@o;B0f~&#t-Q;FO`1UBbo@=~&|OJ2itCcx2WHoI5)iw1trAfAY zceUY@mO)YA8^IW@>nZn=*?NgW4a?#gBotlER|sXCfQE?QdE!^;zb~E0%2i$$?U1!? zeUUK{`r!Hp%OxgpSrL4StpMW$e{tWLuN0tJ+hogCVz@$({UlpXF$FL^j1(w&>vtXV zXrcX@amph634={K+nLdEm`~;9b=aTh<3sk4eeM*U>~P2gRnfi~sFtwkbQ^m*p31?& zbn|igT|)9nWsu3Mg2ivufDObESP~V2$A&DIp$9in_H}(+L=czt{PtTnmaX|bi{U`J zlIc}&z3ZzW=4cSKm+0-8pqXy!gb|3bcbItP!Fka#Lfpl=oBf*4=~c8YU(k=$E3Ss& zrOAqV4+oFnFD8uF1r#5DHl4d`>Zz3$mW1}Z8@XOLVoBTN0yRqSUTj^A9NKC@%~igq z%U5ImMyB_x0>(6R32O;AKE@o;#&bU1qxc2RA$*4|iz!-m;OZB@ryab=;0 zAT$4Sx69uC?G$`RpB*%F*lTV|z2=1v{fFVQHN|V24>pQUKQJr=|&FC(c$MT>pX}be;gpq zRue5CpLsclS_k(DVk;+Ri1pVM>|cP{VvEhS24Rn{jya1@Ai_wh1@s}Ar_&(zAPgO& zO4YM}oMT^GO_f~JiGN0C<3LKPKEz*+x|V+XneruKLq0udEtMt}sYMp#34S?1&;nc1 z1CEAqxqK=v3ImOXsvF~eG9Jj_N3VsFTGWG#6r5k;R-xM$vxjFYP~3G+T&l^%$Q+tP z4z(iRvZ*GYC4(k6Y6WqmOjEHZ--)|oh1QCKqOG?CpI?8iE?fby*}w>}V0di)UATCdr* z?V)Ms+2^yvn~xiHPDYO#=)d0U|CI^+e|3C!H2Gz&b=Nu5MvA9Kj{-mKL(BNLtnLfg z7-Ns-EQzNLVAf30E38mz36zCgPrHs{djZBX`D%}R`c4-jMz82!nw}f4s9k-YssAfE zVA(nTNy^_)tI|cl2Ne?~-eY&75>1M^ut)kW(wg0S{GO{tGoHh77|eOEERR#`>-JRQ zYlanI|Ndx-t8Vr|I0@FBtJrsELaI4^l=`UeVSOIKGXdCunA%Jj;CnE*WaKrwQ@tSU z8~Qi{fT7t~?1pW@e^+=37)T!*?CeBXQQKwbtm=KVo_a9 z>cN;|Mw%~--BFt@i;w#L(mzAANBTHczWx4W?ONAX*IKjquux`i z(dXE)>{K7Fp!1g;5{<0nr3I4OTAz~gSlR8zz{WiKx|5O`i(faTHVYq~)HqVG~dX45*PicMz)>G+%Pv?Ib7Rhlh_Z9Kn zQJM$7>`c0VtgS`fOye zY5~R+(>Rwp(`7X7Nop%81>%GzdgCybA7l6`;D1j(n_YD=yXGnMz?`??f;FG6E&H3% zdC-~2p<~6|AkyK3)LvJCF=AC}7rH4k$?G9p=EHNHxS}joW34lvOL(;&-1=NhStbez z&yE-v%fJGqgQ5wVH8C~HsVJII$+)I3H4O$QDAhOs9J>{Qn@pWd8uLW$7v*;R zwZxMGmGyO8A)`Io23;k4^guz0$qEIvb)n$XVWn3e7phYjCBFtqt`(TynIH19(h30N z_vwZ7|6_PIC$zU0m1Bx%~E;gxebZJs)Yi+1Ay;d%d2Uw^xf?8HcYe z?q-j{c*PRY&q@Aoh)lY2+(?C#LXbXq_I?$6;p&>=ITNwmXz3A&!G^o%_(v3XtV`2j zV9dVjCUv9fPnnuPh-Jiq_42loVk=wfe11o=4WKQtCz5YWQ3*V2$QNY0+(KGUSYc|nY(5dHe<4DluqpA5m{2TVNGV7E9dYP>BmgYGZ zJOx!#jAoC16`riUZJTXl@e$b^&5A5n#@?%Lj4=@VqrzyKa&Qh9*th1K^R~7=E0j$< znYn6@!2>-aat`J*>`pXDqUpJpw!Kq}O;BA$3g}qgkWF7V@l0WkLu}V&c7z;XJezNzFw$ze43EWOr7c_`$}PtCZqt)=IIuMj=6SKFRx;d(+eWkHa#X z+e&=P&stTOn$mjY?*(2lzG*rF?4T);*t?s0W9Jb%w@FJGBcW6EG6>Lst!f9 zPG@SO-nX-tiPGwSr!K#|m|V^H(I~>8EOBYP@?zHXY6-8~#&B@?@A#L&*_ZFgefBuZ zrh|Sx0}Z45-p}*6pi=LExGjh?P$|W6jQ6X42j5hWdL@oe`QDd#i{To#&IK-CHz8Kf z?o`^p%EoPmuU1S8jBA7gXtf}c*$mg?`yE2w3!k+=<&1E49v7>f zn8rw_n-xMly;NMxIGeSbj;sZ%N-z%Z%~D`zAw8()FdmBb;oTN`_r}#e{$ku)!HFa} z-gWKJ-p}h}N*^$Gm+7zjJzPk}$9;B;doSEBgJ{ZHlvuK+e|6el72#|*v7zYNOBly7 z%_SMv+^X4y?h07SrwS@u^_RHWR}8hMS!8P>y;!ogkaZjj0E-qiXez~59y(d~_gJd1G+> z-syV8zYJmj#)_eHU{b?>I8ekLBING0krilJn=DTLIg%|w<}QBy!1!lLXL9=#H5vId zFCHdBZHg(R_pl!UUWPbplF1I-u$fxS>xGLXoatbY|!-44wR7cq0l)j_n2BLa|Ocl!scHkkN33)Xb!;96DIe&=PU)d z@bB5q1))W%Pmx8=-eGkp(vCq~2Ee-s**g8^!sbta^6B=2^aDOa7c>5wsiWyCH#k>} z7wm?^@Ne6sDO4g)hiQu!u#^(sk?N^-)YgKAs)efZ1lR3oBYp)Bhk_J-dL}iRIsWky zs*d29%a4Pa5x3u-@}PAAteN$3YB~}vjWM0Go5Uou-EVO?Gd_pwmhH{ZnWg%Yu$m8x zD7qC0engkuSR<2sP;~pf4s1su;$Xs$e><@Ftxc~!6DhR|#Ex4<5X%;kF}PU+f|AJtE4Oe^-Vv`&0$@+_#nzgPg@^ zUS)Il9n&4+GeBt968J`Gt_zBL??~eqe|z!Z<;&F}ie$*HS=k=8^HvjAHTDoC_{IfBbERs5xz(e$9ad> zAD0ioU@N769+NJq50eMMsL$y^zER zt0P_ZQeX9K_=jnp;QlYOmhwviC3Xasx0X7&<&9~d7%FWk&SX#z{u)$#-tPB4ijfkv zZrYrHw;%^+`>1%M3Exu80xmTMD{wqFtP5mz&%FL2jIn)NU`>RxjSZsI6IO2bdo64v z!!II82(Em$+Kl)4rSe~Q7f8%kw}2+WOSrA!e;BaSFe+$p3;Q1KEBm4%dyp>rrn7+D zfP%%H2=QLnWx5RK8TcLMu7-L!V9BeG^E{T_vcIzP^14L~8K z+fi}(96XLwQ1MD{w1RrRPcddCEGube=#q2)QZAv9`a&k>z}Q9gh3T8e9PkyCH$J(k zveVDL>mUn5pU8_B;W4g%2P!FbT9S341dfjz1^3`rmj;FUpY_G{Us@oR(C$*`IPore z>Abfc&uteeIJWU^OaUyn&9bhJR>uDY^q&FB6q!AI7U(E`*ZqA>-Q|Q2M|cCTLx9?` z{k;*`vT;Jl>m!{{=Uu=v$%l zFkCKi0j>iPWOR5PIC4fZieaB_CFGb^r6+#bxn3tlrkxb47ra7;Y>YX`pI!8fYezrS z{sKv3?c09nx}yF|U&Z+VB?L}ZB2M9BX=X*R=d0G&M!i7XIe~}%R|j@*8jps+OQGqD z*DZe^xoIu6)(XCki16uMH3kl!oSZH$m1kRA!DP-af|^f-|6xebJYgn^qEDqD#hnjU zGs>MVAV1gmzU#}Bn1}_5vNdON2~y3GHS?@;kI+!ra-SxB#7T7NEXLwOMacqv+TxM< z{$2La-{!h>HVu~fABY5U{w10PxfzmAoKCH9^2W5H-yt?D_L?KOnf%~$mMqos%Bt0x zN*|{}UKeESOxr9jK(UVazQnJo%@ zoFd-BjngEvKTi{%wufV50B=T!b4M{oq7~Du1?0qzHYp08WIQfp(P?6dRsfe-n!B;6 zmt2ck69E1mX2)E6d(rz!S~`OwS6qjh#lZ{8Q8~?Zdb_+*&42|0A5jpSc$oc=RIc2^ zodcHBjL@Fb9pCzNHbz<|;IP0Y>5z`2&=>VJV1o((!@rw&fb|S*tHu?!)I@v7r?&h# zIaE6wEpBX&{2pqU^LJVBduTSOi=A~w=U6KxwxvcOm80#_ot*&(i|A1hJTgo~PlP_5 za^r_gJ)uN^-3cF|)3S&ujpM` zO-hwg2LT-NbP=+qXJ|8xctv&e`;}!K8n#;-0JzB_X`1iFS*|$0%$CVkMzsaP^-{im z0j#-lBgw{}h|al}=}K0RkB-mHI~QWH3r*_ZU@BtD;x~`}>PJFZY+IEMzL4)74hM5xYibpM^U3n!u5QqHbAohl1@RgIbx7w?1elN4_-6^l?-yRgbi~F*@Pr z_Hby{_0af~I!F(XIan=Q84kx^gk(IVc#-BiHNqsa>`i*O(HWt3zeTI6YMO!lnE8#a zelx)Jcf2M$e3Ckl;Rt5er(|)WBs~pb`#dH4eil)Id)=*e0?e#4Bo$@$wK-#BtZwFk znSjLj-IMCC)oey#5jwLLG^^9PWV+Hb^126n&kelz4+G=klWz&KEdOyl(p_Z9^Z9E? zQ_$zH9mQj}x!UNrzV%i51n^Zv>}j4fy$N0$zIMiW!(fkWaB^t!2ewLtQMbzf|MN+f z7$r&b?T}@ypbYD82@L-_|Nj(0DVW*~Vn~hS#+;X;Zit&tZ|o->>3iiYn4yNKT>?%1 z%fc}Y2ArqEWO|nLgK1UPXXq~UML+)K`*i?$GoZ(DxYgZS;Em9<{+jAc8$K5g5mOW} zn|u|j{d-ea4&mypJ-Z#Ry6&L3Q6bDNKTb+{9OeFu!Bh8UzgSCtJDyuNppb6rv8p5r zJqNM)!n;-`R-1U@UVzkWC(bWz+R@ePIHR;B?=L1ENm0(>!XLhzOiyQJpIUl5@9ov+ zA$5u})4EV; z!^!M(SEg(Io)2~L_vXyY>9HLXYMs}Eo6gxG-vLC^iL!{ytUF7F*8>Lu$|_A6#I|Cx zn>hvV1^ViX)?WIS$C)y|Z9OiYh3Oh@Zftm`)KfMrYRHo4|Ep)_El^sM2yb3YSpRl( z$1Dus2waS)&DC4gPp)5E_F69kSseLC_!vRsNg4C71gH|p?Qu+aA-bYPgLDewF#56n z$TBDh{3=^x?2T^B{ibIa1LFnS?5&li=b1&7n>K{uV7x_D^%OY&oU8Q@00vtg=u-Xn z&-qRg)Bne@{l9-7n%xI` zcPd7y`m)+&oJfo?xcxI_!@9(A4So;)uC0VKH4A5(vNZcP{O7N)q`3=E9{<$J=hDAQ zpQw0sV%eG5`5+x(9mraX>V@*sxrH3>%QuU2ns2^1|IXvQPJV`u1Sjwp?I~3iN6G2Y zm&n8R1Rv-Nk`*h9*PReaaFg6fMTPTcE+Cgbf4e4z(dU=FoW5wjHIIh!eX$_8UGj#j z9>2p?>gRj1mc!rChE)&cMS61wp zD`wV|=m~{%0+g4AdVTP-hX>i2o}(YHe^A1R5B29cD>-lP0(4wQ>qiC_kl(QY zbt*m1XCJ>}%^-v0r zNcJ0aarGs-&N#;3B>r^zmD>g?n`+>dd!PgC*FGwyJ0R=mB)g$LooZdQ4H=X`6CsD2 z=R5!jz&g!f^Sec6*Lt^4IgV6ilM?;K7*DP=^({8;{KF6_Bv?NLpQKpQIk?7T>x33r zHWbf0yy-Ms{*p=2AD6C|Q9C0$5YqO`b!uI;=%N)_NkAPs#_H39s1#`%^9lPtzHhgZ zn3n;M$URY#J2?Oi>uil$(VZjw>+Sg*$W&hZT9{xx=3V}uMY?g4@!+1l#Gr%*4{&*C zd9-&-df@f1Tcg6Q@@Bf$f(b%G=GA#`g}=49&AIhda#GOU{4r_sTJo>lu9&=_pCN(g zw4W}h-I+l} zqAKe7n}2N5|Dyzynk|I&d`LVGV9~EF&$*7{CW$1z*wB3{gi3*$h3FRt5Ix2}bg}-8ODpK}-nqFmpQ1VS z@cXm7JKY(s+D%J{r&L*}T*m#Dmc&Qow!)%;GFKn8!Sg1##WFA_Gv!6})&7NU5z#LJ zQQyz%NgudQ@8`_&^g5~kJpPya9r9*F!N#tQ3jWKXhd_4JXD!t5!KEA04Q|$OUhzzI znvz<4q-VRmbTV`rfUTpWqpSwm3#P)t4`_u2+lBk}USuW0uY69LPN8>pSnQJwsCul1Lg!P{$IA(_z25|+SSEkN~Cmv=*fCUo*|0*n)^;NFo=SvSunUPz4@ef?!hS3oLJZ6nRFL^OBl zbOJJyHdu=FtmOOV$$7lbKv$ivu$Y$cK~>#w@d_W ztQJG&yWqjO#V#{GA?jGW@S#mrSAiDiS|<6|x}L>JSH=b%HlDCe0Bk=9-;5=`RX=?O z>tyu#Qp)G`=a45X(K2a4s9nM=tN%<=7at~oz(}-D)E9y5>zW~2_i@>!819|oe_6@B zqWCgdge}T_{L}t&=gmlt>@M>jEVY$Hc{Idmxr1MUS;}K?+~GSfOsr3*8+;GZ)5E}{ zitH&ta=?d2&d$~r#gt7P#=9p=4X%+qsjPWpFg)5-jnsS)+Du}P)71(3LHQ1BGI_hV zo-EyE$6S1pXJHWv*FE{>rbavB5=&9@bZCDNSeOF*1M&g_wU+KF>;CsEd_-=+V5X}# zQX#)LpQRFMu^G6gB)Gy8mh}Z}>9(0w|0vE+@%+J0(9`E>x;xk--yggeB(BDk6Ut|s zl*a97`EV2vSx(m%4xKWV>}@4~qvTtS?^CvQE&NK?H0-lgRzp-)--!LBZ#5=wll#Oz zy^X+zsidq=w_w}-N=TzaplxNJ;+=q)O3oo=poh^?|wkJr5BVHije$>U=Tmo&bt4ghfN63)o zHJdTnS5~KRGyImdW$F`3I8}Wm?=nfhtibw!Y2VU($$5z$0dzy|(uS8e_bmV5nFi-e z#CwmsZq224XE@%990@swbqoXBANbM(#*X**3_w)L3v3> zt6~Tl^q2O4k|sijdq214h15-}{hZR(Nq#gazJ%y$9j?y~VuYvn2#7b#WHOXVL<5_C zNJ0PghEBjKMX@O)GaBxYqOmT9Lwd_SUF{Z2&3nH1 zzm{S43AKJeo4WD&-{2|Iy0cq{-bBPZMWf+TbsTQu3kA? zdE=6PF=19>h5zEGB7x?dgX+4Jzq2^=@zGmG%xxymEN=ChNlRq7$v`0|-&G-z+7Roz zu|(q>0?>e!Qq6The2<*r+-+k4|%dKv%AwY{Tew+Xl2 zSx44Jn@lrA_Ch5HV8AXuHh+>uYbFQLGc(i!BD0$#J5FW;kbuTr7-lDE6D88+EG$G} z4Jo4e!s0ZJD`J<+?H`nS|BHdy)M947$#WyBw}F1$3+-F0I^`6$KVWXbmLuA_;h z=2iZV30nUimieRq6<0@`wXo2Z5G;85o?r=1a{l=!+^5+naL*6Ah{B&Zs?J&yGa?>k z?AfCi{qu9jkK0;YVKb+GHUy1Cec@T(04JXicsC{`E#z$I@L9caT&d+%hH~z}H5&=l zCui?Sv;03{bia>z8&QD4S1yB_WC25I{`sr)CbwkKm5*XbM6nso=ej`ZjdOb2WPpC? zGQyGgghCcC4cYUtc~>Zbdq^NDqo(?Tbl3LVHZ>AYd$`*fXUeX%CFp)_LYDL z3dHi1>webw-@)oOu4?QDXxHUiqiM60oNj)ljbu_MjC}ws)jE$bxqte*xu%J`rUmmh zCOlml)5|$BEb{S;huvV*;J5m{S;O0{6q^bsNb^H7?43Xd{8y9TNztcyg}XaTYpe4q zrtUEmWq?a84@YP~ZgG>suN75x!OnJJ?pleO{ZMs;^k6tnzQF%-w|1F0AaC6hQA& z?vzpLdyeV#p&n6yRlvDse5=;yVpYMvWo7ui9IMHUvt%+;;Uv^^iM}7~;+@}uJ>TYD zwIm&&yIgh0Z461S*{r!KpXhwRp%OX3w+$+gx6-fUz4YLh7^W<_s6aiXx#ZLYRs7pi z-a2w&cebl^%5db2#8#z@M0%OH606u(+vkERa>DxrH>4|)Xib+aDo#r@kP#L6#gGTC z5GNS3+Mnvw$H^7K&)arjA+<8}Nlj|y-;Lc{yU&X(9|o`Fz(Q0`0NEjbUs`>e2u`OS z*&hK8tczkxX|5DzKeD0an1_yee2{ARnB{R$-uO?0h01}`{SR#a-Q)&-l6scmF)KW)ii&jN&)7XC^} zczd-+fv4|OoB|*_+abwZQ;Vm$r7rzxfHfYCZ@*=lwDN}_&KqTdL3h}mxz%W4hLvlc z)jkZ!m&5J2+MDBp>+CQV1IT>kOnvZ7*i7>&{2zwklhI;emtWf%O63ExTX)&$70QHZ z+wYd3V)`R!*1;!I1U|BcQHwEscI&U&Mp=UdqTyrPvomKHxwdQvpa8N0U4nTHGU_&Q zO~0VfyFQyvU2NcUe>1DOJ(wbwSC;ft)(qfy(r2V5t=X$MvNC_l1?8A(;UG?u*;v(Wk z+Kqt{p!Qiy&l{>eSz0f$Z0&86$~{1Q*k;ks_U}(eFvW4TZV6?CA?j?{A>Gh=1Ky?H zS}tT(l&i>nEA%Q4NKEJ5hkOYx+nolZW&NXhJi9>xRIW!t3TKWvT4dwUuoGGQXn5Pnc$nbcXVBJa{b@Ra;yg6vm}l_ybTGEl zRb;}Gbc!nWK2`pS7ICFBynfs1-SPn1c>7IvYT5WT)MMV9|N7oA6XqqPj3?a3 z@r3`W*k2u<+=Zkpb?}=hD*Ceyk36_?Pg#e{+c2p&cvA6U~yb8NGbP){NAZtP-C|MBOTkj?M%{ zfRRnYH6rcUKf2O$Am+LPuYiJ8e$wHw zlicEkl}~Z>Wi?FD@zT>NM2mAV=DXdfoF5O5+ta2uEq{pbucQw&3ai~9?q$@Jvws`@!5U2Z>wg> ze?;H!Pv6jOp;@lwtQueSY9rlqPJZeOYtfPyQQ9Z? zujO*1n|cCW)viwRRe-)Sz}x!32lu7&?xl+E3w%%R*b@YQVLMIOEinz${oF(8o_>Gz90$jOcU+0?cB$C*zU z>1WVx_3;Z^h9!D$*faOliShW+d(WQQ_3IC}Yd_l_@8k$6b=!pr!+Qh)hTr7vP+qkj zkifCfv4O0C_ctp3at*Myi`ovrcSbt7n_H)fCtf$bBy|OyFA;U1{p|QuC}RK(T;>zk zefC=K2EBk5pgtpRVSv>9va}bwvAcFc#t39*2XU|wvQNkFvH>TTExa1!15h|X}+rAU& z^@RCe;H>RWLKEk$yq*7P9x>m)M}Pi&wSB~FjgcDArq`l47O1tFvN>&De8&v%g~x=D}(y|Y^o=rrkMK-p3-3gh$a72Q@^ zD)ic0e2%OAW9?DpH%pzpwrmvKN@+~LKd{LhQx66y(2Zp4NPNj&YjOMN`9Bm9TT+tt zz34krin!A)p0J_`rrLISfHEH6K%=b1~;ymM%9%iLj_G677qX$K zlyAk{7FHhG$@8^vi^E-QCBHi5swW#K8zhKYEBkRO+?8rg4#*v5@SAgQ7L$)Fl1Z5^?*#>)pwnsRDKR+iR*EC( zluuO_`uMo0geP$?yxGLhcYN7%a`Xp?lQpe>EpbWpg@hv-xW{$t5V1?0&JSG2tWTCu zD5(w4?VBo}3ys8~}@>Fegtj42>K6)LQ&f7PMzwY4c(Gz$M~ zC*(%2y3Tb^%`;Iy&JEO7#^b~+*yG_Lu`MRz-oIBcl2m|?{;g)Rp@WGUEzNCIXtCRz z++R2N%l+;U-|fLW5|0v>eD$2Hx0}%mNLW!pc5}#>dfQS#@$d;QOvKM8Qf?fH2yzi=agbO%i!Y@Ghv+R%D$`bW#jywNQqB5n@?SowkwJibYwCQ%4#-`US6E)`(Zx>L%RM{RZ7sm+N>{1 zr!~F#NC|R43QLgv8XG1SIFUcqP4hgSSWykjK+?W9cNWjT`MkWb?}}2*ann;>Ym(Z? z?bSUtutu32(K$)JZSSUO0$(A`wxp#RnBT!rD;uG*c3_-nm7V@C(b@6;^_AK{fg}dwO5)u z=mTH@eq=S`xEOu!S>don9stD(W5y3IxBgkhGl30~RV>-#R5$I33ZFt@Rf}Z#>_>m9 zQr!o*qOAtby^!EB{Xj^umezqQwWH2a!XlLY4DL~N;h>qwK|C_lqCzPRV^ zT*8YB&g%z}R2<1H!na7Q8>ctP#@|3z325%)tVS>xB#qhIG}}_Smd>RW?#+8v|8BnG z>~J&hOox891+C33JA76yFFl{)90{K_Xg;wuEL;o;Tqmr_=`?ULF7sxj#$0w@0p0=!BHP+HDr7N&`i$)W_cGZ4k|%rmZ=oanj%NyE;1;w)s*8W0t$=^GD)o z&T>`3jqPg11XQdDo31S>YPlB8Z?V2)TFiW?v{wNk%@nWXGy+Smtg{FfQ+j z^m)Il+$XDg^7N%9=CCcbV0_b+;JIb=Vq(uMkcKE-%R*P}jV9JAouxgVlpwX0g#v&0 z!X0)OC(6P0^)2;gSHA9^=ai4qFu7sdX3uz@!={5Yd!16Ar+hm0!4UMdBHru!y~68n z`C4WqHDBEFtCA5g{+&E9+c%oeg+LlaM~c;uTM_-gH`J~gWc!4`IFT*s*gl2!6i2@q z_l6I2T!7L4zy-AapW_0|oJLFW(<7&Co2P)JZE(dX$fB~@8M=qAI0-fB+`a^r9Bc8b zG@A==Row0l*4YG=Z>klw`YCv1{DM51EdTNU#(or!CWr%Z0fMV^H_=>NF1 zDFp|Pj*2azq;I{WkLyvBEjol8sx^nt!Spk}J;%#V5?Sa9-s~*r{^d_0?>`DCxj)5w+Em;644?3FsNZDAP)$1hP8C??#_vQ`00;wqQ~^nYP;dJoEXk zhT(*0@8DHF0CFjknG#OY9woTNJKsYikf{u=ztH@J`=dz}NF5n^`Bo{@0AITUl1Xdn zedXatb@=@H?&)t~sVYh?RR?Mr-I-y5sDtnNgk`pVR}RhjlF7Q*@aRk9&SB-?r5AHv zYP;KU;7j0EaiUk8DgtOhHi(s))Rm9!u~Uyy5M%BKxf(a6tI{EUb??f@A2ys7>r#bC zdJQ!t9H}FAFjys3q%1;_ifRSz;;1O8(H}H!`l6DiGAHN_KaB+wV4MJ5%OA7r-%h~c z@jOGGz>>n?hWDj!JAySIXtUVR34){Tu2)F+wH5a2o1RBUARNqehQNAQoD0)8XGk#| zbc-e0c(v&}4P*xC$#0rw zhy9GvbhLX(oey??ZN@bX`M(GbNJ#|P9VAA=RohOu&_gYb-g(0V(0%(8w`D9|_rDHb zeY)PJI1UxzPH9>y;7Wq$E@*px%`iYUy%$QWV6?-)Zu~8AlFCBUUZ5-@&xFskZc+gM zA=v!cdK=rgu4RdL$Z>8lX6uzjO~ex?duEm;>BL`uF4dYFQ= zjv1b>yyQeFa~pfn;`sCx8qW1@>1Q2UK{c&KGF{}>{TX{OLm5ln#oJ=|q)OU_a!6TM zo&Z_6>U%1uVTf?$cOM@9^seCDd%W^hor1crJt-Zoje_<`k&|@M=c$nEx*MrvAH_k> zh&rx}Q}~+iFonco!;NbvvTW)HeH`y+G|S09@fte%uGty83rei|tJIB((-t{st<{&E zhT43^d55WB7+WXj?i7des89IQc{tumNjZ<-v?g-(|Ft-d$sKfMJ<$^UDI zWjhzlfiuOsM{>R>MEUj)3$pYL6eczF{@msS1m^LMoB`F)*N{$bFlAN}p)T9Al4J{f|g zEz(zzrviAW$YgCi$O!#n7AH;Oz9vthdgDOAG-COv+R7D^WvYI%M56da)`&Q(Wi2~WexjVFaIr-F>r1P4yYLEY!!V*Kp@HZI4HXVtt^n{>jk8E!eg=(Wla zFL(Q)<&f1#KYcTuBfZSguRMhOw=}akzpKXYXV;r96!z2=yX#HYfJ4Oxu6=_@YFNuz z$go^X9%)ZHmq~22PF5oM(z%g(#oeP^tNeCfRuI5~wNqI?eSw8teK+ONQPVAZNIR!^ z64|5^0iv%&FP@%*D9S(qPs=GUTloEo8jD5rr+3zf5eZ^DA+M8FG9NHqy2;eJ)WD7j zp+gY5DtUZ`<;EhVj0pJnhmZ)-38*N^*(Eo`V|;)`jL}f#C|yPBiKpj?8L`53(+KCcp1+U!aQB}najiYJxJ1Cn5vfFEY4vk*TN~9a7^*5T~bUM3Y=S1%z#^Z3TEA0yR z%h}#)G1}iy55c;ijJ!%068?3h!g`ThS&k*hmb8~=IgE_?v35Y6ap?dHDzof$Buvt6CsJ|8*SSZzqa`mA!fqrcqCmB+pFSRgR>~dI?fnAyW>z!c&*hg z5cfPJ^G_Y4fXR!z>Lr|9bLf*iRXEdw?`i#CGbfX-+*!M{=@{I0bF30-{{!O7O!1nHq@=XN`MH__cX1u9LbY z&Xw#KI1z{(w?3v0m-NBEC2wHeidWrpDUt-(J105cU8ovr0&Jf;G_mbjJ&k`1v`q}j z_T^iA-@)%6@v~ye<+BWe)r8>onx;l-b7PxjQWatR(~`Zdyl$`V=i6)dX_4ymtGZXC zMjhJDpg9-dDnwNT%&gkbO`k!d6QS}b-}PrT>9KYh zQ2kW0o?JNf6HY5L_7{0%Q5gCi+Zdu-4(apPyKE`a9<0wbq01D3P?$=o5@2TbbL+l3 z>xyT^JzuGi`c(YBo40m#`46ItBS+n8g;5ohY(o@jt1RGUlXaYwL-_{M zd3yu$G%Nh}FPp=9*_(i*nI{y5$$kA>QA0g$Of7O`xlaD2+!CLr=NV6w6gQO$8fqJW zj_Ib4ZvVjA@eMWi*GRP|w@RM#R8VWf%}1su(rnXL3u_Hw&#`U5up*RB{aRjg%?w!S zJVdT5yNzM2IFBCu>gN>RDm7x}@kt5vOQnBUx0=yM;!M#8o|ldtppOZArAN)(xEuL> z_)BekvwgbR@I`#G>dK~n%gPeZMch$x_wS)=0TS<#wHhoHRYfcP5-jjzZuXAWj{CnVtC$(>lr?KR7e~pG+_=`uuw! zWx3G5k64;KlmDNwBL|>2DQH4vLf!?^VZsZ~Y?FGlRQmXIm(vU=7JeWm3&4hTW(ur<`GOL>XsdR0`5isn#_I`1)u0m+xZPj(3ZF~1o@1H|KD*uhU_Y7+CZQFfO zlp;;KfPgd+X@V5#q5>j91f`dVbV7(o3lNBa(mM)Dl`b{XJ4grVH4w6HjHNpulm=MhT*0Jv~^WT@BfOVsb z{`|Y%8{sCuUnwpKu@7K??x4|52B8jsX|EGrI~rkvZVFtR{*=YjRN1h=P?K2RJ)y5Q zba&T2!(PBq!ZJ9)M7%bRf`yzB?L8AJQlU?NYLPaOGz^bu<<+QVVP-A0s(cfhHvEkW|;!9Bcd zod-j2Qx>~I`>AtK*=?c-)@iav03aZAv|Wns=Ho?^nVCL#{myBs3zKAD3CMC}?Ju*n z_25Z~czTwzrrS|sangb0>GH!N8MLtLXwGi`k&h)>D=&TBD}Re*__*Bq%%`dGrEo?k zeo)8AX)SrZ)Z|mM_(eIQthRX^zy1Wp4m~_){kw&!LhBt>?%HsY!b`UPp6(P2_)xOuilmcST~kXK`VoX}Kd0W=MBAsiG1z}zo5DzUBTjq_ z(xc%F95yWABa6QlSC$X6S=~)$*2TsDRFu6|wn9~Bp0H{YS4;S1ukR?ly&n~+0I0|0 z`{#O5`db0=WuqQY`YKJ4>Lw~@hr2ei+Ry4HOoehfgvQeL_gI3){W-%B4#=G0-0f5a z3kQ5kdI{ustkX^IEWKTNhp5|RD9&+^GpNH;@;AC6&Z8L1MobN95h2VyY@ZyBmcImH zh>!5@KqN++#7*rs-u;I{Zf@=27jXJXdOQUhD03t!tps}FE*eWM-~onHFn|^&c=7}- zFpUk@uBxfue;yh5W>T!-v$;Fh&5rvF#2t7Pv0B%}@J#XC6gaRhZ#qT6W*7D#KyJeq zl~S*U8O1FWjP?v%@DG36Hc=cb8Z7Q8vFf=tYu&@soka8HnZ2;wA9d#`?~Vth7XOD# ztBq3v(e7d@jSofLwcxYe#o^Dp7%kPxyO<_P9*o30utZ>X4D2-N!p>)QR$PCqx1h03 zKbdZjz*YpUg^h5#nv0Q(&NKUHPxohmSzRx~R(2ajA@iLrPAdDdl*}2{o4ugG?2D@+ z$fn`bR7nxJCUM{zrV@CfGs$-o0Q^bc*lLRV`VF3^g_f6;uLGNJHhoZJwW#xmG`#Ui z=?dcKJMrBAU<=Y6`xC-rRJm}vhjrset(4fcl(g5ClL&o}!ue6C$A-N~ex2LM3|7$z zF9BAOS!`WAo`#iGNPhWjp65Kk`%s9@VC&}i^c)er%U!P$jR6S5#cTeGy@8ZZN&3{W zlU@7fE|{78!Gg`E=gud}WdOxWa@APf;&1AeNOi}b0LF@z7uki3nBiwRv}~T!6UX1( z>fN_i2DWum<1FjH)>Yl39>)DK;4+g{5i>EOrD}P6!5$uQls%h{-3@8FdBS}Xytk9r zFu@0^QqSW}mal1hQt^UGL2^R-?6a=efwbWSS%{Dt33v-05ErW*mN(aDj`X$lL3#wl zX&KnrEUzwPW^vIncVAoBzRtvPahdaMIA*%k#&;IFo+{aPHn?l)ZE%6F5BRI$zrE&A zx`!^xM?50pmx3j4HPjJ;T5;WP_wJ2d^fOf5y3&&E-0;DU`hCHdWug@VT=OxkzQma$ zh?RhhM7X8(MniElyN986^e@aub}DPKAx**6pWhf8NAzJ|l6`)t-yzhtMHZ6tSZW8bto!=3UiGp)o9H-FK zIJxaXalJrBye5tDP~}K8o$bLD{pLa{4?|v_b+qxp1h~Yy)!W2hSKP(>6g2^$7V^}C z$AhF@xHsZas$vWUWK)H8EX(Pu7)vW%>dzawwu2#A2y>~ZA`LM&?muq`afzJK?y3fL zkn?7>PyU{ZoNtz#bo@B((PHvED<4o~-OntIrzOo2?whqV)DfA&%k0rDI`2xUa$kpo z9xL?Ie0Y}pAr7v(7Mtsrc@gGM|)%@%5b^17lM0YUy=_{eLtqQKrCB~9`@d-O+x znf%zL0OR66@|Ca1MgAiCyX%W5DVX6JgJR*C9%098biE#TdwZFs*SuAvfVP?3Xx#m_~{E2Q{ zq4_!imuef|sJ46Lo5ZE?CC&J*rjEuGUI8goUY5#X$g8A9M|Q|UAZ$e@$&;P)e{zjeZ6;M{7duIcjcCR z1Ww+u8MLR)1I?Bxo73!*>p;)XX6;g*nvNCe1cp_k>H`ozp=6yLt9g;(p)~7($`stp z%)D6I`eTN{Uc<-5n7E0Xgb(ZW>XPtM5CImtLgXmO*DTvi^TwB*P(d53V}6ymPswJd z-CA_A`}p7%Ivy^4jA)y>;3P%sPs_0p)=DA1+*x>_a&p10CVGCJxeMudwS~Fs zBX8g>!{4xR12-9duJ~v(GPt3+R#5z?_tL6dg*t^+t|vAE)3|Y59Q8F@@LgT|%<5@o zK}U^`K3+{KP~><}%0JO$M<`mY=plz-GR05i0pS0ro zm0<*zmr>^oFpgNj#T1NNdY1@U1L@*t8) zkW9mrQ~^BwQLqH}?!2^V;stv3s!*Yb@CJpGV2eKxfT60#?algPPd@IpXvkj_FPCd~ zstjLqm&#%k0D3pDdSzO(w}2L>264S0MN~0oJwf@>wO3DPkjK zPJ2SyYqS;)e&%dPFQDsg^!wSqa9Q!qn8dI3h&i8>3yihGfepUxtoc~aVQQP>bZ_;1 z`+UZx{75~Tgj^{X$2o<1wX-^x-*;(>(SF*~?Wpu3?Czhfplxw+F}@Oiv^;Jp<{jJ@ zN3YKGFFq=WV+(4_SD@OY@>_?YhDp4;V0d~;IJ1BLmOC|87mJ=kFcYe~II=iMeu0ox zUFaKx67oBy$HUIefmUMTS<>Lmn8+q04sEB+%x-kn1z?h3*i{#hU#LhV6oRh9tDyIH zFdA`2rz5|et5e>^*}J5ysn!XC=6lbo<1OA*Wu;rAx-NJu=ez@=lzT^Y9QM%{{G(0~ zUQ|@Ob1YM*jpUS^-yzg}x9?d)P0jStnVGEuw-s6i@##xVsaSuK>~oK%Z{8BGxaSBu zJ?BDwSprjs(c*e%E)w1G>;Sr0uq94uaZy(gv-4A;vgwVM_JjT_H__qbw%NVUPS7Te0a(j3qqp~~4hho_Oa6d?=X*28kgdFnvwlcmTsi$1j&MJCiu;#)v^5(#vhl_M_Awf5%u72RI zt*vXkV%WMC)APZX`R}`@3P}xj&+Lt~SFNJ{0HFUfeUb~-ce}JMQUe?c@JzB2bBjs zRkcKt?YW}vMLpyH+V1mZ0|>of|DoVSn2X)d!~;Mm!OC*kivp zSel0uogY6XU9SHA=H63Gu3lI&#l(U7S6TH9v-c=wklQB|7~3u+`~N9 zuOxZkq&fI$;aM&%N}iI1)mYNEI^~a{eb>JZea(1e`s)rh$;62Ft?q;JSkOvJc)?Z^ z!>1IBPkF{6BWo2xaT-zsQft@e{ALqmq{U{>Y6J6G=1kLwbNpd8VuU_GY>rNJX}dff zSFj>^Y+?E_Z)#_~xw6h$LHq{yx?jbI=Sz)?YmSLOx~}%iPr%c$VWmW#Wgtv~+8~VN z<-pOJlQWB>dd*2)c#o$F!tuOApj#$)$_$@u^Kcm6O-x{^oePlCIVl=S&8rW9dbB24 zw!u0|!D~V2#RdLC@6@spg~M|0l~rrH-Emih!(M*n z{66e6eK5Tjs;c{)e4pfj}3uj$Qhbx-kWe;{ThLBMtDFM2mi*diHnWB#V~BUnXLU91Az2 z1rhxn;X_d#8O$C1!X_Ve?l;|$>07V!ckiLl)iOTQ8YQ11S2cAoQpgS9K_YmKqW1qM zzty2@E|%3RLNF$BRyh!VU*DaqP_irifT}3!Vko2^sj=*g2 zfIh=*i+KzAxbo#Ms?GZQ(wCnEe63@5c>6ms;%|`Es%aj)s>^(;h9?@yv^D1J-^_l` zW`6d(LRcVJ0Jr!tm?AX%<%lL#ONhp7yGhEV+0-uC+Wtae3h~$hr-2u@zTQ>TCto7r zQS|#_Y~+t~?2G&Sfaj&H=I;mB1Qz&wzS_UA9|z!CI7xj5#P$!jS|<)m zGpQD5{0`9*D4{4Nx7GU3Iq<0Ny&KoY6BJ~u?5JA84vwF+X*!6%+$gPEWdAOmS6(w@k{R0d$raVrm59h^!>ll2J@eN-6;C=Ty-f@ zI<_c+EbUU@%IzC|C_blhR<36~MV9;yskD@nbQjXL)Wz)^f2gWVdR#+uX4;5mAWC;G zDiBq5p;!>-X4Z|GG+N0$V0r&YIVO(hJ@H;cg!WITJ&CAyvMru0>*IZ=;}-?J=##i= zt@ij=<7Lu{5_^6%JHgwp5SHx$Nen$%7(*EAB0gzT6bG=;JgW6#q;0YMrB#RJUQ z9A&2l4??FH;WkkQH?4wn{i`QKW^`nQ?{4a>2F=f)KlDV#Z=94u6y!3Nce}7f{C_B-{c1MIfdYhze6 ze#J1#+O1Uqt@Ae%-I*PI3=BFqgQ7yXr^VzUSKfh}ckJVxHmy^4@h15w-$Z36w_+Z< zdPD2!;Z6%nJ=G%S?vKAuDP$r27aU}&CZNX#YS*iHniVd2oa1z?^qR03e4fZYAREch zgZaDv_kdWVyZ`z=y=Navj|IfqCm60{R92Kj{XN%r^+a7fNO|rM5SiN0WTtHJN_K=7 zY<8p5=6;r?nD^d|#kq8jDWLE4$z)EmPs;tfeYKgl>Q-YrGKv;?s#HL;XSmv>9uUJe zi>vG=+CMD5SRq^FB9W)ZC6MPadp9Su_RdZ)wjtt^M2~^?%1o7{`sFPqf?e0*wN&%4 zE+axYSy7n^MX!y0V`&~)egBCM?7ZJ>^3U|VE?*;7P92-6ph!Wj6$|t|LIxvo4PU*sePhR-Tv%rg`c>Sdzcq{I+gm{~ z!M9E8D?zlRzUPFUNY);o)07bJOES0KJfZVoO6j84)r0; z@+f`<&|1^R+P?*joqkwiZm&Md)Y!4OdX%@YS*cZM)jFSu56(^cH}HtoVd!n)9|S7n zz-E{6p?X{GDwwU^$y&V6pCK+gUm^Y5fcsBf;Zw6F^C`<3=H@#|vW}5Yo%1CSbuWC< zwj{m*;6gTm`UBRB>ni$JUks}EH8f9IxH;Y0y(IX&?jgP7jgiS%usEXf;&Xa4xivsY zWGK%Maqj*(bqT3eKKpxqUZkLEJA{0hht*XbZm;9{y;Hjp|WFM{bq zL2~g^!7<`oWHp=y^B-Ssz}?Nw7a8E`AX%`0t@}D0WiZ+Nb8NxE2%J7>B$FimyG4gk z@ef5Gk_lml1+l}^^o@Ak*Reahc(=T|Wc}bT@-5FU32J}*bR9ovmuQCJ@6_P>6N?M< zLwFR-!vVIof`>XvG(3HW+0v4Vxl{7$%t5xSiQ_?sq<|2`)nYU-kHUbSUdeQEJ;c({ z!RAKOgYuWhABa+!gh3FIFSYnjkTmnWbx*OYPY`cgRS0eF+}f-@M5Jrban~#?%V(Bv z%=}(w^IjC75pGPhF+%tr5T?5aI6e$~9}%s#w;VLHw!z6Ayl^*-hMT_(dHa|-4Ma<( z4%a7~L<$iZi#b|xP~?kj{z)1gk=EvF_zUD3>0f1TH~v?==~5415+VY@-A#EABPqG- zf;$uxWm*1cqnld-^C{@rZQLs@UsyE-i>_qiZ`1dr)Z;J2y$`Bamn9!}r^SaLc#9!H ze2p_3ZD@%NJ?s_6x5JPae8|~@3Lhqdex3wzLZ5D?D1Z1N zc~+)w2Xd||1-Yv`%&q6vI5htB?5KE&3-90s_3}x^cL(tj8MAX-nQ}qc?e9lW&NBn* zn$KOyEVMZ?{WM#8;`CIHgU#EZ(DMKk0ope)Tr(4|HRz4pHa-RKnYe7V@^o6XF1BN> zd0w?ou1>&EZaIzFpigV~d@uTO*7;%)4on8ny3vaBNL?!La60BIao=iAC+9V1`LaaO;bGOA6OGQ`+)c{``F zY}FCxZX|L)(u@q<7Wo=n9Vr#iaOJ*2Q@VWr74Lc>IvxKBk6(q6r~ga zzvI{(?x&xUSKWIb+RikI4C0gHyM{?%avM_RewOO5&jmzF;zgUqUSp>w(DN93p~6pH zv&ap!mD1Av!DAmJJ+9DsBRQ03?4pX#(!GUfdwS!wDiAV}^y0g8UQSk90Ss}@r_a?n zh}uc=FKO!AroPo$B9}Df2rPCDKIb^g(HOCgc!v6SyKx$`S*IGD!Ye&KU7f|adScY( zcHWBN8HB@*tM%ZA?R?+bzVFUoMSS6ZXWBp7%7C?{o~7RZs{$Mj%q2yMqp1)T|3wq9 zJmBDF@;O6&)ZoNlNd3+PJtwT!7X7)ev;KRVJA4)qeK>FG1!3D0^ctBpQs{HfZ7v0% zxOFLh211el6N;NimLxHFFM}d5eKD{jHAYMPqtTsr9oSaNGePkr2Ns^_+Jm)&jTPtQ z)t`4+G%t4Qs44>=`>>^Ca-t~Xv-F*X5Bz{$!i!tTC4?yyCj~;&T zA_=o%Mf6^1KYnN3*}FInvzNXs`>@SE_YMt@sqE8%nwt$mxrDtqRhN~~@atPPJOI8s z`(;g2L%7LQtp?)~<}q|F=C8i3_*J zlJsm#g4Q48)pBeG@bq0SlW}-H0(N11RxRefc;39Zxr@2TItcWK1Tw9)kD?b_+xK5J z`SaxvO#4iMhnzY)9MZ#`m6c|FT;wd8w0PSwX8&Q5>`&e}3hz-BE60|(71LfIHcyGm zOo>ZdjK%WPA{`(5d6aLfvroyJJs*8btbqbZY8i{~#xI-PWv4gm`FM$+(V>Uw>_kT( z=p3@RwP-o4r$4S>Y4tEEEa?jHuF>1r+YK(P%NY4GcK!&GAWLG%}lysmz?Yh_=%-OZR8C!jRg zp*fz}^+G6KpQzvV8H-}&eXuq2(gmnA1Xq^1@XxxF*RHR<0kyIfSb9oLKypc6&(zJ~ z+g20ays}$;U3jQm^8C)^6_hYa*udS^+iSlfJh{f;aYEGtjOP=#3|Q^LIdA!iKRKyM zWTEHK2kgJJHwVicwfYXz>Zs1Jkkj?DgNYOx_4D(Ky`~X%;-ZLJlM`B@ZMzLGI1;$R zM;C<|;pMw^eNc|%)A%Xj9p`%Fgh~*G}sKw&$-m{jF&K)zbE?qt+^!lgA0!Hk= zSSVjFOzmd60k;Il%^5~N4qG%Vno2W@H#Av!#$>o2+_|2|5x|=mn&|l?}FXj(Z z^@c9rIQ&iabn?J;XRcl>wv2uLmULfSws)p-D1+u#p^28L`)+=eNEv5PJ94m~UA(^3 za6-zx^U4>}U59LL8$ z9J)P(*RNl1oV%&1=|guWO5Pls3Tm&xwLe!Spaoy;9GTaSSOys@PG zg2Ki8DCE0^)lXhK*)HX#Fky;DYA)gT)U6aYxlwi??xBRQY|pmL?IaYh%Op{M>nHb6 zsQ=s6W$)Q1-XAHb@?_51IQY(6My8w&Y9jm9xKf7_T87NE*59#pjUmQ?MPthZ=gZUo zGP1F=pFVReHRzY+W!?VmzT zRo{F0Kl-b>mt~vo-TirZE3QUM)f2YAYOWxojroapUq0og2>f3jx&N(j)zc0c2ECYQ zW7wOtpW(m-`fArY*Dz?7!Q2H?4eR5czR7fm=$#OfbsEwQwjTLFlJTr@W#FO@6 z4Hfe&H+5Bb1^5zXFhCv^hIo29bn3>kL=rb^fLhS3u+ZYY3cQ*r!YhU;+$M`?{QnMNpWPlhNdWWJ^$%vwV`BfxArNa=RQb&+eev-1t zU7LwnYJOm`Tte7({Q?t8Z3{#Eov^Sv-+nQY}`cqI`I!#Gu zd$4BxqXSiy73XPD;QJM7Qb&YEc!|eTleoJqCOG=c8p3|@WMZP52VMF5zi7}) z<6iiM7~mp03X}rcIh*xVpz6S#VQ-PHUvf5$z3Ic~+5*(&=I&Rkr0yh7Q?eF8u`^4I zSe_j%{SQTbd%Z-5r3+M5uh^pNT0-#WTT$3I#xP06JI~eCU%NbB7{?=U}7?Ny%!7gR02ICOz>>$|i>ux-ZFgJo)e%KRP)*O4f|-O69wX#*@LGE-qL|G}pK3+igu}5_E2VjP&<+ ztwSNq&2M}CqegPwrIyf#=U!to$%aW#B$%+676UZ;(rK{6sk<<(VFgr`E`GN3#evWV znWDewefr%*|dFwhP`Dl zg*i|E`*`UqBOn_H+MRqA+kRaitN+MS(`%788E>IFKHty5;mpmuB>v{1oe#e{cEo>q z%D*-HgBSlzmqnpzDK|h>WpTY1OVt@LwijqZlv!BFjvd zJ@9A~SkWF0YgjAHaNbu$`Yh!kMM?rM4h9g0%h7g#)AZ5kBebeH%qIlsEZouQNz16y zUhrg70GP`Gp_lWM2O92vAF2-0YW&6<~CP_MuBlZV$tV>ZfJL?lm4j6JBSX>3xTwB|e z)Gi2fLS;5qoLf)Nq(dgAG@R2JcrKV99I4q^5w$Z#+Rg%1K!mzV4_nXgHiNSH@22^vp1mNYqu_KntHs+{8!x8FGNqmS7U{)% zCTRPFE;-F_`_^yFgyMW$@JUvG!$)NB!+}~?20ZBB^=|#y;xcD^6(UQc*7Np4jBxV4 zP1FXspP8DZHpM06I!(%*8{#YQc3xuPX2RwiqS_eaHf8YOhnFS;dVDBP&~{fQyZS$lOCcQ_&Lo|O`C;yX#SH!tvQ2V0gmi(6mN<;>WO z%*=QWMcBhvoL=#o1!tJ{Bdfz>$d($h>Re#gRLt6xjnbIZk>tazwDvhk_k!yfD_Z>P z#m-?tzqR`P=@<8|Jr!jD+oC@qqzK`i);ztK!z}9NiCwaeUE9p*$fN~iW@1KHweO=O z#?(^Yg=*y{uyJJiPzIKn| za*}C#Zu?eCOZM_hAHC_h!=nSRr@AEp6eABT5y%`Z4TRC|!FHbcKS3gB$)YK!e6fvw zTR?63eDdQa88(k%uMnTOMJ?~E@9DC^q#UB42LHuS);%J$3}RaX_vJ45q%&+89gq%_ zT4$0MiBiz}3FC;j5rs&(k$ zjudxZ{ffJr+?q;)UPya`kF0I7p5Vp*4z%ml!^RIS{$>@?eYK69^+|d0?9>BPp#ki! zm_YH~{4~!`%T5CeL(BTg^kD8Zim^A1Bs}g@@w`weO$=>*ePcuZx6$I>C}Hw~X<=Qs z+@UQyt{tE(-21OW;5xE~H$EIo@+=`mcbod3*A@mtg?Idv#(O94^eqVolBkyO;Gmfc z|MK(B=wv&G13^xmBMKDZpOsnw_ZQ!WBq~&Xj~2J{U@Mgn$jp3fJ8o&m!UMY%wbmv* zN9HBoBb@iek#+K2P=0hh1=jp53ZF%GSg$Uzz0*>Ksv6ne=QHP_OMri`K2!ULg3ly% zWcT=DtJ+#>+rWLP1V+$D&W&vN5l(s(Y#tCef!3qK$lmNymzk3p+@E#Zn~9|r7og#g z`vV*D0REunb|MZ$2OsUSks0?EAC`Wm+?&o=UNR3aBw1Q?p2&2LRU|Q&ud{^F*o&B3 z=r47p9jlCnEpoSKcP6W+L?9$ZGxJ1R;^}>2S8Hwx-ZL@esrvZ(;lCIY4IoyM5SFTd z=-wlbH<VK(X6PtOgu@Zzu;dy}C3$?KEFvT4TAkp~O|5KUtPvx8DBiq<#TN z_a{4w$?NtUlaZz77u-Uzo!U>`P@zz)f~?$&i$65)eZTjKkb*v zOxVrCftqvHoqd;fhcofO3ZMOpVYYG2lZ*{lt0AQk-__Ok+HWU5nHDYB7o6-JwBhSL zXY!Z4@T+^`foQOIovzgXSH}KE47Bs9LAB zO-^$?m05jwqU^T>nj5$u_UJwNYzDZ~^FWQhQ&5-}Z-C(h!rCbZl2%wJlt2g_)9LQwf*`=E0*b6dSV}^qHLpf6@ydc zK4-~wB|N-FM{nYdAzzwQzwYv$G^aI=p!7rv13u&ISo^BD)1AEoUvS?$l!s8z0o>(A z9plbQaLr%LY|Xhk4+Jx)RZ%X-5wsOhiUI?R4VC8l!qn$~kPb+WtqTPb&o@{9m4(6Y zW#~Fb>&H~}&C&G5B*=ID$?hMM_1DG+JeqIp?@f}r6%OIwE9Lg246iV;uAiB`)I; z#vr-`p7K^IjrGf#C1>_v))U)AaqSTrY9B2(NEQ!)si)Pe$J-O=U25Y?9%!ulG0khU z6d?XZDlbg4@kV#dn=kAlmoOgBIILQekv`FTd+U3xKuuqB0Zs3+P4&0))FkmVR5`CK~lX z^2n(bEhTULXi<=BIZCG?&HT+HI@v;F1KrC7Qvh9wsS8ZEM!=6HYZ3yZ2Z9cyYCRoi z${bv>z_n$A5_SRiorWkVj6k+_%1KM z0fF~?x7_c$V$2bbcqWq$QL78w2V+b@r(oXB+}77u|-WKXR7O8&LsWIeXu$w#v7cG_CA8Gfn)&!KZvBs zJ_)gqM+Fy7A!~(*GQ=klqlkBftFGD32hV@gw>f1tQ&f2L_<(X4;D>)kgOzW&t=4JaJxz#V<;4SLMVht^8%XF|1WfyuI2+qP@2+hdpp~ zq`+VD98nX06GdJ5^D!*2=rLb*fQzq7;oPXeT)y1s{*Q(0CR!$U^BZo_(@Viib_)nxPD2Ng3R9y76Bb|G~sr@#ADDNP|a(q=2J{uwjT-*1C0xu{_`^ei3h>Bp6v(8}C)Qb}=dQ;$`g8?T(tz{lva!TPRfs z_FVnYd;yZ?T|4{Wx}I0m>M2xIzLKq2&Xw5UWt;Hp9nDd>2yKt32xO}r)!!>GHsyXV z3F>?V+zDTPuANMq(7Wgky5zwN;C_2vWT-i41r6urSJL(h zLsUM{O1Rt&qi3IDPSB*zT7IuHk*zC9J+qNMmuauXrsv_$vbC>AgRzw`5OmNvc=fbr zKgWS038YpTEEaAJ$uzt`BdUwP$T80ivGuGo6e`huIiWdqdojB`>^tg0Ip6notQXB( z{OS`B?meP7qF4?xmekljy=V&V**vwc=jM-EuF~s1d)ZJ2S(3L$%=I=7Wh*FNmJL)Z zNojpISv2Q{N7ZZx1UQ!$*wUx$v!I#Z$lbVtf3YH-c(BZxcF1i%rt>ykB-Y`17-oy6 zUx}Jz}ueg8Trc7V<@2bR`t12mXzTD*O|4Y-MnskYM$?zc`3<@ccaNK=*Bv1@0 zG8gk$kEt4Z@;)Y^yr#I%ibL)9RDG8j1cG(KvHH8XH3b$LeA-@#XlRXYD+Iy_Xub{xr{-GV8(WdP>y4mG?_&|J6lVN^S5DMFMcZMYm+1vX~rE zzTnrtRb_1k8H%ZTV-K*a2u9UIU*}$Om96!6dFa~jl5Dh{5+vJr~`ax_N z!afNovJQJFYqFp+g*{?gqLW6x(PvCET_!t^kzM%m*rzycSf>y1`f>ro{M`&!Ol7QF z^dvkeZ&v9Dz!%IB^=_eCG^|&NA^;;S8KgeiF58}-Jli%LNJa-)b0Zhd+TKvJ{by8!JAj=h>F-NVPzIF>L>G?1n!;~b@Q z^-=!#!Obz2XSg0#t0Un@ku+HU4o@w_-tU^tCI_Bf`I*I5N!SY^>#b^{rL3`(oEBcR!g@ ziuM3#+iaT6lj9ixe8bqTq=_B9zRi)~D#j8nLF{8ome{$58sN#|5eYWBi~onB7aV?S z(kc}mu#(MSzHRtCWglQHv^H7gS3GBhDeD);cL9orKC+ddEMv(tLb zAEC~{C&kC6E;2zKMIZEbTz!di?m2t}@Ob*1H*7rU6%Qw%@ch_44tT}r({Q~RH#d1P zl;mM%(D99A(6mUc zSG?o);z#Y|3X9de0_`jd$BW_j!vhpo0oq#HuUrG=W~4=sa;bM^`R%Fc&p-@d)1OUM zgvj%3qD73bHEN|7&$P8D6r!T9@x0YI$?2=)P&t>g#nTSvUVJ8Lg=>`kUaxp(*dtd_?W8nie4<$ir$Qrq=fs<;W-FgA0||^S#N3?G)g~Y z&*Bb~l`>K^K6n`*1u22HJWIDIwpo%dLh0cvhSmy`y7sj;2O;FN_QH|5%|@)cU@P&> z#SEC~2*r$0MPpmCySd}Ci@=hLUydUS3t~I9-}Ee{-EZ8rU*fJ~{CP81Y(t$2qAn2H ztA439yO;0Il>X+p1#Za*EmcvzdS;VZtk(M`=~0n1l#)WH|K-|TV|}2l6VZKQ0|!{R znbya=OL^8N*QS=IrQUpWtH8zLbgw)Yk!9ib;+Jz(Ud}Ksr(BVKuWDYZS@0)IiLh8* zJYxN(YVs(+Z#^M5?CtkBT}(b}@XuRmz2Xb8Ep4k9WI!u<7b0o3x8BNQ5jarl$0u+m z4rtJ1&oX!rhy@jongl5VT=ogFxp&Fz)5V~%Wb?tZR^{al!5dZ0#!b-6ZC~4U^D7%H zeQ4i~|A>uu?4v#?1M2$zTZyb^E(U4Q`1Yle9XY(B$P%pCbLCu%9$7isF}u+7IV|?O z^C*tCLg2uz+F5wkL$cGO@lDf@1UT$!38xI*oY^$%Z9A54rUL7go zXot8O&1|vP*6F1;*u^7vR~z-hdFUt`5)9r2So5r?2j?ShhxTq_|4P*0Aw~`phYPpm z0y0hXqLWhg?`BIP21A}?-+xU#sBtB(9oyawVT$yZT)-E6KK|@&=O>kkEZB06^mOp# zQ`EYp<0VR8#>M-oRP0Kq#qB*KJwG5fp?5wL51l@}rfWgF)6U~*3E)VrGn=wJ&h!w@ z1mCVqR-KxuJ%l0zC(Wv*78mP@^UJBP4!Lc7e^f3h;mWfD+8Evm-Y6;q)8RI8cqlf! z6BedH;Z?a3&p6;zw5@y#cv}|?W-azW@3FhaC$GEJjm!-?5=Wn%Qm-)eG=oCbxvT?T zjCq&CK<4aHJlav53MFRJUq#a;UQ3yI37=A$Yp^1nCHb*&^}dCpIgqB6O`xuOy<9z! zUYhT#Z6kU#eJDl2_Dg|^Ad9+?U45aWbc!x&9rO>yrIYf%b>szdW=5V9xg5D{0p8d3 zt*uA*2g#Igbb>gwR))n*>+8mA&E0Kg<8o-84`1;#An1!U-__TcwR?*5Q118m(Uk<* zq=Vgbo`gsBb)nz_=-tDW5}m@K0>?Ausyw9|Kf9ERxwasbT}W;@V9PXvWHh&2ayS0c z@x4DW+OqJ&BS$mB`oouaX8Wc)Ds4u;%BvZ@oeFB{ zH~B}}?wqHX=%M%!cPgu=CHIb`8A*;8lie19wO_&lIUc1%iqvk0iL9mUE3I2&b%7Ho zq7OguC`9+?%_PX+1@FaJ6H_yB(&8E6grx>G;%-!Pt#4J)*4P`CSm9Nz03JD=* zJgclUrDsFQVrkj#&`b(vHwZBrk&V9iXJD$iI>d3Mn(V&qf*4KpmiEU_M3&y4Z z=0cz4_($ZipF7)kDMah`ehAM4u303WIKXN&pnqm&S_O%#!#Fzi$!_O zKBgaUE4=@-XRtF!$a;O@?c`FLqH7>AfgTrAe2rB3(p!2_Z@kAw;R6 zt4Qx4pdcX9qy(gf9_byVOF|931QKef?su*4`{wMKea<@Hntk@n`Gd(H%sdmA_j%s? zzMt#*T})H8G0v{jm@wr#U_YI+_@+BLZXR5@MK$LG<+WvX5zFJCNs~)X$>p64-jq^9 zdDn##cndsnHHTOS0(>5bo=Tn(s2)~)>};2*uMq+=%g(qi46*C)&DIuYThe^_Jm+DX zcNb@8dnqIIoSNttaCfvaOgdfiMPpUQTt8ao?H6NRY>46RGxomCH~E*@M!*bpKT5kH z%k``2TZebd6-r|CNG`L2+1q((>=xURm6_MtvrB4!wA%>ed*AxvXgj#Qm2=G&_I|87 zvj;82Zb^)Z#luyfHH{W0@2sfR%VK@&tM7bmBYCGiY2{3K#QwfxGm3+4vS2#^klm2>3v#PIT)Ag;3RLOfjF-F7 zm!+j&hQ7Gbu{jZBMh3))O&89-0H!{hP0vf=IFi)s5tS!Hi*^Xbb(SaMoiQ{9&h1Y!Gp3 zX8KftWQF+-(qQi`S{;Ul+#Z-`66ZK9dp=nE>9viVFh#!(9uwm^LrGGqPA^94w2jv( zP9)cPlf{W``xcpwUo6GS!Zb7W|q(AWF z2GFUnWr9w$%Fp{_P0_H-%8S3+mqMp8fb>LEatF=9)|hvyNldQ5r_ECR0e~esft zx*-49$w+M2h99&(D!_PI70*?izCn+pYIMEX{riTgiz^U%338l9RFL?)jb!-XYwYXqkYNp<6#I8ZI!Bec}0epGP*_v@Y9E z7*r;ik74I69?m)jw|YlBk?UIinbOIIh-$c*wJ{@0#?S-}1mWRN6vY?pKlF@AcQH+8 z(S6L2dB0Q9^+oN%X=B$_*sN46>tGI&-gqK#-V7P~I71;NsX;M3Uz&iczql(lxs}2F zB|Y3B3AAMG3+SKaN&T^HgBQqw?WwlRGT4AB8^*H+*@z+uz6e(1mWq3?XdYQB^@9$- z&Uyv%O;?>{BXo!4`5CbrV?t}>*$MkMnN;TzQjp7vym(BQ*Uen3^$OKjf-(jzqPeMa z%`l#i7l)4KI!^R%F1tQvT*$0u&}j83jpnZctYTh;$l2x%An%Iayf)rLiPYE}U3uaj z-QTNVNnz9HYfjrc_jVhQR|H5{_$iFL_p^!7z%m|;ml9mWM)ZDXzw72KzH)g)K~GAP z@p)NvR%Ds;gVbfeCnrW^kVaVrW@Ingx2?`OgFCX_)GJlD5+rpds*4k~r`KWU_!zet zZku?0R;*ZijPn?LBle?dw^BI07PS7$$DnC zBejPdc*6ZWh;%_d zQ#H(+_G=5m#I(8!DpDp~-t9i@54^~KoVYODvpCeF{zn&WykLGRXNT~wHLj&xp4_*>M<-W!G0A4wyca7{lt#I@7%(#H~xN_YI_CD|rA z*rc~3_Gze}1Tv%&f-7ZHOmxyoszUM|D^ahbT&}6KK>c3-aKf4fOGox!(>ceD6A1jt zow~aTVpT70e_ohbe6ZzAl|V6QAvJg%%BB+Lk?uL%Z9TRh%Lf5(*+V~SmGdvg6@L4A z(1U$TQ5bQ7W-;h^0-tiY)82VhI3zZAEb9=2Aozo%6ua(((I)ba?x;j6(1-C`RAfd zbhhvj3yz)Z>x4J4bs;8&q+po#BKFz%soam7*QH@Fxa;+7-(J?p>s!1#i*a!NH^v8D zIniCUjH}FtrRu(GsIP2_!2_CAd#yWK7qj7J z4x-z4`_eAT?0uq6T0dt#%Yys)689BB^N=g#qPB<1xo`8LWv;_{8mOsaVtGZ{+Z=yr zzpZWcYkhF*aiscLzTf@Q&Tl$4&>~$#p|_4y$MsR@AV1Fh`wOb~k?fz479&8vyFDtnRqwnwJ1}H&k`VY1HeDZ3aWg2YN)@kpCa?QwaNK(gq=COz0Te>* znCO{%)s1CdygkU%W_c;d^`G_Rk4g z-GYt2kj+VwHSn-l2~`mHm)a6nu|Y@ToxHZ7o`5yWy~{+6CSNza$J8kUacm-7na{Dv zU)-lr(KswrPTw_*Xv9B$#+DJna}#&H4i!?KGI-r z=Pi@VfsM(h1*b6W=(*kV76^4QUVg(S$+pd}>nqmD&E*tGt<1Wfac5YV?)+eKF3NjI zOW0(MubX{JJN}fwp-m9FvRKHap+ATeS`gM~u@KEKBxkqI$j4fYmu!wjh z9q&~C&my6awkVC>ot3*0PQ!chEjY5`EB(Aqo=juO=;nANLVMgplaB;M zG!j%bE6~-l%)_9`zJ*?>tUlFR~oT|3kCtdzJ~=6s_SM+OPKS7 z4_5o7!?U5Xe<}C|lcZGMXWSb(7qog()Qs;Xa6#C#@bz#F-P$4g=j^$=yt`5Hk;K)| zZri6lJvCGe1S+CK3xplo3tUFCm4<@!h61?2>nSU3>O0f(V|1FLS=DAX8WKf}b<={W zVGJElQ@W)0J*0M3I=H%D0-yG9s6FBBu+~if%5z&1o%+M8iNeLAsiLsXD#b2q zt-8)L&EaZhKK9v95Z0otOpVIv;g7IyS6)(3TzGfIq9h_=#mg!%60t84QGQ-;25hUo zfB5tMUuUAF7taa;sE`Zzkm!PGq#k(Kc%HbNpATA<&x<=$_HS&+L+R{|cj zu=V7N3AqBm?^)vhNk3H1O%%KbT#J>GttUd4Nuj}&G!$#h7Sc@^w+AA5=mK-?el1uhN_3B;vI;fD6~&+b?pCM7O>C_kK@2@Pn6E*`^QDM5Db6phPRZ{O7I*rF>5P^l#)0 zG*{(K(@u>^E3xOC$ZU1Y2R&*hE~e3{dQL8CDvPN=n5Fk-0bhYi+S0OO_x;#zo=$qY z4P2pLFRizqF0R+4a?fRU#U?p5w#>5r-nrJs1|lV?$KwAaR&iYQL>(D1+mY`N2Zs!-P8+fpRgd5=Q$oy8O0b@53_kj|p^62-3dG#{eYC?Qi~pHr+wg%X)LZ`vBm zxgle9D+7)&8nqqW!(1CIY(EH&P;eB16PD`4_^j1+?CjxN7Xlmp^uG*@3Ai(aamb`Z z4tP7~PBhr~x;TUMBhTnxgb=W1eMMa3?jLXovR7R*kuF^X6^HGJi9kk0*ZQvQ2RCR{8vn`l) z$;jAm0-ZBghd>Cr`Q*%Y?#orW*IKP;M*Vb)*Ebj-_CM)(Qt>tX6v$8XM?_)yB1EG)@ z`O+5M8`Fwnc6J3Do4=TvD(gM9A+wjvMAFbY);W$h7<$mD7NVk@l{>Fd7|vW*KuWD@?h(@e9@|*88Z7tntny)@b@pGm7ZpE zH4!swGwSB4I9%fo*J+6D&>=GIZ4D(hl4mPSy=6*qrnR?J3v(1dV37*<3c zD{U144rOH&YK_(cx&f!}<{NI1sQ6OC7Yh@yhQe9BsK< z$&-%B7k8UeZGG%INEr4!ZxBJAc%x=^sa&O37l8 zxhny&b~b+UA>v-nKQcux^!VvdDC*7+URXZ_#<=q&NO*SCP0o9kY7|M6AGXAVzQ2AV z`ovC{nc~BfQr6!AbjYIiOls67e7=b&7IrO+cS5>8mIHc_nt#eiUl=mWXTlakCw{lJ zN^uM;G9;6=6cwVCUGvC^^&O+N0UWwlN;{h&qoWd(9oU+sHyi{ka5&wzo^ zr$QAnOyS&V8`I6qQWyOs(A4D2o#3|O8yE_q_me$gA(u9@g@$70g-8MA(rH2<>ML;P z_Byc6o1tDIORG=!bhZ^BuEi1cEZO3RLNxT`@7!0A*2}nGUTZ(5z@W+5HTjS%2Zm$+ zT6n@#MZ0jBCapR#Z0YmuM{<`2uXPMEBZHbVuU<{SUX8`8aIJkm_SZ0~O!Z|q0rA%$ z7v4KWPk*jvU{z-{OTfMHo_0G(A{lr765+4x$)6IKyh zNJR_brZvLs!6eP5@Kly` z6}a7fR$E;tfd@!>gG*P zTeK^VznI5PPhOI;3%y*xXX|5{fh_G*79p;5A}%5QL28LDT)aA6lhlTo zp^&L)iQA&3VN^Qd*d9FfeVEAUjXMM=0!^$)jSenGa=qKFF93H^@Eu>KK|^EpM2R~t z&7?L1A#OJ^a}}1m%bDW^in>P~Oc7Cw7mKtK2Szrfx5(N6BB3(cb!~wuIfDc9L5KY< zRf6|+fnqf5{#EI#jK+aqX-7W&dqs=8q0>Vx=UU7T9M~<26DE~~sRJ&cdX-wGfQ{MS z?u{*vd9Itpq>f0UYP&M^xF7x7h07!R(NDQI%`fYhpkUGqmK~&q&G<()N>O=LwHfW%Wi_>twoU>X zQ!`Jg+ey8gjZ7%?HGa`<6(J}Y0{%;JN6pO@SLC@1ls$xnEWSr{YNl@$uSRa4Nh1B_Y(sQ%pF2stHwtpeQ>YCo?SU8`k}OM>~ZtxkgXo6U=LA ze-|dEJRNO{Er{?J0g)CfdO8EnD&1?LVEo(r^9xLH08lkqD9PjqI_U(d=&SXI1;K zfMX$Vt1}u{Y3j9)eyY|oP?EONa1eSL*DMEXb!$+~g&_ge1X z3$Q=K+GLD$D6^7cJ7PZ)*A8j1z?9vg$y`k8O#4o8vq+_3!kxG$itGt?&+KpVw3MlS zOVJxCoHc4Pln97 zDJH4<(D}8lP!+e>89zT6G;p%v_N#$*PLBhrFdrJ>F2+T^*o)82L*t`gZ0WLVR994) zpp%!DwNe#&$D@aEutL^L9XEm+Kc&JmHxE&RM0Qnl-4LpBmdWuhp%TunBlE-k4_~c) z#Pt%-`@7lIcNN2#?sEj}`Xs=<5d{f1OJISN`?^kmia27UP1Pdp@$Vt`lXOuf`?v2o zK8(**9;~wW&6JoGHuEk-QjXW{%&7n|~$6UlxloPg%? z?hT70{qsG_$S-b7`TYh-1je(mz!NXM^eV40NS0~U?AJ8N!2w2SEiVR3aI-pW&fpsj zU_E_AjGhVSPZ}brRV~gKS6&x(cHYA7$kd!X+@Ukoe63D<8lv@LMiTOL;~w!iaCBKbk%Ctn(W!S!#bKO(I0waBWvXrEo(l$o)c;Zh zV{Vu0`iRv5XrZ8;pFm*~@(=#>|NLXEuO_`{)&c$yYw-5*tzrZN*`%`_Yc$SGEdJf~ z(6~d*$-j{Mh`@ceckjeVWf!Y*vTCT?KJD`Sd(Kzek}tc_e4tk|sEo4lr^kI$4f)A7Td<1%T0sBcN8n0j;NkC54cad@Z{HYgjD7lE zCo?EWnP~UtMoBf($Jc^me3_>-@gj`O*sQ-48AE?5W}n7ZsRG9GY+#Zb?4AEpM@{X& zmdPf({7W$ibk*BgkLiZ=|CcZP@^YmS#3}ULR|817yrDcsb9xYae#~&;|7mrK|1Z{N z7>&1ORi{#V(l7td)bM|q60Q*`ES;|Nqy|kqM_mmqDl@!7=+DIBpR&+#iL9F4zW&HV zaq%{+i}x&T9}1U_ezDy?uD1Z!Xp^zxJs++bqLTM>v-sZ0<{wL-+C;Jaw04-%ad`8K zvEamV9ephJOYAlnn!=V|wStu*Cso2aDjV#4S80duC;iTrs1WrkomrQk$oR97dHbBo zPlB9@NA9Itjp#<;yBoy6k+&`6s}|Lw-oCJ_(Fy;fzBTv#%hwXhU)SLGwifF^#J7Rz z8R;PhI*k)Esi$cc=~9piukd1(vVC?MZVNTMu0k6&B|KnK`GalMm#z5(f%o5-AGAo* zanK4ZRJzhguXEox=gw~xyJE2lLHHXi>b>>$3Lu5`xA(VgD-K`2{PC)mWpMYeU)>#g zl=xT{U{JbTI?sJQ+OP%9B}h_Q8j3WQm+mn)w`g|Z^MD0brKB3P-MeenL8+iV#U(Mj z&>5r52eq(|%(5T{YK_mTkBe1>@3Yv~RK*vU^e3u0GimmO+fucti!GH%#OOX>*_j@SviiE@(bSo)A)ta-29eDaikfDfhUK zqSF`IZ*Zx&KIzx~r%$HZbXk$jq#+xS&c|(MTWMrXJ7(OkOKC}nrN>#YQ&Oc)$<)21 zb^6xQTmEoI0d((EaS+wY&C0|CSn$|69ddbh!QoU;W=85a-^iU91rR5%F>&-zV(s^B zjN|R+DWtpI;@2c9pFh6Q@Pp@s(T{NQ6w1X%L`R)UFK{v&Gz{-$7ejxpV^N(Eh~TPX zF*gp88*6viW?tWW74pBCm>v zL{&G*jtLNn=<>RlgX;jIjk*(2FenVI=C2#2DjsD=sT-_hC(U^HY4C!cuFT%#_ADN@63>|7{+DqqQV4Iz(qd?597|>F`lb^s@P7^OO zzuk%<6cX98oddh3x63=?Ec&|UiMNBdBGRkI;|ynh+ch7V5}l@OM-6T7gq1dRmu3{G zNIU#o$yv^H2qd0#pH{-XrN&qC{rlo@ov(RC=>`zwP93`Y|ki z$J<(~loRY`^7HId6JvLt6);yo&Kiu^jscFS%nGnB|87-(Pp8b2@I_Z^nz)ybnhh>+ zC7y`EeEPQ{7ev_2{MG(a{8W5=z*JBK@vqyuo-vM{!}YqFNA=TBX>orx55F$aiE>JG zE#N9V+zkMtE*}waAgG6+-L;|hMx~PggQ3IWW1QClrj>@)4@8EkeB6trht6=z*3%sW zY*P*2-MjZq&anJ_cUa1U63lFPIXeYOdItt00pjFxdsQTfn#nva%B(ueoq}AhiiYV;NTX<84)vMJ?5tuxjjR3XXTz(~ z-}h(+QsLzm*|JacR)=-$;4(o#^rzXW>L@Q07xVC*zda1*S$`RwDP8w4#w<=|&BD&( z`nL-;?`#S&re~njkmO{H-uhpP^nKZY2j_$GUa@;RLLxH0N&?GFXJ_MC*4QBM1()A^ z2^HE2T8giOG$iX8J*ZUK);9k;gv?lG5_9yaF)0;P!)@0|U}S_co#hTTO1}pJf1( zy;Q}s?-1$JBgVTi(spLQUb^PS!2Tc{U@*DmXA+8UD++HFKOPWTRGDn;2oTBa$fPQ* z(jmC6i*c+n`y_w!(1j3^(FS2~(aQ%v5wGpz6?bq)PCr<#@X?3lQri=T0us=Rr(-AJ zLO>kp9r$Zs>7*_2#`dWf@vuXgfn*w?DUJd8 zlX*~+0}J?m-7GRfr~gHFp6{o41G47Mm3X+#C+Z_Txp%nV&rm(vRTwccdsCx}F}AV+ zaFhnri=^mqMb9HXdNIx3vL<4}wiH#o_@fY46TRm9DJ#cDZnrNLJx!RhEbpncS-!@p z!Kt-m{qJ!wm&au@q=9bQt~A5~aG_9Jub(T)+t=2>)z=)Lo{5}O6TEkyT-Oj#NYS*N z4t@JRV*(%A7IqqVB7p>GV`Zq2g83<@WPNUF7m+P^L;YvP(d2uRoE?2Xu9)+&gU8-0 zsKn^?Lx!Z!yb1n<8}d(;t9Jq?rg4~JGqXUAr}&OLtrNBN@!kCk)C(owiH zUn+3ot^8Ai@XxEfVj3zHMfcoZz~u?!cdf_$hc z_^5fd#1tR%p^1;kRAmp`ZUb2a*$v-UG@B9z=Cv;Z!3}Zilq?FGj~1kZy%@D$Xs=_B z+4T)P?tb6g&RVVphQwi@Ila&(E~mQLAVoESh@T);FBDR|WOK1PU(TGvTtRMiJ{=nGfq{;sxoJc;cY7u4``^-P;Wg7!HSTE3-+EZX zQapX>$@n>cKo92D_T4n&@dkBwYWZC!a`}m>?_}zX53Q6Tj zrw6Yg7<@`#!kr{ybkkrSA>1>;a;mzF4JBh3=8Zb7&2#jK;*|oVgEmnh8n~0D?ap5f z-~h^pURc_*+7zSem_O|t@cTS%&DEOVCxZ4<6tJ)4aAF0pg~5(!ctSE+WcOrySy1Ak$76d_j<-CERr_F{sjY1FW^X<7qP3g23wFWh4BI=QTaZ4Wh^DKi&UQNGQp) zEVF-PPWDRd4D}6#OjpHUd*Bt0oUB;xU>k`9$ujRkrFX{yTg@8MELkKu+UhRkWynMcjov<7^eJn4@$ z+sUK}7(Dg|{^naq_V(Df-mv+iuWP10$|$e=f%vDkIvSFo1_R;9n^#`3lsr)+#n`J% z0)F9EK=#~VNE~V9l7t!J6;N8+(%^ub{g%0nQ;Cuq46arMT8wu3%ZNLZ=94!ybuGy{ z^WlCDCl8P=&r)~b5X%umaE-5s{eo7)`tgFS#MOHA*{1fercu)6;Du`}KEi7@3Ge!BUX*{*) zn3uM}@TCkx)`UAtDZ4BM{O?w?W64V=!eoa9Tehk`^zlG&snbI&PUi>ocsz?rjDg-e ztlXm2jl;mk{T-8Kf&Z_?8@RL7eUzaU`OZ8j-OZG3xiU@*?vJDt9E>oH(Coc^vs{9u zcMfq6L@v5?5K&nVJn<|Wp7H?x^|>M_Z*ADZY~An)>@8I}x43;Ui*PLk%~#r(F;~;V z@B0l2m^3HZy}glu%`j05I7);$KX)24e_GGn5JCmW`n1nr zuwqfxB7r_`Q8-2h@GC}mrdXJE&VFP|Dyc<1;mZFpy>{qh>+`AiwUkyqxJMS?Qdc01 z6+Y9+MA2HhL@oel(y1sJRb)YqSO>-Z`os z<2%VvBEjuG&!SjA-4$n!Y})3_8|0Ce@$*MS&-(e^Zu<`-rezr|P&m#xETN;TFTaAK zE!GEHoXIpf>!i#uj*{xOpw{-i|Gg^7R{T>^_Nyos*Ba|v8rlQ7jcK|NjeIxlz&2e4 zr>afw&X&>-^5_qW78!RY8gy(>VC?EkvS$ycp!#`Vu!xQSe6FEXlWi@<#TyTzaW7q@ zhc)J7IYG#)N-`=OlT*ktL2ngwsT98>UsO+DSyi!{;6;goq`^{lEx1f=$wuKA*O>sD=a zMT}7{=>eR~D3sl*8&+epHqVLHY8HtDk10OE`}SMGq5I$)=ZBBY1QKS1tmF7mT^>=Y zH$=HK1ef`~4wE4VpkYNZqM$C>pX>VsHahryMPwV>M+307M9;u`yQK72{#h1R;*GK% zh<}ZhBU+(z$f5It?cS&QKag9v7OpGfd1=p6H>}3o8;!93A8y7zfYLc`*&gNk7roJW z*TrIMtTEi+OH66zzU85$VL@WbJMertvtFiG2(i?H@md~D&?E3yFuHq6|0V!bdJaRUXm{2)ml<;YsQh+o z5eSLatI|Yem|#Ukn_4|Cd6%{G+~H4jRdh05ShR!96jnJw@PYo>rPGL-bkRl?^8oZo zOT@i`|3xL(me&B|Y5nnfRUJC|@-oCO^c)GWi~J&#dudKdvFAsOA`BOpCtm~eIB{Se zcQ>{iI9C;IfO(vnjv9s9fmz^6RN!G(_U2Du8pi@mMMV(r}Ct67?TU$AGwK61) z#jf(~%J(UbrfM#l9Bv&jB{4r5(rmNW!$0d-NR*$n+ucrKtorwgmO*ym_kMpMAFivu z^OtNc1|TWVeoJqEJkG_0`#o-A8vdR!yt+a6y}@L--HqdCW#L@5#`4{()WHv4Xs#=J zl4LtF8Ihs$<5>dfMaV$?-zY_}!|OuwcB`!0YMo1VepP{2B0>LV3L1AWgic!8;{@c3ZqU8We< zReD#`h}l$_LOHoGz4~ksXyFy0OBFj7N8HcJHmyEhL4I<% ztMW?dKYQt-<3d0->gKtT_>X7cJ{^vGm-rIbotnosuNK#tH=^0n9=oW;C;2VhewFw5 z%fRn8>OUQtj63ZB>&{&4oZI{qL=F`QpP%&O0sz ztyKBE@*4NP0^fM8MP7_xGY_@_lb8x}@=){Ry2>yNgVkiSH!{T&$MbA=SXMZP-h~;K zprtM+o%^+rU$8#mq!z=pkncX>&#T#7S*>+;iuqtLQK471*UzB;@*moSe>q9~pMP(f z0(7z4tq>-t{EH-#7#MuF1y3;PnLgEf`yDmiv2-u9HbYD<=<y&D&*Rt^bGqen9O;kR@{%V9|CkCpRMyC|EKx zmpm*CfhSH+MA)|uvIQ~BPKGS>IJ_)uZb(2Y*Oo5latO6DP}R zLe4&l$Zjo0Ni6iLziNB;wz->e4pWH*ntw5585?R~K)d*|KM_bBK> z>&y2|2A3GAf7AqX+TdtSIRhoib>N!{65At z2VV0Nnd7uxFd2rIE{^5}ZDO3pq<#C<@Iinvj@L~%!^roir$&|5rI;(8Rlr+%vvh&K zo-|N4f2640q>Ry3oB<;K5Pc3mKq#C>OUi#0nL=T8C!pQlR2Y?Kt{lAkO&bHN{BiB4;d#NSMRArde?nu{70f;p$izUk!pOEKN; znsQ5q>DpVKuZzRl`*&yJPvN~syB_COb$?uRCadZLJ&rJpF6qb6z{lj3`O*-d?VwG- z&SEj;BN&?V<<5uur@C*mHQWR**f>NWqo%{uMY^uBQcS2UaDRkchXcBz~^^$$u#Cc>j`{GtnT(kr4A$RiQ0#r-cqHrU(TLBQ)h8^k{<6dNz5zo zMHD#sJPpqoA$WM-Z9+%?&f4`WFagIz83(18(DYH82hG>1lFWhHB>P^-tSXppP%~Ux z^}3{Yx%Qhs);Q~zGSOM=iS@CU$Q|fRN<}bptQ5*>2B$t|SL`|k z2p6$U>_Zw%=Xz9_#JPxfl!d%bpk!QUE)trw6sBTwW553A!1+y}JURQo++4K5oA|wB z&^-$0jzCdh)R?uj<5>==Oe?n7#Mw^cC(ksu>&3!e9PF!JO|(#9HJ9 zn>WNlxbD^yx5Ejb7Sr;;wu!dn0I`~6-7ZE7|L2xp46Q%DyqxWLq<*2@A>cQcKp8px zM8Quts3lPcRh_yE^+{^Xk?>CYmPEZ&WALqZ=diTry7Qcabf+H!i93`gB0!j2|B_@H zT9rxRAE_O)&{o7CYj@^4p5HFp-6rMPDp^A;Z z^OFsP@|oTX;;mp}=6q(W0;jC%gfnLT>V#hWD-S)o-XB@- zm-t2+w1|VhtZoyZ%Zu!x>+FO4QD#~Sy&g)q8B5CVAY^_jLj1JaaXyyn*OePz^xKi# ztxDWlvva@;6!5;yBE zm{HBMV3j;MDY!7nHs=A)-$C^Re}}>YJ9_ufbIgrtGqb_78*cy$!9U6UCSp;Fy!qsf zRK zu=8tXA5l1Hy0MYob8oiSZ43q2Uzh_~XU1nCzw^mKT;Go069LzhgKZOS!Xfu$Kwz>U zWwL~~LZ?W^KGyNSN#oQA$C9f%o4Gl^5BpJ)xWpK($f50~)OeLk*HL-?;%rR2C0^dWoX6a>nwkQy9k*FM%G8sO8nzisBy_tcSXR7p5#e3A zr1T*6!?_?*b?$HzF$cg28{x^bIwJ((nVTbQ6;4Gb`B9Z0A84(FU-0A+>+5@bc}$EF z*{egmwmE{2rAL}V0El9w;*(OR3Hql^Yz;Bnda)KwNxUJ$0#Bb9dEdQE1W_$g|HzW^q4X zzF6aTPC)k3X>M4-Pr&5uW~-a^Wb5<0q^m($>ng+)Xvza@`l8IPf3lob@{`8*JEQTn zO{-n~Mks=HVx61zIXpKIWPVr`4y0guRxG*9Xm8D=)a@%k!XcFB92?3!#F8>R!PIbM zUq}o8bBIq_eAInzLb#-`EXPZ;{Kfh}&Ro~JCVwbnMF|q|OND8FgmJ7t4_!$1(Gyt4 zwi5N6(D00Od8qhx>^m zgyuyeNwQs8Jk2;`2W*cW=wYY3(e6NXVSr*vRkER^{pTaBg6$r*1?^-A@}S75)C<@&=33rgf0;gz#%n?BC5yFPicZO9mKy4MD#hj`%cx^?>^YAU_w4{`f+coQ7rJl05-#kjF^e_1mCKn zj<2<8zRrIsc0gzA1TW7?B?f~vl`HOI9}OohH{_`?@0bmDMYl`#Pn)IpOfRaRsNwqS z%%7F{;Wey18DFWZd2xv}5d*_E;eN6u>#4B(Fy6lusujNbA+X;M=TSXBT}_u>4Sg+W zzRbaPB-A1g1YP@ypRoDaR8Q<_o?gnCTyNqpAK@V}_DArfFV@mNp^foo?l; zXU$Qp%>s*XgE29!1w+?xrv!>kPRoPJT-@0jK30g-390(R7tD7fy6W}U`i4je=j*uP zmoi4H+y0FmXZ%dF({=!XbK(2UuFklG4yit%Xndx(0Lj~I#nDK-8m)L}B93z!nO>7| z)#2;*4S1cQ4qb{<(Bh$%`d0s8;HF=jX|8Gi%)f2qmf9o;?Il7!0>Wcbs)LtP*?Nt=5S+z%`MxzvoObj{Jp2ba>r`i#B$}*724bLJ@cuo!ELeK zr0-4jbSqYRtW12PC5I`#0t4U7+HBl)ujyHHFI-BBxHn(ZTp!%Vx-BXGUbk9Ja8;U4 z?PKCce%n+vir@fo!si3P1MKFeYO@_qy)ZO$=#MbDcWts)J&OeX=_@zHV0CME3-mhX z_UG)6Rdho7Ofwtl_x8K)2I&M4qEv%2X&@zU>b~#tiK0F)=$Ygh@}&Rr^-`A=^SOZF z6CeYA!K-ZzNGQuEdir4HBD!G@irGLSeTUT4Ooxw`qxVj24(yULO@B;Z zgo#qF6w{L>uX41iI)@MY=s<(^dG(i;p_M!|KCeAm3$eohiqQItDcHjrz~*n6^uFeL z{m&y-4<4>M^H#L=isgRfIWHgciE`GiGHXTLazQkuxUueDq?3a_zZCQ~F5SkJp{=hA zSVO8XzwwY{XWM<9Ma*dZQUCi*#?)b&#ap3zHaMNckRr%uoz2ZUS=_Fuv)B9JeST#Wq?Ole*rmfVr;V?kBr`WaG1Rcvbjq>bpv-m^v5a zL}!#`Ylm^Dd-AnNQoc*KeB?Q8R~x=fR-d7J<>?wXVkkh=_$l!RZmG#s?bhoLAGCwZ zF3(-2z1b$jlL52kveX!>a_0EfChKEqpY*VOb1GI~DyLrMm7cIY&$CNJ8Dp-(uS2)e zri9~DT!zQkMD#VjB$;n$>WqB-NwmFE;OEmj+LV0h(x*L@XJxA17j_4oOsQb)o&R*s zHK=x`P0cvj?!bQ=0LR0M;k|C~jak4corHg|{E1fOB*ji^43({GCj`JE3L3wE{7ZAu zt@Mp%@a3iq!hRuXC6CnDHmiX~Wpq58*z+7MKAZV0`FeC{&BQ!6-AuMtl78#LX`VGz zHjjL-Zb)XDm(HJ;Vd!CQny8fevr%3ik6BoRmfA(>HxDr;`^YGfNc z`iOQvH5(UJL|O0W=UdB>tvl<}ubuL<{}nXB`pgETZ#=4x(`tDlZ}j283`*N7V9Hj4 zTHj?tu2piHx`hg6)=9w=+TxaHWxbMRJSr+O4iV6M-jPRTmzS8D#|5I1vsxJ6hvI~ytUBc1P^Z! z;$vyVRjP|D0C~MPx_yE~MyXl4tIk}OV+sJd3p8(zpX9#&@O_sP>lw^BqHu-~y+0 zSo3nEru2|j|3Nd^fBIYf*U6;?9ocbV2HDOl4;X{~VT4$It8a+G0pmq)gCu_>_pqpU z8<^7Dn(<4oZ1k|rOe6%2EAfd;oZVE4o`=$|2T6^|$W*M{DW0BuP`7qzTE5D|!;NFs z@)u)ff?L)#u$mFQ>_q7xX#nZWh1R;N1*8Fb z%4o83+q{44_N?%SNA0Ep0>xUpy7`rKUDiJ}z zf7_8hYAL%+ONHk=9a3{5GM0b8;OCyN2HytS_GZn`tv^;?zqkI3Kv!%4klP=1Cti7Q zU9nN#LOXJsMJldf7&K^nd)kjyL#4OX)p(2defBw~gn^mvrknd*S<3ugeQLHVY>W2@ zNgr-T$2P0l8z3A3J>ukMFisHOX@q;$M_6T3krc2sD?el{B8x zfaQQ&0p&0Hx)qIlM?6p%pez1XUq;!y7 zv2()_7PeywybvH3r(+l~@-=+0ERA8sU}fMYLmDrgMljB?laPp)X(!r(G&b~csI+P< zrcTkSHcBx0zIaD~7~YW}$Wm``w~CG@YWjWa;On39^$R64!wbl(+Jzl1Ugc9j zjS%?F=6Hvu>(d`I_@z(#EaOgk4(M)Gx~~8wsF_A74U3JPhBdckXW|re&~Ek3*~aAN zX-7vc=O0a0Up$;~5;(i1liNF%ZGS9k^NXzt+y)p={R?Gbn8WO7SRTFN(S!xjlF$ao2}iR0c%*Pt|MF8UU~J+ zR9xGo`keSvmV5KN$x3jQKEcF2AOP^HiV`k=0*Guy7S^JTo$gfFIh${347(00r}C)j z57`v_C>|BEMsXK%lr`*)G$N^XTB#?d=~7Fz-q3;P$#AtGe@|jGJ)Ia;e~6T!_#8j{ zt$F=K8(fG7`wxBQD+Vi`#g;ffdqnMzsC`;0jO!A9L(kCdCf?m;bJwJ4qHq?-FknFZ zm1<@DKS=)pIE}_7-}Lq%Ig*-l3s<^UWLv}i?rfj3fzrYs)-!DpcpRt8S1X$i{-n2@ zVFh{t=TPsH;_h{I=}hilc|;i=tQ zuvsM)mqPuD?f(iptw>62JK7Fa1n~P#oH0u|J^^k}v#e_EE@jrU&hl4m7pz{(-yeAo zP1`ZbE1tLK_YrucbO;0?PyG&E8@@LBPaW;d5FAIcZ=Du3DFyNSI(97@;NlY+?UhRE zTAuSe)zrSfppIoyb`_egx19X7x@^A6tM|`38p+!cxmUb$*@6Z&_(|!pO#(ht&$F+q z=&@7|Lh3zv+NMl4=26D1L5}d%j@9^uk8CO`8@BNCzFn9%a1M&r6gClbpBk|}lmUxlCakSd3Y^Xn!~Io6RJDPdLUV! z(JC(Hau9qq+8JV*=r)36(yWxYL09q7LO1)omF3$*g5m%uzvUjH*?j3A5Vy(CTAAy< zrCyBUAXi|ZAdX+Pd+45ey~rm+NqdP~4wIZdui{abS0@a~l{ZQ$FTfm{SBofx`F6?a z8P83A1vkFy{gLk88K-oqw;DsLkRDT)pb)W!Zx`y}8gA}aW z3WeT$n*GwxUFdGx9l2-&4bOY8IrwTQ%|Rox6(T>S56r^ znx-m&+lFpmbMQ9!x~cPy@)C*BkH0%N#D60GB+#db0)I+Y%KZvysLL#eSd^DqibhLJ zsf!dkB|xtpOO2erW%;R0{ze#x#n`O=)#=m}*)CFQZ`(EfZfr`SAIu!Hw;Gh)qK9zS zqcg_Vfm5tt^$`EpTE1%@} zm(f1}9tzZIY*o6-3VhVuw5xBdt@2z?m}o-RI>&6}XBXSVi}3APOb_4NV0uq-@^6VP zyIW6Q$&=2#{T+MJJuiYcS+Na#q4Gj^h}_d{Td{e^R_bQb&W5Bjvl?k%dp2~|e*8Mt z!}W1sPKRm4;#R#FWV*e{QEf-=!u5}!MV}cN4050{7FF9gYW>*!H*(hkyXrv2iMeHp z)bEk`+ihjXy?Esxkac4#ovk5nTWYXgS6oF=Zx>m<%HDRrau~d1@ksxffy#wsBVP{c zfRE{>uY709noItMKW)c=ltP<*5hY{Ms6Ge3IMi!s@OCmA(e65ow`}pYZw_cCmJzIM zfU@E>x+BS|!MeAz$L{5qqZg6W)KYN{6jhdEItUXZ%gw^WM<^ZIB%RZl%JKk(eQUeO z#qdfQ`((#S&slpNF3txm$y>5b4$);F&PE&92oVSwgaQ-u%*KRuEuPEQM8Zk8Nam%< z?xVIsW@@-#MO!AlwvuP*!Vu-n_5xD}VbW1+R9$}V7~$A`BvNb3$NuDHu*?U7BczLi zr9HW)N_`x(fpL%T5Ka)YqVP&L&J!XtQYYGp9zgU@MWcr<9_`dr;pj33m$7wY#a3gg z2=(!Eoo3oj3C+7Nf)}3l_31rFKs!ivXD+};T_ehk%~Xiy%p0BTGoHT=VlYj&8rj?~ z##a7`E{;?CMK}zCUEI^#nZdrJF55o&->{+W=8wmo@KiO8ZME=I@;g--Q<3;euRt!4 zd|zy`(%bJ%G3G&8l8$P@7jrIKv?GMN;q+&86zCHTZWoA^k@kR#fS3Nnv0OW($HDI2 zPIX|~%$SRbk00=P4&&PsJ5oZ`BGJATFRYY#BAwCh+(7J0rP(3sWkN5v`zRfUFz}o| z-`Tn-BxI46X-il=5F1$^Phs(XdbD=1eK=J)(WnpLf`Z?&$WFa$uYe2yjt+RMjU> z^kZzjB~jTG9md1kQGy_=%_US7pFFH4Gll(0B;G@D7d2wu#I(KOkdV~9+z>r}p)<-= z(?8}wtL`N^Lugwu2yu!B9^vuOG7-;gv;n;B_|#|T=K)B*ol7@yf!7gKTWyI7wD04X z2>6`wlI3l*pRjm|*tvIdKirZH#xnq-Vkr=faVgYg%f)YesNUap*DquKF?eK&e3Jd@ zr&Psq=5uNFQqe3mKN_#;l9E-tS!MIDukx3qAM~1a1JH^xkdNr9&Jzg=5<<4df|41T zEJ=H~*ixl*AA3X({nP5Guj#jRBx7p}dUf^hYh7S#ULXb?bXW_c@$g7RO*Wf+@pxtZ zRaqJ?4(6KEd0}K2#kQ|oU3B_*Q+e(yLv+8T#K|}?g z75jvTT;C^9J*d%b*D+>W3TC)4g^55PHO*dg(d?>f^Tf^asK(vke68zt{&23s{s(=0 zAGgok4kL?qF{jtE(NuHl1i|yVMP}kbhHbhx+{ZNKr=}CQ$#8y9+XQ>YNx=n7@xB7F zxbSj34G2-t=hE9{C$%k?s?o`XHL-Q;sOg^;ok&zxhKXdSA%#^xN54rTj%sdB3O7O& z66UQsT$OK=uis3$WT0XryQ-+kM6Fw&XKIzXO0jSAXFP~*GIDQIL>L<747y6a{?s&) zUWFL@*&`e;(b6RoY$JP>wG@dwVUFzNr}Y%dqnJJ?;p)Et^)qY}O9o%z-%F1&DcPa) zgu{lst}Sw$R|eB&U6H(`>D9QNVzG+_`YG3)Hr!}1A3Z>)HP{W!W)rK{HJR0oizxsb zlEYAbftDZRR++;0Jw1sPaE%dn#oKxlH=z(ZfsZRX>+&EJY%gGLn8jK1>KTu(?)IbP z{@;|Z_#LP?{vx}Ql8=iYw@u(DKRStmyZKknw1tbIoX9+XRi3ivuId3UP&f!w`|At^s!+{ks_*l5*! zOO5}rDt*vg7tTErx8B={Q&fMW5l@e~no}H-yC}YVXe-2~y%-0yEjo8vOHlWHV1L$b zcUqQm#%=VM=JzwZ6HfSk8QCrz@70pGAW@VvdjBC1q`I|*A99#YG2)TzrKdvg7hR_a zuBsLm#X7?K4VWk~65>7AU`s6HI4ZAJH}u`aL(zs~C8anqZg1k5x`!QNqPN;v{@qdY z{;|6?+F`$L>xe(}<@fxkD%Mmx1pb7uv&E!e-kgtdm}nTK26|7FGXK*2{DPs#{iPv3 zhj)N6r}PX|U|QyE+%lIGMe*M1>BRYDcvXH5ER~M=*2{bg3w1_B zO)&}U*q4*RBQB1JefAMF^bigJ`aNT2G5f|{^q{0kXKo2yH>v&-@G46bb&m>m z-7oJ$DVMy9(^tDf!>2|+<8Y5VC2@S;n^HIf4>%q;LLC<7x+%OU-fNt(n~MZ%E)!

!P>B#q zyj#)Q?6se7rC9I`rGLur6vo`ApR`PS@Pmp$&<;-PbKYDPe)BcLEajEBZq_@b#R+z6 z0xknk%XsB6E7(4oYPD!uefBtDQlr#ok1*prc&r@!mj=QOVE4`G(Qo-_XL$TN^k%Uk zQiaw&{Nh`i^Y7$`tqo-8=eE+1~%%WKu;lK|_7)j0+|| zuC85RDRN4zX}h`mHQ5p|YLKk_Jyqxsuab&4XXtybo{<0J1Vm=&Y(Q>yEZW`toqIz; z%FuLFk{VCvXZB?u<*y!CYPHdpw2<=tvOP3M-2g8?)d@)arx@}d)GFkCU?LsN2zvrR zc9gCYm|67q)CR?#9$#v@g^zXEc>#8~70xNSbI*Qkjz;WxAti__D6mnjoJ*@vMH#jC z=d3wM_n>G#j=bF^ zVRwG^cY}?p5@G?)4B>-VIpO6x_M?TO?xdMu2Uzu^o6kva43uz+u?77qix9VmqvtDduQ74?scRFdGA78S1WVYtz?(JlS-+F!I z5`NlJ`qCu;EIthJ;2OMIIx-S0-tjAx(HGhYnDkTkTWAEZP8-ehns&;8pMkAEw%j4T z_eV9gi+WqREXsG|wm-k_on6;@_B{1+bB|F~;u9?5?SlSd%&c%U%gs)wyazPm39N3d zJoJ!xA1aX>PFRE9-(ChRnBxgwPoWgp=E#5Ga`^XNhyUn(7?F(J4<##ioWgtL)*L%< z4g4XVKqIJ09ZoQ{67~G^$`9{R$&zn#51*?EH)e+tZHv?hcy|aPo+F66cKnYELkHe$ zFCo`UG(X?Y*3d`shwEPzgC9eM(X1||g~0IxynksU-xMG}6BzdZ~HiiN7p9z*M;o5?-q z06bHB5Zy>cMWwo;hnq+7`q9VvA79Cmvugsu5%fyp0w2CX_xkfMDyh2AE~bSou#>({ z;hm{z_I863XH~D_Odk7zxtBw%r`h{S$?o+RvfR@#)sErtc?Vznt|Rk&l4 zxd;fcB^Kv6GW$fl(nx(qK6N^3E zk(FZ5qnu{=4&uZvJwMu)1r zH#XqrNDKJ>OzGtIw7%;lzrt;z%R)S)-6BYIL<)94Z_>$@5l!|kx383}9)N+iSPX$+ z6U$DZ_4ve~LyDBJ{RG-~lXcPagzFTKPLi)rm;ZglcyLQ2R$~f!zqvJW$o#$G zb;g#P*_IltNP!T+?*NXVK6YCHlJff#P6+&K&zIi~KKqc_^%ma5XXE?dI;-(BwJ}n! zKd=v6AzWOdc~C)=28tmL*7hq-IU2pj5op(x*l~o^?hMGk*(C4=;ni07v9>b{OkdTz zrua1WL33>YLg{wxKzd{H{cj2;y6aZ1O>L6(39K_rsk=FlnUG%E30zak0(x_|kop&@ zQ&T;fkojeR;cVaFPnBm%NyryH2NvBYRT7tD9`+14(1I&7`k8KOlOJ-!Wi{t7)c!QP zTC*7t+pJU6?S5^JMR?#fP4~+cX6rAFjBRh9hln49fLj9}5Jn*0-pa~! zV5sB{j~*9?{coB+WR}L|Gi*jrC!CYw-(wa?4`E^jW1W?}GoG7@;1ULSIIaU{)OrWx z4O~2UGDqy`wLrzUCPwADYjnvC5qg)b0Pi5(EgV6^%0cJIw2`h}{L~6veNs-gTLHzn z&GRo!O&=I4ua{eExPr+QEHLh_D8EU6?RnMUBMs$26h`j|PspzIk%5hy!)XX>&| z0M+nq;_X{k(++TCV0k0X0GyDbo9Ovw8fHq^KAf}h!v zQh8|e>$hq-fK#7E-)Q&_7&x-kBCFH>XF#a7St2&a}peM6H|4;qG?}IN5Fic4W~uXy3!f_5HKp z&(DgSo%?TN{iyT*Y26%W#(pxCLHym*E3f3ApWTGfHj{GOB{Zp78yl>;p7dwd4`A!= z+^?B*n_)t?lVE+To&BJwq{pNOQ*C16GLzKgE`V|m;Y-hGVU$d;ZHXlQrFmTqVX%~p z3YRvSrA`X`grfUINTW}-yWOlC?-`CO9w7LWH%19+{ng8d1nkMN*9nv7zu6tfyD9%9 zAa2@B$fZ`)&UIE+pa z0!v<0>c?L7)+LAMm0Cy;c`I>|*O~!9(LMnbQCwG)pz7-l^3KLj3a-91U$|^?Oqnz1fT9r668mA5XR z9>jqq^~1}odGBO*BfrQv%{Q5Z8@|PzDO0|cvppR-R9)&R#H=bse;3)+^(FXLUDNKz zhwp)U9XbLgHl?8H>!Srw)ZX#L8hmH6;EdLVcnY@-*ow`$MOiNEI(|SgZL?NPP3?)O zl$Neg41D}z#n?9u;Q6gp`_d&g=3YzXqh;ezZnmZi$3RLz&za!JpHDEUg?M4-qU+GW zQZ45?DxVe#o)hkQM?CIC=iW38E^8&z-G%Pzi7EJscq@{q)+# zJfQ0s9$P17idDad03Tf)IjmFd&fB*WJXL0!;p$TB&6clw9$z(8Qn|h|FC|r&L#?EI zxEV;pskj7Z2)Y2GM)8%Z7P(IW+G)qnYL0on`9dF@^~NlHeiX4lgdM^TDR|+-yPOgC z7AFd$#iAFJwDU{NO@D@s8LRCCfM9Ap5?^2tt}EMv;->8JV#- z0>$XuZU}K&5!~W71mhemk`?1t^yszSJ1#dCQ6v_quNjV0(1;_wSRzc>>ovTFxaU=q zy!!(y^dV}&!Rm!)(ib{zIeq$*E}-yrWU(*Pf^OtowwTT+HOqP`xYhc}457{|_bc{U zOjUPUkFjLjCr#yy=Sz%P?#qyYbQh@uYL}o|<7TByP2e78@n~b&!DXfq5P2pL+m%B% zqDU<|(d@hv;G6XjZdUy48t2#fG$&Aj+PY**(wMEl2yLVn zCG+XAsm)}0Z~qIpqIz_2ju*?_Bsa0?{;0%o;Bcn zikkU1-ytDJ;s4Qh2<*zO;h^sugh4`WcnUV6F|RInb--P6U-A`qq9W`Axr^1^ zHQ`AatdLH$PD32bJ%KOF%PgHwgzGQySyFzyB$!dT;Ay z{zU;h9(GQe61_bXmPvk=#2K zk`?NH4zI%Y>z{TQe$NeqGc?(gb~|nFj;U)e5Scp7XN54T6=_F~kww@Xh7{Jjr0oC| z+0Bq3llp0cnZz`-J*g)F(R6_?lRx3@mbzt1Q|{D621_lAuEQHkJnmjf2kHPb9jq*(EJQLh9R;GO)@)S!;nefMEA>c`=g3i0OaI2U^$ z2|=pEpu)A}3qPiR061&2n+Q|oRpx+)S6Y9PCe;@(-c&j7rbp$2Q$q0LcNP zYJrF#tkIa{wqypKXv7wvMt90VOZaVy0mFWOMDIPewY|lW6UOGS_Mpq8MrS1-E{!B4 zBlOwM7szYDd#}Hv(Wo$Kp=hzY!Qn%{7U&rq6JO>($}LA1=0m&Z5flHIcK)NhL7O`7 z{1++TzuwA9biti-uM#LJt!ac8A(g_ljljtGd$01Wp3=gbbK9HCJ}X?G)-`!_t++(6 z@vd*_TorRC*22waY`2DoQGI~Ul-3NzS+3?9{^fY=>?6OqlqWqsW>%^YC62qO$`oc+@&Jc ziU*^M&Z9nQd@{wL!~y&HIzIZ5E+a@_+UY4^_+CmU#~sh=%0ls($2B$*(N{8bW>5r8 zzlfDK^^=}4>^7k@qvWG)UAeEwN5HM`avF!{iPt>FtGWRy3z&!S`J1Dd)o%Xdl|+iJ z0`ia?d3Kn8>Wv6Ewbe}A8=*bXeye8gTY5Thw0^d`yFRqtrRp_<8n3`NqsD7(pWJvj zn%~7Q_UhZRMM~>p3ZQljX7w__%Map#0NR4iOG^tRJ0!pGk9;{0q5&*jmi-mTc6R zKYc0W>#tue6si2^=I3VkL)fB9dDeLR<0Ru{a|3>tj_Lc_ktI*#IqN;t>WKIm>I5B` zt5NZ?f(`or`F`!V4^bTagpG8kn^qN8GGVMokapH9wGbzU zQ_tc-3Fh7bU}$dh0e%C}{PtbgVo%I6iMIk>V@+}FOsp6#YLlX-(owMy<6jbFZ{K9l~P@a&1g}i%9vLx89nBUKRj|TYz_( z;kM>TDja4t310pwly8L4oFnXDL9n6Bd3YRffxiPvPx)&2(oqyuIOQzmiunG>A=;+o zeydd61i&hXlQf$>w|x!le=P~RZzS=gE@5bt5TKdP>deOB!O}x>F4kO(qf|D-!3e%qNgyH99;eR;}}=A3m9BV`%s8v%`^^8gnMkVR3IYZI?-FI^IVKXEUET ziFLb2D-~RSb9VGh7{*gWL6%QV`mVM|F0hx8)_gMFkuS>GskPeZxk~*gO#0ZAD_C;x zmA+{0A7t?Cl@S=oWdU+u)OQa8%q$;xg-^w{xhVF&d^ZBDExLU&xM?X)V=xo9Yhafo zQip*X!YVM_&McJ7jpAj*na?5k=d#A76Kk?eJWQ(u%qO4RqeH&v-MxKw9lpsuz@4eU zvMCHQlIRM&*&ilJU!E*?`2q4ZCY$)yRWw0fqK8}c%X;V45F0AAbd3URRqg6V6i}Og zN{whIgT~984>DrAE)!>;>#kU-mmIvg0eBgXP4({GXu)0e8{MV z(SNsn5rl1$B@Hdk;YtfZ&ew-YXG2HC*;V6ur4PTzJ3}txZ-`5!?+s9bz52d?1=KjXta zP4RFOHl2J}Or-xkC1_asMmFN`1a(nadw0Kv4Cu~LAc=l1k2me|uPc2?I+A*9n9Jxa z_40Cx@0;u!&s%hoE{(ZMs-2`7^fLEFJtaXl?BdRND2{IPL~&?TMtoXG(XWA}U+Wgu zOBveTTj5s?+><*gp8eI(j*w7BH%->s+{poh7OcWBmv2h@*^Te9`Z3*%8UQvbvmRLd zO=t+-GoHyd_L-iaEe>=mi2YdIp{t+2DEGPlqf0d<33OxomR-;s#2RmML* zNhmmfrTiyMSwBcZEDlraKxIxLU#-@z-_d`08LDLPl=~|1?c>WMwLJ>rH_=-&j(m4f2+~Y@pCyrJ->QBnXbd)m)9~sLz4h0U6)+fEEcl$ zrxA1g*EXu9rWTj6SBpvp$j%#jr{V^!wZ8Xob#q4O3qRL3s5=fy+-<->d{C8kOv6Xi zXZ2~+@ncH)bCN|Xfqnf*Y6^UmW!}fCTy_p(FI;;4)9dRWuUWpfm$3CBg#mDink=x# z6u)v?3i8P)VzYK?!>fhNnx2uJ^5o!-U?C}Ki9rzGbDMi7$KxFO%l>UDoT{xuRm>}( zu>xFt7&jkyg;s$4z}0P)zcl)9c5f2RFKor#-n`@%bbUmF$qIIt%tnZW0i0)9np4h1 zTZC)Kft}T^4@xeqzgp_gWEBh+;nwTFdrh3H$zkmjmNL1w8MB!ls98z$TpP(Tu=iU8 z+3j0X91!5yTR3UtBDM&U88POX=A90fof&kO6J~zzZeQ;iVf4%ACMPk7y4p{q0?svJ zaq2(_)%%je+g<8#eb;xp=AKKx0^LY@C0q1=3{(Cg511D!;Viv}CA{OkZ8P7ldSq;h zD>v?*C{P#q2Y#XE)dWC~9R>dB-%s{-`0@Si3-HO^9@X%inhGZ$yl#?Dsr8IgMjTF_ z?rZ47%p6g3aqf$i@ANqbJ#uS{!l41AqABx4Q?e8~wsNj+q*iC-#JCh(Dzb-_eAko& zw9&10ah&aHQ!}n_MOso)NLVcjY95k!%{NW$@DC&_nz*TGSbjk+Rop}movh?35Ouq= zbn(`l#huDuFbN8nBh-^{gAy}f7$`{G^1MU_L>Kzu=aUwvkzV)wen}3M@wCizxZah$ zBbL8Dal-BG;uPajuJXs^R0=MPYbnm7_!y{O4k-dhY^|JDAo9`5KFPH;Ka3x4C;dwP zM~iU3~F}zZ&jcVLI=7s3Y3z-IOl2`{%Wb#V9EHU2YTtD zj$%l~P3pf&sHYw41|R=8NzUJXs2}+{QJ0ys}l-{e`_RxbB8x_8x=U5)KN=+BP9_(`!Qlsg`Ukc|1r`I(3+ z^#LEh3ZsG!u)0M;yULcmlcgSOTk73TSZwH&RtXAu4aQbWaVxN0pHy~6Vj9iT|nMC{h|(B@H@K; z**D+!?>Y!S;hzV>uiN0)G;U{V$+SRn<%~#oAkpW6G#?s;?cg4-5do>lcKG>qI8&46 z`~a`?Qkn_CmDa5BG`_h!~EfwM`T6lsn76GFaWI{K&C z&;Cj zdF3R2w`fumw%ajy6f&w3Df5SDvhpZq`&oT6dx*}+k27F9!NGDdE|IFI5m@P znsVHlOKZ`S@hIo+*k%Q4Mt6bs+lZnJ%P=)9p`c^VlHr#9;`3P9x#OaH7n(hfRwGU> zkf9)1kPRNKbjjd>ca~{9^?JcCaEOpzbhKp% zNG*jrolVZaADze&SCTQ^}gO)bd_+_10y}IiII$JNYq*4=Mhh zxzRay$Z`0JgX9e*ocFU|AIv3GUR-;3d4V%~*;^{4dt^$iq8=TyHESsrFBJQj=8A&LMqO%3pDVcHu+D1=u#sam=N!S2AaP`*%(mCYjIs%-4008}|IJG@ z!}Ggk*U}SEQ`8o_3G9QJMgKfU7IkM3-y~n`ThoD7;zG^l+rR7_1r$0@sJ`yySQ*ZB zp?!-NE?L|l8B*I?=xfc#S11s8(K?bpy?A!O0nt)|D`(%99+Mvs2vwLqehDh8 zS24Ceu)RnU3vRiwp2O9|{5$BUuJWvoHEtre-PgjS@6AutP=`v z2UqC%T+U*RwQTu`e(1&BD|EijYIm)MGUDK^Tk<7Qe?JMsX@5WE2mxbV_6`#a!@V6%8yv_d0#r`8NZdk{an)`B~6 zICJ_BCx&-Y{2KLAYrNr!DqR{eG>Eg$e`%g*lGin{nro3~_syNY&=b~j|4?UoeEFyH zXP@AU<8S`|{b{c?T(K*gGr4m^vVecag~%s+{GzZkm&OZKlYF}pe{9Iyr~XEt`sz~a zQkV9u-||9SaEH+M4IBQ%6EP1`*dsDWw7Ny5UfD*F3hvJ_q_LTOAAV&m?bU6Ny*x-y z68qr&9VY30$2ajb7tS{yFDNJ0{-ue~U}`3dmNE`@#pkyt@@AAjev({Z82lrDTSYAA z^TsHO>IA(n)^2@&2vO;r=C73M+_>5@Ce3n1kh2>!9d4XVE7ojOk&8S8Q(R3Z0CxR7 z-9hw}=)eX3s}yr=*O_edNE0nb3~Eqg7g$49>@T%6^@?LEOS(Gx$k3opSKMJ|@nYS( z+0|q2$0=t+d^eKSJ|pdYS{&-@Z>BwiI(B?-JxnxhU7E<7rE+m^GUpAcds0f{LlkLn z#0-rq8p|`h7t3?oAolIs4{l*Y5{^0(3IrDLhjmrmJV+UM*LA0Iv&iae%GKvYFJT^L zt+w?m2${m>8+(rjG0K?3El1oLa%sxnPYN^eYLC^}VQ#+bIThF)uD;H}uf1t6LPOEN)0gT9 zEnQOYke&Q^!$+g6bX1X>C9mslTv*8rZRiM~;h}UcZ&UAZpwD+j3+f(PJ)f`EhECfrm;lY z)z`m{gZn`7SuaK(lgy{=ddw(_>xlJxFe5K4sR15;$~&@){y&%OeLMLt0yK9DRh?z1 zTk)XsZ(`tj>EpnuT>kaj_NNI!*M_zvD->L`b>YlLZcLfHnOdR)(XKb*2iQg*0H!{NVu*E$bb8U6Z2t6mwT;b3bkg%f$chb%@L?v6h80EF ziwcjQ30cAnM_jjEB$2h{<=lOq8>cd+j|?*8!bMR{4Kcr;U(9o#?H1pSv;;FIjNfL10?9UDSx~-I90B`^pYz}b| zJhva4`p(a$wF>ybpR*MiKuICL!OoAbBo&KBR`})v$JzZ{JmcwAZ0GaB&5AyKE{s1f z`iE938T2qhjO{(4)-|$}8aUKwVf(qa&0|ynPJ$S+hhMyfpVap*z-C6jG8>>hcTV~V zG+fQ-iPEz5I_935#s*wMwDFtI%s%{U0}Y?e>v^7N^!f4NzL$fp zyKG2=mx=rsPBfZ|%uK)n{O0xFZZFgaPYc>TNs5I;-`Z)SV6WM*?k@Csn%z4zd@-T( zPw}WZY;tzt}dskR-7@Q~0U`uCo^>5(vbICw!y1OG!?BUZrhQ8Iq7gJoyHk7rk!R1n{JUc-KWO+ZU{?wWz( zGSxCh-Qk017c`?>6dH|Wtgu33L{ZF$7vtkA$x~K!(c{7yzY5~ow0ws|EPs_p&|72( z6ik6E)}&gD2<3*T9+}9aEllCB73x=Eq86|`J%1Kkd~!8lL0_m1@)j z1J4)owbTVYz!l#NY$q3Ovg?VA+a>gVtmHrghfzwcWTmpl3c0GoMO^0*Dy~_xcx}>i ztKWap?=MZtp=aadzTU9r)jxq^iyWJgbH}+18)^QjLtzCozt$ZMni5v@=aU!!PCvyp z4w6g+zNyR6XjyTn|Czl-3HKt)?CVm(r3T0R(<&E_Rr^=E*CjHopbnOdW!3{c#!JFj z-A}iw^%DxGTo>0m_A^M6l)(91j(9O^B~@Y^{wx0QS>V^9QRb_=Pt<3=C%P}&JKE=_ z0<*mM*_+IYDavhRO03D~Eu80|>OL+YOlf+u+rdoPnG-O`43)2H&tir+k{_R1+a^#4 z4nQfd>5pgZ4WJ`xHfz8r9^uo42ZSw*OxTRQs!B0!ihNQl=Xh{na-mwWb$tr33`~u$ z^wrZNE9HHqV%dg%9$#lNiT7uwn1^2ZHVuB6Pw3-eY+dx$y~g-<%m)MS0O0=81Sg}C z@q`3SA%^+u|5Wk)<3-bASQ7>B(!A*VYKZ?LL=_XE0ATTLX%Aqfc7wZJYHp4XIdsgP zHEG=6IuxLlUh;gl+0wX3;0F0h58;3^bQ`lW-?RH;jV`OXlY`}~@P1!)_;QsDelxnE zsb`}%SvfSLCy)XPpmSD;G(MAso>0#4q;?kmUJy21EjYNYu_QiM@p9qUMHy1Q|G>f{Z%JL{kUls=_yHa`$SiX+QIas=+4ekdG&-x!(B!pxTX zD&_3{@<6*y#<9H)PVz)M-_p4O*1phwG^;Lcji57(L#Gv*Uk&)BE=_(9f$vl~~=ZX1?sgi5wz47G)mc^?cm`b-Xn z=+h3MRX~&f7jy3!*3=tj{bBHlS9*6qrWxdLboCYpX& zZyb?hCo568p*Foa2A4Q4nV-XL5gBJdy{17+c`nF3Xl7V=L{4y93sDEyO0BXUG}u3i z${7L8t)pqwnR+rYRTCcU2g9dkVo6=tAEwbsQ7=PPZ|mkgBl+QVquN}`H#A;E;*&TX zCmW_`!refJ-#7PmaY)#i;y_)9#Ot)l?2zcm^X+DqUIyy;0>d;=Z*MGF_%u4zM(e3(6 ze)g3lOxZy{Y0n9jl}8f0(40VSa;pV$>su227h2;WE}UZbWT2qnB3esE1q{^o%1-a% zCnd&r8>CG`AGc5J0Pqak3~nD5<~NVu*8O;j^x0@ulTg>jQH*~!A$On_B}Z=W2&vu#+nEIBjUE_;41e`BQg z#+*}Sv7?E)f*(lgabt%nJP_PGDQ~hmvz?=ce4n!t9(IR5o#6$-`@R4EhHE| zh3T)Vb%$5;)PbizN?guprVhV%)9&00Y`+Ltb0eXOM1B|5{hXY}Nu z`Najdk4vH%el`G=lR^CW(WHcf1D&M6Dn96}wm07H;~5ogbx4`X=RyfE|42Nz_uGdj zht`yKi%cK@J=Cflik}UMiBa*Eb=RvSJV6xrpMpO&C3jggP&ZDjUPyL-d+QZ%AXo4^ zsb=ouzX%B^F92;}*wJ#5gO?Og%jbws9^RGRel+-@cjb5I-S=r%hg?!`Vd@cFVXwz2 z5?+3uV}EnzV^l6YqpkvjtM%9*!7jI+@pnf!rZW^(Uqj5Wr8RR3w=l+bFaeJA=zLxZ zdQx_0H!)km&i>Cjv0~O~c33D`(uIDo_0vcCda1~CBdVOgm9uDR>yVn36 zK^LufzV0WBH48^u@3!y$ZTDQ~HTCI$EjG48AaWAjIH`iZ`Kt)-xjwDzoR#?`<-glL zZGTresc|Ps-J?TnuSHWE8`S8lYfJw2i_+(u8vHDX6_A`DOH@M@*DS|{+5oAG$1$WZ zb6ILp>2f07Q}dX?=a@`j|`pjHqis zSS=Go%7&W!By_~d9L9_WvJk=g~#TkIv-ybIiN??8@u<#u+TY}u_ZSC86CwnJu0_svvJAzyIQy8(ONpmnd{ z)l9;CuK@eBmiCezS?IWf_;}O4mhe(-`uXaJ^q-3c8{;NT+fpY!I~C$o&DowW`EhY) zd|gk^KWhmwR9c(Nd#A~6WBrYif|VriMSj2-TwO@yK@mH8QZ9z+5ztYKQxW{2FujQ| zvvL&p>mSGUESp<6WoBccR{iEo%jd4ichhfK>CsRrNK8iXLhB8Q@V5C)EsFAhS;kn4r@-0`O;=mHy4ysVdpPOl@pf(|$8KLeYBKi{v78hWziua)?VMb}yhIXLq4B zc$+s{B2xhVl4NkaD{zFxl2_@1$_Ar}?}cjg9sCJcDlP z`;!W;R;BgyKX$^5mudu|O8A=h%A4z2D&^3i?v_eScvH5E_iKSt?`wLZ9bTR}*2e5% zjn}`4(yhwzeZJKGh3q>Tb{EIy5Nc+iOEOkrFf?!2cS#KPwd3wCd7$1^0Mw!=n%fBZ z#aSPjXmb=q*(_&l2kusN*-t_$(8TPM4oxwz%&U%ibd=UJAnk8a+>}!FSCCPrVd6@1 zF5Wvybv4vGMe~NNSpADYIBl!0=kt@946%mWk=OZD@d^B3OvJT{STRn$s`a__Z*&vhM?B>k*6vPZGB2*aAoGvDhJh zXQS@Ktij*d*^59^RBg@|U>*jBkqDFy3-R;bjA;$D_h|M{Qz7T6cpd|`q4txtcIEmN zxTp{PiVk5e4HJQyXRoG6K_Rvomf7a5lgi=74JYyku{dedzZkA+p`Y<;?yigs-S@w~ zkn&X%Xe?j#Et_|SfB+fv46@{X);&1z2r;Lvy%`24%Y(1GFO?$C-5rUg@7h3~i<&6T z&S#}SvsjBjUb&&0Z;I%2wt`7r+t|&=Im1tSDNjf?RAoeMi=ziX(CmQDRxXT{Go>-K z=5ps(tRohrM;~&%h7Ybt_rpEy+7~sD%xH9GSmFDV&3PS7vddH$4c-Ds5Z~(l}>t% z*9rd0;8nYfJu)(KwAmF zvKw=_d3*c7SUAT_8|;t$B4IK1w(hOr&x&30=iGr8x}JM^m4KZd|8gR5NRgr~pxg*E>$R1FE1`S`Gm^&lgm%prh#=r$>hsz z^JkVFJ<9(X?@C*ShbPIlZ?4~ReV#i)Xn;_^m=ddHc_n8<&6?>F*kiZ$Mc#}{oL%&* z;yGVKJz!9O-@&EjKxEP7frbFDOi3x;NtoB6f+4WCcSU&s|7kVp9kyXu$YZ7P=e@XB ze;(~EN3y`BE#PF$!`+EGeeKMnj*Tjq=(Z1{gb`^GrGbj?cu~MeGVQ@JwG!51wbWR{ zvryJqUCP1|t@jEZdZ4Opk5wjWiqgu=myjg?rd zpj7N#cs}_a!QQ)oS{Zh^^GpW>jWIS;Pe(qOf;41ln2pCItLEjeN52aC^vu(CSDm*` zdIlj3eC|GNakPS(J|-L`iRv%6wiDSuqb)^EnioZK1kP=LQeBU3GwuC{s!=WW05X#c z(AToR0yZ4N=M4?*-Wq{if7j64+f|co=m!^U6zWbwsr6)EGf9+v_q;uzv$+>|BI|Uj z*`6d40o~jEhl+;69pc%%r6JarK)&@gOm|$a!6WQmu;?wj&C}xI-`1ml>Ks@mD(_zs zZYXCGrZOPBqEcBDWI<4oQb45lc&vj3!*IAeKHLZS_+1ZOxo9B9ttElIJ>J_zsx6(l z^hRu@xsrse!1aiV#t$fO`a~-CJAM4>?8}1eVZr}m_HvdSt69iE+Q+>1ta?ru4nY1+IQ09mQ94^e~c*P`e&banEN&0`DoA$>%(9e>wKWteHm0J&R;V zxYVn|iPmBc3SRuPty|ym12H2!i%lebLzFAU9q~$%CQHvXIe9$y@*nVkoB`fD?EwB=OT{4RTzg}|{D{6Wf+pOI6_mrLUDP5#@DyfrCmKbIl`m)e2Vm`i7y zQV@}5ad)f!%enwJA$+=iDVzl(&qyb(@Qbq_Qs0PAhCY^}V!3f^7fRijz2|cqVsKzk>-gdf@A34>nzHWE4S}j+xoj>x15es+wsNU2n z-0nwx@eGm;Ex|b6zOedU^lAGADcYW3&TUS7N9;T!Sz9+&>qa%yS<-N3t4i6)1$nHN z%?Y1)ARMofule_Gg>tO!RP96Vx)2&bk(dW;c-pe0Dm5_T`(R!%gFsS*U`fh~&l)EY zxnS6hVkT|qvOwLtLm0Mf1cb(XtcNT|UDKD$8XGbl@i2II4NG&@i$nwdp-hG@{BAeu zt;R?Jk-D#2hiM$vC^0FQv-{AG;jKeLW|qh$#~$G?P)mls-G|zv;E!Q1DoO8_K=J;J z$uL5~2IlDfB8;II^YYg0JjvqWPbU-4&i&`f=RQ`rxr8hC?bwqYU!7k^zm2%Pfrc?3 zq8Om1795Pc@IO_u{zEklU(W8QiCc`wxp+*8sWF%L37jYTDw!0tdi1rz-dgNgv>#6u zIUbr@ZDpL`<^#@kR!27Pl460&Dc}|&e9&;+e>)0-jII9;GbFQOhtUCeQEn?JQkk(LzS${ zTutGIa;xyGO4|i^ZyTUY>dqo|>-%K8*O}5E7Tvl}Kk}H#$nm05TCwrI0mEUvHD!O* zdb?r*o|(Abqdy?lIPnh^+(!D)+Wrux)YTv#=`1l8CR+imc{mZ~8MKrD;J%ug-Oe-1 zJ!PwG%j!%OC$z9HZCgYjahHVS*o4F`R53^#J)KfUMFujV*x)+qwLd3S9H7>CS~;Pd zz3tL8D`J}M@7iX^7$a90qB15vBVy870YUJWFLwN!t#*VHPcT6c1@ad241vtJMeYMN zg0tbX%lLp<9*{zW1Jwfyq0b$ zRHD1Z70&Ddw?8k*{ceY|Q)V$^m@Rmb)HL|kXhuApb32nE)mM7rZ~J6N_wJ_0JOPg9 zFs{)*b_Nk5tN~5tKcTM0Z;aDz2Em)7HDg7&pv4adk)!WA%e)1Fj*(F<#u5g@?vT~o z#dEEGVa|}^*}ySQq14vd$n*IJv5 zU(2H%83zWMRJfNt1@xWvqW^s4J#wiX@y7nWkc#AQGiEcY?S3QmjsD7|FPDClj_FhV zXWXIx>}yKq|4^j@^9H)D;FQJs@ag*e9v|rRIO89xyC%3V*9Dg>khv-Gz}BvcJUzxM z%>!JD~X473#!S`s4OEmNn?0eG=%~EJf6YB0=D#?u#;3oD&4lHlzufl zQpo`|R<#bPpiaw*lUQs9hXi-;?$y(oehNO{A?#AbR=nrUux&5TBkWC^TJ&hs*_BxYTP*OR44 zVFmPIte#)a+YcSy=iv8tC`R2c@PNLZ?|Jd!-Akg?cb6BQx-O|Kp9b|=@Ofc4W0XIP zgBvvY)u z73+hB`N8DY9o(9d1Sa&MhCL|>I_iYA6tYDz^d%`!TONsH0 z*H*{~ZeBx-M{Hh-^dLVV_9Y?2FnKMqlf_ zzC|t);0+oq+{Mhn&LEV#prD*~)7kb@*ii@ha|F<)5DI4BpX=y@aq7n5rv&EpfvHLu z#H0HbZQ&5y(U7H|_T|j3V0drvQz~V1u>W4`Z;AMg2rlS;55O8M=^)-mI}1)|F>rkS zZfjVo{kiwtYTEOu(m=pkZ139pkA2JJa>Eg$;(#I4IGW5#E-F9`{E66H*$j=4jd%?U z%IT;Y18IcGk`Tbc-nuO%2`7E^i)CbdN3{QXVeB2#9% z$ic<;0iZhY`=)Gy`QYFnpjBvwgY2Stp-iH$g*USGo;-ddX8VO6r$c^d|hco zAY#Bp5w#6U;4w_BK4es75HZc~?R&8IN;oIzo=RuAGv;(5LD8b>QlCXpJhVa0$lw2YTG+ zEL_)$E==ii79Kyyy+Qrs`2_xbw)uA2kkvy*kP zQ?GQ|b(sE1Q#P((vrxO@F3ET)Phts3yqFg7{N<^N{wklI6%@gfyWM0DU3t0C%-N;P ziwJ}XW&7CjN!mwnoaYCC7)V1%wY8p6M`$QAr=+UvkEdzThtIi-w_LN9P&e9|@|rS} zpo@{Z3jIX2$nD&tY(hZR`)X(tZaZQMA08-zA2xRo@oA3dh1F?yMp_v!&$7E*0@uHx zs?wz<`=ZVUPCTtY!1Pm^Q;(%bX(js&8xk=jV4>zZ?_A+|1>tB2v(g z$-6<5zSEFpVAz6~$*#O;1-Ni9fxSNo~t+BkD>Y={x|_#Qd2x<9lI~tQw#<70Z1w)1c5~k7NZ4gda{X zRBvcPfbznf@xB!Muq%^-3jorLU9O%Xx_HrnZm%<*_GMYFn)+ zI=Q;J!Zuz4N~S==Y`G*K+RN_q@Q3`S6p(1y7$y?(ez95JuRb3F+R_3 zY=3dAPs~cH)cFSdg#tpMbdRTM5OYD7Exf_6z$?u%y)WW+L{omtbVqT!Og9=gKTdfP zdeQ+XexC!ks~TD|>fZf5bk0BTxao1W9u6K2Gaxxs0Y)yuZY)_H57pAdw+l>f`~o?x z7l7gUxCO&C>APaF(ew1?m4er7Qh%j6@~>^>if4Q^EF$Ul6zVm#5ZWSGrbQ=}JHusH zzHu96sj~D67pSr3bsH750fdIY`0S&oHnd7d9FCR{@XotqCOgOHYuPM7u(8uH8}3fC zaJyE}AYJ$}POkW4FYnskB-0#YI?Ri-bdndR|>_hs^%rAyV)k{FX+qo#8m3`}}1FsLK+NzS-NLv}UMdunU9rFIv@ z0?=9b<$LpaC;W$n&2;tC(O$;d(;ww}x~f|Bcz?1O*n514j<@P_k<>H`2z&_b5{zfU zp2Pu{R{oN=-`xsaW}P-vYGm1e5#YbW~Qzfi3q)$wa!28 zJq~GHdr^6=tNrA6J65u)s@fl@_l-Cb8C2#)^LUCf;~gLwJ-%S!O?|}S$%>Ng$SRL~W2JZ7 z>!Y`tPr^ZFhs0_PX4?@VxH&|jDci+~kZfQ}9WjVogwpsnGx4(PaKBeaL4i${SmTmk zzj<(CRB0l22d@UM)b>{bAp!DDpoG<-2Xok$(9mw$hap=}Bds&;`u}K(p0mCGcGL;t z1 zDP}%s)29_t92f!11x2*VW~cojE07pN7j>hoBWQ>Q%_Y!eC&imWI9_rfHQQz&w;>H7K%L)zdx<&-SCqN$=qm`8ktGRU!J$yXA+3 zP0=!W>YNpX&PU_MGr|%o+S_3@t#Ftv@h=gECG$fT#o*CnjjlMi{eH!01C@qHSsc5Y3!&$?Ewq;G(CN`hZlew z&8M&+5(qD!z;ivf$O9oUaX@!QuOSN$zegl4ZGixP*|dQl{EaGU2B?(Kfmtg z@$n6N@I$>rM*7Y&EY!~=#{OsWu5I$TX#3CifkpLE7Y!``a130+As&)u@F)(bU|+uI zbSaFdx|Sxk!Go;aM7?`<(M-;;F&?kp^$*pzPu#c_quk?}Tft$^0n2jsfb-7TX+wv@ zsh`gAT2dxIduCQXAaPi<*ovx_R=AV+wb(fW(>I7_5#0+B(k<`maEv#kt~iyfoa8{@+5A<&+?c{OzY)R*$Gk82v+@oZ zLUVNAMs!eAjnjK=XbsgnCNc2l_brq_yNt1x1z$Wrze!zwl)$$(&;qEZhJ}Rg=7~;Q zQkU@uF&Vz}0T}0;l+L!==6XH3F8#^zHk+};?~1?5n?;0NuN|45;HG==9!rXCuj|XN zSKjGoD_7yM$?nQHYwW0o^R#~729!Qt`BqZ_)%h!7Ai=FOFk#+^7+w>-Q<-uUj;L<( zbq>yP^=0OJP$6?YUHO8@3*!r*j}@+O1@WMFI*LTWU)qL?U8IVxmIvsjI!}XRGeQy3 zUaz!GTvtl^(RY?QDroAUO=58Pru<^Wry?LFRu>c92w^4x6aU@%N&jU4?NH%~`H&M< zC3DjJQ&!8(iLEX1`t+kWR3gQ}iO2aG0)hn2=+c57c#BOeU|yggEhw0Dq+|2Mv#9pW zO7yU>Nn#MU`TFHtOUr>9k{fib?e%zZVhs{SGHbJ0 z7&cjZxOafM4l*vwQyZ9-s}9SGr*WTEjU^B8@|rt5>&K#W&y=7kkBPdS+louJ?YWN- zjct?4^YB?Fx8`KbQ7Cl9S7%@Gei7W)c-Jv|+iu}3+{SRi!F)Q*vKTr9LNpR6>8EP( z0opoT`G&Hht2o@EupgVWR#6?x{il41*LZ!rFmpBSnE9l(479HZ+qdZfafHb+S$?D8 z&xVc!S&V4W!#m?&G0*kK9!ia0oyIR2#@rJ>?^YDex5x1&l_wM5!d}^4yz^}O8|l-T z&J1+ZaXf%uaib5p4OY&W-J0OANw-OtP*Z4W3AVX6fRVV-Z^s8#uVElfED3UY;N(eh z73d%3t?-+oMj>?bAnlg*}s;_ zHK%|}wh-$tvP26Q_lgg1To#~<$Bgg`2r&&QlLDJRr22BM#$9Z>ZW+7qHsqMz@-0(M zG9CS!@E(n3d7#ArL2&O%;odxb9y94*#j$rfV;6Ku=9OkC|A}Rfo0SQZZ>Z?-4Nmp% zRdXHvD+mU%95DrnVum6g5*ZU3beSEX$>fbX0G%d~YYvrr@DPb?-&*LayK-W(_F5uP z$G#rV7==M6MsRj6Zin`){zKK>loJ(FTpGJH;S#0)ngNZGje}cb1|zLhd7}>*2DAG2 z8E3GMi5v;UCt>Qa&>H9UmhAVBY+s-o-<2s(6nFK?3qRjpHhfi=vHZ3iVRN-@?JtDg zgPBx_JZ^D>!%xG4B-yL|hv1Dr?_AXt{eJJp?Or{a%hQd*>s>mg7IHQaCqDnpCe%Gi zV{^0PDFEUFL@DC4Dj-kFR&(ULTp6t<>*sg&!{A1`!PC(S zs)yTixMqv!9Pvh?#E~K4arC||T7~Gsw}HYBZ3}y&=?BBkNNU5!##+$>mvjnI$|-E5 z{RafWNU|9W&u{1uF1FG%Wi`0-H^Er8@g@tNoA)^~Y%3D18#cIHn#pX<%Lq;j;rn@3prU{{;wu<=M z>3&zQI&LQmeQAOo@en|90*VwI!2${6v%C6is;bSQ6jNl4k_K~ff}XWRw*A4UGNlhR z3#dpD_Y@mOC$Rr@l(MY}IoSk+;R3=Jy2npI|NCL# znK_mc@(sVPvdG(hZCsaU?$6BnJ*&DbnvV7}Vcz~VW7#|5 zLs|Ql3ZG75d#}H?)hI>IM*|T!eSE8(HOdnGsnWcjA-PQ0$=;Nu9%IsV^OD1F_BcBm zm(0H3ceqwy-t+VhQ1mZnu2Fef2e3tty;EV!`7+~8C*-WV$gIt~9?u{`XtyBKOR`-A z`jpbF`X&JU9O+bWr=IhX>em~=HHFj;p|6;qUolvUMU%>1+_}}pt(|>F(L^+5Zokp^ zb|Yk^^`Na|X#eE=nIFNhOAW`8fXohTNm_|gyL-x;Q1#*n7{zIf(}^W&R~KBV?JiKI zeP)s71Rx(YZ8X7)47_GUPO_-42PR;d_GMNMrDnAeIota@-TEy|@dfM4XQIVTOb)(p z%)*Ypkj;r%UkUbeZSqyKUIolxt!Ljoaaa&HD?2N+v%f)B>lB+ zJ<+5&`EE#@GxMmw{|e8S z2ufD?gi#o8?&n_P{>62oxpcioH<6;CjL}Ux{CmJ#u=Nw15)-EHcctiuS?R_^eQBD| zUAyfs5Cy+IdOp_A{W_`}fe5-u@L>v<6NXe^+e&a+{p5ij$!2vJ$$8ZRWnu~|fi=Wk zOKO(mtr624B+T>_@8`l+cUSzSvJxx;#9-0}LkA#ji|;NZ-i423-!R7(t#|fkmK}ZA zYMp+$#0a})j(qo2iJN;T2xl^k-PDBu>e6tJmv1~XrK<>SMQ1tWKAx<8A5}M4?3P*C zhN?eFrk$lYF9AFcl|Ad+3Q{~4f6)ust0S4#0Ute2cJf`{z6xO%kcffi`z?7pSMVci->m%PKLH^V>enGG#Zgkj#$+~KiTrZ zk&#{Y*8Z_Oc6p!Lik-Sw>by(8mG7SJqxUEO)9nrbm-in^Xa4Q@I%l6o0dk(6)PdAk zOE_#7)JK0~o$;viI{k*pR(i=&Y1#TdNR!F+=2Kd#H{E=UtfWVa*9(a;W*H8M@kjW} zII~Hf`z_devP<_eE~B1{(Y_knvocJIcUci(Uj#L%ww6H|mx5Bl?uOYh#;AZu_qTKx z*(Q8sWUFC|LO+dt5_3X=`+s*9-HJ}0Es7a7kbTc?y5xi?c3E=r?m9IuTnt6AK!>_p zs1YrGPW?EciRCDa^yMboeh+r=C(7u!P?Ei6V;uL*vk?K{Lwca9G_OWJ6Yg18+EUck zasO~{O1`hqbX``kSNGZ%QhVi}b&3pOxqwAQG=AjF#LVYr0JTFvkP+p#t4t*O^7Hb- zT@Lk4YybZWg3MXo4L>A!nH(HrM3CtAXtN2$3$i<^#ggfTn?4o}7p4fv4#Dxnb;EHxup2sS*H~B+=BjIb(gMDS_|#!sRQuB*5O`mIB&EFQquDQ2T{1Hb7hF9$IlXA} z9e<4I@SR@{w$58ji4GE^ZZwnHe3%T2ISNAi9BU*EG(>meTZ5jl|JwWQNk4esSK2L5 z4t<-tfviSYTI`EQFgcPS_Z+}}zjC(xxYz2Z+*&B3EVYXL=(pDChA&23_vu*Y;|byV z^srh~=geGBW^~eb4-}rZ`o_LXy@%+`YuT>X&CgpN?zvM1zci*|(G;B7?d1VQB6!Ib zLRC__j zAF?EnB~LOU_~@CuHTw!^D6B(;o^YJRo<9mGP<8P9fiReiyR@yy6;I!CQXFveMUimX>k^6o+!Pi~{_rnWgkZ&$mf50Fc2+#c>@4TD%% z{IZ^2knki3#s&)9u08)0#_L%frb}cvj!~ASG0X3Q%E_?bXiML|cwVBuQ~1eVIy_B? z`Ny1QmWam`@_zPYPv+97+c>$NX=+a_F z9W&LwC!wK39?N%Qpl|h44gU6+zy}u=-b!@~GK>qx4=l-VbKh4z<@uWxa^J*?OD}gg z{ZTZM1IrTG0KPh#Hw&fdS)M{LlS=0MQb#aX^2?9uo7B)pn_><1dVFZ9^pCE|nsGeS zx2J`DtPHpe5=7DCf&Y)W*@@(V!_7^17%1=`D!ESil~DhK7I;b11S02fzD5#Aoij$Y zs3kS7YcU2D&ykY*!mAGRN*d5r#g*y7Dc2kts++vV8G3I|EBsc}6K=oz-_OQA|5{}l z2r9(xpwY{-4Kc$ZFsF9LimxQMMFvi!v*h^vgD8)X^PV|M?UgN*j8fau`HkG)E{(Q~ zEeeG{*n3EZ_et7r1Z1~O1V2%`0%~$(LCPVeqk4{62AVH^VJu6?kb3zrp6UZwQ1aL9 z%3aK*?rB}t4tGf052%%AK+}XXSl$ZW5N)C4&+@eme(f=*j`QJ(9>Zt))t|d>^F;`Y zz`$9+=_MdVII3S@to5{jf@S4_2Z8JLz_-VbV zpvT|FdJYJ=<^;v*aFgh|1ZZRQ5n0C%Mj?$Av=YHSrLCbuk;2+4pKCvkF^gqh_pOzi%(;STui-q zt-pD#gh4+H$mHGv?nzqHK+z`)E?!1zIKR^x} z#GiMv0-i#6ZOjuiK&4P)`v42^;#pnyEHPTVP6q%C+-V9_ud@bNGXyxmDn zER5Gr7JrKLq6zR-2zq&mZ?*Lr5*_ALMADJv4;VI(=2p?{_aC|^4ub^ruWFz? zX@1yXCR;fJgyJL5?arFfIaz3fPDF{SrG*-HzG~f{PhM>O`8qgNd(EEIIv>q zS2^F~$=l)dwaD%AnawA%^BAy7hyf8gVzt0abm4T?SyS5wrdSxScZ=Z`eG_E0_U5q~4%Bw71K6%Anv?KIfzmMhW;*+%dv7A)&t8?Gh zyk{sWX!3TbxF>sHVJL`YkgSA&s(QZoXHmtajJ^dT+zshZa zlvsIxh+CaV1mRYH%CoC==AsgFTm^)Q&dFggwo&cu7zo*F5Er5QTG%s5H_NT_3TRM% z^xZf-jB_cE6De?w7=eFWqb3oX@3^}Q3qO&wcW;2GN?fknsM@y6Faw>QvV5WHcL?Xxo-6MM2jIiyd$R4TJ|wMe z)2e*#Xq~9O?t3Fay3wIzRC+!4Ajg#VJ^X1l1Mk&8M=xK&H2i#&C*flh2&CZ4c)KBI z`{7yEKU5+;0fIDS;Yq|b$94PFpU+4^&nhhx=WyPvo(dQX_N5=mnjbM7%smQj4{iwx zFEovMfi1cUXgC+Ola9*pwQ6s_weh@9i93|%#hHx${?#kMU}ah;<;w9u^b6DPTl;+b z0(5}GjPc!kZAT$wgLP!_osZSQa(h*JjTR33yln9r{3yJm3?Ha-g_$T zJ=U(Py*oaJ_$xOr(>iYxVyzcgkFLJMd$Z@52{5=j#E&~Z&?<#4 z2D7OPu`x8HWl<&dX@R!R@a@uz1zY=z74yvI7`aSX#jL}+sY=_MSf9h21_fMqioScS zL`e!=x$I75qjqLaDljDl;J5GfgvfXIw`mfk2AW;Hz-Wk zO<&nOp!IOBq^92Id>4XpJ;fG^OA$RGU`_j@Gq`u@r)jc5>v;jHN79v8kf0@93dPJ- zT>>;l2?up?5A1N`nSMcjg}U{tv6I?bt6$Z(tFexR`8N@=n27a{dcFvfo<)-esA+ZVFnfUr`L+q=s zcgL={bDa#%)y#$M6cm#`NkE&chbY|SN_4s5s0Jr;q6jd*SR}rfa&ODFYXS&d-EV8{ zkW_we)R=znO4q);u9`ff?KJz)(cX-G+)%8(P^L`s3is8Mcb9r^eO|4X4SQAyU`Zbm zCOL`5)qfoB1qZk`&$B7kSNcGExq27;$l{G9d)@M@+j~uUXOtbyHSKaS!byZ6+Bdsr zt#rAHWV9*k$VQqD>srH~=LGa@CpVmy|GIIx_*?(pz;aQ@5gU*2aDjc@%JhIL3)FT*`dXgn$8tT|4f7*liq zBV45}TO~mP5$h%h>cLvf1iD{<~aHPt=sHHILEP7f$24|`5?}jPWPWNk{KGQ$SjN0A4nUs zGQhoWxn%JUBiJK**W=ry%%gL;9i!nkjpFu_Z^Nat120`<>A`!m4RKPJt;NzGU%1rL zh(l#|7_yNMo!E~s%dNPffZy5W8K8j7-FZWoxkLHOP-$c~f@-WZf1_6uY#-V#K4R?p zNgncBsef3jj>c2;LdgftS9YC{kMRfPC$0bW%xqp>aYVW|A)#Xll3HC|g+0tFh{64_ z)Mpp_KL!xFK1{!A_w0*wb|PYX;5cqVk-K?bRke=~2dmVp!B)FuMwh+Ff)IQ!elZeh z!D{KG54u-F8=WQBCaE_w@aLf3xA14-w(*8EzJ95Og%$XOrUrP`k##WQV_26T-r|%? z1w>{5OoH(>C|&e#H8Lq1mjVpfkAR;-O&u$3TQo|1|KCo~K{5|B{)g(`|J4bKDTrqs zYu$(g!Hode$thwryzHpxkS)ASTR+U^A1Yc@obKPJQ!s+61w|_5|8%aJkB}3hd^^o) z9*7cA(_G;7FOL)SvakfP18?wu_)QBbtOYEv(a!RJs1{xpke>){uJTgCOn3DtrNHwP zRP4B83V;V1Vr(XDdSHalqzpn=D@@aiD(O8i5`Zg_eE?0^-p zoQ=ZD0JVN?%V5=b0Z5B{k%+4;gGTQ@n1rk$BCCG)`#82rQ(GbhUfuIQs`)CS zZhux;p;(&x%vvhlQ?o)>WZ94J^4{z{k8%kUU2vYGGXCELLm5?~gPq~3#cQqc%7;BX=@;D7#te-3 zmQxkIxybhL2{K(dDZ9Jn@jPRCXW|P%ZjjwfujBVytN5G6QPN6R==m0wT}0SJ@gqqz zv04|(J&&DPb#B8y8iRp#_m+g1??0#Z7l=64P?7Laj_ymC*akx^5 zNHrdz-NJRCA+6{~m>L~n4;=wQU>;tV)jLN%#{cZ+SC^omvhJpQ*Z6DY<||J+QG*6* z*YA{{X7|cd{l$Ru3%^6>ky#qycw@1~xsfUe-=iWQ76U`MOIHG~8ELUxSVkxIeI>P^ z^-1#8jwBkZtlcgqvl1NX#p;!2x^<2SE~40{z2&5D@T|>;@4qH17|B%e`7Ta^G3p|(XuNRCr32+=*d1{Ak?9)v zcUkCIicYr4KUCobbLSr65c;v%uJrCr87oOwe|2k`R6Txuou4U@6oE;y2*LeVi$MK_ z#;Jq3>w_aZmxKpa{s(RE9n^IDuKQv`L=mNTq^mUPN>!02B`>HT1c*u}geE0GNE8L6 zHvs_w=^{;PD1p#IFVZE{geE1BPy>X#JHNBmd(NJ<_nfuXoU{MPe1{BS^38<%d7kUO z?$0&I$57t%Ir%;f5w4e0KK~V;C&9KIpF0!j;h7tGm+_yY!+-U9pP8NV3Q0VQ#u4i> zab-c6^j2i-?x6FT{Thffjwgu3P!YD$X|#nv^95g=~4qZROMq$1rvB)D63Cj zctS}-_4cK#FT3CEEwd~WzHB*oS4xGx_#7`U*@1QjyyHPf!iAvyz%dvL%qAGGaefRi zqVzt<;|g&y@2bhA={2QU!Ov;l57MKCT0jEXC^aoDX!~l&* zizsBYvmv8Zw_m93BY94aAumSNKCu_x=ZVjbzCLe45A_qLfPtGi8nW)RC{~-s6PJn_ zOX>&@|4t06HjgrrBU7eAHXKJ1jc-?6ujT6CtTPka`*1$C2`j|yaF;JjvbmN74~4nu z)YTSQ^By~f{L=B~4bIg^1{4c05A{|aol9)uz28=m={x-1YO<1)_cY3IjAEQmR_(wT z;``HNYt`sSMeEPy`_+c2&!vebw>W$$HB95y)h|X}B5rn_(8O(l_0%lZEGEGYo}*75 z<+WWcwf6u_3s~%QZj6Kc*W(Q?`8&_1!(QG-&lOT-$yX1&tLA%MJX4ZfAb-=*gK}d| zY?>i<^XmUiupE?g>j1~h1y$%wT8HFq243#ymsV`ED8Mmxg)_4cw+P%i7(C zu`gTv?3#m`2|pGrt$}iPt93^bRp05}&K!S`SAtxsm{oH3=d7i1il6oEdqkhmClJ$_ zd5glAtHx@}3{nlpe7=lq`&!D#JO%drfYt8FTw#F?z7v$9u&C158GqB=EY%eIgY>UU zvB18Vh*@s*rKT3=iVY?YKvddXP(H%_yYKag%^ys4&^X127qYaEo-^ujvAn_!y7pmMgWRinnn=VPWj%>2DlU6d5vp@zojlGID)2 zmxxBVfasxWxYTfz&#vu8qh`xwhe?#lJEhKcH`ok+s_VB0-kp#(zDl}1G{2_crs`UZ zIzRptaAh9aa+OJQ2hjYBMgUfm{2U-&(-86^c{So`{-kTU%5Ft_{Utymxtb>S6nGHd z8jaO}ZaS-A%gR646=ZU3>IV$4M0GYaf`jsc81GhJoNJM{g#0>xr8su}2+;v*IRrw2 z=X?BuXd0Mo<5Zx=RO;Hvk+h>As6Au@76QW zB<-I(M-kf-ug3>t%A#)$n4K49W@?+3MrVB=s8%1Pxj=us(gjbH!i8w;8&(`qavdT) zZp8@Z#3nbzRnJr#ujpMLRY<*~aA*J12!G-!C@ZE8Rfgy{H8gC=PAL|p(9RqtnU((^ z0CW3I{jURDG@4jS?kd{He!CfY?6G*cm1e>LblF~pSag{Exkpyfs~;;ru=G`IhyvE% zo3ZMUt}&GSVz(>(;hxW@oma|8DSfbVK`mPE*fMZ_a^q`F%+$dd_)oClk_Wd7f*B@K zLc?{ZmF<_w%x$~#-)MrEjy(4sFrmAUsL$RFMJ#<55E@14uj80mt)L$s?iw$IZr`e- zjyRoccjVK88{c-rEnMWS7Rx~FPID{E)YwB*p7+F^!2RPk{|L&DQu4;dd| zb#PHr(;Myq0B2eapWL*_`+7e2p0>8K^d)uQr32!nwGWCt`~5Mto1LmCD|ORWoRQ>; zfmBjmV6Lc9H?e%ED6eknjC0*%{R+qgLZBkb+G+i=*Z(*Um6ozO7oeK$SEHD(Y{F$z zER}g}u3;lYxO)9m6x!dzqXD&^Q|Owk6m_NoXons#H4+8DZT533UD;7A(U*J8W?!1+ znd02*aCb5PvF@?SIj*6Fu6mOq-1~fMihKq>B?kV~*2UED&3QtHh`dcd%e`6Eu)cT( zdX&jd2Z(iF>!T(;*`HQA^-PJBy6U1&tnDgTH?tY5fU;#5uZ*Y+uMezJ-X>{krXiH! zW+$MUhp+pL2Bld;)-SlJK`Iqb<=s0r#$n1!`M#QQX?|$(RqEE z#rN7?&#gUMUpxpa_>hQ>kx}H0k$qWTQ`uBKcAx>D=ZeiPoU?(Psw>FF8#JJL8uaHZ z2938G$}&%Zc-YJOc)~>~&f6xr&Z@L$Chfa;t6ktsQxk0PrxQ5@g>)7Ym_r8+hmmHvEeI7LN z5$hGBk71dY2YF)aCD_3a$^DDW9In4y0wrQ$Dy185Q(uA+tqV6cuJHI^u;224X*p}jY3rO{ob8w z8vG%@;>Z$hR*+wqs^d$;BNlnI{5i4f$i7JtTjIAc9aCgxb8cKMD6H$>)KK{=3=cLR zlDjLy5`HW0%XJ-}C!rUU-aQbVdJiAAy+I29pl+axBCwx1Lh zS3vy%JR&a2Z@3&A!EX<}M2)>fO+WiSy8UL6UEQ8!>w-hjL8}%HgUNOt0gE0ye{3~~ z_Hye#V=kMGT*5O~Gpr_CxvbO$`Q!8v^XD&Iy!5C@QpT6xYz3J=k?LuQYNKl`pY1kkFUe)-^HbXPl6l-O@3%pJJ#m# zLmBP3ywFkDUHlt!)e1W(35hluzWcNH^0wqxzj6SM^P{>cQEky9%ttMlft+DO{Ur~= z$`nQ&+BY=^I3*2#W>)F7{E~Gp>2>R+P@U7K9r!XG6cck8>PjiO!Dr{DCD_7y9T^Cd zWHaA3lsI`YF4}Zqmv9OC;B}YF^z+E|fu#LLbgx8x%FomwWi&5MeBoD#Yf)??E?9=} z+qGj){;*gBt$OiL^NG@1DFGb!C&q^)wA6ki{`|?kyd+-!-@wU%`>ASB3u|&(nKO}Q zaAGi9>a>EaBu~aIL98Ik`ik+Qkp@wBl8dg@`%d$@mp&8;k%lDMQ?kdC@ z8MZT)v)%Y#!pxN@W*Gs*Cxd?gU9VMkztR9F{Av+qLg1YuVPe^ ziOW#8u%zM8Z*HKki8oGaat|VkZ4~Q}SvTWMt23ADTp`eFS5I%0q9TU^HL}n> zuN2a^Te=CmNpcflt_+#MZJk{HkV8v<7=T|Qb``i3ZA{h~J5{PT)jP)CUr`ZTdClJ* zANN!L;qO3ct;4`6Co*S2M?fL32Zysa$RKx}WZ>#fxRtV=r3C)@t@hVD+?S^!iP~vK z?^2{SGIr6_^BIP#3-GD!Y=<=(#YGLYoC?`-3f8E`Uyh`~j;?HGU27Qk`6MgUJfWK^ z@O=d~sO*yG87BUv=L+j+8VC_ABJ>w z4G}6tz;V^gLSjB)Y_A1->^~Ehl=h|7oOk9blJidLBsZG1c+n%W!{2Eb+OYpPHv1Ym zr-WOmp`rC61hQ!lys4Qu(Fr^q+UTdcGaIj|i}31-*@)G_Uw(@B7VMFj%V|ZYtmm|*_EqJJRi7MbC#bcq4hCL{oKf9Tea6v?d`cZ)4u*3?`$-GsG1MD-aoH82Z z|4nBTr!^4#`xGss>OZKg#qp;Rjx2IbcmJSKdWbai#v5x7ve3Onp1qtIhQpJG?~)&! za=$I;!DFi@Za)J!&-h9&q&E*vNEvCA1^`#h4CY*RjLVzTsR==Nu5oEoT>Tgg=laaM zU1H1*$@#$JN0U~ejU^fd z<1G2Vp0d|Ns)Q4ZLgdw^vb44cC7Uz{#ulwZ4awYY#;dJ}WtrIW>7R+}ij|csH?_iji@C{NHQLL2q3hhM>z#r;QaMENm{O=+>%7QRO?NC_ z;^zpVOeo3lR3X3&bJtk?DL236d0p$on3@~sBuU+iFT%S(WWDkg8}3?T#sG&J(2YO6 zCBOBn3VT_Aj?jK5Ms3NwJElWdF2|50wlbMN`NT23%!s=p){^WH;3A$Z`%xrGkJ(GO zhn5@!BgUK5-%{g-I`)VrzHSgbwg8P$_7S1+6BWFF(^>sZXAD`%#wK;eWioTT9t-4>Qb|E~HArwbsZl z31Z~2YiO2Q&?T2FRuTs?IKQ^&FyVmRSa zF-3IZ51{`&tqL~P0jrZF6=S}us0mwgon>=L&|L>l!;}_f$$F@u zC0UlUCW0U_fmh)WB;hel`2<5Z*Kf~Fed{&}oVrAuTFj0@T}-tSn~G9@P(r@Dx?@_O zh=D06@)YO)oU84~ac4zNOpNPwNraCE*V~xTc3?IOJ`d zWaYW!U^m+*M{h?BGj%5Bdv!NHR3B)czV`Utqq!*o$wK(>yvR=yH5{kmZ=2C6a03v2 zf)11RC{s)VAKn)*yuD<(vL|XO!=t2i?`GJIL=i4K8~*OCemuW-cc)=F`H;2^fBws7 zj{c66qIqrD{nBmGFWe%hhBYO!UuZes8J~Y%dW-opF^N6dQ94>@mL}xzQ1)r#5rT82 zl*0`d=mNg^C<=ZPX3#JqN@DKF<|OM5fzt@?i z#DXnsYG_SRf9jr$j1(@#qLnu-p2hP%U)IDjf?3P)s^FzR3oJgpQ;>7W#pbd7wMyNH=Xc#> z3NzMPS3b}g;4BK190t%G9yT5)0axaBHD{+N(#HygH1+=J|E?td$LBPmWaO#-98?cK zKxL_#s6jBlT<73u3@a;kJ&`^vEO1Jgcz%|a01@-?67O^RG8@otzO>*i+blj-k*gq*hd&>nrzwA^Sfw6_xz_uPpG`G!@w#8I2F0% zaI%0jhUDU2pM9@&B9{nc4d)=*RU{&2Iz$X!I#MTIurz;1D+FYLQrt)Kc2PecQ)Y(e zDxUv>KXBF*@?ey1F3c08h^B|-6M6m_PO)++g+$9^8BK;Fo zo!p9f?U1H66NBwmKtuJSo1LM*Pw3>qJyZ?~eIv~*46*A{Imrr_KFo1CeDI1m|Nbn2 zy;nYh34=Y}J>DbL80@}Mxc0WY!jJ{P+ASQeXI5I2bNPF&|MHQhBwTf7p9iVdGDTs| z1}$ZO^6{8nYqwR%0^af6;hLYx;23{HL-vg2H2I|{-0iN-yj8ax)AqG!p0`R3Fnx-z zUBCdCHd$Qd*QWx*;)<2C#i$9q7?)sPZT3#io4Q+D?{W2?K1*RtbeATpq&3fPFca-M z<{48h!#d1}WfT=v`ig71xHmw{h-!r=ejaB%FP9&7Jb;wSGyEp5wc7->&tIS|rs2^d zNYZeG%-Z<2x>35>m)A`REQ(NqfUbbGS2xcO#b|xomx-=p27~*jTF25Y2=+7ImK0-9 zf1IS{m5Sy7U zW+iKzjS0=D!ovIBS|tYILL*<$>`&R9nJkkS=>ua8F$9xJcrB9&es#}p3JL5Fxaju- z$WS%NZM_>TIYAYHNl&6oY1Fz-I3Gz}fS+U=2ljm>TfRSBH#6+vDYBQG!_ukk5hh{I z94guKO27Qs%d;$)e)ctjCLh$J+Yc)n5*w*#_`5<;tIm~GL{e%LH!k{&}FlJ|J8uR6(J+c3T!R?u+{tA!9j%xx%gCleO~L=R%y+2sbbWZmjOpA>CUKA zb7}0w#D}6W>Cg`P*ncAm_#FP1hyoZ%7s2GyKXVH!o73DIsS4EEr}IGzBd&fbDpy=Q zq;~oAcsT9;(5R+gHR-8i>D`?OZgODvi(p1DGn>nptFmAE^Jij{vHg6UH-U3%J^cy> zkhgppBX*3BY>IA8jmmdb>yjG7^e$NfJi%QpobX{4_&FNB)!kQ+EyiD;n3d_3Pj+qd zq5c2%x_Bt)W!D(4qO*nbO}k1)BHWWkX|2vYp(T z956Z3<;YvrHo6<_bdWOsL#eSQDKE);Lu^gu`hOH?xx&b$%TaJ)y(IuM(6xnaoMM=- z4fcIq{`>cAVVe3g^^u=Jerfqke04|$FBx7&xOKc5;3h>9O1cLEFFg#B;{FsaIS?jer9jYvz+EyKzAPRSA0nCj#XFklGn+R zcU6ekn&#M@l`F_rv^;pACK%zqUHP!xpL+NFv7Z4ow>8Z<3_h5Rq-C)YnIYrQZXnpdOE zr#_#UQj8<;=tuSX2jL|jl;`Rvziro3)Htd1q~m9T8X;JqZ5&EsANV5TmJ zJ}QY<`F#JYBpa=1u@_v|x?X?f^StDETzc%^l1;ukHKLr^h@DZ;>Gwr!z9<9yO=%k3N+Icr-tfa8zZrR*6D88OE_fo2;?~S)&9Q$l=d^PAWAs ztW)i$RFt#+0^D62Jb!zcgLs9lYx$#^r3%lVq_{d1{3^8sA!*}J84wvL#)8<h0|bD}b6)qQ0PPB0U$g0;1pM@;d|= z!Ax^xcUoyCF^8f|^JAX3OYhX)mNs8F`em$s@yj+p8k(}V#OP_)0hw*ei#v2`iNo~X zC)xZY{(zQI-ZTdOA&>N zZM4kz~~rbs>Mm5%KXj*7!g-%i~cC+t^VHR37>{V?ax~HfZ2A0eTJsZl)K$?n@Tc1*3_j0 zn21DNt=y-xpGu%elbL)M0iUql(H;jH*MWq`!&}~MxaJF|!jg#&EWA;*dn)#rbZN*3(#D!JNGc&M*MMYM<4IcP7r-cD|4i1b zsXFoGQndkN+iH8&!vQ?gkB9Me9S=l43^;?TC!3hJNUw=`)}+wb94EKB6c$$=zsSR| zCF6*^J?TuI4N%r|J>B?wTuGnEE4>ml0tcZVoGA)eiNbKY55~(hiX7LwazVnK(L3P<+LQ0M6`TNtWy({Ry3G!MK16ZGsCfV?`t6;a+cfb&K!R$C%$g+OkVXcf!a{tF)t8d?zNC#(fFQ zFEXquCuNr_rw(p^YgF#KM~94E7mn%E{Vw)m_RB?YNj*Gh>&Rg#2CP4>*k2;ZTw6M7 z{1$1%(kogaYU2LdH$lCma+$#TGd?t-Yb^b*ptn2z2;Dr7M1Lf3zj{1<BDlp$C~#tXc|4p?8c+T`m$2o2>#r)YCt$R&ET>S z+&NzI)e2WEgG7ZVhD(T%y!f-{^6c~GNGpW_I6ZU;V=JS1e~e;_Rc|Z`_dOg)py1fu{1A9p8(4z-*1U^Or3MBZc@K+(=sW4}B@(d4bfAn26m>>!_e^qRZp zg>sVz>}?cYvdxlUc$P3JEC)~HB*2C1Nc_hW_Q)}H4dt&y>n}Os;X%`FarRetlZWso zTd-QEPBgQ6Km|+?MMQya=GBsH#v0L)h{|_o9Y!y^ZbdpDX5apVEPWrQ62i+R2G>8T z@z``gbnTdAoUi)aWHU3my%3sOk+6JHK`_KaSld$$EnIR{BDPVMzPKnG_hDmHeciT}bEbRORGU@e?4J{KxUcM8$wBv-r%gy?{g*i| zrx?Q?cq=hCRc@Bkg^Mfo8k2`aYurtmCGQT7_cdONX?#i(j1Tn+G-zh%wYS2rQDO%Sd>>NIHK#=K#Gzse_vu_W=JlBBnuxzf`w z`FqVj8tm!qrbw**m|h41uGu&GG&T86ADK%8D>qXuAe60A)RdzuOrIPPfICxb(-a#M ztB=xYoFXXBJs#Ta{g&wLzI>_(se8_%+p8FlcKh!3A;C7=R^z{k22eCOX$rJbrnvLW z1;SMj1tF2Q^Y79219bn>BHug^+%$2R3-0a_{J(!$k8(@3n?e+bw_avSEp*W8m!&XhO@8ffJdKOj!kwGcT6=efXpbX5 z^o=`S)B+JJDrk$^zrDRUvK~Zh?MvRjzsVbjvInWYM}oHy?H5WB!{<)^rVG7Ir2mOn z`PW})X`$_){KlvOku))}?0ImeS<6l3Z)ps~>Q1^L4UqLVcn6mH+VlLu>jMvFV@31;w4?>#}D- zy?@i?h5#?`fmTDIuXopPU@*y5!jWPOEx)P3JHntB(6%w7?(5AI>Wwo+&hP2a{>=r9-ybn-9+Qw>PX3-{je;{UU^fx3X66hz3kp z7GVX0B$2M`DA`kKvbWYp+nY4jMm`#4%zT7XgwU$B5%Tm={prJ3b$fa!J(+6`yB5wO zmzB(22lu_u8Rp*mXh?F;2w_4kP?@k0=kpl1_V~!a)di3#Dg%1afjkFzxdLY56$P=~wH0PjABqQq2QH zF;)FHq}m@W#p)RRQmDLox8AJt-IY?X>^u1Ru?h-*um|qX=uZ1ppGMIxydzkr!K^yg z`uP0^0YOjxO0%(*D!;%AlM)iLZ?hOubVsEcxzI;5=7BN*IcsPp_&w+;CNMm-^uV7u zR}Eci-y=soN$l$K=K%^Kg?W>~ar+5%+X~IsxNmn{xD|t#mf!12uV^o^(ZHO>^T4Ta``$+%+cVK84bLFCJ; zMBP}Xp{>p*Kdz{+MNZW3n=cHPAZJS}nkuvEruzE}(Q(GHw37e*e?h$bgL?Uo*JIhG zidc31Y%X;>Nt)lHg_+14URv9f5iEgW{S)pMY>SVZ)XpcuixG}o-_WfNmt#8$ttfbt`sz3J`U$Lg;~EI;>*_9!F!{0K1npJe`G z=$J_jISS8z*^neXSK~*b4S$QNx-}+Oem?dxbg%9{{2rY_Oq49dw_ZznNfoIfU&yC8 zHaCw`Z%spKsGT`Xu8m7PMZ0mVApQ9Z4pOfjKVAe@@G!bdoBoeK-{%_70}s;7a4y^m z^x{n+YLej57CdLQS6z3s=nKMX>5;Suc8eLjfCwJ<%=~FEEnQErLk?`JCpO*>zU@+8 z))Drg;U1Hig_qQCOS-mjon>{*L>q4mwizq$H0@#3vv=>*Rd47JtS-cR zUk9G^mc!bB(br)%BniVYP$!Rg-clSb2Y9eN_4<#n!;viUNL{dRaHpG}plZBA7PI1@ zPM(y*Fb-PrwISKvbg8hdkDE1DDTL~1DnaWKAP@wZbSwOBUnM{G8s zi=Hs!A^tNvXVPOfL(l$yAgHnTkYyKlLQxk=tzEIkwUqSFzk*Wd~#SfML!Qr9B_2b!s}Do2lOAMXt-|wpX4RfY^{ziIAAGtAW1ckrrO%ZM6|Gn zQ$=P&`I*hcWCnP2R*!Hcpd5@~MX@e!(oZ3(WfkYNpVcFYhgCiq84DDbHy)B>-YQM`a?2e^Do+ffQ78E>b0xc=N0r%XSLVcNV= zs2ln40v+APrJ$WI#C4cLJ@GJErPElq|KXUM!>*gJWSO_I(31b-Fwq8sH!xKPOBGye z8+}9eC(ZL@ZkZh187oG$JT+el^O=LkBYXzK>M}WX*yF_pdSm!8c17)yy=UOt(Hu#c z7tKef3k!}Zu>@c2cvf+*&&>Ae#KDBs$;+i$#3x%}vJ2_yrcHATY7(XKz57;(i+D3o z!kk7%9S|@qHMr(AG^kN!m>{$l9rP9iXp^_r=BsRMR1GuNPNvObFnd{RMWVD8Wh0Kt zaK=wJMo-N~Erx>SJDvxDn_kW_-q^etCEWFA<|(qLm4-za_^)9R%xXUe`oPVDj-~*Z zwCMtPbFe$w5Z%V_utk74srQ8G{W}vF0}DUB4;{u-lgDt$hsD9=Habh0=XcD;uNRk` zRZVV~G8MFDO)_msjY?zz5z;R%(B+4CP8{+F=(X=Kz{EySkP3svpk|q!PCqq*M=3$6 zuo)t4pELk9n=A0GHjj6g>#7WUop^;UCx0fG=6jw4)iz-#WW(3a5Utx*X9chqKcJF} z9#Ou*9E6%P6G{6>-U+(e%EafN2ILv~E=$SFC0D4D9~Eu#WJl0?q3RPAFpEML^Fu4h zLE{26(s0nEw=n2)f*DzV_fi7H?laRd;K93@UqNhI*HA*aWH5#YxE>$F461KJPu^-y zTM6_3ipW<8i1sTvccjbacfKCY@F)9nX$u!La;Kw3qSoVzEp5}t3SzM&x8N;RZaqJD zAm{C19C`C0OHAGh*JXAlTlsxzi^0~(p^hKTT0Z61=<{b-v=T)suQ-*4$xqho+yHZ0 zkk-eP20L6;4Al&4zHF~HaI-Kxejqp7s4c#w7Jcmy&88`;M~lLlPU;enu8nb10$g8u zUiIm?#LMF?WM#H(HNR4~XIt6f;oaU_*zM9QTc`^;FbT2~aj4t)*TIY|`BClI@=N|_ zomCsksc&9jg^`J?0q4xQJH6`05+C*|?y$fVh&1aU>-DH$>Fu;{3t)2=lifLh-%;v$ z=al(7s~H0b>Gb<{9rw;XE`06<)NtIHDDgLIq%``@^n`+S$`bweKvTWJYsKxWFYWwn z3O=*6e0{bV_pe}$v86e^c2><2&1{!%n{t~hGczGfOL}Zkd`4|yD*h$_nuQA_petNk!mjsAJ-MNnD3|-Kl zyq>+PSNPky!!u-eXctP{@!UMEtGcyfLXzcGhRv7R6~V;DsfvS&nzzPkL7(azR;b;n zomDo5Z@M(%M9rDhM45iZ+Xz1kd!YVx+ZlbH!i-zr@2xpw-P(0QJDf@em;3Ef2iZm* z9erJMwH|md-sh<*LYQsM}Ev^|d*wj3-U-kB)cZ8YCX9AAcCqF9k+V_=WI zy9g*GcI%KX`u*#c5L>+cycvotTcS3`l9*twz7JB2C5iI%4=e1^nFKd?@mK}Pf5%L} zw~h;;UMB-vRWV&<`7@y2V&lux@QEL;N9j=*dcyp3-BnBV??>8X;*#}-YD~exC&kXmot^!XUTK-Ejpeh#H`Th@ zg*MaBTcJylyE-I949tKCK6EFpX1@L5IyEmf20JrZqq1Dt?~fW5?li1}JmN4|y4D?} z{;CIae(hRkP*`daOl^$H&_=SWraD5{g#kgJ^?9ZDc%Lmf~ zi+;(tfqG9Le(i+!5kzV_j*h*{IFko`(`Dl*dc-DtT^SfJS<-)|_PilMxM^I*L+6{R z&s(h&B68aN{%QzcC))9CZMuUaCeWgjpA}~ZdqN~EL?4t$dawW#LDQ-x#rQj~!Zw^2 zG4PkOKkYtJsd`zCpoT<*yoJx#^ef zSmYvKJ^y^!1#{cd`_T9dD~m>Fv>aY>P2Wykpn?j+gN)L(R3+lximymET<3DC*R1XN z17bEwA}L@rJ`ejf~0_u}P!pnOHl@m15ZH=Cyl-#Oa? zeA#&|HYohaFoo7l0>rMIuN_MBJ47;^yS$vtU@yVdb89huFFS;W-T>yhW(iMd^3m+b zfvfn%!t%`}-Z^(Uuh5HqN!EQO$|5gj9|r&6ypM03Q@{bj_O;F#EDp-gpBnz+>w`7% zn{8bEDF1Y{!SM5EqmNB;ef6#Px%f;KS4`wK`%+5`Bq>Li@|UOfRQ_am>;MIHh+|JC zTSTcnRL1yuFwbg>lwGDJ#WFbGI1rZx$VSi~x2#LVPv3GH4_HmWXA8JwT_9>jR8X|P zZ+hgT>aag7ZRT#eh}9JcTWgEBz4XY8moC~#v;MsI#*)9%HbUhi!fE&%MQNk z`J>RdV_4TN;?vaRCj035#F-ACshlbINhZd6x-hY~=>+wA@arJT{Sg>1j^NbcugN+w z`O>GvM&+d*XK5lw!nb761<@$+mp#$eS_3BAt{JGWOLIz_v}EnfsW(g>4w1v=-kVP8A`x-yOVBqz}-qg3Q7_p)P_<%OC=y0NEM>!lWv z13G~(ctmNUbl-?MMR5N5k?|pNNp_E>(V6|zR;Bx=y&Ca933mSocmLV-KN0p~J$X1U zo>$Idf>kdDgC74vEM{kMH%!53zG{XVL1MJTbj6ru($ge@u()%wM6CXdmxJJKkBo~q zbajHcm(aZHTGwF`-^p>nG~AfmUkb;jX`+6P3<W%_L{7kr z7o*CwFzLp~4t_mDRK2k;45AgjB~y{2pCU#bsg1P3{MBPF+*j`J1Rt#3m)^k$(z~?W z{DAS#lKE8zlhsr&a}R#XR+S|BzRr8*y8f+KhvgpK=o?j62o>qfx{Z$PoB68=HJa5X zU5d-He4{N?-x$$5z``P;w|G4tU(`r$bB7BVFaa)%f3cYZk#exeR1_Mv)_G?N{T^zt454`scD5EqxqL29$u;~w!T0|1Tp*x4M zD%!-%)K3)YyjilWb>gnGz~MZ<;j;-Ww}>)PyI$Y)N$F^BTxQ6(A-?E^a6xt%4_e|y zQ+qY}$Hr!-95ktuxM^iKk4nlnJ*$~1=wpq~ffYt;U3&4P`6Q{80E<_Cn-FDr)f zC65{djvXbDw&Q7K%|&?&J0~v1#51fdIyiN%fMz^8uy0Y>J*_=GXldU0)8coQOtDA8 zA55xiNkmw5Ppuba^jEO1!=)N0-10rNoDE7>&gL^c-zHqMwD~co{2pYqkU#ozR=Vk8 zz4LF~?rOUa$CHBLnj&PwF4gk9LT7>7Bff)%#*oy>K=&M`&dIW`Z=UOlU4JBak1z61 z1mhJ6I&Xsl#Uf*!!-bs*KOVn~#2w^Z1Bsd&ruBPmacdjj*P%PE=tR0Wb>p~lx`xHA zJ_UDww$kQyfWkZVI?;Dm_~W*YmY8H)7nrCLhUqm7U_i%~cA>Nd_UiI!bF1jRIz9nEc)56uBM&hnMO#=Q8rj*6SE)LUZ| zCEV-$4ZqFOjl(F7uNLwK_S9#RRo?>$c})>l9`xU%OR71hd0e6?{CU)2>N&F35~bG8{R&XX1F*HaI`u^uw;^n%l+WmMU@*$#D!fv6@gM9$rdI z)|uc4fcV5P-nYf&guW>Jl2#CNUVQ(ubfV4UFbVcSXw>1C5k)P=^?9jU;#7q14*l%9 z*CwZc@R+m^7|3hUD|@&1?$f95EE5&|Ovj(=_F6&^RD)@>`2n`KY`f2T3Q&gT=ZM} z{7f8c8O=86KI|ay^(gc=a(ZN5C;GhK_t`BII*I5tSA0sZMq?ll(6u1l$i05(Pm%RS zw9|q+PtsLNB&V&Vu5rBm2V+LgJG8WvK?M+VyM~`5(c>sJT*fKjrpw{v8}5BeMk`vt zcR5#T3b1D_%|BGs{npJrL8QUIeYY!Q$^NI0|7hK$XuA|(T>^*F1d3dgs8Qct|9X;w z9jq}a7M;Y1Ix=4E^19oFvA$=98VyT=_=585j#c1gGE;X!%m;G_!^%PGF9GpNa^|TT z%=klll3`h0`kqT*a&x|XK*qOZrj>fLZwT7DVJ5mpcbvaV0ZY9o@ z4295zAxCV{K&#LP0`_-UdHSZqQL|>?7IM!)BNdEp(%an&((EB-hXraTk)jT@iNV96 z2mKmP*B*_!Z}}?Z&+EMv_3Q~Ps&Qr)4C*Pq@mqkK?pA43R-nfX*yBcW>Zu6qamwn3 zhM|SmS*F8VJ74y}Qe;&r{e-kb?Co6^mpr2zmZExQ1fOilo$Kf%_WnwiL8V+wf?XD;fMJide$xf-%ODaMV}SSape3D+TJ{x?f>8R)pTRkMiHOobFRPeoA_ zt$B*n7-LfNJkL=gF%?B(3?ko~-@g0ad)8U|?sN7z>-=fu4=XF{^ZC3#uh;YOcosUh z1+pJbwH_$v#SAnBI6rgmu#D})J-CjMSz}wuVW&9avkW?EY1WqulA{{g2Ksk*4986gFxF;5Zua?}y> z6%niqQ&X=}%tb#BO1yY1MfXhNuV8nC3`a(V3pwSK}-`tPZAGX zYzbKx08=%+=<&i0g;8rDAVjSv|J9!vwb+mwjQ%volv4|RySgl9WOdZLKT}fnupclf zf9hnTz#TYr#cHsaq9x{kWZki$}hrftXXUHH<{z*={+#lvs>N5lunz|r}r-_ z<8H_8$_zbR)3_j?bF+0xfNo)UiHX%kWd9Gs9a;iTGd0-_MCwmuFUjcMF-)rq{wTOC zw!8Iv4*fOZFmgnQ^oO{1X#G=S&IED}ru8=FRK8X?wg;l=pY29GXVs-4?I(`$WgB%8 z;DVha3|cTsKO+r?A7tr!Y_g^1l+6vxg|R*id*-1(dHdUqa~G!`SQAhtIsJM_EyOOk zAkMM8AZTizTz+PVff#Ojc4*kcxg>G%-U&>-1Y^>{iSz32X2%IAMyeJl#xsm%0V~6O zi!6DUHQ7yFP=N2Iv@y33b2a;7jxX@1roxN^7g?$Z$6U%W}q0I~7cfvD31tMIJC<)*D8H$TT#1hhSf z=Pa>V*q;zEsH!uzKhJViUNz3a3?EKkPvUJe`5bmbiY-EI=LmV53@2bBtnZwCRlmHE zrop*Ce16`~XG{n+34u!3o9&qVqA5Q@XbyY4Xm9LCN~|p+kP*)md5F@uwo0l?SUfF} zfVzn`*Nn@_MO8ov{xgz8ydA*>oL{%yF4$Enm({^}_W&_(!l%r> z%>PVz5`Qzg^JV!!F{{j_uXN5sriaO?X)Dlv-xEyJGP6aOxYcw;|H;mHLl|**N$@#Q zvrS!`9Pz%84#N7?^3I_J7tB)lGvKgQ?M6{#OIZJ`9GJ?TXOFt$caVT$T zhpa7syx*M&Q+BE=-0P63dupLYi}UFpnmm5nUIIHr`jZ2ffYMR?mxxEv`!jBm`)6)# zj9LrgwZ`d!DHn?0m;ZXeAbU>m(V#o-7Tifv03VO?Egb#36`0Qj}c>ah)a@n0ol^8NNYRI(H8o#7DSuZe{ECY=t`#BSBOULl~>*Gs92FLjE$x#&VO zz`D=pXbA{O->E}fBLqABP&YsUS&)ZZK81q)XMI%YL7Yz0Xz>X(`Ad{H;5zn^1uJtY z-%f_|O`dF-aG82u8mn9mk8KPpE2Z2ab*6?3L0r}}Tf#e@zF+kT?yw0wVve32SG$w% zM7=ZRJ*OueV>+dl80g!OUdSxcla}bO0M;F~cu*TX_wmfcl9y(dbg#bjdE}Z-cKR5(Z3oMO;PLm;X_)zK$B- z&H+@jkIv{Gew=EA{gN3VYDryiYiD|3Zz`ycpe;5cxg$CRLJdkOlC@}(#YP%vc1t}` zNMZWbFA=ATm542m$Jy^LCYpLOWr97fsQg36psutIzg4ST^JgCbZO{1lGt{K6Zu)k# zVsqu!pYa3@Dc#Fsrk6mj(K^3l60IVlYv&SZga*f9qevb>Jo50GD% zN$hg!`v;6(#;{&7sekBHd+TxrYfg@Q8^VNgapGI#=R(xwjP+TI?tYC6u%IB0pg|30 zBt%Uh5Z^Th1C*Qp!2(XPm@IF}y-tX0?zI9gmS%TFrJ)z8ps$;-$7FF@!k^}9l9yE2 zQo}JiPvu}y5BnH=_u4&B*Ca-d{f#(ebyfSS3PmQ86_`yfit`LWaj-bKc zUJKISRMn4my|$h_ou@x;U(#b?4S!hXUVYRu(PHEUo;unaU$c1r;?x>IS*hN4=^r}W zB~So_5=tO32A}oYv)8}Eiaf3#Keag&Z}T3q^Yu4?p;DVgmmaA3zHgJ;EjHwFFESq- zMGhv!X|#P<3#I8T?$Y-_yO7)vw_oT6t;gA==}!`*#EAj{_lyj|LpJo6zI)@!HFUzBqn^;ROXLkMuf$} z%tqa-dG(&6jInL;l&HBYPH=ID_wG%P$}&iGO(emtAjaxUU7{8eulL8>@6u#*^}3=; z&g^*^e}v+oaPj*UI@vm(9aT*TTf?aiOm9M+874FV0~A`Vs{~GPq_X26)3(kMh`j7w zPN58A^LN6pg=rVGS>DPrp+4svb;JS{h30`hpL&@Zq!ik2DhY-VH0YjR2fE0dWhmQa zEU1>H#ym)S+dlmMr5!^~YVNugG~WtWx`Lr%v-VMgsmaN)0qaLND!s|n+8zAPry!{@ zb?#ebe!iVY{6cl4!XFjuzTCpp=*EKFfvWqzReIiZ#z<6&ccbVU?&v?Xn!@0bk*1jI zXiktzhAenNhDfb+EZt~yNRzB#(JxKvBFOIW^1o#%V{Uns0x9CGAO+pAGKZ-VyJ~4w zT^;I*nMO6J6r1s->ge^*Hi5yk!3<8rN55cp_$TFD7-v3~r5D7ou9)weXu%1!S11Gf ze6wx{;~2dIy@$~0OOh6X=W~H(s;H_a9RMP<-k)4<8AKL_o{56=Q)K}FPq=K_yg2H( zDX6Gq{Ych&251Rzmj6`MRdJYhiQ#jS?Ca=B~OKaZRPiaZc>cg&HtZ z58|koxA=53oo41cr_uRH9SYHw_MWAI(6wZ_og?Uv?JuTZ$*N2F)+(9CfugP1_tIXB zyCMJDPg?UB2X9LXCn?{JQy*PzaERTq;UE7@FK%~THfYR3A*G0%WWem_*_?QiQTRBC(O@`0<{^&8HTJ_l3`qIL!X3_3jCI$qx_T&s9( ztw}T`1l5x0yK`3 z{Dtp-bj_dt4{BhTzR_5@@U)zpwc=mBq{EV2_+?UuIOP%|O9C9-^)ZP8$n{} zIj*nk13L{2El~*?jx9m(XzTkEHMpyA7qH59Q~t(@LHzTTM#f8bfKJ<=%X&nf_Ped! z%q!0|HoYy)pLzNW<}r!ps7SQO$M8t%nIs){9tgtta;tu39E!hL78Kj#Tj_ZFjW0>2 z=P8ro5(X>XVSS5a8jTDlo?$-0qzQb|?b8|dziD||7wgPc*Pu~`+*gqtTbiy8q6>GD z*Q`6Gqop3<)&$NPI<3Hg^T})g!_}=5jZwvl)H#o7+L{#M*S=H~=DjHpcGKwtI;J$y zb7}CYmHKPm{uNjP2@-M z`c$^7dxDgnpwgG#2qnWqU#ymH-|`K2F=>a)*|M{Bc>5d}3<<*5uEi3?YceRA<>P#FK7CBA_V<)eW*9d$(hyKL0*tOI?tAH-Wy4B8FPRyv> zAGi5}*uKgwM;*xq!S{}0N8x|t#}9+Z2w#$6`-hH<*hcS#A9)*qqF#8u7RdsnHPY#R zspEB%>TN%_yTbG`hKcnD2-Alt`X_rIPSa?kUpNnFLB!-m*5Ug44sZp74{-9-_^&G; zFL~0yAASyU;J*S5VLKnabJ)oW!+P;h7E3>uQ-CJibCq3zv#AMXk7Dfqd9kO-@S}AL z?Lz2!WJ-&+5~suWRnO?3qbJ|ix1<@9*qWymM!I?Hh}t)!#RT8&4*Ooej0@w5V*FJu zrmjXtnG@ly|Il%+vo}hnMiCG%+5waMn=>M`RIsQ=(p6z|W}`bM?Xq=#i$$&TW-XqbwGu9RcZXKn4asQ?@Lj%d@h~gdS zTcot$Ib-5PBZE5Q=(gNui-{FL>fO{fgTmC$eh5q7Dkml8?Ev3h)fq-azd(tb8_U>& z`La^HoK*M7xnz(MjPa~UJ|U9&ka!SZdcr!ze{Z~S!gWioxKIzlm#QRn7uD#SnCg&( zT~$`h?P3jS(tlFG6t{agz3p=b8Ke5}qS@e^U}~Bw?N<3e?|uBNN95?_p3I3>)3skb z#V`TRe*CapN3%~HL~cuMziHbBmc>oe%N56oTfMAYwPcLS`iKvTB=?gp8-(RBz@88y zC8l3D{`|-T)((u3+wSLmG4L=r}>{kc6 zx}i8#cOq3qJ5on)T8-~-7LM-_0`vmPWGjyg$~)FQYX;>Kc(v{V?}UBm?u19}t%I(} zC-C~Y5bR=}+D$m#nyr#tDVIXJbNd%7igz|kl(Z_{i5vvioyxPG$xSyl)}9<&cStVv zH@38RkMGPC<+#OkTWf&mtZB~ugqQV`+s&F~oq3(SH(1-yMoGP@j-_{>4ZyGeU9apB zjR?a1)zW|a_g$@crw-u9u3aHzn_d3np}Bs0crS8V4;%nAeBs-YNApHu zDyCZYv1(#?c!Y}lY8Vt@U0b8@f%?P7lMR9X*6X78GV!KbhqKXbonOD+zo+Z; zgLdFTvlXY+rD-|XqKH~Pt=&zFF=;P=2cI>k7g$^mF4U7YB`KR3Wt@sn9JuuW$KM@^ z6fht=8s%6LS5r!?%z6v0tSz3qOU8CfyKM(&D5#g}URk#eAYgdOsP-JbW=8x0gx%n( zU!1g{ox(T|c_rj`f4$ynWLt7Ds$Z0^(@TE?rG^4vXfKNy>kHG>%8SVb{d^gEC#Pp} z*tO5y#48;+#Ok{>>OASL??`3jujJ}e0Arb1`oaMu44K(%8JG(QYyPgla2&C13=PYn zQ?tbfdbV=-X7k05pQ@hHZzV|%Hi#VtFzWLpG%jZ$J+G6#?ez38Xy?oRsR1T+MC_j(e%6zOuNiiZB^r32h3VY8ZP^ zjB3)SCZwatsEWRM38S;@<~vJFm;I6gC2InqkIDY_gTwJW_#%BLF%H5ePFPv(S64gV)2d2?q~|DBl`7fk_(8D2h(p3)<&->A z)iK9x_jn7IF~wWpq~Zn}CpqvSX@o;`apMI>ntA ze8IkSN*Y)&o++B*zXY(}9&Z1DiIQ-m$jJ?I)mS+&s*CgND46<82qrmECtYVz>OhJ0 zU8M&}v+8)FVV5tC8O=l%?X<`@A#PYK#5ouL?sh8wS)SBZ(R$(G%?8_#x=P#NL3MeU z3rXOky4bAKI^xd)GcoPZPBD#pp&D1sQ`lh8OTWOfFYT=S3&{*FXAsy9A zHJ~y!&hJz`EF3HF+p?z}#B}#j1Is}VDVIo!@wK}95QY8@g}jd5LX9YSRX(jYzRTus zWg^n?t_%L9iF74b&2Y>SN|zg%qHq_z<1}ctrAU}SKHPb*cwT<~RL+E+uEG4>1NjO4 zVtye__tOc}M#gHU*lEwD1U|p&b^%T*zq7({t{&}8k>&TWtsE304l1vl{cU+g_gU@( zOvCG%u8!qhrSgP}6!*yrs^DfeH%UKRf?p9@X<@OR26BFK7lg;U2;J=`myRuY(AjlEOK(B@Neq&&^hmn}y8qGGqn9t?4m!2WPw z(Urr`^<1b}d9JNKh5*uffjIsZkfqVP@02@hGNw|zizEie9!f}Z+Wu8^Xmn{Kt;O6X zas*Q}3E<+400@tM{e*qg4X&ws*r(r5KG&Du-#t|pgE)G+z!pZ8px(9}L39*p(v36{ z7sU|a_hzZJ7lJ3{_d>i&wPDk!s-ddi!_2yNvzOn+-c5>Fh0Yf$?`Q7ZBqtb?ZRDpa zi9_JC7wZMJ&BWLbao=`?D--iX2NK7wF!A%|$r4v|dn$MqHM$|b-PUpq#No(oIr`J? zF|9Jz1^y7HH)GGfJomJJkgB~vl1mIacoF_P=y~^}>286@X*feOLq>ciTX|#cVV+}w zc_7Ajb}5zM6{&mg4QK9U$NN`>q|R~mEL6iP-^~6&dwlRAS2X#I4YZ|6;zSL`#P_Rr z0uLJ1VKIvdF>i^1vFtb|#}PF?rdWq5TcUKhyf{F^q_ztDC~9TCUCzB*tU+4*>Ne}A zz>uex+ee*gO!-uk#;+DBp1}ume$|>uCjbtEXgxz;=ZmP9q%y0pOBJ84FtokqP2%md zyI2X0z&ImW$@R|aJYz_h)WTgf>D-ttNz?4NLb80dbKdK4f~uiPFIp$-)iXzYC|?KN zeE`#8Dd}=q0U2bP(XJPvok0bNzs7`LTdirRkEZJB4k+Gf1K!UM`fv@Et{FMLdNBoJ z6oAE%uj`&oO>Sh0wJ78Cs2<~M6X(w!Yt@Xpr?ZyL07UO zxBeCBnxI7ayX%j=l^%hkP_!`UA1{zV--6sIu+v&jrW+1J^CrrXZS*`Z->u=u><9yj zXlGRichQ|6-tsSX;nw~&|1?n6%0jhh(kmvgk!{c1dXP6GrN-kq>!y&Ff|tX~<4AQk zvm4-HwM!L#xzgugkg=tJd67UCka~AJ=4L^%RbzEESlCB*MX$56t>Yx2A zz$tZA1|no_d+Dv2m#P*WA`VJ`tc6o=QdH{46HG;j)Gz~`&TI}?G zq-F0OcK>Z!P&|FiEQGU-{BnQAEQvE-d{6_-gjEr^P7eKbYXRq(scvr`hCD{>#L>DP)C`yUB?IN^M(+~!Lze_@IKSzZlWjSe zYgx$gaE!OP9e0x_^2?pL7`nk0KUS^nT(GZnpR+Y?DZtxhM^7#*Nw11Rr~w^rSWr@% zP8f2kD@p{S<$57%pi7uA2{m)+m}DJ@P)on%3t`zdHyBZ^T4-14Y-&;G;%6~5>)AJX zCVIcLHX9g_u{(q!T0FmDArliMgrf zycnnTIuGclK28AtsXG+EnD>gwCGA!Eg6$ox8>HUIju*x+vbK7VWu>%R)-_!S{H%^q zqXDVs>)Mzx;T~v$A+wHy;PR)qUVd;Pj0d8MmyT*?$v7KeLKu_{$ODxq=?K8Vz#b*k zeN{t)t7qsprQz2efamaG?RY}8Fn3{SzCpQ}^VCRD7)3V2zNL0BW!y^Bt9Px(!SfDa zE>lQEk%6?2U=#B|+hY!>Z_id77yhAB7^jS>?cm|<0LRvBAyPI5=@&f*nrPPZguEy8 z0~8(rX`8N_kqEi+C-hCi)k$!sXSva3ziGp~{Qprj^IxA|iKaN8rKrh}q7HagC*%j| zEhj|A7gqNSrsX-`rbs&KvRO=te^I1d#%|_(_RQM9(|J9 ziPtM3A{LV&@y;r(A!$?Af13{Ql7tJ4!lv5|4Ykp7at? zfA>ns&@O8(jJj0FQu_k$q&!5w{e_iK{xa@bj1Y?7pK+yTtEL>>&2a5FBAu*95RDWj zw*(N7EVG+${unI966Gd5EpJBDEEUbrfD<~M-X!Svs8wDT!i zc`tU!Z{R+tgQ>-e4OJ+-tABf}JHY~WX+mJJJEECQs+uvp01qU)c`;gZ#Av>+XECz7H} z@KaMIwD3=D`~q36=Ri=|3ya!o;!lMm!*LAy)5@NLp@@VWN$>TYz!MLSBh1uHxs_T+N0wiC%(s6cwR=0GPhc zK7BJ%F+c-Ie_O{qU~t!oe!FXBtHUBTTU4j3=^rs9vo#nfd9l}>!J}*A&jy$I4Wkg* ztB*RFC&elnYuCWQ<+}9%IWcG-)u3;ks&G(UUCnMb<+Pyqs%NoGY@5+@*Sxx9K=4=j<`?JgYK48dxz%r*Xc;&B7gw@*$S;T?F~#0(A)!mW3$5lheYg!~ z5q~ock_T((UB`9IVeNuC3^^JZzg#)H0jDp>Wky8PUpV%S6UjN}lW!)ytcs=Qz_vmq z%wO%8zuYjhvlR!ay6{VoB0g#09}u4@W{ZqVI!>y|UxZ^ntWU^`U)@1Epz3vYEVS+S zY*E^lXj?@!$glHIhcxn=gGRm4zdKEPVid`iTIe{kba&QY;4vAr_)NhcyI6-A^E>%2K`&s5O;X-816vt`)3*=(?P$T=7AAmNrK)AS)yMt}~hfl5p1O zh^>FZMhXAWp~xz@Ha#d~9+)}HZS{TZ-SH$bu1TZMdQFj^*?M`R)Gk=1$a2WG^Hgm~ zd{5aYrI?%qH<&tbMQo3?gILw^$<)RJ-{nYx81!b=DVf4fNgjWQ$# zY)ss_txuAiV^i$|1RJHSxf78Yb6k^gyn(@&&pr~BI}yn# zSk76dQjhe;(f?aUSR@CTg9PqC%abnu$oCu40ZnR2Y?CY0tWwLKM5_%B#oUp5jmkOwe!#zFyq-^PyaQD$ zV+Uoc-{nd+h#WPY?o;>|0KhyaXDz8YA@mkp5>(id;5w2JD?&~uq)mLf=_e%)R`T-u zs3|dy$F9D4(gVs6x;~?^2WJHFD%yjS?63&Jl@POp?T;-2Ugk! zcdKZ=4|BuRaU393KJ;|$(02j~;~=Q=X_^tIBVpPEfQ@&c#a@Rx8wb{|{Aa!xuaBh1 z_50eaPxM|cvk4_Pu?6)Xf=A_WpTrmEoFPN`l?F6v!)y(?Ki%=$u={E4T%?_c=F`~k zKV}nKkvT!UWb9EFh@qzb9SIkuew%CyAPrT52>@W5N=oNy>7h+UP)BRI*>4F3s@}QF z>QBF(Ew~Ja!{AE|lw0@&4oe(4e92%;o@Y)po)8eZ&5j*6PL-U#UbbunkfY~o(EX;v z$q{AuInmHQ-+EAcxxsYaZVWa218`O`ct#nfWbV$o1q{G1K;IH`f}5qVaammn`QUco zgjzB8?A3U4Daoy^xBZa>-GSS8JnNJ~LS~m{z^y?-e*LEbts0CJce^_gCUqpY9G66( z8luM1DlmNG@}6z<*gJ099L55(ib_UqqHsVU>zB@VIHISM7Ee+&oqnwo#A0VE#kHkYAN?kUrK zZUT2vOkFWIHSC#qbiEPa58V;_v}ss#5btDowNH}uI$?t?ZowxQ zq%qC~rSDhuC!U-0+R&YLe9H!B)}N)Ld4JdCV4gKF&zyzj2FnLbhAj*`&f2BQJTT9X zea^|E!xj1X+U<9=H}C4Ft=5G%eiAu00jar~Hq zIF<54wq;1gF$EuFo~g_czNie9xfic1s+zt|^=Zh$Zrbo*6(dW`wfl@pL$QV2T?*J~ zRBuq~fhE!bVK`cMg#CvOz6sYJoCNPf*mkGG_IFs|ZOP=sr50&Rk`!tw6O3Vfp8rGv z;VIv2ZM^c^ha{Z+51nwIt5xRbpGvQMKIYwO=f4+fI7SsTCo>TW_Nli(SFyvQG4gTN zkQb7oCcf!*|Iq0q&gMcT1a({JwM^EObqqH3e0JF-_~j+)JsoOp`#Yz5w;_R@qK`_(V_I=H zEjAlXLlNa_{@yM$wl2oI;b7}tSS7|lS=up`OX@8Q&zGfq+v75${3zUa+h>||u$`Kw zhhL?2BP?d3@awbHe{-&rF;7Sf5tpm+2&g{kMuM`(+kVN$ z<3Ez^R4VFVy8Z=oKOnN3c1dc6T8TZoY=+W(gH1N?ZeXDYpcwETC^ES*D1kU}`ZSsd zW&R^T%%YA=sqkhClNk6pBU;Noq<&Cn@^(lba61$llwc9Byq1`SQwW00u&r@N#Ccy; zms$X#V!R`%g4#>{R_fhtaIE1~VdC-br=+t8-&B(SWoL`!ANGmv&9)F`%OM#mn0jlO zauAs3JJz1on12c+C1}h;q!;=T!$0q5Bg)O3xM3)4)9(h6N z<0IYkXf^WdrS?*qJ^M-3crygC86?v{>W-A~9=x$;R-YwVz z6Z1AY%iQEt>{ZbVv9SlGsp{fm$bGJZg2q$zG&je+3aBmEKJw|`q-y^kFN*KsWEO+# zX`~M+=U2`xYjP1GYyq)!=5xqMPh|%0u+y4>GGefCD&1yD(|u8*3x(%cu=?jrtTO9w zPunjBf}fFxPX)+x&&DZMo`f^nhTC1H$S)}%lbE|oG%v5#mPwz?dv={KGV$@1_XWm% zS&FJI$v#^Sk{R|5CzIy5;%rW;5W(EYtJh$>#m#A5bvv^*glIL%9 z`PJh~f}Zt@?$e2x(;udCif}VC^B^G58dqIVP&0mC_kCZ%j8^_i&nvUwZ*&GPj$c4D z+N>qWAfifl8~Q$a(yNzZ>c}_0k`pBbb-SH*D!bfISMQ&#bCvTF^XI~)8nM+X1p4+m z`r5QQ%K~}4v(ns|TOdx&G+(5a0C{=-=QGs%xpLG)1*?!xkB2%lF9;=Dv4q4H@%+Ue zOM#{=e0MiLIjnH@&E->}Is5S~IYBv&qoDgR7jg+c;U+on1*ValtdK}tU29iFeNaH* z5&l~U72(}@(+dM=5tt5%LH*WLMYHG9qkz^0_0aZ=OI@J_ zW-gwz4T?*WktU3=pkSQUO#9vqz+U5vuBY8fQgAdbHVwDEFrN|V3YPZW{)cWEek=}Z zbUMuq^clig){OBLa7@3_^JFP!k!6ibv`jz$*D&4idI$UR$ES6|sL8)l2Rtq6^mG2m zzisk^ave|)H zrms}UmgM*gRZQa@NRXN@F2{eROzO2-3lMX{;9w+gHyr7rpmJCMDYACv_NMXF((c6$ zsy{T+AVR02My+iS`xx?5_ZRB>wo*qF9#B&KBWYTF<&Jt%F4~$*zjPbJ>z%_iNU$fZ z{&=4eH_=#^LTDcf zl)NP0>co_uza2Bq@;Ss^=!zH9qbi*{S$VfvonJ>UzKy-UbMJ>U*lT+2H-FPN9@q_{ zq2C|B`gImKgdT~q-I^A#J2g#PIV<{`?U=as_ZWtg4$_Fu9S^$b$Mb|_kBOG_YWOiJ zvF)}4nLiLOMAfD)KWOm#?)>ugapufVBf8`!r2HRw#`zAD)MNuyNQjW*=frRKxYT?J z1z4F~y_4e`zlTq#joEs{-08)$)E|gV^mUSc7J7gUv2EU-fqVl>+pUyVA}(Q z!=S|7)1~Q1is6;GCDKvYTcY>ghN4bgLLu2>)a~wPtnQ(osehvH&18Sk-kQN*^lq{Fx_D!Khcc!Fbh=Fj0YFg9)E*K)dDv|iHa;dAC)4s;7qW-e}PMf9R5`8upPPFkxap;Y!;alF^BKFd|NkJ8i5gOVLPo ztPiBIMbByo{OJ^{-)G3cNB5Hd9Acj=zPL-b2g2;`jqnvdnHCq#jtDY9TqDFvCELIYn7;r;$>nL zbaJ>(XMP>+bITA6zsVG%c4;ZlLLf@PLMwu)L-%&bjtR{qmFO7gROc>G*Wl9 zGMe<>XYoAm^5~v4X8ee@kX{V(T?q(VNIGo^JSxg*S^<~MoTzqdTvL-I@we;OOuLGv zcoA@R*?m+(AK_+i<)lN%vt+PzUw;#W(p&HsauG2CUznFi)}QXE-1y*-USMT0ySVUKFgoXrybmt?!T3~<%3;4)FoE)H=F`(;PcFBx=9w8)|!agnLDY#{VX~$niU+C~y-zqIgNIh2Qza5u9AJ?pUugi1z<(oLH;l|ECborZmx=n{Xb(MdT zD3fRacD|#EbQCVH!MZ?p81MK1n2wUjW0`j%f}S0HzSG5pXKc|HOuF*g8cy76$6clf z6LN03@Uu?rzbi4XZ?Ud0`K2w!O6WKhlom#6`sU zY16v2t_;&y@BS?VExp@m8D9GG8VcJ4tIiCb;71=5Ji=6OIYaRSWO=*+*x@;>-GJD!QY z8PjJ1BLQ2;9ODy&K_yIP5qjPQ%{VQ&4-DXQZ0DD#Z^WQ?x3P!Vvu&lUP!5~g%DCdE z@xP3%G!~hH<#5+F)p@PBfry7@iNYB0+0)hSRo}_~9=h4qw+RO9TRg?cx0OnJ(O`hM* zvjnpdX~M*QuW%}BLxiecr=a%jWv0eI{bXZvkHR|VzbiED zi24}BNKv1JR4iAGP{i+~LiG2nC%C43(pHl6GBeX05;O1U=`1G(XVaCRiQ@%=YlrEc z6#nsLnfG&i)m%H7y^1poz?b*~c(nKlCW-lxk?$-ocXeF9YAv34hTsN@3q)j;HzMjn z^PJIwd&t_%KNk-h!SZpxeWnfbWJ}^z({Bq^-@D>0WP3O5mjIWu03-FvCiOn)^7lsg zCwr1Yp0`clp)d0zMCY-YeY`xwGmqXoUcb6#&#e#Mgbf*#qVWCpn-wutn}}~?3HNu% zpo|^jvj#V}n3nr)75Jg``>?smbJk9OYeb}Ju~dhE~K_i?W7 zT)d}Mc( zVM;BWRZNN&o#jq1VX8PT7`n@?ZW^Ha87%eC1I1PYFf`jw_thMhnk`4|pCs#0f}{zL zG}or}tr1B%gu?C!z!zbq?K+xolEi+=nx5bg7#w*~=K75dqq9GtVYgF++7{cHJR{gVtm(jYB(N^*h;A#)0Ho=Nn!aeRHzO3<&f0g zapF*fbjgrxnL0CkX=c4Wo0yVk{zT?y&$*PkpT8TpOt|_!tJ{#nZWIwY2y7L#d`F2k zrH?kBxJcjKt83{n<+y3loZ2DZ*)iq@bYOYt3YT*FIA@Kd10%7GtH1}aGSFph{`~gB zV`+1Q(P=Q>?1^Fv^HGD^4qPM|9+!0BvtEv}7xSq_L!{ewwjX&!caKqQ^1#-2rfS6b zo{DDypYOtJByXv1HBU&hZ_Di66M9$oZz_m${V-j!^LL5`v9>!xO^=UqB0u!(@Q}M; zGC2K3ikRSBFOSO24%T-@q|figGf4t}*B6oOWNj_t5P}wa;af<^*Ab~rpt1~_3oi(n zGv}wfJxgZwHqjJROsjF7i{8hInFnEfetB|$7;ET{o5kekYX?`l!c9H>))uZZf zi?u-H04qWferKde1v9=`<>KaR_?(bwv>-sb%_2Uc1dQf8eE?=uwNf-?2RZ0|j~Ix3 zInczuYJc1W7=chtmZ+Z+i!cX zH^w@@`()3Zuack`;%Q0N$pv~eaX1dtRRwG<(G&?TqPMvcaW2evSSFh73)GAG*!PD1 zy6?SX_ZJzfpH$`kq1!bXqofra>3EJA8uYK`C;Oh_mKwJC8OV?DP&zONcU=lJ=p^<( zGEu+3Vi#aaGf|hgQf0_jUXOVr-b{D%^E8gjX~vlx@>$H|f2xsGBRZ^@aDoLM8l03j zIXBfAVDCm@w|r@!HAE1A9xFhh;-4ppk2|kQ=zUg{F@F{EX}A5`bQsyxk|qUcW?quM zx9l4% z4;r*TS&lPx#A&k~ZEhgy4v%&sETr2pJQRr*+R`BuW|l2X47C~-z{2DS=NU`v})k_jP}`@9Tfv|1bCN{vFV((=o|6Zk!dQmx!Ak5x$6Os6+`e zH)9GLvyju@1Y92Z#IB+~Cr%mq_Rl=c3ej~7T4zQyJ>tTE2Jld)t-#Tf&(m&KKRd z+AI(!@=d%1LUwdP5fypYpr>)GdT`w9=vPz6UjR@`w{>ctu-Ky4aJO-k;K_C1WwRh= z+|fs+CGAq&!Wg*RTchO+TebHLBVri=_J(9{3{%IozQ@FN+V(5;6>Vq53-~LB9m$qT zDD8G`e-nP|Cmqi@=#G_FjK!rQe=I&8siy4h$~U#IRZW<|>Eha*!e|w&Vm6U=26uDY zjEG>EJexjr!oXW<(rcB3s=Z{Jh6pNk%Te`9O$pOcVTdVR72;W;Y}m>V<1A3}I4H+7 zG2Fu27?Hc|#^*~!)5m`-_7p{qy?y{ntpQKfyqc}GG~|}Qu~g-bviW@i2JqskWjrg- z<@VI>6Hqa?YqL6&4%V7q_G`p0T}Uc$*5};lvhd&aeEkPahjd|)Ti@Vnu{Vzo}ib2Z9K5$!i^=pM&jq@sL`~!YTt4 z5vB6&z^foR=HLl<#ck8 zG8ap$z|JqbboPI$t4ewAZe;)7OieV+0H(IB%nof~Nh@z(X0HuDB3lXUnq=E@?pIwT zE^M@hJ4KwFYtH*7Nnf4N^XN0w!mElr_Pcq|XaQDdml{w*9wsZt$2G}Bg(*018rn4u z+1yqS4ZSWlYo%N2SXr$@?2bIvZRt7CHuR(eojJ@|3DMn}Kcts~_i2iWizm#JDkf11 z?CO=o&>uF9LpMRZHiuXr|JJg5?d5$H18YTsAT~CfaTwJ{1 ztU)Us^Y{#MG^{cddJQB?tm8YH*mGUD#I4yD5}Ys39`f|5K>Q>T8OH_f4$;Vqc2vkd zPYpRVW)JT7NRYI@oWCL+F)z?~o5PlCw_62{QxgJtKR?eR;4@Pn)BcX>Ud8_i8W-PO zL)cpm4VF^-s-CtJz@}iGfWUME6ka7+UA&H^oMI`=#w9Opn?C1GQV9Amsc^H=sjW@gM?2zhvgs{Xq3;(7_xQ zWWsuZn9_KTKzE>-?H{8H#PJ2y;|zvO0h3A}-h{2T@pRTK(N9^*=nyhY8m-AO!O3F- zaQ%<1W4P$WGNTsX}S!a}qVQVX{b z(<_hno3;jaK|cV4JO%P*??4!5HkE~}(-^{$Myf&(vzSKr9vR&e=CRL8oMlsD#C#DY z^%pNp>3989v*U%1o4cFb&7o^Y@9}CNc%OELLh+5taCiQIdTpU=4#+%ZH9^GYOspNb zev_fk4k4MtdMbG2@$9F)-{Ova=w1JAnLmJXDT~6axHY_J-7_|C3cKExRIGO1q%&Px z5@-dMeJJ_hfG>eYQfJEvVBdOJ7c>vH2`t4thMU1!k=c4aR(v)F>c!hff)*F?JBZ8V zheX)$+ThUdnIWA5o2}nNrT(9}5Fv6qQJ&v6^9%4Xv+18j3IEmm{f)JT*$3e$MGA8n zZIWF^6?_#mZ4i=U%G_>iP~YR_0Ip{lhuy>t(gE+ZvY^ZjN**Nfsky} z#1JbFq;|c3DQDb+Jvrd&mn!S(a#s9QCirNPd56s=Y1K-vgB8!aP5%pM0$dg4>|lam za{AA_A)T!=ia-?}Eb+MK=uHy2<#g9~k7sa15 zO1X90!%~j@Bx?$`dwvUu4rqVEnZR3;6ST zj#W2i`};>nM&{ zbRKCewqa!4NtClL$)m>E4t~)*gM44BQ z7F9H9-E{!#94W$TOa56k(XZ44o*G8Kd&|>!sGD_XrvnCV59r}48_5*PMNi#xtvp)z Kdm+-lM*jucPD8E$ literal 0 HcmV?d00001 From 7862d3c98aaba44f2811eb4657ff2c174414d7a1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 01:08:32 -0700 Subject: [PATCH 24/67] Fixed errors in GPU crosstool in cuda_clang mode. The source of errors is that 'crosstool_wrapper_driver_is_not_gcc' was not included in the GPU crosstool, but was still listed in build dependencies. PiperOrigin-RevId: 167107311 --- third_party/gpus/crosstool/BUILD.tpl | 8 ++++---- third_party/gpus/cuda_configure.bzl | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/third_party/gpus/crosstool/BUILD.tpl b/third_party/gpus/crosstool/BUILD.tpl index 7d8b6005135..98cb326572e 100644 --- a/third_party/gpus/crosstool/BUILD.tpl +++ b/third_party/gpus/crosstool/BUILD.tpl @@ -12,12 +12,12 @@ cc_toolchain_suite( cc_toolchain( name = "cc-compiler-local", - all_files = ":crosstool_wrapper_driver_is_not_gcc", + all_files = "%{linker_files}", compiler_files = ":empty", cpu = "local", dwp_files = ":empty", dynamic_runtime_libs = [":empty"], - linker_files = ":crosstool_wrapper_driver_is_not_gcc", + linker_files = "%{linker_files}", objcopy_files = ":empty", static_runtime_libs = [":empty"], strip_files = ":empty", @@ -30,12 +30,12 @@ cc_toolchain( cc_toolchain( name = "cc-compiler-darwin", - all_files = ":crosstool_wrapper_driver_is_not_gcc", + all_files = "%{linker_files}", compiler_files = ":empty", cpu = "darwin", dwp_files = ":empty", dynamic_runtime_libs = [":empty"], - linker_files = ":crosstool_wrapper_driver_is_not_gcc", + linker_files = "%{linker_files}", objcopy_files = ":empty", static_runtime_libs = [":empty"], strip_files = ":empty", diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl index b85e565f362..4a0f4710881 100644 --- a/third_party/gpus/cuda_configure.bzl +++ b/third_party/gpus/cuda_configure.bzl @@ -971,7 +971,6 @@ def _create_local_cuda_repository(repository_ctx): ' ":cudnn-include",') }) # Set up crosstool/ - _file(repository_ctx, "crosstool:BUILD") cc = find_cc(repository_ctx) host_compiler_includes = _host_compiler_includes(repository_ctx, cc) cuda_defines = { @@ -981,11 +980,14 @@ def _create_local_cuda_repository(repository_ctx): } if _use_cuda_clang(repository_ctx): cuda_defines["%{clang_path}"] = cc + _tpl(repository_ctx, "crosstool:BUILD", {"%{linker_files}": ":empty"}) _tpl(repository_ctx, "crosstool:CROSSTOOL_clang", cuda_defines, out="crosstool/CROSSTOOL") else: nvcc_path = str(repository_ctx.path("%s/bin/nvcc%s" % (cuda_config.cuda_toolkit_path, ".exe" if cuda_config.cpu_value == "Windows" else ""))) + _tpl(repository_ctx, "crosstool:BUILD", + {"%{linker_files}": ":crosstool_wrapper_driver_is_not_gcc"}) _tpl(repository_ctx, "crosstool:CROSSTOOL_nvcc", cuda_defines, out="crosstool/CROSSTOOL") _tpl(repository_ctx, "crosstool:clang/bin/crosstool_wrapper_driver_is_not_gcc", From 8b20ddf3e0eedb52a7ae0f10a55658e64efc4d1a Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Thu, 31 Aug 2017 09:46:52 -0700 Subject: [PATCH 25/67] [XLA] Sanity check the list of called computations for fusion nodes called_compuatations for a fusion node should only include the fusion computation that it calls. PiperOrigin-RevId: 167149669 --- .../compiler/xla/service/hlo_instruction.cc | 7 ------- .../compiler/xla/service/hlo_instruction.h | 3 +-- .../xla/service/hlo_instruction_test.cc | 9 +++------ .../xla/service/hlo_rematerialization.cc | 3 ++- .../compiler/xla/service/hlo_verifier.cc | 19 +++++++++++++++++++ 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 3bdb67ba923..75b88aeb128 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -793,13 +793,6 @@ HloInstruction* HloInstruction::CloneAndFuseInternal( } } - for (HloComputation* computation : - instruction_to_fuse->called_computations()) { - if (std::find(called_computations_.begin(), called_computations_.end(), - computation) == called_computations_.end()) { - called_computations_.push_back(computation); - } - } VLOG(2) << "New clone:\n" << clone->ToString(); return clone; } diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index 5688fcc4255..e393e05c344 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -797,8 +797,7 @@ class HloInstruction { const Shape& shape, tensorflow::gtl::ArraySlice operands); - // Returns the computations this instruction calls (if any). This includes - // computations called by fused instructions inside of a fusion instruction. + // Returns the computations this instruction directly calls (if any). const std::vector& called_computations() const { return called_computations_; } diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc index ea5749581b5..2e1eeee36b5 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc @@ -758,16 +758,13 @@ TEST_F(HloInstructionTest, FusionOpWithCalledComputations) { auto* fusion = computation->CreateFusionInstruction( {map_3_y}, HloInstruction::FusionKind::kLoop); auto* fused_computation = fusion->fused_instructions_computation(); - EXPECT_THAT(fusion->called_computations(), - ElementsAre(fused_computation, computation_y)); + EXPECT_THAT(fusion->called_computations(), ElementsAre(fused_computation)); fusion->FuseInstruction(map_2_x); - EXPECT_THAT(fusion->called_computations(), - ElementsAre(fused_computation, computation_y, computation_x)); + EXPECT_THAT(fusion->called_computations(), ElementsAre(fused_computation)); fusion->FuseInstruction(map_1_x); - EXPECT_THAT(fusion->called_computations(), - ElementsAre(fused_computation, computation_y, computation_x)); + EXPECT_THAT(fusion->called_computations(), ElementsAre(fused_computation)); } TEST_F(HloInstructionTest, ComplexFusionOp) { diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc index 278a1d7efad..20152cf0cef 100644 --- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc +++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc @@ -1248,7 +1248,8 @@ StatusOr HloRematerialization::Run( sequence->at(node.computation()))); } return Status::OK(); - })); + }, + /*visit_unreachable_nodes=*/false)); // The peak memory usage of the module equals the peak memory use of the entry // computation plus the output size of the computation. This is because the diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index eb6fe5d2e3b..c44be716cdf 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -280,6 +280,14 @@ class ShapeVerifier : public DfsHloVisitor { const std::function shape_size_fn_; }; +string ComputationsToString( + tensorflow::gtl::ArraySlice computations) { + return tensorflow::str_util::Join( + computations, ",", [](string* s, const HloComputation* computation) { + s->append(computation->name()); + }); +} + } // namespace StatusOr HloVerifier::Run(HloModule* module) { @@ -290,6 +298,17 @@ StatusOr HloVerifier::Run(HloModule* module) { for (const auto& instruction : computation->instructions()) { TF_RET_CHECK(instruction->parent() == computation.get()); if (instruction->opcode() == HloOpcode::kFusion) { + TF_RET_CHECK( + ContainersEqual(instruction->called_computations(), + {instruction->fused_instructions_computation()})) + << "Fusion HLO calls computations other than the " + "fused_instructions_computation: " + << instruction->ToString() + << " instruction->fused_instructions_computation(): " + << instruction->fused_instructions_computation()->ToString() + << " instruction->called_computations(): " + << ComputationsToString(instruction->called_computations()); + for (const auto& fused : instruction->fused_instructions()) { TF_RET_CHECK(fused->parent() == instruction->fused_instructions_computation()) From 059c68457a00463146613c0751cb9b16eab28888 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 31 Aug 2017 10:33:32 -0700 Subject: [PATCH 26/67] [TF:XLA] Implement SoftSign, SoftSignGrad, ReciprocalGrad, ApproximateEqual, Rint, IsFinite, IsInf, IsNan. Enable L2Loss test case that apparently passes now. PiperOrigin-RevId: 167156124 --- tensorflow/compiler/tests/binary_ops_test.py | 19 +++++++ tensorflow/compiler/tests/randomized_tests.cc | 51 +++++++++++++++++-- tensorflow/compiler/tests/unary_ops_test.py | 40 ++++++++++++++- tensorflow/compiler/tf2xla/kernels/BUILD | 1 - .../compiler/tf2xla/kernels/binary_ops.cc | 25 +++++++++ .../compiler/tf2xla/kernels/is_finite_op.cc | 43 ---------------- .../compiler/tf2xla/kernels/unary_ops.cc | 25 ++++++--- tensorflow/core/ops/nn_ops.cc | 2 +- 8 files changed, 148 insertions(+), 58 deletions(-) delete mode 100644 tensorflow/compiler/tf2xla/kernels/is_finite_op.cc diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index e349aefd4cb..e6862f0d9dd 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -52,6 +52,12 @@ class BinaryOpsTest(XLATestCase): def testFloatOps(self): for dtype in self.float_types: + self._testBinary( + lambda x, y: math_ops.approximate_equal(x, y, tolerance=0.0001), + np.array([[[[-1, 2.00009999], [-3, 4.01]]]], dtype=dtype), + np.array([[[[-1.001, 2], [-3.00009, 4]]]], dtype=dtype), + expected=np.array([[[[False, True], [True, False]]]], dtype=dtype)) + self._testBinary( gen_math_ops._real_div, np.array([3, 3, -1.5, -8, 44], dtype=dtype), @@ -82,6 +88,12 @@ class BinaryOpsTest(XLATestCase): dtype(4), expected=np.array([[16], [81]], dtype=dtype)) + self._testBinary( + gen_math_ops._reciprocal_grad, + np.array([4, -3, -2, 1], dtype=dtype), + np.array([5, -6, 7, -8], dtype=dtype), + expected=np.array([-80, 54, -28, 8], dtype=dtype)) + self._testBinary( gen_math_ops._sigmoid_grad, np.array([4, 3, 2, 1], dtype=dtype), @@ -107,6 +119,13 @@ class BinaryOpsTest(XLATestCase): expected=np.array( [3.97322869, 2.99258232, 1.99817801, 0.99966466], dtype=dtype)) + self._testBinary( + gen_nn_ops._softsign_grad, + np.array([4, 3, 2, 1], dtype=dtype), + np.array([5, 6, 7, 8], dtype=dtype), + expected=np.array( + [0.11111111, 0.06122449, 0.03125, 0.01234568], dtype=dtype)) + self._testBinary( gen_math_ops._tanh_grad, np.array([4, 3, 2, 1], dtype=dtype), diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc index a342e37e0ee..49c1699b6ed 100644 --- a/tensorflow/compiler/tests/randomized_tests.cc +++ b/tensorflow/compiler/tests/randomized_tests.cc @@ -888,6 +888,16 @@ TEST_F(OpTest, Any) { }); } +TEST_F(OpTest, ApproximateEqual) { + Repeatedly([this]() { + auto dims = RandomDims(); + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ApproximateEqual") + .RandomInput(DT_FLOAT, dims) + .RandomInput(DT_FLOAT, dims) + .Attr("T", DT_FLOAT)); + }); +} + TEST_F(OpTest, Asinh) { Repeatedly([this]() { return ExpectTfAndXlaOutputsAreClose( @@ -1662,11 +1672,9 @@ TEST_F(OpTest, GreaterEqual) { TEST_F(OpTest, L2Loss) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); - // TODO(b/31644876): scalars currently crash. - return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("L2Loss") - .RandomInput(type, RandomDims(1)) - .Attr("T", type)); + DataType type = DT_FLOAT; + return ExpectTfAndXlaOutputsAreClose( + OpTestBuilder("L2Loss").RandomInput(type).Attr("T", type)); }); } @@ -2165,6 +2173,15 @@ TEST_F(OpTest, Reciprocal) { }); } +TEST_F(OpTest, ReciprocalGrad) { + Repeatedly([this]() { + std::vector dims = RandomDims(); + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReciprocalGrad") + .RandomInput(DT_FLOAT, dims) + .RandomInput(DT_FLOAT, dims) + .Attr("T", DT_FLOAT)); + }); +} TEST_F(OpTest, Relu) { Repeatedly([this]() { return ExpectTfAndXlaOutputsAreClose( @@ -2250,6 +2267,13 @@ TEST_F(OpTest, ReverseV2) { }); } +TEST_F(OpTest, Rint) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose( + OpTestBuilder("Rint").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + }); +} + TEST_F(OpTest, Round) { Repeatedly([this]() { return ExpectTfAndXlaOutputsAreClose( @@ -2402,6 +2426,23 @@ TEST_F(OpTest, SoftplusGrad) { }); } +TEST_F(OpTest, Softsign) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose( + OpTestBuilder("Softsign").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + }); +} + +TEST_F(OpTest, SoftsignGrad) { + Repeatedly([this]() { + std::vector dims = RandomDims(); + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SoftsignGrad") + .RandomInput(DT_FLOAT, dims) + .RandomInput(DT_FLOAT, dims) + .Attr("T", DT_FLOAT)); + }); +} + TEST_F(OpTest, SpaceToBatch) { Repeatedly([this]() { std::vector block_dims = RandomDims(4, 4, 0, 5); diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index ca2a438005f..b21f1998a5d 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import unittest + import numpy as np from six.moves import xrange # pylint: disable=redefined-builtin @@ -161,12 +163,17 @@ class UnaryOpsTest(XLATestCase): np.array([[-1.7, 1.2]], dtype=dtype), expected=np.array([[-2, 1]], dtype=dtype)) + self._assertOpOutputMatchesExpected( + math_ops.is_finite, + np.array([[np.NINF, -2, -1, 0, 0.5, 1, 2, np.inf, np.nan]], + dtype=dtype), + expected=np.array([[0, 1, 1, 1, 1, 1, 1, 0, 0]], dtype=np.bool)) + # Tests for tf.nn ops. self._assertOpOutputMatchesExpected( nn_ops.l2_loss, np.array([[[]]], dtype=dtype), expected=dtype(0)) - # TODO(b/31644876): enable this test case when fixed. - # self._assertOpOutputMatchesExpected(tf.nn.l2_loss, dtype(4), dtype(10)) + self._assertOpOutputMatchesExpected(nn_ops.l2_loss, dtype(4), dtype(8)) self._assertOpOutputMatchesExpected( nn_ops.l2_loss, np.array([[-2, 4]], dtype=dtype), expected=dtype(10)) @@ -198,6 +205,12 @@ class UnaryOpsTest(XLATestCase): np.array([[1e-14, 1e-15, 0.6]], dtype=dtype), expected=np.log1p(np.array([[1e-14, 1e-15, 0.6]], dtype=dtype))) + self._assertOpOutputMatchesExpected( + math_ops.rint, + np.array([[-1.7, 1.2, 4.0, 0.0], [-3.5, -2.5, -1.5, -0.5], + [0.5, 1.5, 2.5, 3.5]], dtype=dtype), + expected=np.array([[-2, 1, 4, 0], [-4, -2, -2, 0], [0, 2, 2, 4]], + dtype=dtype)) self._assertOpOutputMatchesExpected( math_ops.round, np.array([[-1.7, 1.2, 4.0, 0.0], [-3.5, -2.5, -1.5, -0.5], @@ -301,6 +314,12 @@ class UnaryOpsTest(XLATestCase): np.array([[-2, 0, 8]], dtype=dtype), expected=np.array([[0.126928, 0.6931472, 8.0003354]], dtype=dtype)) + self._assertOpOutputMatchesExpected( + nn_ops.softsign, + np.array([[-2, -1, 0, 1, 2]], dtype=dtype), + expected=np.array([[-0.66666669, -0.5, 0, 0.5, 0.66666669]], + dtype=dtype)) + self._assertOpOutputMatchesExpected( math_ops.is_finite, np.array( @@ -335,6 +354,23 @@ class UnaryOpsTest(XLATestCase): np.array([[4, 3], [2, 1]], dtype=dtype), expected=np.array([[1, 1], [1, 1]], dtype=dtype)) + # TODO(phawkins): these tests fail unless fastmath optimizations + # are disabled. Use more robust IsInf/IsNaN detection and enable these + # tests. + @unittest.skip("test case fails in fast-math mode") + def testIsInfAndIsNan(self): + for dtype in self.float_types: + self._assertOpOutputMatchesExpected( + math_ops.is_inf, + np.array([[np.NINF, -2, -1, 0, 0.5, 1, 2, np.inf, np.nan]], + dtype=dtype), + expected=np.array([[1, 0, 0, 0, 0, 0, 0, 1, 0]], dtype=np.bool)) + self._assertOpOutputMatchesExpected( + math_ops.is_nan, + np.array([[np.NINF, -2, -1, 0, 0.5, 1, 2, np.inf, np.nan]], + dtype=dtype), + expected=np.array([[0, 0, 0, 0, 0, 0, 0, 0, 1]], dtype=np.bool)) + def testLogicalOps(self): self._assertOpOutputMatchesExpected( math_ops.logical_not, diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index d09e721c936..6e6c5dc17f5 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -31,7 +31,6 @@ tf_kernel_library( "function_ops.cc", "gather_op.cc", "identity_op.cc", - "is_finite_op.cc", "l2loss_op.cc", "lrn_ops.cc", "matmul_op.cc", diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc index f9bb1e2fb1d..58538b45137 100644 --- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc @@ -102,6 +102,7 @@ XLA_MAKE_BINARY(Mod, b->Rem(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(Maximum, b->Max(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(Minimum, b->Min(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(RealDiv, b->Div(lhs, rhs, extend_dimensions)); +XLA_MAKE_BINARY(ReciprocalGrad, b->Neg(b->Mul(rhs, b->Mul(lhs, lhs)))); XLA_MAKE_BINARY( RsqrtGrad, b->Mul(b->Pow(lhs, XlaHelpers::IntegerLiteral(b, input_type(0), 3)), @@ -140,6 +141,11 @@ XLA_MAKE_BINARY(SoftplusGrad, b->Div(lhs, b->Add(b->Exp(b->Neg(rhs)), XlaHelpers::One(b, input_type(1))))); +// softsigngrad(gradients, features) = gradients / (1 + abs(features)) ** 2 +XLA_MAKE_BINARY(SoftsignGrad, + b->Div(lhs, Square(b, b->Add(XlaHelpers::One(b, input_type(0)), + b->Abs(rhs))))); + XLA_MAKE_BINARY(TanhGrad, b->Mul(rhs, b->Sub(XlaHelpers::One(b, input_type(0)), b->Mul(lhs, lhs)))); @@ -147,5 +153,24 @@ XLA_MAKE_BINARY(Pow, b->Pow(lhs, rhs, extend_dimensions)); #undef XLA_MAKE_BINARY +class ApproximateEqualOp : public XlaOpKernel { + public: + explicit ApproximateEqualOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("tolerance", &tolerance_)); + } + + // Computes the max of the scalar input x and 0. + void Compile(XlaOpKernelContext* ctx) override { + xla::ComputationBuilder* b = ctx->builder(); + auto result = b->Lt(b->Abs(b->Sub(ctx->Input(0), ctx->Input(1))), + XlaHelpers::FloatLiteral(b, input_type(0), tolerance_)); + ctx->SetOutput(0, result); + } + + private: + float tolerance_; +}; +REGISTER_XLA_OP(Name("ApproximateEqual"), ApproximateEqualOp); + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/is_finite_op.cc b/tensorflow/compiler/tf2xla/kernels/is_finite_op.cc deleted file mode 100644 index 788dcee5443..00000000000 --- a/tensorflow/compiler/tf2xla/kernels/is_finite_op.cc +++ /dev/null @@ -1,43 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/tf2xla/xla_helpers.h" -#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" -#include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/literal_util.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/bcast.h" - -namespace tensorflow { -namespace { - -class IsFiniteOp : public XlaOpKernel { - public: - explicit IsFiniteOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} - - void Compile(XlaOpKernelContext* ctx) override { - xla::ComputationDataHandle input = ctx->Input(0); - ctx->SetOutput(0, ctx->builder()->IsFinite(input)); - } - - private: - TF_DISALLOW_COPY_AND_ASSIGN(IsFiniteOp); -}; - -REGISTER_XLA_OP(Name("IsFinite"), IsFiniteOp); - -} // anonymous namespace -} // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc index 7b39f0533b4..6b8f5ec7b33 100644 --- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc @@ -73,8 +73,12 @@ XLAJIT_MAKE_UNARY(Exp, b->Exp(x)); XLAJIT_MAKE_UNARY(Expm1, b->Sub(b->Exp(x), XlaHelpers::One(b, input_type(0)))); XLAJIT_MAKE_UNARY(Floor, b->Floor(x)); -// Returns 0 if x is 0, -1 if x < 0 and 1 if x > 0. -XLAJIT_MAKE_UNARY(Sign, b->Sign(x)); +XLAJIT_MAKE_UNARY(IsFinite, b->IsFinite(x)); +XLAJIT_MAKE_UNARY(IsInf, b->Eq(b->Abs(x), + XlaHelpers::FloatLiteral( + b, input_type(0), + std::numeric_limits::infinity()))); +XLAJIT_MAKE_UNARY(IsNan, b->Ne(x, x)); // Return 1/x XLAJIT_MAKE_UNARY(Inv, b->Div(XlaHelpers::One(b, input_type(0)), x)); XLAJIT_MAKE_UNARY(Reciprocal, b->Div(XlaHelpers::One(b, input_type(0)), x)); @@ -105,6 +109,12 @@ static xla::ComputationDataHandle Round(xla::ComputationBuilder* b, b->Add(round_val, one), round_val); } +XLAJIT_MAKE_UNARY(Rint, Round(b, input_type(0), x)); +XLAJIT_MAKE_UNARY(Round, Round(b, input_type(0), x)); + +XLAJIT_MAKE_UNARY(Rsqrt, + b->Pow(x, XlaHelpers::FloatLiteral(b, input_type(0), -0.5))); + // Expresses sigmoid as a rescaled tanh: sigmoid(x) == (tanh(x/2) + 1) / 2. static xla::ComputationDataHandle Sigmoid(xla::ComputationBuilder* b, DataType dtype, @@ -112,16 +122,19 @@ static xla::ComputationDataHandle Sigmoid(xla::ComputationBuilder* b, auto half = XlaHelpers::FloatLiteral(b, dtype, 0.5); return b->Add(half, b->Mul(half, b->Tanh(b->Mul(half, x)))); } - -XLAJIT_MAKE_UNARY(Round, Round(b, input_type(0), x)); -XLAJIT_MAKE_UNARY(Rsqrt, - b->Pow(x, XlaHelpers::FloatLiteral(b, input_type(0), -0.5))); XLAJIT_MAKE_UNARY(Sigmoid, Sigmoid(b, input_type(0), x)); + +// Returns 0 if x is 0, -1 if x < 0 and 1 if x > 0. +XLAJIT_MAKE_UNARY(Sign, b->Sign(x)); XLAJIT_MAKE_UNARY(Sinh, b->Mul(b->Sub(b->Exp(x), b->Exp(b->Neg(x))), XlaHelpers::FloatLiteral(b, input_type(0), 0.5))); XLAJIT_MAKE_UNARY(Softplus, b->Log(b->Add(b->Exp(x), XlaHelpers::One(b, input_type(0))))); +// softsign(x) = x / (abs(x) + 1) +XLAJIT_MAKE_UNARY(Softsign, + b->Div(x, + b->Add(b->Abs(x), XlaHelpers::One(b, input_type(0))))); XLAJIT_MAKE_UNARY(Sqrt, b->Pow(x, XlaHelpers::FloatLiteral(b, input_type(0), 0.5))); XLAJIT_MAKE_UNARY(Square, b->Mul(x, x)); diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 0a96258dd1f..1ab1f1a7366 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -1945,7 +1945,7 @@ Computes softsign gradients for a softsign operation. gradients: The backpropagated gradients to the corresponding softsign operation. features: The features passed as input to the corresponding softsign operation. -backprops: The gradients: `gradients / (1 + abs(-features)) ** 2`. +backprops: The gradients: `gradients / (1 + abs(features)) ** 2`. )doc"); // -------------------------------------------------------------------------- From 8dbd2b91f9b387e6f3b0084671e46325fc9915be Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 10:51:13 -0700 Subject: [PATCH 27/67] Update ops-related pbtxt files. PiperOrigin-RevId: 167158999 --- tensorflow/core/ops/ops.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 63b7532b334..3a28ce3767d 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -24686,7 +24686,7 @@ op { } output_arg { name: "backprops" - description: "The gradients: `gradients / (1 + abs(-features)) ** 2`." + description: "The gradients: `gradients / (1 + abs(features)) ** 2`." type_attr: "T" } attr { From 863163067ad38e093ae7c4557c5065f589eef70c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 11:00:15 -0700 Subject: [PATCH 28/67] Introducing tf.contrib.receptive_field PiperOrigin-RevId: 167160384 --- tensorflow/BUILD | 1 + tensorflow/contrib/receptive_field/BUILD | 71 +++ tensorflow/contrib/receptive_field/README.md | 164 ++++++ .../contrib/receptive_field/__init__.py | 23 + .../receptive_field/python/__init__.py | 19 + .../python/util/examples/compute_rf.py | 90 ++++ .../python/util/examples/rf_benchmark.py | 460 +++++++++++++++++ .../write_inception_resnet_v2_graph.py | 57 +++ .../python/util/graph_compute_order.py | 88 ++++ .../python/util/receptive_field.py | 481 ++++++++++++++++++ .../python/util/receptive_field_test.py | 221 ++++++++ tensorflow/tools/pip_package/BUILD | 1 + 12 files changed, 1676 insertions(+) create mode 100644 tensorflow/contrib/receptive_field/BUILD create mode 100644 tensorflow/contrib/receptive_field/README.md create mode 100644 tensorflow/contrib/receptive_field/__init__.py create mode 100644 tensorflow/contrib/receptive_field/python/__init__.py create mode 100644 tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py create mode 100644 tensorflow/contrib/receptive_field/python/util/examples/rf_benchmark.py create mode 100644 tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py create mode 100644 tensorflow/contrib/receptive_field/python/util/graph_compute_order.py create mode 100644 tensorflow/contrib/receptive_field/python/util/receptive_field.py create mode 100644 tensorflow/contrib/receptive_field/python/util/receptive_field_test.py diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 799f2a72d0a..d14c593ade9 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -315,6 +315,7 @@ filegroup( "//tensorflow/contrib/nn:all_files", "//tensorflow/contrib/opt:all_files", "//tensorflow/contrib/predictor:all_files", + "//tensorflow/contrib/receptive_field:all_files", "//tensorflow/contrib/reduce_slice_ops:all_files", "//tensorflow/contrib/remote_fused_graph/pylib:all_files", "//tensorflow/contrib/resampler:all_files", diff --git a/tensorflow/contrib/receptive_field/BUILD b/tensorflow/contrib/receptive_field/BUILD new file mode 100644 index 00000000000..ed2f3af08cb --- /dev/null +++ b/tensorflow/contrib/receptive_field/BUILD @@ -0,0 +1,71 @@ +# Description: +# Contains modules to compute receptive field parameters for CNN models. + +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +load("//tensorflow:tensorflow.bzl", "py_test") + +# Transitive dependencies of this target will be included in the pip package. +py_library( + name = "receptive_field_pip", + deps = [ + ":graph_compute_order_py", + ":receptive_field_py", + ], +) + +py_library( + name = "graph_compute_order_py", + srcs = [ + "__init__.py", + "python/util/graph_compute_order.py", + ], + srcs_version = "PY2AND3", +) + +py_library( + name = "receptive_field_py", + srcs = [ + "__init__.py", + "python/util/receptive_field.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":graph_compute_order_py", + "//tensorflow/contrib/util:util_py", + "//tensorflow/python:platform", + ], +) + +py_test( + name = "receptive_field_test", + srcs = ["python/util/receptive_field_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":receptive_field_py", + "//tensorflow/contrib/framework:framework_py", + "//tensorflow/contrib/slim", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:nn", + ], +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/contrib/receptive_field/README.md b/tensorflow/contrib/receptive_field/README.md new file mode 100644 index 00000000000..f7539ec1450 --- /dev/null +++ b/tensorflow/contrib/receptive_field/README.md @@ -0,0 +1,164 @@ +# Receptive field computation for convnets + +This library enables you to easily compute the receptive field parameters of +your favorite convnet. You can use it to understand how big of an input image +region your output features depend on. Better yet, using the parameters computed +by the library, you can easily find the exact image region which is used to +compute each convnet feature. + +## Basic usage + +The main function to be called is `compute_receptive_field_from_graph_def`, +which will return the receptive field, effective stride and effective padding +for both horizontal and vertical directions. + +For example, if your model is constructed using the function +`my_model_construction()`, you can use the library as follows: + +```python +import tensorflow as tf +from tensorflow.contrib import receptive_field + +# Construct graph. +g = tf.Graph() +with g.as_default(): + images = tf.placeholder(tf.float32, shape=(1, None, None, 3), name='input_image') + my_model_construction(images) + +# Compute receptive field parameters. +rf_x, rf_y, eff_stride_x, eff_stride_y, eff_pad_x, eff_pad_y = \ + receptive_field.compute_receptive_field_from_graph_def( \ + g.as_graph_def(), 'input_image', 'my_output_endpoint') +``` + +Here's a simple example of computing the receptive field parameters for +Inception-Resnet-v2. To get this to work, be sure to checkout +[tensorflow/models](https://github.com/tensorflow/models), so that the Inception +models are available to you. This can be done in three simple commands: + +```sh +git clone https://github.com/tensorflow/models +cd models/slim +sudo python setup.py install_lib +``` + +You can then compute the receptive field parameters for Inception-Resnet-v2 as: + +```python +from nets import inception +import tensorflow as tf +from tensorflow.contrib import receptive_field + +# Construct graph. +g = tf.Graph() +with g.as_default(): + images = tf.placeholder(tf.float32, shape=(1, None, None, 3), name='input_image') + inception.inception_resnet_v2_base(images) + +# Compute receptive field parameters. +rf_x, rf_y, eff_stride_x, eff_stride_y, eff_pad_x, eff_pad_y = \ + receptive_field.compute_receptive_field_from_graph_def( \ + g.as_graph_def(), 'input_image', 'InceptionResnetV2/Conv2d_7b_1x1/Relu') +``` + +This will give you `rf_x = rf_y = 3039`, `eff_stride_x = eff_stride_y = 32`, and +`eff_pad_x = eff_pad_y = 1482`. This means that each feature that is output at +the node `'InceptionResnetV2/Conv2d_7b_1x1/Relu'` is computed from a region +which is of size `3039x3039`. Further, by using the expressions + +```python +center_x = -eff_pad_x + feature_x*eff_stride_x + (rf_x - 1)/2 +center_y = -eff_pad_y + feature_y*eff_stride_y + (rf_y - 1)/2 +``` + +one can compute the center of the region in the input image that is used to +compute the output feature at position `[feature_x, feature_y]`. For example, +the feature at position `[0, 2]` at the output of the layer +`'InceptionResnetV2/Conv2d_7b_1x1/Relu'` is centered in the original image in +the position `[37, 101]`. + +TODO: include link to derivations and definitions of different parameters. + +## Receptive field benchmark + +As you might expect, it is straightforward to run this library on the popular +convnets, and gather their receptive fields. We provide a python script which +does exactly that, available under `python/util/examples/rf_benchmark.py`. + +To get this to work, be sure to checkout +[tensorflow/models](https://github.com/tensorflow/models) (see the 3-command +instructions for this above). Then, simply: + +```sh +cd python/util/examples +python rf_benchmark.py --csv_path /tmp/rf_benchmark_results.csv +``` + +The script will write to stdout the receptive field parameters for many variants +of several popular convnets: AlexNet, VGG, ResNet, Inception, Mobilenet. They +are also written to the file `/tmp/rf_benchmark_results.csv`. + +TODO: include here a plot for receptive field sizes of different convnets. + +TODO: include table/link to pre-computed RF parameters. + +## Compute RF parameters from a graph pbtxt + +We also provide a utility to compute the receptive field parameters directly +from a graph protobuf file. + +Have a `graph.pbtxt` file and want to compute its receptive field parameters? We +got you covered. The only prerequisite is to install +[google/protobuf](https://github.com/google/protobuf), which you probably +already have if you're using tensorflow (otherwise, follow installation +instructions [here](https://github.com/google/protobuf/tree/master/python)). + +This should work: + +```sh +cd python/util/examples +python compute_rf.py \ + --graph_path /path/to/graph.pbtxt \ + --output_path /path/to/output/rf_info.txt \ + --input_node my_input_node \ + --output_node my_output_node +``` + +Don't know how to generate a graph protobuf file? Take a look at the +`write_inception_resnet_v2_graph.py` script, which shows how to save it for the +Inception-Resnet-v2 model: + +```sh +cd python/util/examples +python write_inception_resnet_v2_graph.py --graph_dir /tmp --graph_filename graph.pbtxt +``` + +This will write the Inception-Resnet-v2 graph protobuf to `/tmp/graph.pbtxt`. + +For completeness, here's how you would use this file to get the receptive field +parameters of the Inception-Resnet-v2 model: + +```sh +cd python/util/examples +python compute_rf.py \ + --graph_path /tmp/graph.pbtxt \ + --output_path /tmp/rf_info.txt \ + --input_node input_image \ + --output_node InceptionResnetV2/Conv2d_7b_1x1/Relu +``` + +This will write the receptive field parameters of the model to +`/tmp/rf_info.txt`, which will look like: + +```sh +Receptive field size (horizontal) = 3039 +Receptive field size (vertical) = 3039 +Effective stride (horizontal) = 32 +Effective stride (vertical) = 32 +Effective padding (horizontal) = 1482 +Effective padding (vertical) = 1482 +``` + +## Authors + +André Araujo (andrefaraujo@) and Mark Sandler diff --git a/tensorflow/contrib/receptive_field/__init__.py b/tensorflow/contrib/receptive_field/__init__.py new file mode 100644 index 00000000000..10745a6a53d --- /dev/null +++ b/tensorflow/contrib/receptive_field/__init__.py @@ -0,0 +1,23 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Module to compute receptive field parameters for CNN tensorflow models.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=unused-import +from tensorflow.contrib.receptive_field.python.util.graph_compute_order import get_compute_order +from tensorflow.contrib.receptive_field.python.util.receptive_field import compute_receptive_field_from_graph_def +# pylint: enable=unused-import diff --git a/tensorflow/contrib/receptive_field/python/__init__.py b/tensorflow/contrib/receptive_field/python/__init__.py new file mode 100644 index 00000000000..217047f92d3 --- /dev/null +++ b/tensorflow/contrib/receptive_field/python/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Module to compute receptive field parameters for CNN tensorflow models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function diff --git a/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py b/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py new file mode 100644 index 00000000000..70a0d11dff6 --- /dev/null +++ b/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py @@ -0,0 +1,90 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Computes Receptive Field (RF) information given a graph protobuf. + +For an example of usage, see accompanying file compute_rf.sh +""" + +import argparse +import sys + +from google.protobuf import text_format + +from tensorflow.contrib import receptive_field +from tensorflow.core.framework import graph_pb2 +from tensorflow.python.platform import app +from tensorflow.python.platform import gfile +from tensorflow.python.platform import tf_logging as logging + +cmd_args = None + + +def _load_graphdef(path): + """Helper function to load GraphDef from file. + + Args: + path: Path to pbtxt file. + + Returns: + graph_def: A GraphDef object. + """ + graph_def = graph_pb2.GraphDef() + pbstr = gfile.Open(path).read() + text_format.Parse(pbstr, graph_def) + return graph_def + + +def main(unused_argv): + + graph_def = _load_graphdef(cmd_args.graph_path) + + (receptive_field_x, receptive_field_y, effective_stride_x, effective_stride_y, + effective_padding_x, effective_padding_y + ) = receptive_field.compute_receptive_field_from_graph_def( + graph_def, cmd_args.input_node, cmd_args.output_node) + + logging.info('Receptive field size (horizontal) = %s', receptive_field_x) + logging.info('Receptive field size (vertical) = %s', receptive_field_y) + logging.info('Effective stride (horizontal) = %s', effective_stride_x) + logging.info('Effective stride (vertical) = %s', effective_stride_y) + logging.info('Effective padding (horizontal) = %s', effective_padding_x) + logging.info('Effective padding (vertical) = %s', effective_padding_y) + + f = gfile.GFile('%s' % cmd_args.output_path, 'w') + f.write('Receptive field size (horizontal) = %s\n' % receptive_field_x) + f.write('Receptive field size (vertical) = %s\n' % receptive_field_y) + f.write('Effective stride (horizontal) = %s\n' % effective_stride_x) + f.write('Effective stride (vertical) = %s\n' % effective_stride_y) + f.write('Effective padding (horizontal) = %s\n' % effective_padding_x) + f.write('Effective padding (vertical) = %s\n' % effective_padding_y) + f.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.register('type', 'bool', lambda v: v.lower() == 'true') + parser.add_argument( + '--graph_path', type=str, default='', help='Graph path (pbtxt format).') + parser.add_argument( + '--output_path', + type=str, + default='', + help='Path to output text file where RF information will be written to.') + parser.add_argument( + '--input_node', type=str, default='', help='Name of input node.') + parser.add_argument( + '--output_node', type=str, default='', help='Name of output node.') + cmd_args, unparsed = parser.parse_known_args() + app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/receptive_field/python/util/examples/rf_benchmark.py b/tensorflow/contrib/receptive_field/python/util/examples/rf_benchmark.py new file mode 100644 index 00000000000..94228dfa61b --- /dev/null +++ b/tensorflow/contrib/receptive_field/python/util/examples/rf_benchmark.py @@ -0,0 +1,460 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Computes Receptive Field (RF) information for different models. + +The receptive field (and related parameters) for the different models are +printed to stdout, and may also optionally be written to a CSV file. + +For an example of usage, see rf_benchmark.sh +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import csv +import sys + +from nets import alexnet +from nets import inception +from nets import mobilenet_v1 +from nets import resnet_v1 +from nets import resnet_v2 +from nets import vgg +from tensorflow.contrib import framework +from tensorflow.contrib import receptive_field +from tensorflow.contrib import slim +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import app + +cmd_args = None + +# Input node name for all architectures. +_INPUT_NODE = 'input_image' + +# Variants of different network architectures. + +# - resnet: different versions and sizes. +_SUPPORTED_RESNET_VARIANTS = [ + 'resnet_v1_50', 'resnet_v1_101', 'resnet_v1_152', 'resnet_v1_200', + 'resnet_v2_50', 'resnet_v2_101', 'resnet_v2_152', 'resnet_v2_200' +] + +# - inception_resnet_v2: default, and version with SAME padding. +_SUPPORTED_INCEPTIONRESNETV2_VARIANTS = [ + 'inception_resnet_v2', 'inception_resnet_v2-same' +] + +# - inception_v2: default, and version with no separable conv. +_SUPPORTED_INCEPTIONV2_VARIANTS = [ + 'inception_v2', 'inception_v2-no-separable-conv' +] + +# - inception_v3: default version. +_SUPPORTED_INCEPTIONV3_VARIANTS = ['inception_v3'] + +# - inception_v4: default version. +_SUPPORTED_INCEPTIONV4_VARIANTS = ['inception_v4'] + +# - alexnet_v2: default version. +_SUPPORTED_ALEXNETV2_VARIANTS = ['alexnet_v2'] + +# - vgg: vgg_a (with 11 layers) and vgg_16 (version D). +_SUPPORTED_VGG_VARIANTS = ['vgg_a', 'vgg_16'] + +# - mobilenet_v1: 100% and 75%. +_SUPPORTED_MOBILENETV1_VARIANTS = ['mobilenet_v1', 'mobilenet_v1_075'] + + +def _construct_model(model_type='resnet_v1_50'): + """Constructs model for the desired type of CNN. + + Args: + model_type: Type of model to be used. + + Returns: + end_points: A dictionary from components of the network to the corresponding + activations. + + Raises: + ValueError: If the model_type is not supported. + """ + # Placeholder input. + images = array_ops.placeholder( + dtypes.float32, shape=(1, None, None, 3), name=_INPUT_NODE) + + # Construct model. + if model_type == 'inception_resnet_v2': + _, end_points = inception.inception_resnet_v2_base(images) + elif model_type == 'inception_resnet_v2-same': + _, end_points = inception.inception_resnet_v2_base( + images, align_feature_maps=True) + elif model_type == 'inception_v2': + _, end_points = inception.inception_v2_base(images) + elif model_type == 'inception_v2-no-separable-conv': + _, end_points = inception.inception_v2_base( + images, use_separable_conv=False) + elif model_type == 'inception_v3': + _, end_points = inception.inception_v3_base(images) + elif model_type == 'inception_v4': + _, end_points = inception.inception_v4_base(images) + elif model_type == 'alexnet_v2': + _, end_points = alexnet.alexnet_v2(images) + elif model_type == 'vgg_a': + _, end_points = vgg.vgg_a(images) + elif model_type == 'vgg_16': + _, end_points = vgg.vgg_16(images) + elif model_type == 'mobilenet_v1': + _, end_points = mobilenet_v1.mobilenet_v1_base(images) + elif model_type == 'mobilenet_v1_075': + _, end_points = mobilenet_v1.mobilenet_v1_base( + images, depth_multiplier=0.75) + elif model_type == 'resnet_v1_50': + _, end_points = resnet_v1.resnet_v1_50( + images, num_classes=None, is_training=False, global_pool=False) + elif model_type == 'resnet_v1_101': + _, end_points = resnet_v1.resnet_v1_101( + images, num_classes=None, is_training=False, global_pool=False) + elif model_type == 'resnet_v1_152': + _, end_points = resnet_v1.resnet_v1_152( + images, num_classes=None, is_training=False, global_pool=False) + elif model_type == 'resnet_v1_200': + _, end_points = resnet_v1.resnet_v1_200( + images, num_classes=None, is_training=False, global_pool=False) + elif model_type == 'resnet_v2_50': + _, end_points = resnet_v2.resnet_v2_50( + images, num_classes=None, is_training=False, global_pool=False) + elif model_type == 'resnet_v2_101': + _, end_points = resnet_v2.resnet_v2_101( + images, num_classes=None, is_training=False, global_pool=False) + elif model_type == 'resnet_v2_152': + _, end_points = resnet_v2.resnet_v2_152( + images, num_classes=None, is_training=False, global_pool=False) + elif model_type == 'resnet_v2_200': + _, end_points = resnet_v2.resnet_v2_200( + images, num_classes=None, is_training=False, global_pool=False) + else: + raise ValueError('Unsupported model_type %s.' % model_type) + + return end_points + + +def _get_desired_end_point_keys(model_type='resnet_v1_50'): + """Gets list of desired end point keys for a type of CNN. + + Args: + model_type: Type of model to be used. + + Returns: + desired_end_point_types: A list containing the desired end-points. + + Raises: + ValueError: If the model_type is not supported. + """ + if model_type in _SUPPORTED_RESNET_VARIANTS: + blocks = ['block1', 'block2', 'block3', 'block4'] + desired_end_point_keys = ['%s/%s' % (model_type, i) for i in blocks] + elif model_type in _SUPPORTED_INCEPTIONRESNETV2_VARIANTS: + desired_end_point_keys = [ + 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'MaxPool_3a_3x3', + 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', 'Mixed_5b', + 'Mixed_6a', 'PreAuxLogits', 'Mixed_7a', 'Conv2d_7b_1x1' + ] + elif model_type in _SUPPORTED_INCEPTIONV2_VARIANTS: + desired_end_point_keys = [ + 'Conv2d_1a_7x7', 'MaxPool_2a_3x3', 'Conv2d_2b_1x1', 'Conv2d_2c_3x3', + 'MaxPool_3a_3x3', 'Mixed_3b', 'Mixed_3c', 'Mixed_4a', 'Mixed_4b', + 'Mixed_4c', 'Mixed_4d', 'Mixed_4e', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c' + ] + elif model_type in _SUPPORTED_INCEPTIONV3_VARIANTS: + desired_end_point_keys = [ + 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'MaxPool_3a_3x3', + 'Conv2d_3b_1x1', 'Conv2d_4a_3x3', 'MaxPool_5a_3x3', 'Mixed_5b', + 'Mixed_5c', 'Mixed_5d', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', + 'Mixed_6e', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c' + ] + elif model_type in _SUPPORTED_INCEPTIONV4_VARIANTS: + desired_end_point_keys = [ + 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', 'Mixed_3a', + 'Mixed_4a', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', 'Mixed_5e', + 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', 'Mixed_6e', 'Mixed_6f', + 'Mixed_6g', 'Mixed_6h', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c', 'Mixed_7d' + ] + elif model_type in _SUPPORTED_ALEXNETV2_VARIANTS: + ep = ['conv1', 'pool1', 'conv2', 'conv3', 'conv4', 'conv5', 'pool5'] + desired_end_point_keys = ['%s/%s' % (model_type, i) for i in ep] + elif model_type in _SUPPORTED_VGG_VARIANTS: + ep = [ + 'conv1/conv1_1', 'pool1', 'conv2/conv2_1', 'pool2', 'conv3/conv3_1', + 'conv3/conv3_2', 'pool3', 'conv4/conv4_1', 'conv4/conv4_2', 'pool4', + 'conv5/conv5_1', 'conv5/conv5_2', 'pool5' + ] + desired_end_point_keys = ['%s/%s' % (model_type, i) for i in ep] + elif model_type in _SUPPORTED_MOBILENETV1_VARIANTS: + desired_end_point_keys = [ + 'Conv2d_0', 'Conv2d_1_pointwise', 'Conv2d_2_pointwise', + 'Conv2d_3_pointwise', 'Conv2d_4_pointwise', 'Conv2d_5_pointwise', + 'Conv2d_6_pointwise', 'Conv2d_7_pointwise', 'Conv2d_8_pointwise', + 'Conv2d_9_pointwise', 'Conv2d_10_pointwise', 'Conv2d_11_pointwise', + 'Conv2d_12_pointwise', 'Conv2d_13_pointwise' + ] + else: + raise ValueError('Unsupported model_type %s.' % model_type) + + return desired_end_point_keys + + +def _model_graph_def(model_type='resnet_v1_50', arg_sc=None): + """Constructs a model graph, returning GraphDef and end-points. + + Args: + model_type: Type of model to be used. + arg_sc: Optional arg scope to use in constructing the graph. + + Returns: + graph_def: GraphDef of constructed graph. + end_points: A dictionary from components of the network to the corresponding + activations. + """ + if arg_sc is None: + arg_sc = {} + g = ops.Graph() + with g.as_default(): + with framework.arg_scope(arg_sc): + end_points = _construct_model(model_type) + + return g.as_graph_def(), end_points + + +def _model_rf(graphdef, + end_points, + desired_end_point_keys, + model_type='resnet_v1_50', + csv_writer=None): + """Computes receptive field information for a given CNN model. + + The information will be printed to stdout. If the RF parameters are the same + for the horizontal and vertical directions, it will be printed only once. + Otherwise, they are printed once for the horizontal and once for the vertical + directions. + + Args: + graphdef: GraphDef of given model. + end_points: A dictionary from components of the model to the corresponding + activations. + desired_end_point_keys: List of desired end points for which receptive field + information will be computed. + model_type: Type of model to be used, used only for printing purposes. + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for desired_end_point_key in desired_end_point_keys: + print('- %s:' % desired_end_point_key) + output_node_with_colon = end_points[desired_end_point_key].name + pos = output_node_with_colon.rfind(':') + output_node = output_node_with_colon[:pos] + (receptive_field_x, receptive_field_y, effective_stride_x, + effective_stride_y, effective_padding_x, effective_padding_y + ) = receptive_field.compute_receptive_field_from_graph_def( + graphdef, _INPUT_NODE, output_node) + # If values are the same in horizontal/vertical directions, just report one + # of them. Otherwise, report both. + if (receptive_field_x == receptive_field_y) and ( + effective_stride_x == effective_stride_y) and ( + effective_padding_x == effective_padding_y): + print('Receptive field size = %5s, effective stride = %5s, effective ' + 'padding = %5s' % (str(receptive_field_x), str(effective_stride_x), + str(effective_padding_x))) + else: + print('Receptive field size: horizontal = %5s, vertical = %5s. ' + 'Effective stride: horizontal = %5s, vertical = %5s. Effective ' + 'padding: horizontal = %5s, vertical = %5s' % + (str(receptive_field_x), str(receptive_field_y), + str(effective_stride_x), str(effective_stride_y), + str(effective_padding_x), str(effective_padding_y))) + if csv_writer is not None: + csv_writer.writerow({ + 'CNN': model_type, + 'end_point': desired_end_point_key, + 'RF size hor': str(receptive_field_x), + 'RF size ver': str(receptive_field_y), + 'effective stride hor': str(effective_stride_x), + 'effective stride ver': str(effective_stride_y), + 'effective padding hor': str(effective_padding_x), + 'effective padding ver': str(effective_padding_y) + }) + + +def _process_model_rf(model_type='resnet_v1_50', csv_writer=None, arg_sc=None): + """Contructs model graph and desired end-points, and compute RF. + + The computed RF parameters are printed to stdout by the _model_rf function. + + Args: + model_type: Type of model to be used. + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + arg_sc: Optional arg scope to use in constructing the graph. + + """ + print('********************%s' % model_type) + graphdef, end_points = _model_graph_def(model_type, arg_sc) + desired_end_point_keys = _get_desired_end_point_keys(model_type) + _model_rf(graphdef, end_points, desired_end_point_keys, model_type, + csv_writer) + + +def _resnet_rf(csv_writer=None): + """Computes RF and associated parameters for resnet models. + + The computed values are written to stdout. + + Args: + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for model_type in _SUPPORTED_RESNET_VARIANTS: + arg_sc = resnet_v1.resnet_arg_scope() + _process_model_rf(model_type, csv_writer, arg_sc) + + +def _inception_resnet_v2_rf(csv_writer=None): + """Computes RF and associated parameters for the inception_resnet_v2 model. + + The computed values are written to stdout. + + Args: + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for model_type in _SUPPORTED_INCEPTIONRESNETV2_VARIANTS: + _process_model_rf(model_type, csv_writer) + + +def _inception_v2_rf(csv_writer=None): + """Computes RF and associated parameters for the inception_v2 model. + + The computed values are written to stdout. + + Args: + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for model_type in _SUPPORTED_INCEPTIONV2_VARIANTS: + _process_model_rf(model_type, csv_writer) + + +def _inception_v3_rf(csv_writer=None): + """Computes RF and associated parameters for the inception_v3 model. + + The computed values are written to stdout. + + Args: + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for model_type in _SUPPORTED_INCEPTIONV3_VARIANTS: + _process_model_rf(model_type, csv_writer) + + +def _inception_v4_rf(csv_writer=None): + """Computes RF and associated parameters for the inception_v4 model. + + The computed values are written to stdout. + + Args: + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for model_type in _SUPPORTED_INCEPTIONV4_VARIANTS: + _process_model_rf(model_type, csv_writer) + + +def _alexnet_v2_rf(csv_writer=None): + """Computes RF and associated parameters for the alexnet_v2 model. + + The computed values are written to stdout. + + Args: + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for model_type in _SUPPORTED_ALEXNETV2_VARIANTS: + _process_model_rf(model_type, csv_writer) + + +def _vgg_rf(csv_writer=None): + """Computes RF and associated parameters for the vgg model. + + The computed values are written to stdout. + + Args: + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for model_type in _SUPPORTED_VGG_VARIANTS: + _process_model_rf(model_type, csv_writer) + + +def _mobilenet_v1_rf(csv_writer=None): + """Computes RF and associated parameters for the mobilenet_v1 model. + + The computed values are written to stdout. + + Args: + csv_writer: A CSV writer for RF parameters, which is used if it is not None. + """ + for model_type in _SUPPORTED_MOBILENETV1_VARIANTS: + with slim.arg_scope( + [slim.batch_norm, slim.dropout], is_training=False) as arg_sc: + _process_model_rf(model_type, csv_writer, arg_sc) + + +def main(unused_argv): + # Configure CSV file which will be written, if desired. + if cmd_args.csv_path: + csv_file = open(cmd_args.csv_path, 'w') + field_names = [ + 'CNN', 'end_point', 'RF size hor', 'RF size ver', + 'effective stride hor', 'effective stride ver', 'effective padding hor', + 'effective padding ver' + ] + rf_writer = csv.DictWriter(csv_file, fieldnames=field_names) + rf_writer.writeheader() + else: + rf_writer = None + + # Compute RF parameters for each network architecture. + _alexnet_v2_rf(rf_writer) + _vgg_rf(rf_writer) + _inception_v2_rf(rf_writer) + _inception_v3_rf(rf_writer) + _inception_v4_rf(rf_writer) + _inception_resnet_v2_rf(rf_writer) + _mobilenet_v1_rf(rf_writer) + _resnet_rf(rf_writer) + + # Close CSV file, if it was opened. + if cmd_args.csv_path: + csv_file.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.register('type', 'bool', lambda v: v.lower() == 'true') + parser.add_argument( + '--csv_path', + type=str, + default='', + help="""\ + Path to CSV file that will be written with RF parameters.If empty, no + file will be written.\ + """) + cmd_args, unparsed = parser.parse_known_args() + app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py b/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py new file mode 100644 index 00000000000..a2384f66ce9 --- /dev/null +++ b/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py @@ -0,0 +1,57 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Simple script to write Inception-ResNet-v2 model to graph file. +""" + +import argparse +import sys + +from nets import inception +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import graph_io +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import app + +cmd_args = None + + +def main(unused_argv): + # Model definition. + g = ops.Graph() + with g.as_default(): + images = array_ops.placeholder( + dtypes.float32, shape=(1, None, None, 3), name='input_image') + inception.inception_resnet_v2_base(images) + + graph_io.write_graph(g.as_graph_def(), cmd_args.graph_dir, + cmd_args.graph_filename) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.register('type', 'bool', lambda v: v.lower() == 'true') + parser.add_argument( + '--graph_dir', + type=str, + default='/tmp', + help='Directory where graph will be saved.') + parser.add_argument( + '--graph_filename', + type=str, + default='graph.pbtxt', + help='Filename of graph that will be saved.') + cmd_args, unparsed = parser.parse_known_args() + app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/receptive_field/python/util/graph_compute_order.py b/tensorflow/contrib/receptive_field/python/util/graph_compute_order.py new file mode 100644 index 00000000000..8af4be16d6c --- /dev/null +++ b/tensorflow/contrib/receptive_field/python/util/graph_compute_order.py @@ -0,0 +1,88 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Library to compute order of computations in a graph. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections + + +class GraphDefHelper(object): + """Helper class to collect node names and definitions. + + Example: + b = GraphDefHelper(graph_def) + # Prints node that produces given output. + print b.output_of['conv/foo/bar'] + """ + + def __init__(self, gd): + self.output_of = {} + for each in gd.node: + self.output_of[each.name] = each + + +# pylint: disable=invalid-name +_NodeEntry = collections.namedtuple('NodeEntry', field_names=['order', 'node']) + + +def _get_computed_nodes(g, output, seen): + """Traverses the graph in topological order. + + Args: + g: GraphDefHelper object. + output: current node. + seen: map of nodes we've already traversed. + Returns: + order in topological sort for 'output'. + """ + if output in seen: + return seen[output].order + node_def = g.output_of.get(output, None) + if node_def is None: + seen[output] = _NodeEntry(0, None) + return 0 + + r = 0 + for each in node_def.input: + # Parses name of input node. + if each.startswith('^'): + each = each[1:] + each = each.split(':')[0] + # Recursively computes ordering. + new_v = _get_computed_nodes(g, each, seen) + r = max(r, new_v + 1) + + seen[output] = _NodeEntry(r, node_def) + + return seen[output].order + + +def get_compute_order(graph_def): + """Computes order of computation for a given graph. + + Args: + graph_def: GraphDef object. + Returns: + map: name -> {order, node} + """ + helper = GraphDefHelper(graph_def) + seen = collections.defaultdict(_NodeEntry) + for each in graph_def.node: + _get_computed_nodes(helper, each.name, seen) + return seen diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field.py b/tensorflow/contrib/receptive_field/python/util/receptive_field.py new file mode 100644 index 00000000000..4e723829bff --- /dev/null +++ b/tensorflow/contrib/receptive_field/python/util/receptive_field.py @@ -0,0 +1,481 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions to compute receptive field of a fully-convolutional network. + +Please refer to the following g3doc for detailed explanation on how this +computation is performed, and why it is important: +g3doc/photos/vision/features/delf/g3doc/rf_computation.md +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math +from tensorflow.contrib.receptive_field.python.util import graph_compute_order +from tensorflow.contrib.util import make_ndarray +from tensorflow.python.platform import tf_logging as logging + +# White-listed layer operations, which do not affect the receptive field +# computation. +_UNCHANGED_RF_LAYER_OPS = [ + "Softplus", "Relu", "BiasAdd", "Mul", "Add", "Const", "Identity", + "VariableV2", "Sub", "Rsqrt", "ConcatV2" +] + + +def _stride_size(node): + """Computes stride size given a TF node. + + Args: + node: Tensorflow node (NodeDef proto). + + Returns: + stride_x: Stride size for horizontal direction (integer). + stride_y: Stride size for vertical direction (integer). + """ + strides_attr = node.attr["strides"] + logging.vlog(4, "strides_attr = %s", strides_attr) + stride_y = strides_attr.list.i[1] + stride_x = strides_attr.list.i[2] + return stride_x, stride_y + + +def _conv_kernel_size(node, name_to_order_node): + """Computes kernel size given a TF convolution or pooling node. + + Args: + node: Tensorflow node (NodeDef proto). + name_to_order_node: Map from name to {order, node}. Output of + graph_compute_order.get_compute_order(). + + Returns: + kernel_size_x: Kernel size for horizontal direction (integer). + kernel_size_y: Kernel size for vertical direction (integer). + + Raises: + ValueError: If the weight layer node is invalid. + """ + weights_layer_read_name = node.input[1] + if not weights_layer_read_name.endswith("/read"): + raise ValueError( + "Weight layer's name input to conv layer does not end with '/read'") + weights_layer_param_name = weights_layer_read_name[:-5] + weights_node = name_to_order_node[weights_layer_param_name].node + if weights_node.op != "VariableV2": + raise ValueError("Weight layer is not of type VariableV2") + shape = weights_node.attr["shape"] + logging.vlog(4, "weight shape = %s", shape) + kernel_size_y = shape.shape.dim[0].size + kernel_size_x = shape.shape.dim[1].size + return kernel_size_x, kernel_size_y + + +def _padding_size_conv_pool(node, kernel_size, stride): + """Computes padding size given a TF convolution or pooling node. + + Args: + node: Tensorflow node (NodeDef proto). + kernel_size: Kernel size of node (integer). + stride: Stride size of node (integer). + + Returns: + padding: Padding size (integer). + + Raises: + ValueError: If padding is invalid. + """ + # In this case, we need to carefully consider the different TF padding modes. + # The padding depends on kernel size, and may depend on input size. If it + # depends on input size, we raise an exception. + padding_attr = node.attr["padding"] + logging.vlog(4, "padding_attr = %s", padding_attr) + if padding_attr.s == "VALID": + padding = 0 + elif padding_attr.s == "SAME": + if kernel_size == 1: + padding = 0 + elif stride == 1: + padding = int(math.floor((float(kernel_size) - 1) / 2)) + elif stride == 2 and kernel_size % 2 == 0: + padding = int(math.floor((float(kernel_size) - 1) / 2)) + else: + padding = None + logging.warning( + "Padding depends on input size, which means that the effective " + "padding may be different depending on the input image " + "dimensionality. In this case, alignment check will be skipped.") + else: + raise ValueError("Invalid padding operation") + return padding + + +def _pool_kernel_size(node): + """Computes kernel size given a TF pooling node. + + Args: + node: Tensorflow node (NodeDef proto). + + Returns: + kernel_size_x: Kernel size for horizontal direction (integer). + kernel_size_y: Kernel size for vertical direction (integer). + + Raises: + ValueError: If pooling is invalid. + """ + ksize = node.attr["ksize"] + kernel_size_y = ksize.list.i[1] + kernel_size_x = ksize.list.i[2] + if ksize.list.i[0] != 1: + raise ValueError("pool ksize for first dim is not 1") + if ksize.list.i[3] != 1: + raise ValueError("pool ksize for last dim is not 1") + return kernel_size_x, kernel_size_y + + +def _padding_size_pad_layer(node, name_to_order_node): + """Computes padding size given a TF padding node. + + Args: + node: Tensorflow node (NodeDef proto). + name_to_order_node: Map from name to {order, node}. Output of + graph_compute_order.get_compute_order(). + + Returns: + padding_x: Padding size for horizontal direction (integer). + padding_y: Padding size for vertical direction (integer). + + Raises: + ValueError: If padding layer is invalid. + """ + paddings_layer_name = node.input[1] + if not paddings_layer_name.endswith("/paddings"): + raise ValueError("Padding layer name does not end with '/paddings'") + paddings_node = name_to_order_node[paddings_layer_name].node + if paddings_node.op != "Const": + raise ValueError("Padding op is not Const") + value = paddings_node.attr["value"] + t = make_ndarray(value.tensor) + padding_y = t[1][0] + padding_x = t[2][0] + if t[0][0] != 0: + raise ValueError("padding is not zero for first tensor dim") + if t[3][0] != 0: + raise ValueError("padding is not zero for last tensor dim") + return padding_x, padding_y + + +def _get_layer_params(node, name_to_order_node): + """Gets layer parameters relevant for RF computation. + + Currently, only these nodes are supported: + - Conv2D + - DepthwiseConv2dNative + - Pad + - MaxPool + - AvgPool + - all nodes listed in _UNCHANGED_RF_LAYER_OPS + + Args: + node: Tensorflow node (NodeDef proto). + name_to_order_node: Map from name to {order, node}. Output of + graph_compute_order.get_compute_order(). + + Returns: + kernel_size_x: Kernel size for horizontal direction (integer). + kernel_size_y: Kernel size for vertical direction (integer). + stride_x: Stride size for horizontal direction (integer). + stride_y: Stride size for vertical direction (integer). + padding_x: Padding size for horizontal direction (integer). + padding_y: Padding size for vertical direction (integer). + + Raises: + ValueError: If layer op is unknown. + """ + logging.vlog(3, "node.op = %s", node.op) + logging.vlog(4, "node = %s", node) + if node.op == "Conv2D" or node.op == "DepthwiseConv2dNative": + stride_x, stride_y = _stride_size(node) + kernel_size_x, kernel_size_y = _conv_kernel_size(node, name_to_order_node) + # Compute the padding for this node separately for each direction. + padding_x = _padding_size_conv_pool(node, kernel_size_x, stride_x) + padding_y = _padding_size_conv_pool(node, kernel_size_y, stride_y) + elif node.op == "Pad": + # Kernel and stride are simply 1 in this case. + kernel_size_x = 1 + kernel_size_y = 1 + stride_x = 1 + stride_y = 1 + padding_x, padding_y = _padding_size_pad_layer(node, name_to_order_node) + elif node.op == "MaxPool" or node.op == "AvgPool": + stride_x, stride_y = _stride_size(node) + kernel_size_x, kernel_size_y = _pool_kernel_size(node) + # Compute the padding for this node separately for each direction. + padding_x = _padding_size_conv_pool(node, kernel_size_x, stride_x) + padding_y = _padding_size_conv_pool(node, kernel_size_y, stride_y) + elif node.op in _UNCHANGED_RF_LAYER_OPS: + # These nodes do not modify the RF parameters. + kernel_size_x = 1 + kernel_size_y = 1 + stride_x = 1 + stride_y = 1 + padding_x = 0 + padding_y = 0 + else: + raise ValueError("Unknown layer op: %s" % node.op) + return kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, padding_y + + +def _reverse_sort_by_order(name_to_order_node): + """Sorts map of name_to_order_node nodes in reverse order. + + The output is such that the nodes in name_to_order_node are sorted in + descending order of the "order" field. + + Args: + name_to_order_node: Map from name to {order, node}. Output of + graph_compute_order.get_compute_order(). + + Returns: + sorted_name_to_order_node: Sorted version of the input, in descending order. + """ + return sorted(name_to_order_node.items(), key=lambda x: -x[1].order) + + +def _get_rf_size_node_input(stride, kernel_size, rf_size_output): + """Computes RF size at the input of a given layer. + + Args: + stride: Stride of given layer (integer). + kernel_size: Kernel size of given layer (integer). + rf_size_output: RF size at output of given layer (integer). + + Returns: + rf_size_input: RF size at input of given layer (integer). + """ + return stride * rf_size_output + kernel_size - stride + + +def _get_effective_stride_node_input(stride, effective_stride_output): + """Computes effective stride at the input of a given layer. + + Args: + stride: Stride of given layer (integer). + effective_stride_output: Effective stride at output of given layer + (integer). + + Returns: + effective_stride_input: Effective stride at input of given layer + (integer). + """ + return stride * effective_stride_output + + +def _get_effective_padding_node_input(stride, padding, + effective_padding_output): + """Computes effective padding at the input of a given layer. + + Args: + stride: Stride of given layer (integer). + padding: Padding of given layer (integer). + effective_padding_output: Effective padding at output of given layer + (integer). + + Returns: + effective_padding_input: Effective padding at input of given layer + (integer). + """ + return stride * effective_padding_output + padding + + +def compute_receptive_field_from_graph_def(graph_def, input_node, output_node): + """Computes receptive field (RF) parameters from a GraphDef object. + + Args: + graph_def: GraphDef object. + input_node: Name of the input node from graph. + output_node: Name of the output node from graph. + + Returns: + rf_size_x: Receptive field size of network in the horizontal direction, with + respect to specified input and output. + rf_size_y: Receptive field size of network in the vertical direction, with + respect to specified input and output. + effective_stride_x: Effective stride of network in the horizontal direction, + with respect to specified input and output. + effective_stride_y: Effective stride of network in the vertical direction, + with respect to specified input and output. + effective_padding_x: Effective padding of network in the horizontal + direction, with respect to specified input and output. + effective_padding_y: Effective padding of network in the vertical + direction, with respect to specified input and output. + + Raises: + ValueError: If network is not aligned or if either input or output nodes + cannot be found. For network criterion alignment, see + photos/vision/features/delf/g3doc/rf_computation.md + """ + # Computes order of computation for a given graph. + name_to_order_node = graph_compute_order.get_compute_order( + graph_def=graph_def) + + # Sort in reverse topological order. + order = _reverse_sort_by_order(name_to_order_node) + + # Dictionaries to keep track of receptive field, effective stride and + # effective padding of different nodes. + rf_sizes_x = {} + rf_sizes_y = {} + effective_strides_x = {} + effective_strides_y = {} + effective_paddings_x = {} + effective_paddings_y = {} + + # Initialize dicts for output_node. + rf_sizes_x[output_node] = 1 + rf_sizes_y[output_node] = 1 + effective_strides_x[output_node] = 1 + effective_strides_y[output_node] = 1 + effective_paddings_x[output_node] = 0 + effective_paddings_y[output_node] = 0 + + # Flag to denote if we found output node yet. If we have not, we skip nodes + # until the output node is found. + found_output_node = False + + # Flag to denote if padding is undefined. This happens when SAME padding mode + # is used in conjunction with stride and kernel sizes which make it such that + # the padding to be applied would depend on the input size. In this case, + # alignment checks are skipped, and the effective padding is None. + undefined_padding = False + + for _, (o, node) in order: + if node: + logging.vlog(3, "%10d %-100s %-20s" % (o, node.name[:90], node.op)) + else: + continue + + # When we find input node, we can stop. + if node.name == input_node: + break + + # Loop until we find the output node. All nodes before finding the output + # one are irrelevant, so they can be skipped. + if not found_output_node: + if node.name == output_node: + found_output_node = True + + if found_output_node: + if node.name not in rf_sizes_x: + assert node.name not in rf_sizes_y, ("Node %s is in rf_sizes_y, but " + "not in rf_sizes_x" % node.name) + # In this case, node is not relevant since it's not part of the + # computation we're interested in. + logging.vlog(3, "Irrelevant node %s, skipping it...", node.name) + continue + + # Get params for this layer. + kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, padding_y = ( + _get_layer_params(node, name_to_order_node)) + logging.vlog(3, "kernel_size_x = %s, kernel_size_y = %s, " + "stride_x = %s, stride_y = %s, " + "padding_x = %s, padding_y = %s" % + (kernel_size_x, kernel_size_y, stride_x, stride_y, padding_x, + padding_y)) + if padding_x is None or padding_y is None: + undefined_padding = True + + # Get parameters at input of this layer which may or may not be propagated + # to the input layers. + rf_size_input_x = _get_rf_size_node_input(stride_x, kernel_size_x, + rf_sizes_x[node.name]) + rf_size_input_y = _get_rf_size_node_input(stride_y, kernel_size_y, + rf_sizes_y[node.name]) + effective_stride_input_x = _get_effective_stride_node_input( + stride_x, effective_strides_x[node.name]) + effective_stride_input_y = _get_effective_stride_node_input( + stride_y, effective_strides_y[node.name]) + if not undefined_padding: + effective_padding_input_x = _get_effective_padding_node_input( + stride_x, padding_x, effective_paddings_x[node.name]) + effective_padding_input_y = _get_effective_padding_node_input( + stride_y, padding_y, effective_paddings_y[node.name]) + else: + effective_padding_input_x = None + effective_padding_input_y = None + + # Loop over this node's inputs and potentially propagate information down. + for inp_name in node.input: + logging.vlog(4, "inp_name = %s", inp_name) + inp_node = name_to_order_node[inp_name].node + logging.vlog(4, "inp_node = \n%s", inp_node) + if inp_node.name in rf_sizes_x: + assert inp_node.name in rf_sizes_y, ( + "Node %s is in rf_sizes_x, but " + "not in rf_sizes_y" % inp_node.name) + # This node was already discovered through a previous path, so we need + # to make sure that graph is aligned. This alignment check is skipped + # if the padding is not defined, since in this case alignment cannot + # be checked. + if not undefined_padding: + if effective_strides_x[inp_node.name] != effective_stride_input_x: + raise ValueError( + "Graph is not aligned since effective stride from different " + "paths is different in horizontal direction") + if effective_strides_y[inp_node.name] != effective_stride_input_y: + raise ValueError( + "Graph is not aligned since effective stride from different " + "paths is different in vertical direction") + if (rf_sizes_x[inp_node.name] - 1 + ) / 2 - effective_paddings_x[inp_node.name] != ( + rf_size_input_x - 1) / 2 - effective_padding_input_x: + raise ValueError( + "Graph is not aligned since center shift from different " + "paths is different in horizontal direction") + if (rf_sizes_y[inp_node.name] - 1 + ) / 2 - effective_paddings_y[inp_node.name] != ( + rf_size_input_y - 1) / 2 - effective_padding_input_y: + raise ValueError( + "Graph is not aligned since center shift from different " + "paths is different in vertical direction") + # Keep track of path with largest RF, for both directions. + if rf_sizes_x[inp_node.name] < rf_size_input_x: + rf_sizes_x[inp_node.name] = rf_size_input_x + effective_strides_x[inp_node.name] = effective_stride_input_x + effective_paddings_x[inp_node.name] = effective_padding_input_x + if rf_sizes_y[inp_node.name] < rf_size_input_y: + rf_sizes_y[inp_node.name] = rf_size_input_y + effective_strides_y[inp_node.name] = effective_stride_input_y + effective_paddings_y[inp_node.name] = effective_padding_input_y + else: + assert inp_node.name not in rf_sizes_y, ( + "Node %s is in rf_sizes_y, but " + "not in rf_sizes_x" % inp_node.name) + # In this case, it is the first time we encounter this node. So we + # propagate the RF parameters. + rf_sizes_x[inp_node.name] = rf_size_input_x + rf_sizes_y[inp_node.name] = rf_size_input_y + effective_strides_x[inp_node.name] = effective_stride_input_x + effective_strides_y[inp_node.name] = effective_stride_input_y + effective_paddings_x[inp_node.name] = effective_padding_input_x + effective_paddings_y[inp_node.name] = effective_padding_input_y + + if not found_output_node: + raise ValueError("Output node was not found") + if input_node not in rf_sizes_x: + raise ValueError("Input node was not found") + return (rf_sizes_x[input_node], rf_sizes_y[input_node], + effective_strides_x[input_node], effective_strides_y[input_node], + effective_paddings_x[input_node], effective_paddings_y[input_node]) diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py new file mode 100644 index 00000000000..44e5beda607 --- /dev/null +++ b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py @@ -0,0 +1,221 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for receptive_fields module.""" + +from tensorflow.contrib import slim +from tensorflow.contrib.receptive_field.python.util import receptive_field +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import nn +from tensorflow.python.platform import test + + +def create_test_network_1(): + """Aligned network for test. + + The graph corresponds to the example from the second figure in + go/cnn-rf-computation#arbitrary-computation-graphs + + Returns: + g: Tensorflow graph object (Graph proto). + """ + g = ops.Graph() + with g.as_default(): + # An 8x8 test image. + x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image') + # Left branch. + l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') + # Right branch. + l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]]) + l2 = slim.conv2d(l2_pad, 1, [3, 3], stride=2, scope='L2', padding='VALID') + l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID') + # Addition. + nn.relu(l1 + l3, name='output') + return g + + +def create_test_network_2(): + """Aligned network for test. + + The graph corresponds to a variation to the example from the second figure in + go/cnn-rf-computation#arbitrary-computation-graphs. Layers 2 and 3 are changed + to max-pooling operations. Since the functionality is the same as convolution, + the network is aligned and the receptive field size is the same as from the + network created using create_test_network_1(). + + Returns: + g: Tensorflow graph object (Graph proto). + """ + g = ops.Graph() + with g.as_default(): + # An 8x8 test image. + x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image') + # Left branch. + l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') + # Right branch. + l2_pad = array_ops.pad(x, [[0, 0], [1, 0], [1, 0], [0, 0]]) + l2 = slim.max_pool2d(l2_pad, [3, 3], stride=2, scope='L2', padding='VALID') + l3 = slim.max_pool2d(l2, [1, 1], stride=2, scope='L3', padding='VALID') + # Addition. + nn.relu(l1 + l3, name='output') + return g + + +def create_test_network_3(): + """Misaligned network for test. + + The graph corresponds to the example from the first figure in + go/cnn-rf-computation#arbitrary-computation-graphs + + Returns: + g: Tensorflow graph object (Graph proto). + """ + g = ops.Graph() + with g.as_default(): + # An 8x8 test image. + x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image') + # Left branch. + l1_pad = array_ops.pad(x, [[0, 0], [2, 1], [2, 1], [0, 0]]) + l1 = slim.conv2d(l1_pad, 1, [5, 5], stride=2, scope='L1', padding='VALID') + # Right branch. + l2 = slim.conv2d(x, 1, [3, 3], stride=1, scope='L2', padding='VALID') + l3 = slim.conv2d(l2, 1, [3, 3], stride=1, scope='L3', padding='VALID') + # Addition. + nn.relu(l1 + l3, name='output') + return g + + +def create_test_network_4(): + """Misaligned network for test. + + The graph corresponds to a variation from the example from the second figure + in go/cnn-rf-computation#arbitrary-computation-graphs. Layer 2 uses 'SAME' + padding, which makes its padding dependent on the input image dimensionality. + In this case, the effective padding will be undetermined, and the utility is + not able to check the network alignment. + + Returns: + g: Tensorflow graph object (Graph proto). + """ + g = ops.Graph() + with g.as_default(): + # An 8x8 test image. + x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image') + # Left branch. + l1 = slim.conv2d(x, 1, [1, 1], stride=4, scope='L1', padding='VALID') + # Right branch. + l2 = slim.conv2d(x, 1, [3, 3], stride=2, scope='L2', padding='SAME') + l3 = slim.conv2d(l2, 1, [1, 1], stride=2, scope='L3', padding='VALID') + # Addition. + nn.relu(l1 + l3, name='output') + return g + + +def create_test_network_5(): + """Single-path network for testing non-square kernels. + + The graph is similar to the right branch of the graph from + create_test_network_1(), except that the kernel sizes are changed to be + non-square. + + Returns: + g: Tensorflow graph object (Graph proto). + """ + g = ops.Graph() + with g.as_default(): + # An 8x8 test image. + x = array_ops.placeholder(dtypes.float32, (1, 8, 8, 1), name='input_image') + # Two convolutional layers, where the first one has non-square kernel. + l1 = slim.conv2d(x, 1, [3, 5], stride=2, scope='L1', padding='VALID') + l2 = slim.conv2d(l1, 1, [3, 1], stride=2, scope='L2', padding='VALID') + # ReLU. + nn.relu(l2, name='output') + return g + + +class RfUtilsTest(test.TestCase): + + def testComputeRFFromGraphDefAligned(self): + graph_def = create_test_network_1().as_graph_def() + input_node = 'input_image' + output_node = 'output' + (receptive_field_x, receptive_field_y, effective_stride_x, + effective_stride_y, effective_padding_x, effective_padding_y) = ( + receptive_field.compute_receptive_field_from_graph_def( + graph_def, input_node, output_node)) + self.assertEqual(receptive_field_x, 3) + self.assertEqual(receptive_field_y, 3) + self.assertEqual(effective_stride_x, 4) + self.assertEqual(effective_stride_y, 4) + self.assertEqual(effective_padding_x, 1) + self.assertEqual(effective_padding_y, 1) + + def testComputeRFFromGraphDefAligned2(self): + graph_def = create_test_network_2().as_graph_def() + input_node = 'input_image' + output_node = 'output' + (receptive_field_x, receptive_field_y, effective_stride_x, + effective_stride_y, effective_padding_x, effective_padding_y) = ( + receptive_field.compute_receptive_field_from_graph_def( + graph_def, input_node, output_node)) + self.assertEqual(receptive_field_x, 3) + self.assertEqual(receptive_field_y, 3) + self.assertEqual(effective_stride_x, 4) + self.assertEqual(effective_stride_y, 4) + self.assertEqual(effective_padding_x, 1) + self.assertEqual(effective_padding_y, 1) + + def testComputeRFFromGraphDefUnaligned(self): + graph_def = create_test_network_3().as_graph_def() + input_node = 'input_image' + output_node = 'output' + with self.assertRaises(ValueError): + receptive_field.compute_receptive_field_from_graph_def( + graph_def, input_node, output_node) + + def testComputeRFFromGraphDefUnaligned2(self): + graph_def = create_test_network_4().as_graph_def() + input_node = 'input_image' + output_node = 'output' + (receptive_field_x, receptive_field_y, effective_stride_x, + effective_stride_y, effective_padding_x, effective_padding_y) = ( + receptive_field.compute_receptive_field_from_graph_def( + graph_def, input_node, output_node)) + self.assertEqual(receptive_field_x, 3) + self.assertEqual(receptive_field_y, 3) + self.assertEqual(effective_stride_x, 4) + self.assertEqual(effective_stride_y, 4) + self.assertEqual(effective_padding_x, None) + self.assertEqual(effective_padding_y, None) + + def testComputeRFFromGraphDefNonSquareRF(self): + graph_def = create_test_network_5().as_graph_def() + input_node = 'input_image' + output_node = 'output' + (receptive_field_x, receptive_field_y, effective_stride_x, + effective_stride_y, effective_padding_x, effective_padding_y) = ( + receptive_field.compute_receptive_field_from_graph_def( + graph_def, input_node, output_node)) + self.assertEqual(receptive_field_x, 5) + self.assertEqual(receptive_field_y, 7) + self.assertEqual(effective_stride_x, 4) + self.assertEqual(effective_stride_y, 4) + self.assertEqual(effective_padding_x, 0) + self.assertEqual(effective_padding_y, 0) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index ae93cf210be..bb6334942f3 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -157,6 +157,7 @@ sh_binary( "//tensorflow/contrib/ndlstm:ndlstm", "//tensorflow/contrib/nn:nn_py", "//tensorflow/contrib/predictor:predictor_pip", + "//tensorflow/contrib/receptive_field:receptive_field_pip", "//tensorflow/contrib/session_bundle:session_bundle_pip", "//tensorflow/contrib/signal:signal_py", "//tensorflow/contrib/slim:slim", From 9b7e05a6f643ba41fae86e2081c31634262146e2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 11:00:19 -0700 Subject: [PATCH 29/67] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 167160397 --- tensorflow/go/op/wrappers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 0781347fd6e..9583ffb38b5 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -20306,7 +20306,7 @@ func Prod(scope *Scope, input tf.Output, reduction_indices tf.Output, optional . // gradients: The backpropagated gradients to the corresponding softsign operation. // features: The features passed as input to the corresponding softsign operation. // -// Returns The gradients: `gradients / (1 + abs(-features)) ** 2`. +// Returns The gradients: `gradients / (1 + abs(features)) ** 2`. func SoftsignGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) { if scope.Err() != nil { return From cf3e3d19ed7e4b2d74f5fddc5785acba7df5b818 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 31 Aug 2017 11:28:21 -0700 Subject: [PATCH 30/67] Change node name index in InstantiateFunction from an unordered map to an ordered map. Use ordering to improve control edge instantiation algorithm from O(num control edges * num nodes) algorithm to ~O(num control edges * log(num nodes)). PiperOrigin-RevId: 167164967 --- tensorflow/core/framework/function.cc | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc index c2d3f37ab30..b788d6b7778 100644 --- a/tensorflow/core/framework/function.cc +++ b/tensorflow/core/framework/function.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/core/framework/function.h" +#include #include #include #include @@ -271,12 +272,17 @@ class FunctionInstantiationHelper { int nid = -1; const string node_name = input.substr(1); const string node_colon = node_name + ":"; - for (const auto& p : index_) { - if (p.first == node_name || - tensorflow::StringPiece(p.first).starts_with(node_colon)) { - nid = p.second.nid; + const string node_colon_bound = node_name + ";"; + // index_ is a map sorted lexicographically, so the key we are looking for + // must lie in the range [node_name, node_colon_bound). + auto it = index_.lower_bound(node_name); + while (it != index_.end() && it->first <= node_colon_bound) { + if (it->first == node_name || + tensorflow::StringPiece(it->first).starts_with(node_colon)) { + nid = it->second.nid; break; } + ++it; } if (nid == -1) { return errors::InvalidArgument("input[", i, "] == '", input, @@ -421,7 +427,7 @@ class FunctionInstantiationHelper { GetFunctionSignature get_function_; InstantiationResult& result_; // A small index for all names that can be used as a node's input arguments. - std::unordered_map index_; + std::map index_; // This contains information about a node in the new graph including the node // names and input nodes' indexes. struct NodeInfo { From 6cdc01c1dcfe9300dc5b1ecede781ce65e7fc005 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 14:09:26 -0700 Subject: [PATCH 31/67] Automatically fill in num_classes, growing_mode, pruning_mode, learning_rate and multi_class_strategy if not specified. PiperOrigin-RevId: 167188663 --- .../estimator_batch/estimator.py | 12 +++++++ .../boosted_trees/estimator_batch/model.py | 1 + .../bias-feature-column-handler_test.cc | 1 + ...categorical-feature-column-handler_test.cc | 2 +- ...e-quantized-feature-column-handler_test.cc | 2 +- ...e-quantized-feature-column-handler_test.cc | 2 +- .../stochastic/stats/node-stats_test.cc | 7 +++++ .../contrib/boosted_trees/proto/learner.proto | 30 ++++++++++-------- .../kernel_tests/prediction_ops_test.py | 1 + .../python/training/functions/gbdt_batch.py | 31 ++++++++++++++++++- .../training/functions/gbdt_batch_test.py | 18 +++++------ 11 files changed, 81 insertions(+), 26 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py index e28adad53ec..f8028acbdb0 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py @@ -61,11 +61,19 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator): logits_modifier_function: A modifier function for the logits. center_bias: Whether a separate tree should be created for first fitting the bias. + + Raises: + ValueError: If learner_config is not valid. """ head = head_lib.multi_class_head( n_classes=n_classes, weight_column_name=weight_column_name, enable_centered_bias=False) + if learner_config.num_classes == 0: + learner_config.num_classes = n_classes + elif learner_config.num_classes != n_classes: + raise ValueError("n_classes (%d) doesn't match learner_config (%d)." % + (learner_config.num_classes, n_classes)) super(GradientBoostedDecisionTreeClassifier, self).__init__( model_fn=model.model_builder, params={ @@ -129,6 +137,10 @@ class GradientBoostedDecisionTreeRegressor(estimator.Estimator): label_dimension=label_dimension, weight_column_name=weight_column_name, enable_centered_bias=False) + if label_dimension == 1: + learner_config.num_classes = 2 + else: + learner_config.num_classes = label_dimension super(GradientBoostedDecisionTreeRegressor, self).__init__( model_fn=model.model_builder, params={ diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/model.py b/tensorflow/contrib/boosted_trees/estimator_batch/model.py index 2d517f78111..8cda5c8f2b1 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/model.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/model.py @@ -92,6 +92,7 @@ def model_builder(features, labels, mode, params, config): examples_per_layer=examples_per_layer, learner_config=learner_config, feature_columns=feature_columns, + logits_dimension=head.logits_dimension, features=features) with ops.name_scope("gbdt", "gbdt_optimizer"): predictions_dict = gbdt_model.predict(mode) diff --git a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/bias-feature-column-handler_test.cc b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/bias-feature-column-handler_test.cc index 82664aed72d..f4c7df7fabd 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/bias-feature-column-handler_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/bias-feature-column-handler_test.cc @@ -42,6 +42,7 @@ class BiasFeatureColumnHandlerTest : public ::testing::Test { example_partitions_({0, 0, 1, 3}) { // Set L2 regularization. learner_config_.mutable_regularization()->set_l2(2.0f); + learner_config_.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); // Create handler. handler_.reset(new BiasFeatureColumnHandler(kClassId, kSlotId, kBatchSize)); diff --git a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/categorical-feature-column-handler_test.cc b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/categorical-feature-column-handler_test.cc index abd72384648..ea82b3f086d 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/categorical-feature-column-handler_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/categorical-feature-column-handler_test.cc @@ -51,7 +51,7 @@ class CategoricalFeatureColumnHandlerTest : public ::testing::Test { values_(test::AsTensor({1, 2, 2, 0}, {4})) { // Set L2 regularization. learner_config_.mutable_regularization()->set_l2(2.0f); - + learner_config_.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); // Create handler. handler_.reset(new CategoricalFeatureColumnHandler( kClassId, kSlotId, kBatchSize, kFeatureColumn, indices_.matrix(), diff --git a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/dense-quantized-feature-column-handler_test.cc b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/dense-quantized-feature-column-handler_test.cc index 396f48e5321..1bc9d733ad3 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/dense-quantized-feature-column-handler_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/dense-quantized-feature-column-handler_test.cc @@ -51,7 +51,7 @@ class DenseQuantizedFeatureColumnHandlerTest : public ::testing::Test { dense_quantized_values_(test::AsTensor({1, 1, 0, 1}, {4})) { // Set L2 regularization. learner_config_.mutable_regularization()->set_l2(2.0f); - + learner_config_.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); // Create handler. handler_.reset(new DenseQuantizedFeatureColumnHandler( kClassId, kSlotId, kBatchSize, kFeatureColumn, diff --git a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/sparse-quantized-feature-column-handler_test.cc b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/sparse-quantized-feature-column-handler_test.cc index db8c64a617f..643d936ad23 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/sparse-quantized-feature-column-handler_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/handlers/sparse-quantized-feature-column-handler_test.cc @@ -53,7 +53,7 @@ class SparseQuantizedFeatureColumnHandlerTest : public ::testing::Test { sparse_quantized_values_(test::AsTensor({1, 0, 1}, {3})) { // Set L2 regularization. learner_config_.mutable_regularization()->set_l2(2.0f); - + learner_config_.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); // Create handler. handler_.reset(new SparseQuantizedFeatureColumnHandler( kClassId, kSlotId, kBatchSize, kFeatureColumn, diff --git a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/stats/node-stats_test.cc b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/stats/node-stats_test.cc index f99b6826a78..ecb7a04efb9 100644 --- a/tensorflow/contrib/boosted_trees/lib/learner/stochastic/stats/node-stats_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/learner/stochastic/stats/node-stats_test.cc @@ -30,6 +30,7 @@ const double kDelta = 1e-5; TEST(NodeStatsTest, AlmostZero) { LearnerConfig learner_config; + learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); NodeStats node_stats(learner_config, GradientStats(1e-8f, 1e-8f)); EXPECT_EQ(0, node_stats.weight_contribution[0]); EXPECT_EQ(0, node_stats.gain); @@ -37,6 +38,7 @@ TEST(NodeStatsTest, AlmostZero) { TEST(NodeStatsTest, LessThanMinWeightConstraint) { LearnerConfig learner_config; + learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); learner_config.mutable_constraints()->set_min_node_weight(3.2f); NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); EXPECT_EQ(0, node_stats.weight_contribution[0]); @@ -45,6 +47,7 @@ TEST(NodeStatsTest, LessThanMinWeightConstraint) { TEST(NodeStatsTest, L1RegSquashed) { LearnerConfig learner_config; + learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); learner_config.mutable_regularization()->set_l1(10.0f); NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); EXPECT_EQ(0, node_stats.weight_contribution[0]); @@ -53,6 +56,7 @@ TEST(NodeStatsTest, L1RegSquashed) { TEST(NodeStatsTest, L1RegPos) { LearnerConfig learner_config; + learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); learner_config.mutable_regularization()->set_l1(5.0f); NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); const float expected_clipped_grad = 7.32f - 5.0f; @@ -66,6 +70,7 @@ TEST(NodeStatsTest, L1RegPos) { TEST(NodeStatsTest, L1RegNeg) { LearnerConfig learner_config; + learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); learner_config.mutable_regularization()->set_l1(5.0f); NodeStats node_stats(learner_config, GradientStats(-7.32f, 1.63f)); const float expected_clipped_grad = -7.32f + 5.0f; @@ -79,6 +84,7 @@ TEST(NodeStatsTest, L1RegNeg) { TEST(NodeStatsTest, L2Reg) { LearnerConfig learner_config; + learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); learner_config.mutable_regularization()->set_l2(8.0f); NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); const float expected_denom = 1.63f + 8.0f; @@ -91,6 +97,7 @@ TEST(NodeStatsTest, L2Reg) { TEST(NodeStatsTest, L1L2Reg) { LearnerConfig learner_config; + learner_config.set_multi_class_strategy(LearnerConfig::TREE_PER_CLASS); learner_config.mutable_regularization()->set_l1(5.0f); learner_config.mutable_regularization()->set_l2(8.0f); NodeStats node_stats(learner_config, GradientStats(7.32f, 1.63f)); diff --git a/tensorflow/contrib/boosted_trees/proto/learner.proto b/tensorflow/contrib/boosted_trees/proto/learner.proto index 06ee223467b..919e7cd8142 100644 --- a/tensorflow/contrib/boosted_trees/proto/learner.proto +++ b/tensorflow/contrib/boosted_trees/proto/learner.proto @@ -17,7 +17,7 @@ message TreeRegularizationConfig { // Tree constraints config. message TreeConstraintsConfig { - // Maximum depth of the trees. + // Maximum depth of the trees. The default value is 6 if not specified. uint32 max_tree_depth = 1; // Min hessian weight per node. @@ -86,20 +86,22 @@ message LearningRateDropoutDrivenConfig { message LearnerConfig { enum PruningMode { - PRE_PRUNE = 0; - POST_PRUNE = 1; + PRUNING_MODE_UNSPECIFIED = 0; + PRE_PRUNE = 1; + POST_PRUNE = 2; } enum GrowingMode { - WHOLE_TREE = 0; - // Layer by layer is only supported by the batch learner. - LAYER_BY_LAYER = 1; + GROWING_MODE_UNSPECIFIED = 0; + WHOLE_TREE = 1; + LAYER_BY_LAYER = 2; } enum MultiClassStrategy { - TREE_PER_CLASS = 0; - FULL_HESSIAN = 1; - DIAGONAL_HESSIAN = 2; + MULTI_CLASS_STRATEGY_UNSPECIFIED = 0; + TREE_PER_CLASS = 1; + FULL_HESSIAN = 2; + DIAGONAL_HESSIAN = 3; } // Number of classes. @@ -118,16 +120,18 @@ message LearnerConfig { // Constraints. TreeConstraintsConfig constraints = 5; - // Pruning. + // Pruning. POST_PRUNE is the default pruning mode. PruningMode pruning_mode = 8; - // Growing Mode. + // Growing Mode. LAYER_BY_LAYER is the default growing mode. GrowingMode growing_mode = 9; - // Learning rate. + // Learning rate. By default we use fixed learning rate of 0.1. LearningRateConfig learning_rate_tuner = 6; - // Multi-class strategy. + // Multi-class strategy. By default we use TREE_PER_CLASS for binary + // classification and linear regression. For other cases, we use + // DIAGONAL_HESSIAN as the default. MultiClassStrategy multi_class_strategy = 10; // If you want to average the ensembles (for regularization), provide the diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py index 51e084b79c6..37595f1c75d 100644 --- a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py +++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py @@ -344,6 +344,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase): # Prepare learner config. learner_config = learner_pb2.LearnerConfig() learner_config.num_classes = 2 + learner_config.growing_mode = learner_pb2.LearnerConfig.WHOLE_TREE result, result_no_dropout, dropout_info = ( prediction_ops.gradient_trees_prediction( diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py index 6f85874a33a..83c88a04426 100644 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py +++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py @@ -261,6 +261,7 @@ class GradientBoostedDecisionTreeModel(object): examples_per_layer, learner_config, features, + logits_dimension, feature_columns=None): """Construct a new GradientBoostedDecisionTreeModel function. @@ -273,8 +274,8 @@ class GradientBoostedDecisionTreeModel(object): a tree layer. It can also be a function that computes the number of examples based on the depth of the layer that's being built. learner_config: A learner config. - print split, sorted_feature_names[split.feature_column] features: `dict` of `Tensor` objects. + logits_dimension: An int, the dimension of logits. feature_columns: A list of feature columns. Raises: @@ -289,11 +290,39 @@ class GradientBoostedDecisionTreeModel(object): if learner_config.num_classes < 2: raise ValueError("Number of classes must be >=2") + self._logits_dimension = logits_dimension self._is_chief = is_chief self._num_ps_replicas = num_ps_replicas self._ensemble_handle = ensemble_handle self._center_bias = center_bias self._examples_per_layer = examples_per_layer + + # Fill in the defaults. + if (learner_config.multi_class_strategy == + learner_pb2.LearnerConfig.MULTI_CLASS_STRATEGY_UNSPECIFIED): + if logits_dimension == 1: + learner_config.multi_class_strategy = ( + learner_pb2.LearnerConfig.TREE_PER_CLASS) + else: + learner_config.multi_class_strategy = ( + learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) + + if (learner_config.growing_mode == + learner_pb2.LearnerConfig.GROWING_MODE_UNSPECIFIED): + learner_config.growing_mode = learner_pb2.LearnerConfig.LAYER_BY_LAYER + + if (learner_config.pruning_mode == + learner_pb2.LearnerConfig.PRUNING_MODE_UNSPECIFIED): + learner_config.pruning_mode = learner_pb2.LearnerConfig.POST_PRUNE + + if learner_config.constraints.max_tree_depth == 0: + # Use 6 as the default maximum depth. + learner_config.constraints.max_tree_depth = 6 + + tuner = learner_config.learning_rate_tuner.WhichOneof("tuner") + if not tuner: + learner_config.learning_rate_tuner.fixed.learning_rate = 0.1 + self._learner_config = learner_config self._feature_columns = feature_columns self._learner_config_serialized = learner_config.SerializeToString() diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py index 9ce434edf8b..16e24d97dde 100644 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py +++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch_test.py @@ -164,7 +164,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, - features=features) + logits_dimension=1, features=features) predictions = array_ops.constant( [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) @@ -268,7 +268,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=num_examples_fn, learner_config=learner_config, - features=features) + logits_dimension=1, features=features) predictions = array_ops.constant( [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) @@ -371,7 +371,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, - features=features) + logits_dimension=1, features=features) predictions = array_ops.constant( [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) @@ -442,7 +442,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, - features=features) + logits_dimension=1, features=features) predictions = array_ops.constant( [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) @@ -505,7 +505,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, - features=features) + logits_dimension=1, features=features) predictions = array_ops.constant( [[0.0], [1.0], [0.0], [2.0]], dtype=dtypes.float32) @@ -588,7 +588,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, - features=features) + logits_dimension=1, features=features) # Create predict op. mode = model_fn.ModeKeys.EVAL @@ -627,7 +627,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, - features=features) + logits_dimension=5, features=features) predictions = array_ops.constant( [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], @@ -730,7 +730,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, - features=features) + logits_dimension=5, features=features) predictions = array_ops.constant( [[0.0, -1.0, 0.5, 1.2, 3.1], [1.0, 0.0, 0.8, 0.3, 1.0], @@ -833,7 +833,7 @@ class GbdtTest(test_util.TensorFlowTestCase): ensemble_handle=ensemble_handle, examples_per_layer=1, learner_config=learner_config, - features=features) + logits_dimension=5, features=features) batch_size = 3 predictions = array_ops.constant( From 569af010a7faf0744fd366648a8c4b3bf18e35c3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 14:45:18 -0700 Subject: [PATCH 32/67] Fixing small issues introduced with tf.contrib.receptive_field PiperOrigin-RevId: 167194226 --- tensorflow/contrib/receptive_field/BUILD | 3 +++ .../receptive_field/python/util/examples/compute_rf.py | 4 ++++ .../python/util/examples/write_inception_resnet_v2_graph.py | 4 ++++ .../receptive_field/python/util/receptive_field_test.py | 4 ++++ 4 files changed, 15 insertions(+) diff --git a/tensorflow/contrib/receptive_field/BUILD b/tensorflow/contrib/receptive_field/BUILD index ed2f3af08cb..e2d7c075104 100644 --- a/tensorflow/contrib/receptive_field/BUILD +++ b/tensorflow/contrib/receptive_field/BUILD @@ -47,6 +47,9 @@ py_test( name = "receptive_field_test", srcs = ["python/util/receptive_field_test.py"], srcs_version = "PY2AND3", + tags = [ + "no_oss", # see b/65254194 + ], deps = [ ":receptive_field_py", "//tensorflow/contrib/framework:framework_py", diff --git a/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py b/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py index 70a0d11dff6..1cf978b90a3 100644 --- a/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py +++ b/tensorflow/contrib/receptive_field/python/util/examples/compute_rf.py @@ -17,6 +17,10 @@ For an example of usage, see accompanying file compute_rf.sh """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import argparse import sys diff --git a/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py b/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py index a2384f66ce9..793ae163d80 100644 --- a/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py +++ b/tensorflow/contrib/receptive_field/python/util/examples/write_inception_resnet_v2_graph.py @@ -15,6 +15,10 @@ """Simple script to write Inception-ResNet-v2 model to graph file. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + import argparse import sys diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py index 44e5beda607..2771389250b 100644 --- a/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py +++ b/tensorflow/contrib/receptive_field/python/util/receptive_field_test.py @@ -14,6 +14,10 @@ # ============================================================================== """Tests for receptive_fields module.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + from tensorflow.contrib import slim from tensorflow.contrib.receptive_field.python.util import receptive_field from tensorflow.python.framework import dtypes From 91617d22fc5868948a361e04a0642a765a092544 Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Thu, 31 Aug 2017 14:45:25 -0700 Subject: [PATCH 33/67] [XLA] Dump nested fusion nodes without crashing PiperOrigin-RevId: 167194247 --- .../compiler/xla/service/hlo_graph_dumper.cc | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index dfb111d1d0b..07b3369d5c1 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -561,13 +561,21 @@ tooltip = " "; } string comp_body = DumpComputation(subcomp); - string computation = - Printf(computation_fmt, id, style, subcomp_label, comp_body, id); - // Add an edge from the subcomputation to its parent node. If subcomp - // belongs to a fusion node, it's drawn in place of the fusion instruction, so - // there's no need to link those. - if (parent_instr->opcode() != HloOpcode::kFusion) { + if (parent_instr->opcode() == HloOpcode::kFusion) { + // Dump any nested fusion nodes. + for (const auto& subcomp_instr : subcomp->instructions()) { + if (subcomp_instr->opcode() == HloOpcode::kFusion) { + StrAppend( + &comp_body, + DumpSubcomputation(subcomp_instr->fused_instructions_computation(), + subcomp_instr.get())); + } + } + } else { + // Add an edge from the subcomputation to its parent node. If subcomp + // belongs to a fusion node, it's drawn in place of the fusion instruction, + // so there's no need to link those. edge_ids_.insert( {{subcomp->root_instruction(), parent_instr}, next_edge_id_++}); const char* edge_fmt = @@ -578,6 +586,9 @@ tooltip = " "; subcomp->name(), parent_instr->name())); } + string computation = + Printf(computation_fmt, id, style, subcomp_label, comp_body, id); + return computation; } From 19680e68f33bd5918594a931855e751cc053ad8f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 15:00:04 -0700 Subject: [PATCH 34/67] Initial submit for TFGAN. Code will be migrated soon. PiperOrigin-RevId: 167196427 --- tensorflow/BUILD | 1 + tensorflow/contrib/BUILD | 1 + tensorflow/contrib/__init__.py | 1 + tensorflow/contrib/gan/BUILD | 27 +++++++++++++++++++++++++++ tensorflow/contrib/gan/README.md | 4 ++++ tensorflow/contrib/gan/__init__.py | 19 +++++++++++++++++++ 6 files changed, 53 insertions(+) create mode 100644 tensorflow/contrib/gan/BUILD create mode 100644 tensorflow/contrib/gan/README.md create mode 100644 tensorflow/contrib/gan/__init__.py diff --git a/tensorflow/BUILD b/tensorflow/BUILD index d14c593ade9..9faa0964cae 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -288,6 +288,7 @@ filegroup( "//tensorflow/contrib/ffmpeg/default:all_files", "//tensorflow/contrib/framework:all_files", "//tensorflow/contrib/fused_conv:all_files", + "//tensorflow/contrib/gan:all_files", "//tensorflow/contrib/graph_editor:all_files", "//tensorflow/contrib/grid_rnn:all_files", "//tensorflow/contrib/hooks:all_files", diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index 47a0f54a023..d9f2b843469 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -28,6 +28,7 @@ py_library( "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py", "//tensorflow/contrib/framework:framework_py", "//tensorflow/contrib/fused_conv:fused_conv_py", + "//tensorflow/contrib/gan", "//tensorflow/contrib/graph_editor:graph_editor_py", "//tensorflow/contrib/grid_rnn:grid_rnn_py", "//tensorflow/contrib/hooks", diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index 315ea943cf3..d1d0e2823ad 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -31,6 +31,7 @@ from tensorflow.contrib import deprecated from tensorflow.contrib import distributions from tensorflow.contrib import factorization from tensorflow.contrib import framework +from tensorflow.contrib import gan from tensorflow.contrib import graph_editor from tensorflow.contrib import grid_rnn from tensorflow.contrib import image diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD new file mode 100644 index 00000000000..b2de2823563 --- /dev/null +++ b/tensorflow/contrib/gan/BUILD @@ -0,0 +1,27 @@ +package(default_visibility = ["//tensorflow:__subpackages__"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +py_library( + name = "gan", + srcs = [ + "__init__.py", + ], + srcs_version = "PY2AND3", + deps = [ + ], +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md new file mode 100644 index 00000000000..586e5ac331c --- /dev/null +++ b/tensorflow/contrib/gan/README.md @@ -0,0 +1,4 @@ +This directory contains the TFGAN project. + +This file will have more details as code is added. + diff --git a/tensorflow/contrib/gan/__init__.py b/tensorflow/contrib/gan/__init__.py new file mode 100644 index 00000000000..a46b0e8d5de --- /dev/null +++ b/tensorflow/contrib/gan/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TFGAN grouped API.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function From 842f3b2c6a1305075529590244e25d4b28eda029 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 15:12:00 -0700 Subject: [PATCH 35/67] Remove unused BUILD dependencies PiperOrigin-RevId: 167198355 --- tensorflow/contrib/image/BUILD | 5 +---- tensorflow/core/BUILD | 1 - tensorflow/core/kernels/BUILD | 10 ++-------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD index e631c243c3c..a27bec48010 100755 --- a/tensorflow/contrib/image/BUILD +++ b/tensorflow/contrib/image/BUILD @@ -121,12 +121,9 @@ tf_gen_op_wrapper_py( cc_library( name = "image_ops_cc", - srcs = [ - "ops/image_ops.cc", - ], + srcs = ["ops/image_ops.cc"], deps = [ ":image_ops_kernels", - "//tensorflow/core", "//tensorflow/core:framework", ], alwayslink = 1, diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index de9eb057e48..cd083abb507 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -3079,7 +3079,6 @@ cc_test( srcs = ["example/example_parser_configuration_test.cc"], data = [":example_parser_configuration_testdata"], deps = [ - ":core", ":core_cpu", ":core_cpu_internal", ":direct_session_internal", diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 0893a012047..6530ecf13fe 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -259,19 +259,13 @@ cc_library( cc_library( name = "conv_ops_gpu_hdrs", hdrs = ["conv_ops_gpu.h"], - deps = [ - ":eigen_helpers", - "//third_party/eigen3", - ], + deps = ["//third_party/eigen3"], ) cc_library( name = "gpu_util_hdrs", hdrs = ["gpu_utils.h"], - deps = [ - ":eigen_helpers", - "//third_party/eigen3", - ], + deps = ["//third_party/eigen3"], ) tf_cc_test( From 6e8d0c632dea30758c7cc343decdf8ab7956e59d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 07:40:30 -0700 Subject: [PATCH 36/67] Improve the error messaging in the case of label dimension mismatch. PiperOrigin-RevId: 167273846 --- tensorflow/python/estimator/canned/head.py | 7 +++++-- tensorflow/python/estimator/canned/head_test.py | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py index d2c5772483b..80d109d927a 100644 --- a/tensorflow/python/estimator/canned/head.py +++ b/tensorflow/python/estimator/canned/head.py @@ -200,8 +200,11 @@ def _check_labels(labels, expected_labels_dimension): dim1 = static_shape[1] if (dim1 is not None) and (dim1 != expected_labels_dimension): raise ValueError( - 'labels shape must be [batch_size, labels_dimension], got %s.' % - (static_shape,)) + 'Mismatched label shape. ' + 'Classifier configured with n_classes=%s. Received %s. ' + 'Suggested Fix: check your n_classes argument to the estimator ' + 'and/or the shape of your label.' % + (expected_labels_dimension, dim1)) assert_dimension = check_ops.assert_equal( expected_labels_dimension, labels_shape[1], message=err_msg) with ops.control_dependencies([assert_dimension]): diff --git a/tensorflow/python/estimator/canned/head_test.py b/tensorflow/python/estimator/canned/head_test.py index 23678013c66..fa3d5b44eb6 100644 --- a/tensorflow/python/estimator/canned/head_test.py +++ b/tensorflow/python/estimator/canned/head_test.py @@ -139,7 +139,7 @@ class MultiClassHeadWithSoftmaxCrossEntropyLoss(test.TestCase): features = {'x': np.array(((42.,),))} # Static shape. - with self.assertRaisesRegexp(ValueError, 'labels shape'): + with self.assertRaisesRegexp(ValueError, 'Mismatched label shape'): head.create_loss( features=features, mode=model_fn.ModeKeys.EVAL, @@ -889,7 +889,7 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase): logits_2x1 = np.array(((45.,), (41.,),)) # Static shape. - with self.assertRaisesRegexp(ValueError, 'labels shape'): + with self.assertRaisesRegexp(ValueError, 'Mismatched label shape'): head.create_loss( features={'x': np.array(((42.,),))}, mode=model_fn.ModeKeys.EVAL, @@ -1692,7 +1692,7 @@ class RegressionHeadWithMeanSquaredErrorLossTest(test.TestCase): values_1d = np.array(((43.,), (44.,),)) # Static shape. - with self.assertRaisesRegexp(ValueError, 'labels shape'): + with self.assertRaisesRegexp(ValueError, 'Mismatched label shape'): head.create_loss( features={'x': values_1d}, mode=model_fn.ModeKeys.EVAL, @@ -1737,7 +1737,7 @@ class RegressionHeadWithMeanSquaredErrorLossTest(test.TestCase): values_1d = np.array(((43.,), (44.,),)) # Static shape. - with self.assertRaisesRegexp(ValueError, 'labels shape'): + with self.assertRaisesRegexp(ValueError, 'Mismatched label shape'): head.create_loss( features={'x': values_1d}, mode=model_fn.ModeKeys.TRAIN, From 73d796423348347702d43b498257f34e41fba367 Mon Sep 17 00:00:00 2001 From: Mark Heffernan Date: Fri, 1 Sep 2017 09:17:43 -0700 Subject: [PATCH 37/67] Rollback update-ability of dataflow and alias analysis added in cl/164923041 and cl/64778750. It did not scale as intended to large graphs when used in copy insertion. This change also includes some simplification and performance improvements to dataflow and alias analysis. Also add some value-ordering tests to HloOrderingTest using dataflow analysis to generate values. PiperOrigin-RevId: 167283460 --- tensorflow/compiler/xla/service/BUILD | 1 + .../xla/service/hlo_alias_analysis.cc | 599 ++++++++---------- .../compiler/xla/service/hlo_alias_analysis.h | 51 +- .../xla/service/hlo_alias_analysis_test.cc | 144 +---- tensorflow/compiler/xla/service/hlo_buffer.cc | 16 - tensorflow/compiler/xla/service/hlo_buffer.h | 15 +- .../xla/service/hlo_dataflow_analysis.cc | 590 +++++------------ .../xla/service/hlo_dataflow_analysis.h | 96 +-- .../xla/service/hlo_dataflow_analysis_test.cc | 328 +--------- .../compiler/xla/service/hlo_ordering_test.cc | 89 +++ tensorflow/compiler/xla/service/hlo_value.cc | 6 - tensorflow/compiler/xla/service/hlo_value.h | 3 + 12 files changed, 576 insertions(+), 1362 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 9d4e7fc254f..610c611eee4 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -849,6 +849,7 @@ cc_test( srcs = ["hlo_ordering_test.cc"], deps = [ ":hlo", + ":hlo_dataflow_analysis", ":hlo_ordering", ":hlo_scheduling", "//tensorflow/compiler/xla:shape_util", diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc index 0beea423798..3dd8ac6dc5f 100644 --- a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc @@ -37,6 +37,230 @@ namespace xla { using ::tensorflow::strings::StrAppend; using ::tensorflow::strings::StrCat; +// Data structure used to construct the alias analysis. Thrown away after alias +// analysis is complete. This data structure keeps track of which sets of +// HloValues must be in the same HloBuffer. This is maintained as a map from a +// buffer identifier (BufferNumber) to set of HLoValues. +// +// Initially each value is its own buffer. In MergeAliasedBuffers, sets of +// values which must share the same buffer are merged together. The end result +// is a partitioning of all HloValues into sets where each set needs its own +// HloBuffer. By performing this analysis without constructing HloBuffers on the +// fly, we can after-the-fact construct a vector of contiguously numbered +// HloBuffers after the buffer requirement has been determined. +class BufferValueMap { + public: + // A unique identifier for a set of colocated values which must share the same + // buffer. This is not necessarily the same as the HloBuffer::Id which will + // ultimately contain the values. The reason is that HloBuffer::Id's are + // contiguous, while BufferNumbers may not be. BufferNumbers may not be + // dense because buffers may be created and destroyed during the analysis + // construction process. + using BufferNumber = int64; + + explicit BufferValueMap(const HloDataflowAnalysis& dataflow) + : dataflow_(dataflow) { + buffers_.reserve(dataflow_.values().size()); + value_to_buffer_number_.reserve(dataflow_.values().size()); + for (const HloValue* value : dataflow_.values()) { + BufferNumber buffer_number = next_buffer_number_++; + buffers_[buffer_number].insert(value); + value_to_buffer_number_[value] = buffer_number; + } + } + + // Merge together sets of HloValues which must be in the same HloBuffer + // because of aliasing rules (eg, in-place kWhile instruction). + void MergeAliasedBuffers() { + for (const HloValue* value : dataflow_.values()) { + VLOG(3) << "Merging colocated values, value: " << value->ToShortString(); + + // Gather the set of buffers with aliasing rules (eg, kWhile) which this + // value must be contained in. + std::vector aliased_buffers = ComputeAliasedBuffers(*value); + + BufferNumber current_buffer = value_to_buffer_number_.at(value); + if (aliased_buffers.empty()) { + // The buffer containing 'value' aliases no other buffers. If the buffer + // containing 'value' already only contains 'value', then no change is + // necessary. If the buffer containing 'value' does contain other + // values, then remove 'value' from the buffer and create a new buffer + // containing only 'value' + if (buffers_.at(current_buffer).size() == 1) { + CHECK_EQ(*buffers_.at(current_buffer).begin(), value); + } else { + MoveValueToNewBuffer(*value); + } + } else { + // If multiple buffers are aliased merge these buffers together into a + // single buffer (arbitrarily chosen as the first buffer in the vector). + if (aliased_buffers.size() > 1) { + for (int64 i = 1; i < aliased_buffers.size(); ++i) { + MergeBuffers(/*from=*/aliased_buffers[i], + /*to=*/aliased_buffers[0]); + } + } + BufferNumber new_buffer = aliased_buffers[0]; + if (current_buffer != new_buffer) { + MoveValueToBuffer(*value, new_buffer); + } + } + } + } + + // Compute and return a sorted vector of all BufferNumbers. Can be used to + // iterate through all buffers stabily. + std::vector ComputeSortedBufferNumbers() const { + std::vector buffer_numbers; + for (const auto& pair : buffers_) { + buffer_numbers.push_back(pair.first); + } + std::sort(buffer_numbers.begin(), buffer_numbers.end()); + return buffer_numbers; + } + + // Return a set of all the values in the given buffer. + const tensorflow::gtl::FlatSet& GetValuesInBuffer( + BufferNumber buffer_number) const { + return buffers_.at(buffer_number); + } + + private: + // Create a new buffer. + void NewBuffer(const HloValue& value) { + BufferNumber buffer_number = next_buffer_number_++; + buffers_[buffer_number].insert(&value); + value_to_buffer_number_[&value] = buffer_number; + } + + // Move the given value into a new buffer containing only the value. + void MoveValueToNewBuffer(const HloValue& value) { + BufferNumber new_buffer_number = next_buffer_number_++; + buffers_[new_buffer_number]; + MoveValueToBuffer(value, new_buffer_number); + } + + // Move the given value into the given buffer. + void MoveValueToBuffer(const HloValue& value, BufferNumber buffer_number) { + BufferNumber old_buffer_number = value_to_buffer_number_.at(&value); + buffers_.at(old_buffer_number).erase(&value); + if (buffers_.at(old_buffer_number).empty()) { + buffers_.erase(old_buffer_number); + } + + buffers_.at(buffer_number).insert(&value); + value_to_buffer_number_.at(&value) = buffer_number; + } + + // Merge the buffer 'from' into the buffer 'to'. + void MergeBuffers(BufferNumber from, BufferNumber to) { + auto& from_value_set = buffers_.at(from); + buffers_.at(to).insert(from_value_set.begin(), from_value_set.end()); + // NOTE: using a union-find algorithm to hold the colocated values might be + // faster. + for (const HloValue* value : from_value_set) { + value_to_buffer_number_.at(value) = to; + } + buffers_.erase(from); + } + + BufferNumber GetBufferForValue(const HloValue& value) { + return value_to_buffer_number_.at(&value); + } + + // Compute and return a vector of buffers that the given value must be + // contained in due to HLO aliasing rules. + std::vector ComputeAliasedBuffers(const HloValue& value) { + // Value is init of a while (use is while). + std::vector aliased_buffers; + for (const HloUse& use : value.uses()) { + VLOG(1) << "use of value " << value.ToShortString() << ": " << use; + if (use.instruction->opcode() == HloOpcode::kWhile) { + // Determine the while value that this shares a buffer with. + const HloValue& while_value = + dataflow_.GetUniqueValueAt(use.instruction, use.operand_index); + aliased_buffers.push_back(GetBufferForValue(while_value)); + VLOG(3) << " value is init value to a while; must share buffer with " + "while value " + << while_value.ToShortString(); + } + } + + // Value is a parameter of a while body/condition. + if (value.defining_instruction()->opcode() == HloOpcode::kParameter) { + const HloComputation* computation = + value.defining_instruction()->parent(); + const CallGraphNode& call_graph_node = + dataflow_.call_graph().GetNode(computation); + for (const CallSite& callsite : call_graph_node.caller_callsites()) { + if (callsite.instruction()->opcode() == HloOpcode::kWhile) { + // Call graph must have been flattened. + CHECK_EQ(call_graph_node.caller_callsites().size(), 1); + + const HloValue& while_value = dataflow_.GetUniqueValueAt( + callsite.instruction(), value.defining_index()); + VLOG(3) << " value is parameter value of the body or condition of a " + "while; must share buffer with while value " + << while_value.ToShortString(); + aliased_buffers.push_back(GetBufferForValue(while_value)); + } + } + } + + // Value is the root of a while body. + for (const HloPosition& position : value.positions()) { + const HloComputation* computation = position.instruction->parent(); + const CallGraphNode& call_graph_node = + dataflow_.call_graph().GetNode(computation); + if (position.instruction == computation->root_instruction()) { + for (const CallSite& callsite : call_graph_node.caller_callsites()) { + if (callsite.instruction()->opcode() == HloOpcode::kWhile && + callsite.instruction()->while_body() == computation) { + // Call graph must have been flattened. + CHECK_EQ(call_graph_node.caller_callsites().size(), 1); + + const HloValue& while_value = dataflow_.GetUniqueValueAt( + callsite.instruction(), position.index); + VLOG(3) << " value is root the body computation of a while; must " + "share buffer with while value " + << while_value.ToShortString(); + aliased_buffers.push_back(GetBufferForValue(while_value)); + } + } + } + } + + // Value is the output of the while instruction itself. + if (value.defining_instruction()->opcode() == HloOpcode::kWhile) { + VLOG(3) << " value is output of a while instruction"; + aliased_buffers.push_back(GetBufferForValue(value)); + } + + // Uniquify aliased buffers. + std::sort(aliased_buffers.begin(), aliased_buffers.end()); + aliased_buffers.erase( + std::unique(aliased_buffers.begin(), aliased_buffers.end()), + aliased_buffers.end()); + + return aliased_buffers; + } + + // Dataflow analysis used to construct the buffer map. + const HloDataflowAnalysis& dataflow_; + + // A map containing the set of values contained in each buffer. + tensorflow::gtl::FlatMap> + buffers_; + + // A map indicating which buffer each value is contained in. + tensorflow::gtl::FlatMap + value_to_buffer_number_; + + // The buffer number of the next buffer to be created. + BufferNumber next_buffer_number_ = 0; +}; + HloAliasAnalysis::HloAliasAnalysis(HloModule* module) : module_(module) {} const HloBuffer& HloAliasAnalysis::GetUniqueBufferAt( @@ -99,10 +323,11 @@ bool HloAliasAnalysis::InstructionBuffersAreDistinct( } } else { // It's possible for multiple values at this index to have the same - // HloBuffer. This does not result in non-distictness. To account for this - // case, add all of the buffers at this index after checking whether each - // buffer exists at an earlier index. This is a corner case, however, as - // the number of values at an index is almost always one. + // HloBuffer. This does not result in non-distictness. To account for + // this case, add all of the buffers at this index after checking + // whether each buffer exists at an earlier index. This is a corner + // case, however, as the number of values at an index is almost always + // one. std::vector buffers_at_this_index; for (const HloValue* value : value_set.values()) { const HloBuffer* buffer = &GetBufferContainingValue(*value); @@ -118,15 +343,6 @@ bool HloAliasAnalysis::InstructionBuffersAreDistinct( return true; } -void HloAliasAnalysis::InitializeBufferSets() { - // Initially define a buffer for every HloValue in the module. - for (const HloValue& value : dataflow_analysis_->values()) { - HloBuffer& buffer = NewHloBuffer(); - buffer.AddValue(value); - value_to_buffer_[&value] = &buffer; - } -} - Status HloAliasAnalysis::Verify() const { // Verify consistency between the value_to_buffer_ map and // HloBuffer::values(). @@ -137,9 +353,8 @@ Status HloAliasAnalysis::Verify() const { value) != buffer.values().end()); } - for (const auto& pair : buffers_) { - const HloBuffer::Id id = pair.first; - const HloBuffer& buffer = pair.second; + for (HloBuffer::Id id = 0; id < buffers_.size(); ++id) { + const HloBuffer& buffer = buffers_[id]; TF_RET_CHECK(buffer.id() == id); HloValue::Id last_value_id = -1; @@ -152,116 +367,9 @@ Status HloAliasAnalysis::Verify() const { } } - if (!buffers_vector_.empty()) { - // buffers_vector_ should be a vector of all HloBuffers sorted by id. - std::vector buffers; - for (const auto& id_buffer : buffers_) { - buffers.push_back(&id_buffer.second); - } - std::sort(buffers.begin(), buffers.end(), HloBuffer::IdLessThan); - TF_RET_CHECK(buffers_vector_ == buffers); - } - return Status::OK(); } -Status HloAliasAnalysis::VerifyAgainstReference() const { - TF_RETURN_IF_ERROR(Verify()); - - TF_ASSIGN_OR_RETURN(std::unique_ptr reference, - Run(module_)); - TF_RETURN_IF_ERROR(reference->Verify()); - - VLOG(2) << "This analysis:"; - XLA_VLOG_LINES(2, ToString()); - VLOG(2) << "Reference:"; - XLA_VLOG_LINES(2, reference->ToString()); - - // Create map from HloValue in the reference analysis to HloValue in this - // analysis and vice versa. - tensorflow::gtl::FlatMap reference_to_this; - tensorflow::gtl::FlatMap this_to_reference; - for (const HloValue& value : dataflow_analysis().values()) { - const HloValue& reference_value = - reference->dataflow_analysis().GetValueDefinedAt( - value.defining_instruction(), value.defining_index()); - reference_to_this[&reference_value] = &value; - this_to_reference[&value] = &reference_value; - } - - TF_RET_CHECK(buffers_.size() == reference->buffers_.size()) - << "Different number of buffers (" << buffers_.size() - << " != " << reference->buffers_.size() << ")"; - for (const auto& pair : reference->buffers_) { - const HloBuffer& reference_buffer = pair.second; - - // Find the corresponding buffer in the reference by taking the first value - // in the buffer, finding the corresponding value in the reference, and then - // finding the buffer holding that value. - TF_RET_CHECK(!reference_buffer.values().empty()); - const HloValue* reference_value = reference_buffer.values()[0]; - const HloValue* value = reference_to_this.at(reference_value); - const HloBuffer& buffer = GetBufferContainingValue(*value); - - // The buffer and the reference should have the exact same values. To make - // comparison easy, sort the values in the reference buffer identically to - // the values in the non-reference buffer (ie, by the corresponding id of - // the non-reference value). - std::vector reference_values = reference_buffer.values(); - std::sort(reference_values.begin(), reference_values.end(), - [&reference_to_this](const HloValue* a, const HloValue* b) { - return reference_to_this.at(a)->id() < - reference_to_this.at(b)->id(); - }); - TF_RET_CHECK(reference_values.size() == buffer.values().size()); - for (int i = 0; i < buffer.values().size(); ++i) { - TF_RET_CHECK(*reference_values[i] == *buffer.values()[i]) - << "Buffer:\n " << buffer - << "\ndoes not have the same values as reference buffer:\n " - << reference_buffer; - } - } - - return Status::OK(); -} - -HloBuffer& HloAliasAnalysis::NewHloBuffer() { - HloBuffer::Id buffer_id = next_buffer_id_++; - auto emplaced = buffers_.emplace(std::piecewise_construct, - std::forward_as_tuple(buffer_id), - std::forward_as_tuple(buffer_id)); - CHECK(emplaced.second); - - buffers_vector_.clear(); - - return emplaced.first->second; -} - -void HloAliasAnalysis::MoveValueToNewBuffer(const HloValue& value) { - HloBuffer& new_buffer = NewHloBuffer(); - MoveValueToBuffer(value, &new_buffer); - - VLOG(3) << "Moved value " << value.ToShortString() << " into new buffer " - << new_buffer.id(); -} - -void HloAliasAnalysis::MoveValueToBuffer(const HloValue& value, - HloBuffer* buffer) { - HloBuffer& old_buffer = GetBufferContainingValue(value); - CHECK_NE(buffer, &old_buffer); - VLOG(3) << "Moved value " << value.ToShortString() << " from buffer " - << old_buffer.id() << " into buffer " << buffer->id(); - old_buffer.RemoveValue(value); - if (old_buffer.values().empty()) { - VLOG(3) << "Buffer " << old_buffer.id() << " now empty. Removing."; - buffers_.erase(old_buffer.id()); - buffers_vector_.clear(); - } - - buffer->AddValue(value); - value_to_buffer_[&value] = buffer; -} - string HloAliasAnalysis::ToString() const { string out = StrCat("HloAliasAnalysis, module ", module_->name(), "\n"); StrAppend(&out, " Buffers at each position:\n"); @@ -290,10 +398,10 @@ string HloAliasAnalysis::ToString() const { } StrAppend(&out, " Buffers:\n"); - for (const HloBuffer* buffer : buffers()) { - StrAppend(&out, " ", buffer->ToString(), "\n"); + for (const HloBuffer& buffer : buffers()) { + StrAppend(&out, " ", buffer.ToString(), "\n"); StrAppend(&out, " positions:\n"); - for (const HloPosition& position : buffer->ComputePositions()) { + for (const HloPosition& position : buffer.ComputePositions()) { StrAppend(&out, " ", position.ToString(), "\n"); } } @@ -301,217 +409,6 @@ string HloAliasAnalysis::ToString() const { return out; } -const std::vector& HloAliasAnalysis::buffers() const { - if (buffers_vector_.empty()) { - // Lazily construct vector of buffers. - buffers_vector_.reserve(buffers_.size()); - for (auto& pair : buffers_) { - buffers_vector_.push_back(&pair.second); - } - std::sort(buffers_vector_.begin(), buffers_vector_.end(), - HloBuffer::IdLessThan); - } else { - CHECK_EQ(buffers_vector_.size(), buffers_.size()); - for (const HloBuffer* buffer : buffers_vector_) { - DCHECK(ContainsKey(buffers_, buffer->id())); - DCHECK(&GetBuffer(buffer->id()) == buffer); - } - } - return buffers_vector_; -} - -void HloAliasAnalysis::UpdateAtInstructions( - tensorflow::gtl::ArraySlice instructions) { - VLOG(4) << "Updated HLO module:"; - XLA_VLOG_LINES(4, module_->ToString()); - - VLOG(3) << "Before update:"; - XLA_VLOG_LINES(3, ToString()); - - std::vector values_to_update; - for (const HloInstruction* instruction : instructions) { - for (auto& pair : dataflow_analysis().GetInstructionValueSet(instruction)) { - for (const HloValue* value : pair.second.values()) { - values_to_update.push_back(value); - } - } - } - - UpdateBuffersForValues(values_to_update); - - VLOG(3) << "After update:"; - XLA_VLOG_LINES(3, ToString()); -} - -void HloAliasAnalysis::UpdateAfterChangingOperand(HloInstruction* instruction, - HloInstruction* old_operand, - HloInstruction* new_operand) { - VLOG(1) << "UpdateAfterChangingOperand(" << instruction->name() << ", " - << old_operand->name() << " => " << new_operand->name() << ")"; - - dataflow_analysis_->UpdateAfterChangingOperand(instruction, old_operand, - new_operand); - TF_DCHECK_OK(dataflow_analysis_->VerifyAgainstReference()); - - VLOG(4) << "Updated dataflow:"; - XLA_VLOG_LINES(4, dataflow_analysis_->ToString()); - - UpdateAtInstructions({instruction, old_operand, new_operand}); -} - -void HloAliasAnalysis::UpdateAfterChangingRoot(HloInstruction* old_root, - HloInstruction* new_root) { - VLOG(1) << "UpdateAfterChangingRoot(" << old_root->name() << " => " - << new_root->name() << ")"; - - dataflow_analysis_->UpdateAfterChangingRoot(old_root, new_root); - TF_DCHECK_OK(dataflow_analysis_->VerifyAgainstReference()); - - VLOG(4) << "Updated dataflow:"; - XLA_VLOG_LINES(4, dataflow_analysis_->ToString()); - - UpdateAtInstructions({old_root, new_root}); -} - -std::vector HloAliasAnalysis::ComputeAliasedBuffers( - const HloValue& value) { - std::vector aliased_buffers; - - // Value is init of a while (use is while). - for (const HloUse& use : value.uses()) { - VLOG(1) << "use of value " << value.ToShortString() << ": " << use; - if (use.instruction->opcode() == HloOpcode::kWhile) { - // Determine the while value that this shares a buffer with. - const HloValue& while_value = dataflow_analysis().GetUniqueValueAt( - use.instruction, use.operand_index); - aliased_buffers.push_back(&GetBufferContainingValue(while_value)); - VLOG(3) << " value is init value to a while; must share buffer with " - "while value " - << while_value.ToShortString(); - } - } - - // Value is a parameter of a while body/condition. - if (value.defining_instruction()->opcode() == HloOpcode::kParameter) { - const HloComputation* computation = value.defining_instruction()->parent(); - const CallGraphNode& call_graph_node = - dataflow_analysis().call_graph().GetNode(computation); - for (const CallSite& callsite : call_graph_node.caller_callsites()) { - if (callsite.instruction()->opcode() == HloOpcode::kWhile) { - // Call graph must have been flattened. - CHECK_EQ(call_graph_node.caller_callsites().size(), 1); - - const HloValue& while_value = dataflow_analysis().GetUniqueValueAt( - callsite.instruction(), value.defining_index()); - VLOG(3) << " value is parameter value of the body or condition of a " - "while; must share buffer with while value " - << while_value.ToShortString(); - aliased_buffers.push_back(&GetBufferContainingValue(while_value)); - } - } - } - - // Value is the root of a while body. - for (const HloPosition& position : value.positions()) { - const HloComputation* computation = position.instruction->parent(); - const CallGraphNode& call_graph_node = - dataflow_analysis().call_graph().GetNode(computation); - if (position.instruction == computation->root_instruction()) { - for (const CallSite& callsite : call_graph_node.caller_callsites()) { - if (callsite.instruction()->opcode() == HloOpcode::kWhile && - callsite.instruction()->while_body() == computation) { - // Call graph must have been flattened. - CHECK_EQ(call_graph_node.caller_callsites().size(), 1); - - // If the value appears in the root of a while body, then - // necessarily the value is defined in the body as well. - CHECK_EQ(value.defining_instruction()->parent(), computation); - - const HloValue& while_value = dataflow_analysis().GetUniqueValueAt( - callsite.instruction(), position.index); - VLOG(3) << " value is root the body computation of a while; must " - "share buffer with while value " - << while_value.ToShortString(); - aliased_buffers.push_back(&GetBufferContainingValue(while_value)); - } - } - } - } - - // Value is in the while instruction itself. - if (value.defining_instruction()->opcode() == HloOpcode::kWhile) { - VLOG(3) << " value is output of a while instruction"; - aliased_buffers.push_back(&GetUniqueBufferAt(value.defining_instruction(), - value.defining_index())); - } - - // Uniquify aliased buffers. - std::sort(aliased_buffers.begin(), aliased_buffers.end(), - HloBuffer::IdLessThan); - aliased_buffers.erase( - std::unique(aliased_buffers.begin(), aliased_buffers.end()), - aliased_buffers.end()); - - return aliased_buffers; -} - -// This method recomputes the HloBuffer for each of the given HloValues. The -// method does not necessarily update the HloBuffer of values which share a -// buffer with the given values, but are not explicitly passed in -// 'values'. Therefore, the caller must pass in all values which may require an -// update according to the kind of HLO graph change which occurred: operand -// changed (UpdateAfterChangingOperand), or root of computation changed -// (UpdateAfterChangingRoot). -void HloAliasAnalysis::UpdateBuffersForValues( - tensorflow::gtl::ArraySlice values) { - for (const HloValue* value : values) { - VLOG(3) << "Updating buffer for value: " << value->ToShortString(); - - // Gather the set of buffer with aliasing rules (eg, kWhile) which this - // value must be contained in due. - std::vector aliased_buffers = ComputeAliasedBuffers(*value); - - HloBuffer& current_buffer = GetBufferContainingValue(*value); - if (aliased_buffers.empty()) { - // The buffer containing 'value' aliases no other buffers. If the buffer - // containing 'value' already only contains 'value', then no change is - // necessary. If the buffer containing 'value' does contain other values, - // then remove 'value' from the buffer and create a new buffer containing - // only 'value' - if (current_buffer.values().size() == 1) { - CHECK_EQ(current_buffer.values()[0], value); - } else { - MoveValueToNewBuffer(*value); - } - } else { - // If multiple buffers are aliased merge these buffers together into a - // single buffer (arbitrarily chosen as the first buffer in the vector). - if (aliased_buffers.size() > 1) { - for (int64 i = 1; i < aliased_buffers.size(); ++i) { - // Make copy of values vector because MoveValueToBuffer invalidates - // the values iterator. The could be done more efficiently by moving - // all values and once. - std::vector values = aliased_buffers[i]->values(); - for (const HloValue* value : values) { - MoveValueToBuffer(*value, aliased_buffers[0]); - } - } - aliased_buffers.resize(1); - } - - CHECK_EQ(aliased_buffers.size(), 1); - HloBuffer* new_buffer = aliased_buffers[0]; - - if (¤t_buffer != new_buffer) { - MoveValueToBuffer(*value, new_buffer); - } - } - - VLOG(4) << "Analysis after update:"; - XLA_VLOG_LINES(4, ToString()); - } -} - /* static */ StatusOr> HloAliasAnalysis::Run( HloModule* module) { @@ -524,18 +421,28 @@ StatusOr> HloAliasAnalysis::Run( HloDataflowAnalysis::Run(module, /*ssa_form=*/true, /*bitcast_defines_value=*/false)); - alias_analysis->InitializeBufferSets(); + BufferValueMap buffer_map(alias_analysis->dataflow_analysis()); + buffer_map.MergeAliasedBuffers(); - VLOG(3) << "After initialization:"; - XLA_VLOG_LINES(3, alias_analysis->ToString()); - - std::vector all_values; - for (const HloValue& value : alias_analysis->dataflow_analysis().values()) { - all_values.push_back(&value); + // Create a vector of HloBuffers, one for each set of values in the + // BufferValueMap. Create the HloBuffers as a vector of contiguously numbered + // buffers. + std::vector sorted_buffer_numbers = + buffer_map.ComputeSortedBufferNumbers(); + alias_analysis->buffers_.reserve(sorted_buffer_numbers.size()); + HloBuffer::Id next_id = 0; + for (BufferValueMap::BufferNumber buffer_number : sorted_buffer_numbers) { + auto& value_set = buffer_map.GetValuesInBuffer(buffer_number); + std::vector sorted_values(value_set.begin(), + value_set.end()); + std::sort(sorted_values.begin(), sorted_values.end(), HloValue::IdLessThan); + alias_analysis->buffers_.emplace_back(next_id++, sorted_values); + for (const HloValue* value : sorted_values) { + alias_analysis->value_to_buffer_[value] = + &alias_analysis->buffers_.back(); + } } - alias_analysis->UpdateBuffersForValues(all_values); - TF_DCHECK_OK(alias_analysis->Verify()); XLA_VLOG_LINES(1, alias_analysis->ToString()); diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis.h b/tensorflow/compiler/xla/service/hlo_alias_analysis.h index 1b538f6d1cf..39554e46648 100644 --- a/tensorflow/compiler/xla/service/hlo_alias_analysis.h +++ b/tensorflow/compiler/xla/service/hlo_alias_analysis.h @@ -74,7 +74,7 @@ class HloAliasAnalysis { // Return a vector of all HloBuffers stabily sorted by HloBuffer::Id. This // vector is lazily computed. Mutating operations on HloAliasAnalysis may // invalidate the underlying vector requiring recomputation. - const std::vector& buffers() const; + const std::vector& buffers() const { return buffers_; } // Returns the underlying dataflow analysis used by this alias analysis. const HloDataflowAnalysis& dataflow_analysis() const { @@ -90,50 +90,13 @@ class HloAliasAnalysis { // output of the given instruction. bool InstructionBuffersAreDistinct(const HloInstruction* instruction) const; - // Updates the analysis after the operands of 'instruction' have changed or if - // 'instruction' has been made the root of a computation. Analysis update is - // not possible if instructions have been added or removed from the graph. - void UpdateAfterChangingOperand(HloInstruction* instruction, - HloInstruction* old_operand, - HloInstruction* new_operand); - void UpdateAfterChangingRoot(HloInstruction* old_root, - HloInstruction* new_root); - // Compare the dataflow analysis against a clean recomputation of the // analysis. Returns an error status if there is a mismatch. Useful for // verifying the correctness after updates to the analysis. Status VerifyAgainstReference() const; protected: - HloAliasAnalysis(HloModule* module); - - // Create a new empty HloBuffer. - HloBuffer& NewHloBuffer(); - - // Move the given value to the given buffer. The value is removed from it's - // current buffer. - void MoveValueToBuffer(const HloValue& value, HloBuffer* buffer); - - // Move the given value to a newly created buffer. The value is removed from - // it's current buffer. - void MoveValueToNewBuffer(const HloValue& value); - - // Construct the initial set of buffer sets where an HloBuffer is created for - // each HloValue in the module. - void InitializeBufferSets(); - - // Compute and return the buffers with aliasing rules (eg, kWhile) which the - // given value must be contained in. - std::vector ComputeAliasedBuffers(const HloValue& value); - - // Recompute the HloBuffers for the given values. - void UpdateBuffersForValues( - tensorflow::gtl::ArraySlice values); - - // Recompute the HloBuffers for all the values which appear in the output of - // the given instructions. - void UpdateAtInstructions( - tensorflow::gtl::ArraySlice instructions); + explicit HloAliasAnalysis(HloModule* module); // Verify various invariants of the alias analysis. Status Verify() const; @@ -143,20 +106,12 @@ class HloAliasAnalysis { // The underlying dataflow analysis used by this alias analysis. std::unique_ptr dataflow_analysis_; - // The map of all HloBuffers in the module. We pass around pointers to the - // mapped HloBuffers, so the underlying container must keep them valid despite - // mutations touching other map entries. - std::unordered_map buffers_; - // A map indicating which buffer a value is contained in. tensorflow::gtl::FlatMap value_to_buffer_; // A lazily constructed vector containing all HloBuffers sorted by // HloBuffer::Id. - mutable std::vector buffers_vector_; - - // The Id to use for the next HloBuffer. - int64 next_buffer_id_ = 0; + std::vector buffers_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc index e2815d6e648..6e311e25fb9 100644 --- a/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc +++ b/tensorflow/compiler/xla/service/hlo_alias_analysis_test.cc @@ -87,14 +87,13 @@ class HloAliasAnalysisTest : public HloTestBase { // constructed. bool AnyValuesInSameBufferInterfere() { DependencyHloOrdering ordering(module_.get()); - for (const HloBuffer* buffer : analysis_->buffers()) { - for (const HloValue* value_a : buffer->values()) { - for (const HloValue* value_b : buffer->values()) { + for (const HloBuffer& buffer : analysis_->buffers()) { + for (const HloValue* value_a : buffer.values()) { + for (const HloValue* value_b : buffer.values()) { if (*value_a != *value_b && - analysis_->dataflow_analysis().MayInterfere(*value_a, *value_b, - ordering)) { + ordering.MayInterfere(*value_a, *value_b)) { VLOG(1) << *value_a << " interferes with " << *value_b - << " in buffer: " << *buffer; + << " in buffer: " << buffer; return true; } } @@ -384,10 +383,7 @@ TEST_F(HloAliasAnalysisTest, SingleWhile) { EXPECT_THAT( GetValuesInBuffer(analysis.GetUniqueBufferAt(xla_while, /*index=*/{0})), - UnorderedElementsAre(GetValueDefinedAt(xla_while, /*index=*/{0}), - GetValueDefinedAt(body_param, /*index=*/{0}), - GetValueDefinedAt(cond_param, /*index=*/{0}), - GetValueDefinedAt(constant1))); + UnorderedElementsAre(GetValueDefinedAt(constant1))); EXPECT_THAT( GetValuesInBuffer(analysis.GetUniqueBufferAt(xla_while, /*index=*/{1})), UnorderedElementsAre(GetValueDefinedAt(constant2), @@ -631,9 +627,9 @@ TEST_F(HloAliasAnalysisTest, SwizzlingWhile) { // HloBuffers. EXPECT_THAT( analysis.buffers(), - UnorderedElementsAre(&analysis.GetUniqueBufferAt(constant1), - &analysis.GetUniqueBufferAt(tuple, /*index=*/{}), - &analysis.GetUniqueBufferAt(cond_constant))); + UnorderedElementsAre(analysis.GetUniqueBufferAt(constant1), + analysis.GetUniqueBufferAt(tuple, /*index=*/{}), + analysis.GetUniqueBufferAt(cond_constant))); // The tuple elements of the while and the three constant inputs should all be // smooshed into the same buffer. @@ -820,127 +816,5 @@ TEST_F(HloAliasAnalysisTest, Bitcast) { analysis.GetUniqueBufferAt(bitcast)); } -TEST_F(HloAliasAnalysisTest, UpdateAnalysisForWhile) { - // Test updating alias analysis after modifying a module with an array shaped - // while: - // - // body(F32[] %param): - // %negate = Negate(%param) - // - // condition(F32[] %param): - // return Constant(false) - // - // entry: - // %constant = Constant(1.0) - // %exp = Exp(%constant) - // return While(%exp, body, condition) - // - auto body_builder = HloComputation::Builder("body"); - auto body_param = body_builder.AddInstruction( - HloInstruction::CreateParameter(0, scalar_shape_, "param")); - auto negate = body_builder.AddInstruction(HloInstruction::CreateUnary( - scalar_shape_, HloOpcode::kNegate, body_param)); - HloComputation* body = module_->AddEmbeddedComputation(body_builder.Build()); - - // Condition computation trivially returns a constant "false". - auto cond_builder = HloComputation::Builder("condition"); - auto cond_param = cond_builder.AddInstruction( - HloInstruction::CreateParameter(0, scalar_shape_, "param")); - cond_builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(false))); - HloComputation* condition = - module_->AddEmbeddedComputation(cond_builder.Build()); - - auto builder = HloComputation::Builder(TestName()); - auto constant = builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(1.0))); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(scalar_shape_, HloOpcode::kExp, constant)); - auto xla_while = builder.AddInstruction( - HloInstruction::CreateWhile(scalar_shape_, condition, body, exp)); - module_->AddEntryComputation(builder.Build()); - - HloAliasAnalysis& analysis = RunAnalysis(); - - // Sanity check some alias information. - EXPECT_EQ(analysis.GetUniqueBufferAt(exp), - analysis.GetUniqueBufferAt(body_param)); - EXPECT_EQ(analysis.GetUniqueBufferAt(exp), - analysis.GetUniqueBufferAt(cond_param)); - EXPECT_EQ(analysis.GetUniqueBufferAt(exp), - analysis.GetUniqueBufferAt(negate)); - EXPECT_EQ(analysis.GetUniqueBufferAt(exp), - analysis.GetUniqueBufferAt(xla_while)); - - // Set the body root to the body_param. Previously it was Negate(body_param). - body->set_root_instruction(body_param); - - // Prior to updating, verify that the analysis is no longer valid. - Status verify_status = analysis.VerifyAgainstReference(); - EXPECT_FALSE(verify_status.ok()); - - analysis.UpdateAfterChangingRoot(/*old_root=*/negate, - /*new_root*/ body_param); - - // Analysis should be valid after the update. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); - - // The exponential should now pass through the body transparently. - EXPECT_EQ(analysis.GetUniqueBufferAt(exp), - analysis.GetUniqueBufferAt(body_param)); - EXPECT_EQ(analysis.GetUniqueBufferAt(exp), - analysis.GetUniqueBufferAt(cond_param)); - EXPECT_NE(analysis.GetUniqueBufferAt(exp), - analysis.GetUniqueBufferAt(negate)); - EXPECT_EQ(analysis.GetUniqueBufferAt(exp), - analysis.GetUniqueBufferAt(xla_while)); - - // Now replace the operand of the while with %constant (was %exp). - TF_ASSERT_OK(exp->ReplaceUseWith(xla_while, constant)); - analysis.UpdateAfterChangingOperand(xla_while, /*old_operand=*/exp, - /*new_operand=*/constant); - - // Analysis should be valid after the update. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); - - EXPECT_EQ(analysis.GetUniqueBufferAt(constant), - analysis.GetUniqueBufferAt(body_param)); - EXPECT_EQ(analysis.GetUniqueBufferAt(constant), - analysis.GetUniqueBufferAt(cond_param)); - EXPECT_EQ(analysis.GetUniqueBufferAt(constant), - analysis.GetUniqueBufferAt(xla_while)); - EXPECT_NE(analysis.GetUniqueBufferAt(constant), - analysis.GetUniqueBufferAt(exp)); - EXPECT_NE(analysis.GetUniqueBufferAt(constant), - analysis.GetUniqueBufferAt(negate)); - - // And finally make the negate the root of the body again. - body->set_root_instruction(negate); - analysis.UpdateAfterChangingRoot(/*old_root=*/body_param, - /*new_root*/ negate); - - // Analysis should be valid after the update. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); - - EXPECT_EQ(analysis.GetUniqueBufferAt(negate), - analysis.GetUniqueBufferAt(body_param)); - EXPECT_EQ(analysis.GetUniqueBufferAt(negate), - analysis.GetUniqueBufferAt(cond_param)); - EXPECT_EQ(analysis.GetUniqueBufferAt(negate), - analysis.GetUniqueBufferAt(xla_while)); - EXPECT_EQ(analysis.GetUniqueBufferAt(constant), - analysis.GetUniqueBufferAt(negate)); - - auto value_of = [&analysis](const HloInstruction* instruction) { - return &analysis.dataflow_analysis().GetValueDefinedAt(instruction); - }; - EXPECT_THAT(analysis.GetUniqueBufferAt(negate).values(), - UnorderedElementsAre(value_of(body_param), value_of(cond_param), - value_of(negate), value_of(constant), - value_of(xla_while))); -} - -// Test update tuple element. - } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_buffer.cc b/tensorflow/compiler/xla/service/hlo_buffer.cc index 2bfdd9156ad..e16413f361f 100644 --- a/tensorflow/compiler/xla/service/hlo_buffer.cc +++ b/tensorflow/compiler/xla/service/hlo_buffer.cc @@ -36,22 +36,6 @@ namespace xla { using ::tensorflow::str_util::Join; using ::tensorflow::strings::StrCat; -void HloBuffer::AddValue(const HloValue& value) { - values_.push_back(&value); - // Sort vector and remove duplicates. - std::sort(values_.begin(), values_.end(), HloValue::IdLessThan); - values_.erase(std::unique(values_.begin(), values_.end(), HloValue::IdEqual), - values_.end()); -} - -void HloBuffer::RemoveValue(const HloValue& value) { - // The values are sorted, so finding the value could be done in log(n) time - // with a binary search. - auto it = std::find(values_.begin(), values_.end(), &value); - CHECK(it != values_.end()); - values_.erase(it); -} - bool HloBuffer::operator==(const HloBuffer& other) const { bool equal = id() == other.id(); if (equal) { diff --git a/tensorflow/compiler/xla/service/hlo_buffer.h b/tensorflow/compiler/xla/service/hlo_buffer.h index cb961e1601c..4873463b2ea 100644 --- a/tensorflow/compiler/xla/service/hlo_buffer.h +++ b/tensorflow/compiler/xla/service/hlo_buffer.h @@ -84,22 +84,15 @@ class HloBuffer { return a->id() == b->id(); } - HloBuffer(Id id) : id_(id) {} + HloBuffer(Id id, tensorflow::gtl::ArraySlice values) + : id_(id), values_(values.begin(), values.end()) {} // Return the unique identifier for this HloBuffer. Id id() const { return id_; } - // Add a value to the set of values held by this buffer. Also adds the - // HloPositions of the value to the positions vector of the buffer. If the - // buffer already contains this value, then this method is a nop. - void AddValue(const HloValue& value); - void RemoveValue(const HloValue& value); - // Return all values contained in this buffer. const std::vector& values() const { return values_; } - std::vector ComputePositions() const; - // Return the unique HLO value in the buffer. CHECK fails if the buffer does // not contain exactly one value. const HloValue& GetUniqueValue() const { @@ -107,6 +100,8 @@ class HloBuffer { return *values_[0]; } + std::vector ComputePositions() const; + string ToString() const; bool operator==(const HloBuffer& other) const; @@ -118,7 +113,7 @@ class HloBuffer { // The set of values contained in this buffer. Vector contains no duplicates // and is sorted stably by HloValue::Id. - std::vector values_; + const std::vector values_; }; std::ostream& operator<<(std::ostream& out, const HloBuffer& buffer); diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc index ea8b239e100..2be1645f1b0 100644 --- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc @@ -67,6 +67,22 @@ HloValue& HloDataflowAnalysis::GetValueDefinedAt( return GetUniqueValueAt(instruction, index); } +HloValue* HloDataflowAnalysis::NewHloValue(HloInstruction* instruction, + const ShapeIndex& index, + bool is_phi) { + const int64 value_id = next_value_id_++; + auto emplaced = values_.emplace( + std::piecewise_construct, std::forward_as_tuple(value_id), + std::forward_as_tuple(value_id, instruction, index, is_phi)); + CHECK(emplaced.second); + + return &emplaced.first->second; +} + +void HloDataflowAnalysis::DeleteHloValue(HloValue::Id value_id) { + values_.erase(value_id); +} + string HloDataflowAnalysis::ToString() const { string out = StrCat("HloDataflowAnalysis, module ", module_->name(), "\n"); StrAppend(&out, " Instruction value sets:\n"); @@ -99,22 +115,98 @@ string HloDataflowAnalysis::ToString() const { } } StrAppend(&out, " HloValues:\n"); - for (const HloValue& value : values()) { - StrAppend(&out, value.ToString(/*indent=*/4)); - } - StrAppend(&out, " Phi resolutions:\n"); - for (const HloValue& value : values()) { - if (value.is_phi()) { - const HloValue* resolved_value = ResolvePhi(value); - StrAppend(&out, " ", value.ToShortString(), " => ", - resolved_value == nullptr ? "UNKNOWN" - : resolved_value->ToShortString(), - "\n"); - } + for (const HloValue* value : values()) { + StrAppend(&out, value->ToString(/*indent=*/4)); } return out; } +bool HloDataflowAnalysis::Phi( + HloInstruction* instruction, + tensorflow::gtl::ArraySlice inputs) { + CHECK(ssa_form_); + + for (const InstructionValueSet* input : inputs) { + DCHECK(ShapeUtil::Compatible(instruction->shape(), input->shape())); + } + + bool changed = false; + for (auto& pair : GetInstructionValueSet(instruction)) { + const ShapeIndex& index = pair.first; + HloValueSet& value_set = pair.second; + + // Positions with phi values should never have more than one value in the + // value set. + CHECK_LE(value_set.values().size(), 1); + const HloValue* current_value = + value_set.values().size() == 1 ? value_set.values()[0] : nullptr; + + // Construct a vector of unique value IDs of the inputs. + std::vector input_value_ids; + for (const InstructionValueSet* input : inputs) { + for (const HloValue* value : input->element(index).values()) { + input_value_ids.push_back(value->id()); + } + } + std::sort(input_value_ids.begin(), input_value_ids.end()); + input_value_ids.erase( + std::unique(input_value_ids.begin(), input_value_ids.end()), + input_value_ids.end()); + + // Remove the existing phi value (if it exists). The phi can be its own + // input, for example, in while body parameters where the body passes + // through the parameter value. + bool current_value_defined_here = + (current_value != nullptr && + current_value->defining_instruction() == instruction && + current_value->defining_index() == index); + if (current_value_defined_here) { + CHECK(current_value->is_phi()); + auto it = std::find(input_value_ids.begin(), input_value_ids.end(), + current_value->id()); + if (it != input_value_ids.end()) { + input_value_ids.erase(it); + } + } + + if (input_value_ids.empty()) { + // A value set which has at least one element should never have its value + // set reduced to zero elements. During dataflow value sets only can go + // from empty to non-empty, not the reverse. + CHECK_EQ(value_set.values().size(), 0) + << "Instruction " << instruction->name() << " at index " << index + << " previously had non-empty value set. Value set: " << value_set; + } else if (input_value_ids.size() == 1) { + // Only a single value reaches this point. There should be no phi, and + // this value set should contain this single value. + const HloValue& new_value = GetValue(input_value_ids[0]); + if (current_value == nullptr) { + value_set.Clear(); + value_set.AddValue(&new_value); + changed = true; + } else if (current_value != &new_value) { + if (current_value_defined_here) { + // Remove the existing phi. + DeleteHloValue(current_value->id()); + } + value_set.Clear(); + value_set.AddValue(&new_value); + changed = true; + } + } else { + // Multiple distinct values reach this point. A phi value is + // necessary. + CHECK_GT(input_value_ids.size(), 1); + if (current_value == nullptr || !current_value->is_phi()) { + value_set.Clear(); + value_set.AddValue(NewHloValue(instruction, index, /*is_phi=*/true)); + changed = true; + } + } + } + return changed; +} + const HloValue& HloDataflowAnalysis::GetValue(HloValue::Id value_id) const { return values_.at(value_id); } @@ -142,129 +234,6 @@ HloValueSet& HloDataflowAnalysis::GetValueSet(const HloPosition& position) { return GetValueSet(position.instruction, position.index); } -void HloDataflowAnalysis::UpdateAfterChangingOperand( - HloInstruction* instruction, HloInstruction* old_operand, - HloInstruction* new_operand) { - CHECK(std::find(instruction->operands().begin(), - instruction->operands().end(), - new_operand) != instruction->operands().end()); - VLOG(1) << "UpdateAfterChangingOperand(" << instruction->name() << ", " - << old_operand->name() << " => " << new_operand->name() << ")"; - - std::vector to_update = {instruction}; - - // If the instruction calls any computations then add the parameters of called - // computation to capture any changes to the dataflow into the subcomputation - // introduced by the new operand. - for (HloComputation* computation : instruction->called_computations()) { - to_update.insert(to_update.end(), - computation->parameter_instructions().begin(), - computation->parameter_instructions().end()); - } - - UpdateInstructionsAndPropagate(to_update); - - // The uses of the values in the old and new operand may have changed. Uses of - // other HloValues are updated in UpdateInstructionsAndPropagate. - for (auto& pair : GetInstructionValueSet(old_operand)) { - for (const HloValue* value : pair.second.values()) { - GetValue(value->id()).RecomputeUses(); - } - } - for (auto& pair : GetInstructionValueSet(new_operand)) { - for (const HloValue* value : pair.second.values()) { - GetValue(value->id()).RecomputeUses(); - } - } - - TF_DCHECK_OK(VerifyAgainstReference()); -} - -void HloDataflowAnalysis::UpdateAfterChangingRoot(HloInstruction* old_root, - HloInstruction* new_root) { - VLOG(1) << "UpdateAfterChangingRoot(" << old_root->name() << " => " - << new_root->name() << ")"; - - CHECK_EQ(new_root, new_root->parent()->root_instruction()); - CHECK_EQ(new_root->parent(), old_root->parent()); - - std::vector to_update = {old_root, new_root}; - - const CallGraphNode& call_graph_node = - call_graph_->GetNode(new_root->parent()); - for (const CallSite& callsite : call_graph_node.caller_callsites()) { - if (callsite.instruction()->opcode() == HloOpcode::kCall) { - to_update.push_back(callsite.instruction()); - } else if (callsite.instruction()->opcode() == HloOpcode::kWhile) { - // Add the while itself, and the body and condition parameters. - to_update.push_back(callsite.instruction()); - to_update.push_back( - callsite.instruction()->while_body()->parameter_instruction(0)); - to_update.push_back( - callsite.instruction()->while_condition()->parameter_instruction(0)); - } - } - - UpdateInstructionsAndPropagate(to_update); - - TF_DCHECK_OK(VerifyAgainstReference()); -} - -const HloValue* HloDataflowAnalysis::ResolvePhi(const HloValue& phi) const { - CHECK(phi.is_phi()); - - tensorflow::gtl::FlatSet visited; - std::queue worklist; - auto add_to_worklist = [&worklist, &visited](const HloValue* v) { - if (visited.insert(v).second) { - // 'v' was not previously in visited. - worklist.push(v); - } - }; - add_to_worklist(&phi); - - const HloValue* resolved_value = nullptr; - while (!worklist.empty()) { - const HloValue* value = worklist.front(); - worklist.pop(); - - if (!value->is_phi()) { - if (resolved_value == nullptr) { - resolved_value = value; - } else if (resolved_value != value) { - return nullptr; - } - } else { - for (const HloValue* input : phi_inputs_.at(value)) { - add_to_worklist(input); - } - } - } - return resolved_value; -} - -void HloDataflowAnalysis::UpdatePhiInputs( - const HloInstruction* instruction, - tensorflow::gtl::ArraySlice inputs) { - CHECK(ssa_form_); - for (auto& pair : GetInstructionValueSet(instruction)) { - const ShapeIndex& index = pair.first; - const HloValue& phi_value = GetUniqueValueAt(instruction, index); - auto& phi_inputs = phi_inputs_.at(&phi_value); - phi_inputs.clear(); - for (const InstructionValueSet* input : inputs) { - for (const HloValue* value : input->element(index).values()) { - // The number of phi inputs is typically 2, and virtually always very - // small. - if (std::find(phi_inputs.begin(), phi_inputs.end(), value) == - phi_inputs.end()) { - phi_inputs.push_back(value); - } - } - } - } -} - bool HloDataflowAnalysis::UpdateBitcastValueSet(HloInstruction* bitcast) { CHECK_EQ(bitcast->opcode(), HloOpcode::kBitcast); const InstructionValueSet& operand_set = @@ -380,8 +349,7 @@ bool HloDataflowAnalysis::UpdateParameterValueSet(HloInstruction* parameter) { } if (ssa_form_ && called_from_while) { - UpdatePhiInputs(parameter, inputs); - return false; + return Phi(parameter, inputs); } else { return GetInstructionValueSet(parameter).AssignUnionOf(inputs); } @@ -439,8 +407,7 @@ bool HloDataflowAnalysis::UpdateWhileValueSet(HloInstruction* xla_while) { &GetInstructionValueSet(xla_while->while_body()->root_instruction()), &GetInstructionValueSet(xla_while->operand(0))}; if (ssa_form_) { - UpdatePhiInputs(xla_while, inputs); - return false; + return Phi(xla_while, inputs); } else { return GetInstructionValueSet(xla_while).AssignUnionOf(inputs); } @@ -487,38 +454,7 @@ void HloDataflowAnalysis::UpdateInstructionsAndPropagate( VLOG(3) << "Worklist top: " << instruction->name(); VLOG(3) << ToString(); - // The updating of the instruction value set below in - // UpdateInstructionValueSet does not update HloValue::positions(). To - // perform the positions() update remove all positions in 'instruction' from - // the HloValues in 'instruction's value set prior to the update, then after - // the update add the new positions back in. There is likely a more - // efficient way of doing this. - for (auto& pair : GetInstructionValueSet(instruction)) { - const ShapeIndex& index = pair.first; - HloValueSet& value_set = pair.second; - for (const HloValue* value : value_set.values()) { - if (value->defining_instruction() != instruction) { - // Use GetValue for a non-const HloValue reference. - GetValue(value->id()).RemovePosition(instruction, index); - } - } - } - - bool changed = UpdateInstructionValueSet(instruction); - - // Add the positions back in. - for (auto& pair : GetInstructionValueSet(instruction)) { - const ShapeIndex& index = pair.first; - HloValueSet& value_set = pair.second; - for (const HloValue* value : value_set.values()) { - if (value->defining_instruction() != instruction) { - // Use GetValue for a non-const HloValue reference. - GetValue(value->id()).AddPosition(instruction, index); - } - } - } - - if (!changed) { + if (!UpdateInstructionValueSet(instruction)) { // No change to the instruction's value set. VLOG(4) << "No change."; continue; @@ -531,12 +467,16 @@ void HloDataflowAnalysis::UpdateInstructionsAndPropagate( for (HloInstruction* user : instruction->users()) { worklist.push(user); - // If user calls a computation, then the respective parameter(s) of the - // computation need to be updated. + // If user sequentially calls a computation, then the respective + // parameter(s) of the computation need to be updated. for (HloComputation* called_computation : user->called_computations()) { - for (int64 operand_number : user->OperandIndices(instruction)) { - worklist.push( - called_computation->parameter_instruction(operand_number)); + const CallGraphNode& call_graph_node = + call_graph_->GetNode(called_computation); + if (call_graph_node.context() == CallContext::kSequential) { + for (int64 operand_number : user->OperandIndices(instruction)) { + worklist.push( + called_computation->parameter_instruction(operand_number)); + } } } } @@ -574,25 +514,10 @@ InstructionValueSet& HloDataflowAnalysis::GetInstructionValueSet( } Status HloDataflowAnalysis::InitializeInstructionValueSets() { - // Gather the values to create before creating them. This is done because we - // want to allocate the vector of values only once so references to elements - // are stable. - struct ValueToCreate { - HloInstruction* instruction; - ShapeIndex index; - bool is_phi; - }; - std::vector values_to_create; - for (const std::unique_ptr& computation : module_->computations()) { const CallGraphNode& call_graph_node = call_graph_->GetNode(computation.get()); - bool called_from_while = std::any_of( - call_graph_node.caller_callsites().begin(), - call_graph_node.caller_callsites().end(), [](const CallSite& cs) { - return cs.instruction()->opcode() == HloOpcode::kWhile; - }); for (const std::unique_ptr& instruction : computation->instructions()) { @@ -603,20 +528,22 @@ Status HloDataflowAnalysis::InitializeInstructionValueSets() { // Lambda to set the value set to define all values in the output of the // instruction. - auto define_all_values = [this, &instruction, - &values_to_create](bool is_phi = false) { + auto define_all_values = [this, &instruction](bool is_phi = false) { for (auto& pair : GetInstructionValueSet(instruction.get())) { const ShapeIndex& index = pair.first; - values_to_create.push_back({instruction.get(), index, is_phi}); + HloValue* value = + NewHloValue(instruction.get(), index, /*is_phi=*/false); + GetValueSet(instruction.get(), index).AddValue(value); } }; // Lambda to set the value set to define only the top-level buffer in the // output of the instruction. Any other values flow from the operands of // the instruction (or from cross-computation dataflow). - auto define_top_level_only = [this, &instruction, &values_to_create]() { - values_to_create.push_back( - {instruction.get(), /*index=*/{}, /*is_phi=*/false}); + auto define_top_level_only = [this, &instruction]() { + HloValue* value = + NewHloValue(instruction.get(), /*index=*/{}, /*is_phi=*/false); + GetValueSet(instruction.get(), /*index=*/{}).AddValue(value); }; switch (instruction->opcode()) { @@ -626,10 +553,6 @@ Status HloDataflowAnalysis::InitializeInstructionValueSets() { } break; case HloOpcode::kWhile: - if (ssa_form_) { - define_all_values(/*is_phi=*/true); - } - break; case HloOpcode::kCall: case HloOpcode::kGetTupleElement: // These instructions define no values. The values in their output @@ -654,10 +577,6 @@ Status HloDataflowAnalysis::InitializeInstructionValueSets() { // values in their output. Otherwise the values of the parameter // come from the caller (eg, operands to the kCall instruction). define_all_values(); - } else if (call_graph_node.context() == CallContext::kSequential && - called_from_while && ssa_form_) { - // Parameters of while bodies and conditions are phis. - define_all_values(/*is_phi=*/true); } break; case HloOpcode::kCopy: @@ -674,164 +593,9 @@ Status HloDataflowAnalysis::InitializeInstructionValueSets() { } } - // Reserve the vector ahead of time so references to elements are stable. - values_.reserve(values_to_create.size()); - for (int64 i = 0; i < values_to_create.size(); ++i) { - const ValueToCreate& to_create = values_to_create[i]; - values_.emplace_back(/*id=*/i, to_create.instruction, to_create.index, - to_create.is_phi); - const HloValue& value = values_.back(); - GetValueSet(to_create.instruction, to_create.index).AddValue(&value); - if (value.is_phi()) { - phi_inputs_[&value] = {}; - } - } return Status::OK(); } -bool HloDataflowAnalysis::IsDefinedBefore(const HloValue& a, const HloValue& b, - const HloOrdering& ordering) const { - // If 'b' is an entry param then 'a' cannot be defined before 'b' because 'b' - // is live into the module. - if (b.defining_instruction()->parent() == module_->entry_computation() && - b.defining_instruction()->opcode() == HloOpcode::kParameter) { - return false; - } - - // Phi values require special handling. Because XLA does not have a phi - // instruction, the definition instruction of the phis values are - // placeholders: either the subcomputation parameter (body or condition) or - // the while instruction. However, the program point where these values are - // logically defined does not necessarily coincide exactly with program point - // of these place-holder instructions. So we explicitly define the following - // order for phi values: - // - // body/condition parameter phi: - // Defined before all values defined in its computation excepting other - // phis. - // - // while phi: - // defined after all values defined in the condition or body. - // - auto is_body_or_condition_phi = [](const HloValue& v) { - return v.is_phi() && - v.defining_instruction()->opcode() == HloOpcode::kParameter; - }; - if (is_body_or_condition_phi(a) && !is_body_or_condition_phi(b) && - call_graph_->InstructionIsNestedIn(b.defining_instruction(), - a.defining_instruction()->parent())) { - return true; - } - if (is_body_or_condition_phi(b) && - call_graph_->InstructionIsNestedIn(a.defining_instruction(), - b.defining_instruction()->parent())) { - return false; - } - - // If 'b' is a while phi and 'a' is in the body or condition, then 'a' - // executes before 'b'. - if (b.is_phi() && b.defining_instruction()->opcode() == HloOpcode::kWhile && - (call_graph_->InstructionIsNestedIn( - a.defining_instruction(), b.defining_instruction()->while_body()) || - call_graph_->InstructionIsNestedIn( - a.defining_instruction(), - b.defining_instruction()->while_condition()))) { - return true; - } - - return ordering.ExecutesBefore(a.defining_instruction(), - b.defining_instruction()); -} - -bool HloDataflowAnalysis::UseIsBeforeValueDefinition( - const HloUse& use, const HloValue& value, - const HloOrdering& ordering) const { - if (ordering.ExecutesBefore(use.instruction, value.defining_instruction())) { - return true; - } - - // If the use is at the instruction where the value is defined, then the use - // is before the def if the instruction allows buffer sharing (in place - // computation). - if (use.instruction == value.defining_instruction() && - CanShareOperandBufferWithUser( - use.instruction->mutable_operand(use.operand_number), - use.operand_index, value.defining_instruction(), - value.defining_index())) { - return true; - } - - // The use at a while is an input to a phi, and logically occurs before values - // are defined in the body or condition computations. - if (use.instruction->opcode() == HloOpcode::kWhile) { - const HloInstruction* xla_while = use.instruction; - if (call_graph_->InstructionIsNestedIn(value.defining_instruction(), - xla_while->while_body()) || - call_graph_->InstructionIsNestedIn(value.defining_instruction(), - xla_while->while_condition())) { - return true; - } - } - - // Similarly if the value is defined at a while, it logically occurs after any - // uses in the body or condition computations. - if (value.defining_instruction()->opcode() == HloOpcode::kWhile) { - CHECK(ssa_form_); - const HloInstruction* xla_while = value.defining_instruction(); - if (call_graph_->InstructionIsNestedIn(use.instruction, - xla_while->while_body()) || - call_graph_->InstructionIsNestedIn(use.instruction, - xla_while->while_condition())) { - return true; - } - } - return false; -} - -bool HloDataflowAnalysis::LiveRangeStrictlyBefore( - const HloValue& a, const HloValue& b, const HloOrdering& ordering) const { - VLOG(4) << "LiveRangeStrictlyBefore(a = " << a.ToShortString() - << ", b = " << b.ToShortString() << ")"; - if (!IsDefinedBefore(a, b, ordering)) { - VLOG(4) << "a not defined before b"; - return false; - } - - // Live-out values from the module can never have ranges strictly before any - // other value. - if (a.live_out_of_module()) { - VLOG(4) << "a is live out of module"; - return false; - } - - // Live-out values of computations can never have ranges strictly before any - // other value in the computation (including values nested in - // subcomputations). - if (a.live_out_of_computation() && - call_graph_->InstructionIsNestedIn(b.defining_instruction(), - a.defining_instruction()->parent())) { - VLOG(4) << "a is live out of computation containing b"; - return false; - } - - // All uses of 'a' must be before 'b' is defined. - for (const HloUse& use : a.uses()) { - if (!UseIsBeforeValueDefinition(use, b, ordering)) { - VLOG(4) << "use of a (" << use << ") not before b is defined"; - return false; - } - } - - return true; -} - -bool HloDataflowAnalysis::MayInterfere(const HloValue& a, const HloValue& b, - const HloOrdering& ordering) const { - // Buffers without disjoint liveness may interfere. - return !LiveRangeStrictlyBefore(a, b, ordering) && - !LiveRangeStrictlyBefore(b, a, ordering); -} - /* static */ StatusOr> HloDataflowAnalysis::Run( HloModule* module, bool ssa_form, bool bitcast_defines_value) { @@ -855,6 +619,33 @@ StatusOr> HloDataflowAnalysis::Run( } dataflow_analysis->UpdateInstructionsAndPropagate(all_instructions); + // Add in positions to all values. + for (const std::unique_ptr& computation : + module->computations()) { + for (const std::unique_ptr& instruction : + computation->instructions()) { + for (const auto& pair : + dataflow_analysis->GetInstructionValueSet(instruction.get())) { + const ShapeIndex& index = pair.first; + const HloValueSet& value_set = pair.second; + for (const HloValue* value : value_set.values()) { + if (value->defining_instruction() != instruction.get()) { + dataflow_analysis->GetValue(value->id()) + .AddPosition(instruction.get(), index); + } + } + } + } + } + + // Construct vector of values. + dataflow_analysis->values_vector_.reserve(dataflow_analysis->values_.size()); + for (auto& pair : dataflow_analysis->values_) { + dataflow_analysis->values_vector_.push_back(&pair.second); + } + std::sort(dataflow_analysis->values_vector_.begin(), + dataflow_analysis->values_vector_.end(), HloValue::IdLessThan); + TF_DCHECK_OK(dataflow_analysis->Verify()); XLA_VLOG_LINES(1, dataflow_analysis->ToString()); @@ -865,14 +656,14 @@ StatusOr> HloDataflowAnalysis::Run( Status HloDataflowAnalysis::Verify() const { // Verify each HloValue appears in the value sets that the value's positions() // indicate. - for (const HloValue& value : values()) { - for (const HloPosition& position : value.positions()) { + for (const HloValue* value : values()) { + for (const HloPosition& position : value->positions()) { const HloValueSet& value_set = GetValueSet(position); TF_RET_CHECK(std::find(value_set.values().begin(), value_set.values().end(), - &value) != value_set.values().end()) + value) != value_set.values().end()) << "Value set at position " << position << " does not contain value " - << value.ToShortString(); + << value->ToShortString(); } } @@ -898,75 +689,4 @@ Status HloDataflowAnalysis::Verify() const { return Status::OK(); } -Status HloDataflowAnalysis::VerifyAgainstReference() const { - TF_RETURN_IF_ERROR(Verify()); - - TF_ASSIGN_OR_RETURN(std::unique_ptr reference, - Run(module_, ssa_form_, bitcast_defines_value_)); - TF_RETURN_IF_ERROR(reference->Verify()); - - VLOG(2) << "This analysis:"; - XLA_VLOG_LINES(2, ToString()); - VLOG(2) << "Reference:"; - XLA_VLOG_LINES(2, reference->ToString()); - - // Verify value sets in each position are identical. - for (const auto& computation : module_->computations()) { - for (const auto& instruction : computation->instructions()) { - for (const auto& pair : GetInstructionValueSet(instruction.get())) { - const ShapeIndex& index = pair.first; - const HloValueSet& value_set = pair.second; - const HloValueSet& reference_value_set = - reference->GetValueSet(instruction.get(), index); - - auto value_in_set = [](const HloValue& v, const HloValueSet& vset) { - return std::find_if(vset.values().begin(), vset.values().end(), - [&v](const HloValue* w) { return *w == v; }) != - vset.values().end(); - }; - - for (const HloValue* value : value_set.values()) { - TF_RET_CHECK(value_in_set(*value, reference_value_set)) - << "Value " << value->ToShortString() - << " does not exist in reference"; - } - for (const HloValue* reference_value : reference_value_set.values()) { - TF_RET_CHECK(value_in_set(*reference_value, value_set)) - << "Value " << reference_value->ToShortString() - << " only exists in reference"; - } - } - } - } - - // Verify all phis resolve identically and uses are identical. - for (const HloValue& value : values()) { - const HloValue& reference_value = reference->GetValueDefinedAt( - value.defining_instruction(), value.defining_index()); - TF_RET_CHECK(value.is_phi() == reference_value.is_phi()); - if (value.is_phi()) { - const HloValue* resolved_value = ResolvePhi(value); - const HloValue* reference_resolved_value = - reference->ResolvePhi(reference_value); - if (resolved_value == nullptr) { - TF_RET_CHECK(reference_resolved_value == nullptr); - } else { - TF_RET_CHECK(reference_resolved_value != nullptr); - TF_RET_CHECK(*reference_resolved_value == *resolved_value); - } - } - - for (const HloUse& use : value.uses()) { - TF_RET_CHECK(std::find(reference_value.uses().begin(), - reference_value.uses().end(), - use) != reference_value.uses().end()); - } - for (const HloUse& reference_use : reference_value.uses()) { - TF_RET_CHECK(std::find(value.uses().begin(), value.uses().end(), - reference_use) != value.uses().end()); - } - } - return Status::OK(); -} - } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h index 7781cc58a3a..aae257dd09e 100644 --- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h +++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.h @@ -88,10 +88,10 @@ class HloDataflowAnalysis { // given position. const HloValueSet& GetValueSet(const HloInstruction* instruction, const ShapeIndex& index = {}) const; - HloValueSet& GetValueSet(const HloInstruction* instruction, - const ShapeIndex& index = {}); const HloValueSet& GetValueSet(const HloPosition& position) const; HloValueSet& GetValueSet(const HloPosition& position); + HloValueSet& GetValueSet(const HloInstruction* instruction, + const ShapeIndex& index = {}); // Return the unique value in the HloValueSet at the given instruction and // shape index. CHECKs if the value set does not contain a exactly one value. @@ -108,49 +108,11 @@ class HloDataflowAnalysis { const HloValue& GetValue(HloValue::Id value_id) const; HloValue& GetValue(HloValue::Id value_id); - // Returns whether the given values interfere assuming the given HLO - // ordering. Two values interfere if they may both be simultaneously live. - bool MayInterfere(const HloValue& a, const HloValue& b, - const HloOrdering& ordering) const; - - // Overload which takes HloValue:Ids. - bool MayInterfere(HloValue::Id a, HloValue::Id b, - const HloOrdering& ordering) const { - return MayInterfere(GetValue(a), GetValue(b), ordering); - } - // Return the total number of HloValues. int64 value_count() const { return values_.size(); } - // Return a vector of all HloValues. - const std::vector& values() const { return values_; } - - // Updates the dataflow after the changing an operand of - // 'instruction'. Dataflow update is not possible if instructions have been - // added or removed from the graph. - void UpdateAfterChangingOperand(HloInstruction* instruction, - HloInstruction* old_operand, - HloInstruction* new_operand); - - // Updates the dataflow after the changing the root of a computation from - // 'old_root' to 'new_root'. - void UpdateAfterChangingRoot(HloInstruction* old_root, - HloInstruction* new_root); - - // Returns the non-phi HloValue that is the unique (transitive) input to the - // given phi. If no such HloValue exists (there are multiple inputs to the - // phi) then nullptr is returned. This is computed by all walking the inputs - // of the given phi value until non-phi HloValue(s) are encountered. - const HloValue* ResolvePhi(const HloValue& phi) const; - const HloValue* ResolvePhi(const HloInstruction* instruction, - const ShapeIndex& index = {}) const { - return ResolvePhi(GetValueDefinedAt(instruction, index)); - } - - // Compare the dataflow analysis against a clean recomputation of the - // analysis. Returns an error status if there is a mismatch. Useful for - // verifying the correctness after updates to the analysis. - Status VerifyAgainstReference() const; + // Return a vector of all HloValues stabily sorted by HloValue::Id. + const std::vector& values() const { return values_vector_; } // Return the call graph used for computing the dataflow. const CallGraph& call_graph() const { return *call_graph_; } @@ -161,6 +123,13 @@ class HloDataflowAnalysis { HloDataflowAnalysis(HloModule* module, bool ssa_form, bool bitcast_defines_value = false); + // Returns a new HloValue defined at the given instruction and shape index. + HloValue* NewHloValue(HloInstruction* instruction, const ShapeIndex& index, + bool is_phi = false); + + // Delete the HloValue with the given ID. + void DeleteHloValue(HloValue::Id value_id); + // Constructs and initializes the InstructionValueSets of all instructions to // contain exactly the HloValues defined by each instruction. These values can // then propagated throughout the HLO graph by calling @@ -187,10 +156,11 @@ class HloDataflowAnalysis { void UpdateInstructionsAndPropagate( tensorflow::gtl::ArraySlice instructions); - // Sets the inputs of the given phi to given value(s). - void UpdatePhiInputs( - const HloInstruction* instruction, - tensorflow::gtl::ArraySlice inputs); + // Return the result of the SSA Phi function applied to the given inputs at + // the given instruction. If skip_top_level is true, then the top level of the + // value set of 'instruction' is not modified. + bool Phi(HloInstruction* instruction, + tensorflow::gtl::ArraySlice inputs); // Updates the positions of the HloValues in the output of the given // instruction. This should be called after the instruction value set of @@ -203,20 +173,6 @@ class HloDataflowAnalysis { HloInstruction* instruction, const InstructionValueSet& new_value_set, const InstructionValueSet* prev_value_set = nullptr); - // Returns true if the live range of the given value 'a' is strictly before - // the live range of value 'b' using the given HLO ordering. - bool LiveRangeStrictlyBefore(const HloValue& a, const HloValue& b, - const HloOrdering& ordering) const; - - // Returns whether the value 'a' is defined before the value 'b' under the - // given ordering. - bool IsDefinedBefore(const HloValue& a, const HloValue& b, - const HloOrdering& ordering) const; - - // Returns whether the given use is before the given value definition. - bool UseIsBeforeValueDefinition(const HloUse& use, const HloValue& value, - const HloOrdering& ordering) const; - // Verify various invariants of the dataflow analysis. Status Verify() const; @@ -226,19 +182,19 @@ class HloDataflowAnalysis { std::unique_ptr call_graph_; - // Array of all values in the module. This is allocated once at analysis - // construction time so HloValue references are stable. Updates to the - // analysis via UpdateAfterChangingOperand and UpdateAfterChangingRoot do not - // result in the creation or destruction of any HloValues. - std::vector values_; - - // Map hold the inputs to each phi value in the module. Used by ResolvePhi. - tensorflow::gtl::FlatMap> - phi_inputs_; + // The map of all HloValues in the module. We pass around pointers to the + // mapped HloValues, so the underlying container must keep them valid despite + // mutations touching other map entries. + std::unordered_map values_; // A map from instruction to InstructionValueSet. std::unordered_map value_sets_; + + // A vector containing all HloValues sorted by HloValue::Id. + std::vector values_vector_; + + // The Id to use for the next HloValue. + HloValue::Id next_value_id_ = 0; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc index 9f3dd539efe..ef0fa1d745a 100644 --- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc +++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc @@ -26,7 +26,6 @@ limitations under the License. #include "tensorflow/compiler/xla/test_helpers.h" #include "tensorflow/compiler/xla/tests/hlo_test_base.h" #include "tensorflow/compiler/xla/xla_data.pb.h" -#include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/test.h" @@ -44,8 +43,8 @@ class HloDataflowAnalysisTest : public HloTestBase, // Run dataflow analysis on the member module. For convenience returns a // reference to the generated analysis stored in analysis_. - HloDataflowAnalysis& RunAnalysis(bool ssa_form, - bool bitcast_defines_value = false) { + const HloDataflowAnalysis& RunAnalysis(bool ssa_form, + bool bitcast_defines_value = false) { analysis_ = HloDataflowAnalysis::Run(module_.get(), ssa_form, bitcast_defines_value) .ConsumeValueOrDie(); @@ -71,8 +70,8 @@ class HloDataflowAnalysisTest : public HloTestBase, const HloInstruction* b) { EXPECT_FALSE(ShapeUtil::IsTuple(a->shape())); EXPECT_FALSE(ShapeUtil::IsTuple(b->shape())); - return analysis_->MayInterfere(analysis_->GetValueDefinedAt(a), - analysis_->GetValueDefinedAt(b), ordering); + return ordering.MayInterfere(analysis_->GetValueDefinedAt(a), + analysis_->GetValueDefinedAt(b)); } std::unique_ptr module_; @@ -499,37 +498,26 @@ TEST_P(HloDataflowAnalysisTest, SingleWhile) { EXPECT_FALSE(analysis.GetValueDefinedAt(cond_constant).live_out_of_module()); if (ssa_form) { - // While instruction should define phi values. The value at index {0} is a - // degenerate phi with a single input 'constant1'. - EXPECT_TRUE(analysis.ValueIsDefinedAt(xla_while, /*index=*/{0})); - EXPECT_TRUE(analysis.GetValueDefinedAt(xla_while, /*index=*/{0}).is_phi()); - EXPECT_EQ(analysis.ResolvePhi(xla_while, /*index=*/{0}), - &analysis.GetValueDefinedAt(constant1)); - EXPECT_TRUE(analysis.ValueIsDefinedAt(body_param, /*index=*/{0})); - EXPECT_TRUE(analysis.GetValueDefinedAt(body_param, /*index=*/{0}).is_phi()); - EXPECT_EQ(analysis.ResolvePhi(body_param, /*index=*/{0}), - &analysis.GetValueDefinedAt(constant1)); - EXPECT_TRUE(analysis.ValueIsDefinedAt(cond_param, /*index=*/{0})); - EXPECT_TRUE(analysis.GetValueDefinedAt(cond_param, /*index=*/{0}).is_phi()); - EXPECT_EQ(analysis.ResolvePhi(cond_param, /*index=*/{0}), - &analysis.GetValueDefinedAt(constant1)); + // Element 0 of the tuple passed through the body so no phi value is + // defined. + EXPECT_FALSE(analysis.ValueIsDefinedAt(xla_while, /*index=*/{0})); + EXPECT_FALSE(analysis.ValueIsDefinedAt(body_param, /*index=*/{0})); + EXPECT_FALSE(analysis.ValueIsDefinedAt(cond_param, /*index=*/{0})); + // Element 1 of the tuple should be a phi value. EXPECT_TRUE(analysis.ValueIsDefinedAt(xla_while, /*index=*/{1})); EXPECT_TRUE(analysis.GetValueDefinedAt(xla_while, /*index=*/{1}).is_phi()); - EXPECT_EQ(analysis.ResolvePhi(xla_while, /*index=*/{1}), nullptr); EXPECT_TRUE(analysis.ValueIsDefinedAt(body_param, /*index=*/{1})); EXPECT_TRUE(analysis.GetValueDefinedAt(body_param, /*index=*/{1}).is_phi()); - EXPECT_EQ(analysis.ResolvePhi(body_param, /*index=*/{1}), nullptr); EXPECT_TRUE(analysis.ValueIsDefinedAt(cond_param, /*index=*/{1})); EXPECT_TRUE(analysis.GetValueDefinedAt(cond_param, /*index=*/{1}).is_phi()); - EXPECT_EQ(analysis.ResolvePhi(cond_param, /*index=*/{1}), nullptr); - EXPECT_THAT(analysis.GetValueDefinedAt(constant1).uses(), - UnorderedElementsAre(HloUse{xla_while, 0, {0}})); + EXPECT_THAT( + analysis.GetValueDefinedAt(constant1).uses(), + UnorderedElementsAre(HloUse{add, 0, {}}, HloUse{xla_while, 0, {0}})); - EXPECT_FALSE(analysis.GetValueDefinedAt(constant1).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(xla_while, /*index=*/{0}) - .live_out_of_module()); + // Constant1 passes through the body and out of the module. + EXPECT_TRUE(analysis.GetValueDefinedAt(constant1).live_out_of_module()); EXPECT_TRUE(analysis.GetValueDefinedAt(xla_while, /*index=*/{1}) .live_out_of_module()); @@ -613,20 +601,15 @@ TEST_P(HloDataflowAnalysisTest, SequentialWhiles) { bool ssa_form = GetParam(); const HloDataflowAnalysis& analysis = RunAnalysis(ssa_form); - if (ssa_form) { - EXPECT_TRUE(analysis.GetValueDefinedAt(xla_while2).live_out_of_module()); - EXPECT_FALSE(analysis.GetValueDefinedAt(constant1).live_out_of_module()); - } else { - // Element 0 is passed through all the while instructions and out of the - // module. - EXPECT_EQ(analysis.GetUniqueValueAt(xla_while0, /*index=*/{0}), - analysis.GetValueDefinedAt(constant1)); - EXPECT_EQ(analysis.GetUniqueValueAt(xla_while1, /*index=*/{0}), - analysis.GetValueDefinedAt(constant1)); - EXPECT_EQ(analysis.GetUniqueValueAt(xla_while2, /*index=*/{0}), - analysis.GetValueDefinedAt(constant1)); - EXPECT_TRUE(analysis.GetValueDefinedAt(constant1).live_out_of_module()); - } + // Element 0 is passed through all the while instructions and out of the + // module.. + EXPECT_EQ(analysis.GetUniqueValueAt(xla_while0, /*index=*/{0}), + analysis.GetValueDefinedAt(constant1)); + EXPECT_EQ(analysis.GetUniqueValueAt(xla_while1, /*index=*/{0}), + analysis.GetValueDefinedAt(constant1)); + EXPECT_EQ(analysis.GetUniqueValueAt(xla_while2, /*index=*/{0}), + analysis.GetValueDefinedAt(constant1)); + EXPECT_TRUE(analysis.GetValueDefinedAt(constant1).live_out_of_module()); } TEST_P(HloDataflowAnalysisTest, NestedWhiles) { @@ -705,13 +688,18 @@ TEST_P(HloDataflowAnalysisTest, NestedWhiles) { bool ssa_form = GetParam(); const HloDataflowAnalysis& analysis = RunAnalysis(ssa_form); + EXPECT_THAT(HloValuesAt(inner_param, /*index=*/{0}), + UnorderedElementsAre(analysis.GetValueDefinedAt(negate))); if (ssa_form) { EXPECT_TRUE(analysis.ValueIsDefinedAt(inner_param, /*index=*/{1})); EXPECT_TRUE( analysis.GetValueDefinedAt(inner_param, /*index=*/{1}).is_phi()); - EXPECT_TRUE(analysis.ValueIsDefinedAt(nested_while, /*index=*/{0})); - EXPECT_TRUE( - analysis.GetValueDefinedAt(inner_param, /*index=*/{1}).is_phi()); + + // Element 0 of the nested while is %negate. + EXPECT_FALSE(analysis.ValueIsDefinedAt(nested_while, /*index=*/{0})); + EXPECT_THAT(HloValuesAt(inner_param, /*index=*/{0}), + UnorderedElementsAre(analysis.GetValueDefinedAt(negate))); + // Element 1 is a phi value (join of %add and %constant2). EXPECT_TRUE(analysis.ValueIsDefinedAt(nested_while, /*index=*/{1})); EXPECT_TRUE( analysis.GetValueDefinedAt(nested_while, /*index=*/{1}).is_phi()); @@ -724,8 +712,6 @@ TEST_P(HloDataflowAnalysisTest, NestedWhiles) { EXPECT_TRUE( analysis.GetValueDefinedAt(entry_while, /*index=*/{1}).is_phi()); } else { - EXPECT_THAT(HloValuesAt(inner_param, /*index=*/{0}), - UnorderedElementsAre(analysis.GetValueDefinedAt(negate))); EXPECT_THAT(HloValuesAt(inner_param, /*index=*/{1}), UnorderedElementsAre(analysis.GetValueDefinedAt(add), analysis.GetValueDefinedAt(constant2))); @@ -1496,256 +1482,6 @@ TEST_P(HloDataflowAnalysisTest, EmbeddedComputationInterference) { EXPECT_TRUE(InstructionsMayInterfere(ordering, negate, embedded_log)); } -TEST_P(HloDataflowAnalysisTest, UpdateAnalysisForWhile) { - // Test updating dataflow after modifying a module with an array shaped while: - // - // body(F32[] %param): - // %negate = Negate(%param) - // - // condition(F32[] %param): - // return Constant(false) - // - // entry: - // %constant = Constant(1.0) - // %exp = Exp(%constant) - // return While(%exp, body, condition) - // - auto body_builder = HloComputation::Builder("body"); - auto body_param = body_builder.AddInstruction( - HloInstruction::CreateParameter(0, scalar_shape_, "param")); - auto negate = body_builder.AddInstruction(HloInstruction::CreateUnary( - scalar_shape_, HloOpcode::kNegate, body_param)); - HloComputation* body = module_->AddEmbeddedComputation(body_builder.Build()); - - // Condition computation trivially returns a constant "false". - auto cond_builder = HloComputation::Builder("condition"); - auto cond_param = cond_builder.AddInstruction( - HloInstruction::CreateParameter(0, scalar_shape_, "param")); - cond_builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(false))); - HloComputation* condition = - module_->AddEmbeddedComputation(cond_builder.Build()); - - auto builder = HloComputation::Builder(TestName()); - auto constant = builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(1.0))); - auto exp = builder.AddInstruction( - HloInstruction::CreateUnary(scalar_shape_, HloOpcode::kExp, constant)); - auto xla_while = builder.AddInstruction( - HloInstruction::CreateWhile(scalar_shape_, condition, body, exp)); - module_->AddEntryComputation(builder.Build()); - - bool ssa_form = GetParam(); - HloDataflowAnalysis& analysis = RunAnalysis(ssa_form); - - // Sanity check the initial dataflow analysis before transforming the HLO - // graph. - if (ssa_form) { - EXPECT_TRUE(analysis.ValueIsDefinedAt(body_param)); - EXPECT_TRUE(analysis.GetValueDefinedAt(body_param).is_phi()); - EXPECT_EQ(analysis.ResolvePhi(body_param), nullptr); - - EXPECT_TRUE(analysis.ValueIsDefinedAt(cond_param)); - EXPECT_TRUE(analysis.GetValueDefinedAt(cond_param).is_phi()); - EXPECT_EQ(analysis.ResolvePhi(cond_param), nullptr); - - EXPECT_FALSE(analysis.GetValueDefinedAt(exp).live_out_of_module()); - EXPECT_FALSE(analysis.GetValueDefinedAt(negate).live_out_of_module()); - } else { - EXPECT_THAT(HloValuesAt(body_param), - UnorderedElementsAre(analysis.GetValueDefinedAt(exp), - analysis.GetValueDefinedAt(negate))); - EXPECT_THAT(HloValuesAt(cond_param), - UnorderedElementsAre(analysis.GetValueDefinedAt(exp), - analysis.GetValueDefinedAt(negate))); - EXPECT_THAT(HloValuesAt(xla_while), - UnorderedElementsAre(analysis.GetValueDefinedAt(exp), - analysis.GetValueDefinedAt(negate))); - - EXPECT_TRUE(analysis.GetValueDefinedAt(negate).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(exp).live_out_of_module()); - } - - // Set the body root to the body_param. Previously it was Negate(body_param). - body->set_root_instruction(body_param); - - // Prior to updating, verify that the dataflow analysis is no longer valid. - Status verify_status = analysis.VerifyAgainstReference(); - EXPECT_FALSE(verify_status.ok()); - - analysis.UpdateAfterChangingRoot(/*old_root=*/negate, - /*new_root=*/body_param); - - // Analysis should be valid after the update. - TF_EXPECT_OK(analysis.VerifyAgainstReference()); - - if (ssa_form) { - // The phis should now be resolvable as 'exp' is passed through the body - // transparently. - EXPECT_EQ(analysis.ResolvePhi(body_param), - &analysis.GetValueDefinedAt(exp)); - EXPECT_EQ(analysis.ResolvePhi(cond_param), - &analysis.GetValueDefinedAt(exp)); - EXPECT_EQ(analysis.ResolvePhi(xla_while), &analysis.GetValueDefinedAt(exp)); - EXPECT_FALSE(analysis.GetValueDefinedAt(exp).live_out_of_module()); - } else { - EXPECT_THAT(HloValuesAt(body_param), - UnorderedElementsAre(analysis.GetValueDefinedAt(exp))); - EXPECT_THAT(HloValuesAt(cond_param), - UnorderedElementsAre(analysis.GetValueDefinedAt(exp))); - EXPECT_THAT(HloValuesAt(xla_while), - UnorderedElementsAre(analysis.GetValueDefinedAt(exp))); - EXPECT_TRUE(analysis.GetValueDefinedAt(exp).live_out_of_module()); - } - EXPECT_FALSE(analysis.GetValueDefinedAt(negate).live_out_of_module()); - - // Now replace the operand of the while with %constant (was %exp). - TF_ASSERT_OK(exp->ReplaceUseWith(xla_while, constant)); - analysis.UpdateAfterChangingOperand(xla_while, /*old_operand=*/exp, - /*new_operand=*/constant); - - // Verify that the dataflow is correct. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); - - if (ssa_form) { - // The phis now resolve to 'constant'. - EXPECT_EQ(analysis.ResolvePhi(body_param), - &analysis.GetValueDefinedAt(constant)); - EXPECT_EQ(analysis.ResolvePhi(cond_param), - &analysis.GetValueDefinedAt(constant)); - EXPECT_EQ(analysis.ResolvePhi(xla_while), - &analysis.GetValueDefinedAt(constant)); - } else { - EXPECT_THAT(HloValuesAt(body_param), - UnorderedElementsAre(analysis.GetValueDefinedAt(constant))); - EXPECT_THAT(HloValuesAt(cond_param), - UnorderedElementsAre(analysis.GetValueDefinedAt(constant))); - EXPECT_THAT(HloValuesAt(xla_while), - UnorderedElementsAre(analysis.GetValueDefinedAt(constant))); - EXPECT_TRUE(analysis.GetValueDefinedAt(constant).live_out_of_module()); - } - - // And finally make the negate the root of the body again. - body->set_root_instruction(negate); - analysis.UpdateAfterChangingRoot(/*old_root=*/body_param, - /*new_root=*/negate); - - // Verify that the dataflow is correct. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); - - if (ssa_form) { - // Phis should no longer be resolvable. - EXPECT_EQ(analysis.ResolvePhi(body_param), nullptr); - EXPECT_EQ(analysis.ResolvePhi(cond_param), nullptr); - EXPECT_EQ(analysis.ResolvePhi(xla_while), nullptr); - } else { - EXPECT_THAT(HloValuesAt(body_param), - UnorderedElementsAre(analysis.GetValueDefinedAt(constant), - analysis.GetValueDefinedAt(negate))); - EXPECT_THAT(HloValuesAt(cond_param), - UnorderedElementsAre(analysis.GetValueDefinedAt(constant), - analysis.GetValueDefinedAt(negate))); - EXPECT_THAT(HloValuesAt(xla_while), - UnorderedElementsAre(analysis.GetValueDefinedAt(constant), - analysis.GetValueDefinedAt(negate))); - - EXPECT_FALSE(analysis.GetValueDefinedAt(exp).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(negate).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(constant).live_out_of_module()); - } - - // After the updates, verify that the dataflow is correct. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); -} - -TEST_P(HloDataflowAnalysisTest, UpdateOfATupleSelect) { - // Test changing the operands of kSelects of a tuple value and updating the - // dataflow. - auto builder = HloComputation::Builder(TestName()); - auto pred = builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(false))); - auto a = builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(1.0))); - auto b = builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(2.0))); - auto c = builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(3.0))); - auto d = builder.AddInstruction( - HloInstruction::CreateConstant(Literal::CreateR0(4.0))); - auto tuple_a = builder.AddInstruction(HloInstruction::CreateTuple({a})); - auto tuple_b = builder.AddInstruction(HloInstruction::CreateTuple({b})); - auto tuple_c = builder.AddInstruction(HloInstruction::CreateTuple({c})); - auto tuple_d = builder.AddInstruction(HloInstruction::CreateTuple({d})); - const Shape tuple_shape = tuple_a->shape(); - auto select_aa = builder.AddInstruction(HloInstruction::CreateTernary( - tuple_shape, HloOpcode::kSelect, pred, tuple_a, tuple_a)); - auto select_ab = builder.AddInstruction(HloInstruction::CreateTernary( - tuple_shape, HloOpcode::kSelect, pred, tuple_a, tuple_b)); - auto select_cd = builder.AddInstruction(HloInstruction::CreateTernary( - tuple_shape, HloOpcode::kSelect, pred, tuple_c, tuple_d)); - auto select_abcd = builder.AddInstruction(HloInstruction::CreateTernary( - tuple_shape, HloOpcode::kSelect, pred, select_ab, select_cd)); - - module_->AddEntryComputation(builder.Build()); - - bool ssa_form = GetParam(); - HloDataflowAnalysis& analysis = RunAnalysis(ssa_form); - - // Sanity check dataflow before changing the graph and updating. - EXPECT_THAT(HloValuesAt(select_aa, /*index=*/{0}), - UnorderedElementsAre(analysis.GetValueDefinedAt(a))); - EXPECT_THAT(HloValuesAt(select_ab, /*index=*/{0}), - UnorderedElementsAre(analysis.GetValueDefinedAt(a), - analysis.GetValueDefinedAt(b))); - EXPECT_THAT(HloValuesAt(select_cd, /*index=*/{0}), - UnorderedElementsAre(analysis.GetValueDefinedAt(c), - analysis.GetValueDefinedAt(d))); - EXPECT_THAT(HloValuesAt(select_abcd, /*index=*/{0}), - UnorderedElementsAre(analysis.GetValueDefinedAt(a), - analysis.GetValueDefinedAt(b), - analysis.GetValueDefinedAt(c), - analysis.GetValueDefinedAt(d))); - EXPECT_TRUE(analysis.GetValueDefinedAt(a).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(b).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(c).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(d).live_out_of_module()); - - // Set the rhs of 'select_aa' to be 'd'. - TF_ASSERT_OK(select_aa->ReplaceOperandWith(2, tuple_d)); - analysis.UpdateAfterChangingOperand(select_aa, /*old_operand=*/tuple_a, - /*new_operand=*/tuple_d); - - // Verify that the dataflow is correct. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); - - EXPECT_THAT(HloValuesAt(select_aa, /*index=*/{0}), - UnorderedElementsAre(analysis.GetValueDefinedAt(a), - analysis.GetValueDefinedAt(d))); - - // Set the lhs of 'select_cd' to be 'a'. - TF_ASSERT_OK(select_cd->ReplaceOperandWith(1, tuple_a)); - analysis.UpdateAfterChangingOperand(select_cd, /*old_operand=*/tuple_c, - /*new_operand=*/tuple_a); - - // Verify that the dataflow is correct. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); - - EXPECT_THAT(HloValuesAt(select_cd, /*index=*/{0}), - UnorderedElementsAre(analysis.GetValueDefinedAt(a), - analysis.GetValueDefinedAt(d))); - EXPECT_THAT(HloValuesAt(select_abcd, /*index=*/{0}), - UnorderedElementsAre(analysis.GetValueDefinedAt(a), - analysis.GetValueDefinedAt(b), - analysis.GetValueDefinedAt(d))); - EXPECT_TRUE(analysis.GetValueDefinedAt(a).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(b).live_out_of_module()); - EXPECT_FALSE(analysis.GetValueDefinedAt(c).live_out_of_module()); - EXPECT_TRUE(analysis.GetValueDefinedAt(d).live_out_of_module()); - - // After the updates, verify that the dataflow is correct. - TF_ASSERT_OK(analysis.VerifyAgainstReference()); -} - INSTANTIATE_TEST_CASE_P(HloDataflowAnalysisInstantiation, HloDataflowAnalysisTest, ::testing::Values(false, true)); diff --git a/tensorflow/compiler/xla/service/hlo_ordering_test.cc b/tensorflow/compiler/xla/service/hlo_ordering_test.cc index ad6070a9c1b..c95e44bd5d9 100644 --- a/tensorflow/compiler/xla/service/hlo_ordering_test.cc +++ b/tensorflow/compiler/xla/service/hlo_ordering_test.cc @@ -19,6 +19,7 @@ limitations under the License. #include #include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_dataflow_analysis.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/service/hlo_scheduling.h" @@ -218,6 +219,94 @@ TEST_F(HloOrderingTest, InstructionsInWhileComputations) { EXPECT_FALSE(ordering.ExecutesBefore(body_param, cond_param)); } +TEST_F(HloOrderingTest, ValuesInWhileComputations) { + // Tests the ordering of values (defined by dataflow analysis) in the body and + // condition of a while instruction. HLO code: + // + // body(F32[]) %param): + // %negate = Negate(%param) + // + // condition(F32[] %param): + // %convert = Convert(%param) + // + // entry: + // %constant = Constant(1.0) + // %while = While(%constant, body, condition) + // %add = Add(%constant, %while) + // + auto module = CreateNewModule(); + const Shape scalar_shape = ShapeUtil::MakeShape(xla::F32, {}); + + auto body_builder = HloComputation::Builder("body"); + auto body_param = body_builder.AddInstruction( + HloInstruction::CreateParameter(0, scalar_shape, "body_param")); + auto negate = body_builder.AddInstruction(HloInstruction::CreateUnary( + scalar_shape, HloOpcode::kNegate, body_param)); + HloComputation* body = module->AddEmbeddedComputation(body_builder.Build()); + + auto cond_builder = HloComputation::Builder("condition"); + auto cond_param = cond_builder.AddInstruction( + HloInstruction::CreateParameter(0, scalar_shape, "cond_param")); + auto convert = cond_builder.AddInstruction(HloInstruction::CreateConvert( + ShapeUtil::MakeShape(xla::PRED, {}), cond_param)); + HloComputation* condition = + module->AddEmbeddedComputation(cond_builder.Build()); + + auto builder = HloComputation::Builder(TestName()); + auto constant = builder.AddInstruction( + HloInstruction::CreateConstant(Literal::CreateR0(1.0))); + auto xla_while = builder.AddInstruction( + HloInstruction::CreateWhile(scalar_shape, condition, body, constant)); + auto add = builder.AddInstruction(HloInstruction::CreateBinary( + scalar_shape, HloOpcode::kAdd, constant, xla_while)); + module->AddEntryComputation(builder.Build()); + + TF_ASSERT_OK_AND_ASSIGN( + auto dataflow, HloDataflowAnalysis::Run(module.get(), /*ssa_form=*/true)); + DependencyHloOrdering ordering(module.get()); + + // Init value is defined before the while, but live range is not before the + // while because of the use of the init value in the add. + EXPECT_TRUE(ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(constant), + dataflow->GetValueDefinedAt(xla_while))); + EXPECT_FALSE( + ordering.LiveRangeStrictlyBefore(dataflow->GetValueDefinedAt(constant), + dataflow->GetValueDefinedAt(xla_while))); + EXPECT_TRUE(ordering.MayInterfere(dataflow->GetValueDefinedAt(constant), + dataflow->GetValueDefinedAt(xla_while))); + + // Any value defined in the body or condition is defined before the while, and + // has a live range strictly before the while. + EXPECT_TRUE(ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(negate), + dataflow->GetValueDefinedAt(xla_while))); + EXPECT_TRUE( + ordering.LiveRangeStrictlyBefore(dataflow->GetValueDefinedAt(negate), + dataflow->GetValueDefinedAt(xla_while))); + EXPECT_FALSE(ordering.MayInterfere(dataflow->GetValueDefinedAt(negate), + dataflow->GetValueDefinedAt(xla_while))); + + EXPECT_TRUE(ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(convert), + dataflow->GetValueDefinedAt(xla_while))); + EXPECT_TRUE( + ordering.LiveRangeStrictlyBefore(dataflow->GetValueDefinedAt(convert), + dataflow->GetValueDefinedAt(xla_while))); + EXPECT_FALSE(ordering.MayInterfere(dataflow->GetValueDefinedAt(convert), + dataflow->GetValueDefinedAt(xla_while))); + + // The live range of the while should be before the add. + EXPECT_TRUE(ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(xla_while), + dataflow->GetValueDefinedAt(add))); + ASSERT_EQ(dataflow->GetValueDefinedAt(xla_while).uses().size(), 1); + + const HloUse& while_use = dataflow->GetValueDefinedAt(xla_while).uses()[0]; + EXPECT_EQ(while_use.instruction, add); + EXPECT_TRUE(ordering.UseIsBeforeValueDefinition( + while_use, dataflow->GetValueDefinedAt(add))); + EXPECT_TRUE( + ordering.LiveRangeStrictlyBefore(dataflow->GetValueDefinedAt(xla_while), + dataflow->GetValueDefinedAt(add))); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_value.cc b/tensorflow/compiler/xla/service/hlo_value.cc index f85d8ec50de..e6cf0d37b8a 100644 --- a/tensorflow/compiler/xla/service/hlo_value.cc +++ b/tensorflow/compiler/xla/service/hlo_value.cc @@ -159,12 +159,6 @@ void HloValue::AddPosition(HloInstruction* instruction, for (const HloPosition& position : positions_) { DCHECK_NE(position, new_position); } - // The shape of the new position must match existing positions. - if (!positions_.empty()) { - CHECK( - ShapeUtil::Compatible(positions_.front().shape(), new_position.shape())) - << "front: " << positions_.front() << " new: " << new_position; - } positions_.push_back(std::move(new_position)); diff --git a/tensorflow/compiler/xla/service/hlo_value.h b/tensorflow/compiler/xla/service/hlo_value.h index 63ecc25020b..6872bc76a82 100644 --- a/tensorflow/compiler/xla/service/hlo_value.h +++ b/tensorflow/compiler/xla/service/hlo_value.h @@ -225,6 +225,9 @@ class HloValueSet { // already exist in the set. bool AddValue(const HloValue* value); + // Clear all values from the set. + void Clear() { values_.clear(); } + // Return the unique HLO value in the set. CHECKs if the set does not contain // exactly one value. const HloValue& GetUniqueValue() const { From 829962bed802cc43f000fa4959e945fd24ffcd16 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 09:45:07 -0700 Subject: [PATCH 38/67] Update feature_util to support SequenceExample proto. PiperOrigin-RevId: 167286643 --- tensorflow/core/example/feature_util.cc | 124 +++++++--- tensorflow/core/example/feature_util.h | 226 +++++++++++++------ tensorflow/core/example/feature_util_test.cc | 218 +++++++++++++++++- 3 files changed, 457 insertions(+), 111 deletions(-) diff --git a/tensorflow/core/example/feature_util.cc b/tensorflow/core/example/feature_util.cc index 6f3cc6c6c5d..f0593ede82f 100644 --- a/tensorflow/core/example/feature_util.cc +++ b/tensorflow/core/example/feature_util.cc @@ -18,77 +18,129 @@ limitations under the License. namespace tensorflow { namespace internal { - -::tensorflow::Feature& ExampleFeature(const string& name, - ::tensorflow::Example* example) { - ::tensorflow::Features* features = example->mutable_features(); - return (*features->mutable_feature())[name]; +Feature& ExampleFeature(const string& name, Example* example) { + return *GetFeature(name, example); } -} // namespace internal +} // namespace internal template <> -bool ExampleHasFeature(const string& name, - const Example& example) { - auto it = example.features().feature().find(name); - return (it != example.features().feature().end()) && +bool HasFeature<>(const string& key, const Features& features) { + return (features.feature().find(key) != features.feature().end()); +} + +template <> +bool HasFeature(const string& key, const Features& features) { + auto it = features.feature().find(key); + return (it != features.feature().end()) && (it->second.kind_case() == Feature::KindCase::kInt64List); } template <> -bool ExampleHasFeature(const string& name, const Example& example) { - auto it = example.features().feature().find(name); - return (it != example.features().feature().end()) && +bool HasFeature(const string& key, const Features& features) { + auto it = features.feature().find(key); + return (it != features.feature().end()) && (it->second.kind_case() == Feature::KindCase::kFloatList); } template <> -bool ExampleHasFeature(const string& name, const Example& example) { - auto it = example.features().feature().find(name); - return (it != example.features().feature().end()) && +bool HasFeature(const string& key, const Features& features) { + auto it = features.feature().find(key); + return (it != features.feature().end()) && (it->second.kind_case() == Feature::KindCase::kBytesList); } +bool HasFeatureList(const string& key, + const SequenceExample& sequence_example) { + auto& feature_list = sequence_example.feature_lists().feature_list(); + return (feature_list.find(key) != feature_list.end()); +} + template <> const protobuf::RepeatedField& GetFeatureValues( - const string& name, const Example& example) { - return example.features().feature().at(name).int64_list().value(); + const Feature& feature) { + return feature.int64_list().value(); } template <> protobuf::RepeatedField* GetFeatureValues( - const string& name, Example* example) { - return internal::ExampleFeature(name, example) - .mutable_int64_list() - ->mutable_value(); + Feature* feature) { + return feature->mutable_int64_list()->mutable_value(); } template <> const protobuf::RepeatedField& GetFeatureValues( - const string& name, const Example& example) { - return example.features().feature().at(name).float_list().value(); + const Feature& feature) { + return feature.float_list().value(); } template <> -protobuf::RepeatedField* GetFeatureValues(const string& name, - Example* example) { - return internal::ExampleFeature(name, example) - .mutable_float_list() - ->mutable_value(); +protobuf::RepeatedField* GetFeatureValues(Feature* feature) { + return feature->mutable_float_list()->mutable_value(); } template <> const protobuf::RepeatedPtrField& GetFeatureValues( - const string& name, const Example& example) { - return example.features().feature().at(name).bytes_list().value(); + const Feature& feature) { + return feature.bytes_list().value(); } template <> -protobuf::RepeatedPtrField* GetFeatureValues(const string& name, - Example* example) { - return internal::ExampleFeature(name, example) - .mutable_bytes_list() - ->mutable_value(); +protobuf::RepeatedPtrField* GetFeatureValues(Feature* feature) { + return feature->mutable_bytes_list()->mutable_value(); } +const protobuf::RepeatedPtrField& GetFeatureList( + const string& key, const SequenceExample& sequence_example) { + return sequence_example.feature_lists().feature_list().at(key).feature(); +} + +protobuf::RepeatedPtrField* GetFeatureList( + const string& feature_list_key, SequenceExample* sequence_example) { + return (*sequence_example->mutable_feature_lists() + ->mutable_feature_list())[feature_list_key] + .mutable_feature(); +} + +template <> +Features* GetFeatures(Features* proto) { + return proto; +} + +template <> +Features* GetFeatures(Example* proto) { + return proto->mutable_features(); +} + +template <> +const Features& GetFeatures(const Features& proto) { + return proto; +} + +template <> +const Features& GetFeatures(const Example& proto) { + return proto.features(); +} + +template <> +const protobuf::RepeatedField& GetFeatureValues( + const Feature& feature); + +template <> +protobuf::RepeatedField* GetFeatureValues( + Feature* feature); + +template <> +const protobuf::RepeatedField& GetFeatureValues( + const Feature& feature); + +template <> +protobuf::RepeatedField* GetFeatureValues(Feature* feature); + +template <> +const protobuf::RepeatedPtrField& GetFeatureValues( + const Feature& feature); + +template <> +protobuf::RepeatedPtrField* GetFeatureValues(Feature* feature); } // namespace tensorflow diff --git a/tensorflow/core/example/feature_util.h b/tensorflow/core/example/feature_util.h index 4004411cb17..19d6d0d0d4a 100644 --- a/tensorflow/core/example/feature_util.h +++ b/tensorflow/core/example/feature_util.h @@ -13,9 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// A set of lightweight wrappers which simplify access to Example features. +// A set of lightweight wrappers which simplify access to Feature protos. // // TensorFlow Example proto uses associative maps on top of oneof fields. +// SequenceExample proto uses associative map of FeatureList. // So accessing feature values is not very convenient. // // For example, to read a first value of integer feature "tag": @@ -42,9 +43,59 @@ limitations under the License. // (RepeatedPtrField for byte list). So refer to its documentation of // RepeatedField for full list of supported methods. // -// NOTE: It is also important to mention that due to the nature of oneof proto -// fields setting a feature of one type automatically clears all values stored -// as another type with the same feature name. +// NOTE: Due to the nature of oneof proto fields setting a feature of one type +// automatically clears all values stored as another type with the same feature +// key. +// +// This library also has tools to work with SequenceExample protos. +// +// To get a value from SequenceExample.context: +// int id = GetFeatureValues("tag", se.context()).Get(0); +// To add a value to the context: +// GetFeatureValues("tag", se.mutable_context())->Add(42); +// +// To add values to feature_lists: +// AppendFeatureValues({4.0}, +// GetFeatureList("movie_ratings", &se)->Add()); +// AppendFeatureValues({5.0, 3.0}, +// GetFeatureList("movie_ratings", &se)->Add()); +// This will create a feature list keyed as "images" with two features: +// feature_lists { +// feature_list { +// key: "images" +// value { +// feature { float_list { value: [4.0] } } +// feature { float_list { value: [5.0, 3.0] } } +// } +// } } +// +// Functions exposed by this library: +// HasFeature<[FeatureType]>(key, proto) -> bool +// Returns true if a feature with the specified key, and optionally +// FeatureType, belongs to the Features or Example proto. +// HasFeatureList(key, sequence_example) -> bool +// Returns true if SequenceExample has a feature_list with the key. +// GetFeatureValues(key, proto) -> RepeatedField +// Returns values for the specified key and the FeatureType. +// Supported types for the proto: Example, Features. +// GetFeatureList(key, sequence_example) -> RepeatedPtrField +// Returns Feature protos associated with a key. +// AppendFeatureValues(begin, end, feature) +// AppendFeatureValues(container or initializer_list, feature) +// Copies values into a Feature. +// AppendFeatureValues(begin, end, key, proto) +// AppendFeatureValues(container or initializer_list, key, proto) +// Copies values into Features and Example protos with the specified key. +// +// Auxiliary functions, it is unlikely you'll need to use them directly: +// GetFeatures(proto) -> Features +// A convenience function to get Features proto. +// Supported types for the proto: Example, Features. +// GetFeature(key, proto) -> Feature* +// Returns a Feature proto for the specified key, creates a new if +// necessary. Supported types for the proto: Example, Features. +// GetFeatureValues(feature) -> RepeatedField +// Returns values of the feature for the FeatureType. #ifndef TENSORFLOW_EXAMPLE_FEATURE_H_ #define TENSORFLOW_EXAMPLE_FEATURE_H_ @@ -62,10 +113,11 @@ namespace tensorflow { namespace internal { +// DEPRECATED: Use GetFeature instead. +// TODO(gorban): Update all clients in a followup CL. // Returns a reference to a feature corresponding to the name. // Note: it will create a new Feature if it is missing in the example. -::tensorflow::Feature& ExampleFeature(const string& name, - ::tensorflow::Example* example); +Feature& ExampleFeature(const string& name, Example* example); // Specializations of RepeatedFieldTrait define a type of RepeatedField // corresponding to a selected feature type. @@ -127,89 +179,135 @@ struct FeatureTrait< } // namespace internal -// Returns true if feature with the specified name belongs to the example proto. -// Doesn't check feature type. Note that specialized versions return false if -// the feature has a wrong type. -template -bool ExampleHasFeature(const string& name, const Example& example) { - return example.features().feature().find(name) != - example.features().feature().end(); -} +// Returns true if sequence_example has a feature_list with the specified key. +bool HasFeatureList(const string& key, const SequenceExample& sequence_example); + +// A family of template functions to return mutable Features proto from a +// container proto. Supported ProtoTypes: Example, Features. +template +Features* GetFeatures(ProtoType* proto); + +template +const Features& GetFeatures(const ProtoType& proto); // Base declaration of a family of template functions to return a read only -// repeated field corresponding to a feature with the specified name. +// repeated field of feature values. template const typename internal::RepeatedFieldTrait::Type& -GetFeatureValues(const string& name, const Example& example); +GetFeatureValues(const Feature& feature); -// Base declaration of a family of template functions to return a mutable -// repeated field corresponding to a feature with the specified name. +// Returns a read only repeated field corresponding to a feature with the +// specified name and FeatureType. Supported ProtoTypes: Example, Features. +template +const typename internal::RepeatedFieldTrait::Type& +GetFeatureValues(const string& key, const ProtoType& proto) { + return GetFeatureValues(GetFeatures(proto).feature().at(key)); +} + +// Returns a mutable repeated field of a feature values. template typename internal::RepeatedFieldTrait::Type* GetFeatureValues( - const string& name, Example* example); + Feature* feature); + +// Returns a mutable repeated field corresponding to a feature with the +// specified name and FeatureType. Supported ProtoTypes: Example, Features. +template +typename internal::RepeatedFieldTrait::Type* GetFeatureValues( + const string& key, ProtoType* proto) { + ::tensorflow::Feature& feature = + (*GetFeatures(proto)->mutable_feature())[key]; + return GetFeatureValues(&feature); +} + +// Returns a Feature proto for the specified key, creates a new if necessary. +// Supported types for the proto: Example, Features. +template +Feature* GetFeature(const string& key, ProtoType* proto) { + return &(*GetFeatures(proto)->mutable_feature())[key]; +} + +// Returns a repeated field with features corresponding to a feature_list key. +const protobuf::RepeatedPtrField& GetFeatureList( + const string& key, const SequenceExample& sequence_example); + +// Returns a mutable repeated field with features corresponding to a +// feature_list key. It will create a new FeatureList if necessary. +protobuf::RepeatedPtrField* GetFeatureList( + const string& feature_list_key, SequenceExample* sequence_example); -// Copies elements from the range, defined by [first, last) into a feature. template void AppendFeatureValues(IteratorType first, IteratorType last, - const string& name, Example* example) { + Feature* feature) { using FeatureType = typename internal::FeatureTrait< typename std::iterator_traits::value_type>::Type; - std::copy(first, last, protobuf::RepeatedFieldBackInserter( - GetFeatureValues(name, example))); + std::copy(first, last, + protobuf::RepeatedFieldBackInserter( + GetFeatureValues(feature))); +} + +template +void AppendFeatureValues(std::initializer_list container, + Feature* feature) { + AppendFeatureValues(container.begin(), container.end(), feature); +} + +template +void AppendFeatureValues(const ContainerType& container, Feature* feature) { + using IteratorType = typename ContainerType::const_iterator; + AppendFeatureValues(container.begin(), container.end(), + feature); +} + +// Copies elements from the range, defined by [first, last) into the feature +// obtainable from the (proto, key) combination. +template +void AppendFeatureValues(IteratorType first, IteratorType last, + const string& key, ProtoType* proto) { + AppendFeatureValues(first, last, GetFeature(key, GetFeatures(proto))); } // Copies all elements from the container into a feature. -template -void AppendFeatureValues(const ContainerType& container, const string& name, - Example* example) { +template +void AppendFeatureValues(const ContainerType& container, const string& key, + ProtoType* proto) { using IteratorType = typename ContainerType::const_iterator; - AppendFeatureValues(container.begin(), container.end(), name, - example); + AppendFeatureValues(container.begin(), container.end(), key, + proto); } -// Copies all elements from the initializer list into a feature. -template +// Copies all elements from the initializer list into a Feature contained by +// Features or Example proto. +template void AppendFeatureValues(std::initializer_list container, - const string& name, Example* example) { + const string& key, ProtoType* proto) { using IteratorType = typename std::initializer_list::const_iterator; - AppendFeatureValues(container.begin(), container.end(), name, - example); + AppendFeatureValues(container.begin(), container.end(), key, + proto); } -template <> -bool ExampleHasFeature(const string& name, - const Example& example); +// Returns true if a feature with the specified key belongs to the Features. +// The template parameter pack accepts zero or one template argument - which +// is FeatureType. If the FeatureType not specified (zero template arguments) +// the function will not check the feature type. Otherwise it will return false +// if the feature has a wrong type. +template +bool HasFeature(const string& key, const Features& features); -template <> -bool ExampleHasFeature(const string& name, const Example& example); +// Returns true if a feature with the specified key belongs to the Example. +// Doesn't check feature type if used without FeatureType, otherwise the +// specialized versions return false if the feature has a wrong type. +template +bool HasFeature(const string& key, const Example& example) { + return HasFeature(key, GetFeatures(example)); +}; -template <> -bool ExampleHasFeature(const string& name, const Example& example); - -template <> -const protobuf::RepeatedField& GetFeatureValues( - const string& name, const Example& example); - -template <> -protobuf::RepeatedField* GetFeatureValues( - const string& name, Example* example); - -template <> -const protobuf::RepeatedField& GetFeatureValues( - const string& name, const Example& example); - -template <> -protobuf::RepeatedField* GetFeatureValues(const string& name, - Example* example); - -template <> -const protobuf::RepeatedPtrField& GetFeatureValues( - const string& name, const Example& example); - -template <> -protobuf::RepeatedPtrField* GetFeatureValues(const string& name, - Example* example); +// DEPRECATED: use HasFeature instead. +// TODO(gorban): update all clients in a followup CL. +template +bool ExampleHasFeature(const string& key, const Example& example) { + return HasFeature(key, example); +} } // namespace tensorflow #endif // TENSORFLOW_EXAMPLE_FEATURE_H_ diff --git a/tensorflow/core/example/feature_util_test.cc b/tensorflow/core/example/feature_util_test.cc index eb7b90af1b2..1036e28652a 100644 --- a/tensorflow/core/example/feature_util_test.cc +++ b/tensorflow/core/example/feature_util_test.cc @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ - #include "tensorflow/core/example/feature_util.h" #include @@ -38,6 +37,16 @@ TEST(GetFeatureValuesInt64Test, ReadsASingleValue) { EXPECT_EQ(42, tag.Get(0)); } +TEST(GetFeatureValuesInt64Test, ReadsASingleValueFromFeature) { + Feature feature; + feature.mutable_int64_list()->add_value(42); + + auto values = GetFeatureValues(feature); + + ASSERT_EQ(1, values.size()); + EXPECT_EQ(42, values.Get(0)); +} + TEST(GetFeatureValuesInt64Test, WritesASingleValue) { Example example; @@ -48,25 +57,33 @@ TEST(GetFeatureValuesInt64Test, WritesASingleValue) { EXPECT_EQ(42, example.features().feature().at("tag").int64_list().value(0)); } +TEST(GetFeatureValuesInt64Test, WritesASingleValueToFeature) { + Feature feature; + + GetFeatureValues(&feature)->Add(42); + + ASSERT_EQ(1, feature.int64_list().value_size()); + EXPECT_EQ(42, feature.int64_list().value(0)); +} + TEST(GetFeatureValuesInt64Test, CheckUntypedFieldExistence) { Example example; - - EXPECT_FALSE(ExampleHasFeature("tag", example)); + ASSERT_FALSE(HasFeature("tag", example)); GetFeatureValues("tag", &example)->Add(0); - EXPECT_TRUE(ExampleHasFeature("tag", example)); + EXPECT_TRUE(HasFeature("tag", example)); } TEST(GetFeatureValuesInt64Test, CheckTypedFieldExistence) { Example example; GetFeatureValues("tag", &example)->Add(3.14); - ASSERT_FALSE(ExampleHasFeature("tag", example)); + ASSERT_FALSE(HasFeature("tag", example)); GetFeatureValues("tag", &example)->Add(42); - EXPECT_TRUE(ExampleHasFeature("tag", example)); + EXPECT_TRUE(HasFeature("tag", example)); auto tag_ro = GetFeatureValues("tag", example); ASSERT_EQ(1, tag_ro.size()); EXPECT_EQ(42, tag_ro.Get(0)); @@ -87,6 +104,16 @@ TEST(GetFeatureValuesInt64Test, CopyIterableToAField) { EXPECT_EQ(3, tag_ro.Get(2)); } +TEST(GetFeatureValuesFloatTest, ReadsASingleValueFromFeature) { + Feature feature; + feature.mutable_float_list()->add_value(3.14); + + auto values = GetFeatureValues(feature); + + ASSERT_EQ(1, values.size()); + EXPECT_NEAR(3.14, values.Get(0), kTolerance); +} + TEST(GetFeatureValuesFloatTest, ReadsASingleValue) { Example example; (*example.mutable_features()->mutable_feature())["tag"] @@ -99,6 +126,15 @@ TEST(GetFeatureValuesFloatTest, ReadsASingleValue) { EXPECT_NEAR(3.14, tag.Get(0), kTolerance); } +TEST(GetFeatureValuesFloatTest, WritesASingleValueToFeature) { + Feature feature; + + GetFeatureValues(&feature)->Add(3.14); + + ASSERT_EQ(1, feature.float_list().value_size()); + EXPECT_NEAR(3.14, feature.float_list().value(0), kTolerance); +} + TEST(GetFeatureValuesFloatTest, WritesASingleValue) { Example example; @@ -115,16 +151,26 @@ TEST(GetFeatureValuesFloatTest, CheckTypedFieldExistence) { Example example; GetFeatureValues("tag", &example)->Add(42); - ASSERT_FALSE(ExampleHasFeature("tag", example)); + ASSERT_FALSE(HasFeature("tag", example)); GetFeatureValues("tag", &example)->Add(3.14); - EXPECT_TRUE(ExampleHasFeature("tag", example)); + EXPECT_TRUE(HasFeature("tag", example)); auto tag_ro = GetFeatureValues("tag", example); ASSERT_EQ(1, tag_ro.size()); EXPECT_NEAR(3.14, tag_ro.Get(0), kTolerance); } +TEST(GetFeatureValuesStringTest, ReadsASingleValueFromFeature) { + Feature feature; + feature.mutable_bytes_list()->add_value("FOO"); + + auto values = GetFeatureValues(feature); + + ASSERT_EQ(1, values.size()); + EXPECT_EQ("FOO", values.Get(0)); +} + TEST(GetFeatureValuesStringTest, ReadsASingleValue) { Example example; (*example.mutable_features()->mutable_feature())["tag"] @@ -137,6 +183,15 @@ TEST(GetFeatureValuesStringTest, ReadsASingleValue) { EXPECT_EQ("FOO", tag.Get(0)); } +TEST(GetFeatureValuesStringTest, WritesASingleValueToFeature) { + Feature feature; + + *GetFeatureValues(&feature)->Add() = "FOO"; + + ASSERT_EQ(1, feature.bytes_list().value_size()); + EXPECT_EQ("FOO", feature.bytes_list().value(0)); +} + TEST(GetFeatureValuesStringTest, WritesASingleValue) { Example example; @@ -148,15 +203,15 @@ TEST(GetFeatureValuesStringTest, WritesASingleValue) { example.features().feature().at("tag").bytes_list().value(0)); } -TEST(GetFeatureValuesBytesTest, CheckTypedFieldExistence) { +TEST(GetFeatureValuesStringTest, CheckTypedFieldExistence) { Example example; GetFeatureValues("tag", &example)->Add(42); - ASSERT_FALSE(ExampleHasFeature("tag", example)); + ASSERT_FALSE(HasFeature("tag", example)); *GetFeatureValues("tag", &example)->Add() = "FOO"; - EXPECT_TRUE(ExampleHasFeature("tag", example)); + EXPECT_TRUE(HasFeature("tag", example)); auto tag_ro = GetFeatureValues("tag", example); ASSERT_EQ(1, tag_ro.size()); EXPECT_EQ("FOO", tag_ro.Get(0)); @@ -228,5 +283,146 @@ TEST(AppendFeatureValuesTest, StringVariablesUsingInitializerList) { EXPECT_EQ("BAZ", tag_ro.Get(2)); } +TEST(SequenceExampleTest, ReadsASingleValueFromContext) { + SequenceExample se; + (*se.mutable_context()->mutable_feature())["tag"] + .mutable_int64_list() + ->add_value(42); + + auto values = GetFeatureValues("tag", se.context()); + + ASSERT_EQ(1, values.size()); + EXPECT_EQ(42, values.Get(0)); +} + +TEST(SequenceExampleTest, WritesASingleValueToContext) { + SequenceExample se; + + GetFeatureValues("tag", se.mutable_context())->Add(42); + + ASSERT_EQ(1, se.context().feature().at("tag").int64_list().value_size()); + EXPECT_EQ(42, se.context().feature().at("tag").int64_list().value(0)); +} + +TEST(SequenceExampleTest, AppendFeatureValuesToContextSingleArg) { + SequenceExample se; + + AppendFeatureValues({1.1, 2.2, 3.3}, "tag", se.mutable_context()); + + auto tag_ro = GetFeatureValues("tag", se.context()); + ASSERT_EQ(3, tag_ro.size()); + EXPECT_NEAR(1.1, tag_ro.Get(0), kTolerance); + EXPECT_NEAR(2.2, tag_ro.Get(1), kTolerance); + EXPECT_NEAR(3.3, tag_ro.Get(2), kTolerance); +} + +TEST(SequenceExampleTest, CheckTypedFieldExistence) { + SequenceExample se; + + GetFeatureValues("tag", se.mutable_context())->Add(3.14); + ASSERT_FALSE(HasFeature("tag", se.context())); + + GetFeatureValues("tag", se.mutable_context())->Add(42); + + EXPECT_TRUE(HasFeature("tag", se.context())); + auto tag_ro = GetFeatureValues("tag", se.context()); + ASSERT_EQ(1, tag_ro.size()); + EXPECT_EQ(42, tag_ro.Get(0)); +} + +TEST(SequenceExampleTest, ReturnsExistingFeatureLists) { + SequenceExample se; + (*se.mutable_feature_lists()->mutable_feature_list())["tag"] + .mutable_feature() + ->Add(); + + auto feature = GetFeatureList("tag", se); + + ASSERT_EQ(1, feature.size()); +} + +TEST(SequenceExampleTest, CreatesNewFeatureLists) { + SequenceExample se; + + GetFeatureList("tag", &se)->Add(); + + EXPECT_EQ(1, se.feature_lists().feature_list().at("tag").feature_size()); +} + +TEST(SequenceExampleTest, CheckFeatureListExistence) { + SequenceExample se; + ASSERT_FALSE(HasFeatureList("tag", se)); + + GetFeatureList("tag", &se)->Add(); + + ASSERT_TRUE(HasFeatureList("tag", se)); +} + +TEST(SequenceExampleTest, AppendFeatureValuesWithInitializerList) { + SequenceExample se; + + AppendFeatureValues({1, 2, 3}, "ids", se.mutable_context()); + AppendFeatureValues({"cam1-0", "cam2-0"}, + GetFeatureList("images", &se)->Add()); + AppendFeatureValues({"cam1-1", "cam2-2"}, + GetFeatureList("images", &se)->Add()); + + EXPECT_EQ(se.DebugString(), + "context {\n" + " feature {\n" + " key: \"ids\"\n" + " value {\n" + " int64_list {\n" + " value: 1\n" + " value: 2\n" + " value: 3\n" + " }\n" + " }\n" + " }\n" + "}\n" + "feature_lists {\n" + " feature_list {\n" + " key: \"images\"\n" + " value {\n" + " feature {\n" + " bytes_list {\n" + " value: \"cam1-0\"\n" + " value: \"cam2-0\"\n" + " }\n" + " }\n" + " feature {\n" + " bytes_list {\n" + " value: \"cam1-1\"\n" + " value: \"cam2-2\"\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n"); +} + +TEST(SequenceExampleTest, AppendFeatureValuesWithVectors) { + SequenceExample se; + + std::vector readings{1.0, 2.5, 5.0}; + AppendFeatureValues(readings, GetFeatureList("movie_ratings", &se)->Add()); + + EXPECT_EQ(se.DebugString(), + "feature_lists {\n" + " feature_list {\n" + " key: \"movie_ratings\"\n" + " value {\n" + " feature {\n" + " float_list {\n" + " value: 1\n" + " value: 2.5\n" + " value: 5\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n"); +} + } // namespace } // namespace tensorflow From 8b05d128c4accb40423eafca7118fe6fec772ef8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 10:36:16 -0700 Subject: [PATCH 39/67] Shard some tests to prevent timeouts. PiperOrigin-RevId: 167293429 --- tensorflow/python/estimator/BUILD | 9 ++++----- tensorflow/python/kernel_tests/BUILD | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD index 83eeeb35b67..167f9b10543 100644 --- a/tensorflow/python/estimator/BUILD +++ b/tensorflow/python/estimator/BUILD @@ -148,6 +148,7 @@ py_test( name = "dnn_test", size = "medium", srcs = ["canned/dnn_test.py"], + shard_count = 4, srcs_version = "PY2AND3", tags = ["no_pip"], deps = [ @@ -201,7 +202,7 @@ py_test( name = "dnn_linear_combined_test", size = "medium", srcs = ["canned/dnn_linear_combined_test.py"], - shard_count = 4, + shard_count = 8, srcs_version = "PY2AND3", tags = ["no_pip"], deps = [ @@ -552,11 +553,9 @@ py_test( name = "linear_test", size = "medium", srcs = ["canned/linear_test.py"], + shard_count = 4, srcs_version = "PY2AND3", - tags = [ - "no_pip", - "noasan", # times out b/63680444 - ], + tags = ["no_pip"], deps = [ ":linear", ":linear_testing_utils", diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 4fa1e1fee80..ed3f02aedd6 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -2154,7 +2154,7 @@ cuda_py_test( "//tensorflow/python:nn_grad", "//tensorflow/python:nn_ops", ], - tags = ["noasan"], # times out b/63680444 + shard_count = 2, ) cuda_py_test( From bfb6e8e6a06ca94dc4792588ce0da705efca81cf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 10:52:15 -0700 Subject: [PATCH 40/67] Add `gan` import to cmake. PiperOrigin-RevId: 167295726 --- tensorflow/contrib/cmake/tf_python.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index 48023099379..1b706159a3d 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -315,6 +315,7 @@ add_python_module("tensorflow/contrib/framework/ops") add_python_module("tensorflow/contrib/framework/python") add_python_module("tensorflow/contrib/framework/python/framework") add_python_module("tensorflow/contrib/framework/python/ops") +add_python_module("tensorflow/contrib/gan") add_python_module("tensorflow/contrib/graph_editor") add_python_module("tensorflow/contrib/graph_editor/examples") add_python_module("tensorflow/contrib/graph_editor/tests") From 046533f852bd7d8bffd1a0f19cc1991cdedfc92b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 11:29:50 -0700 Subject: [PATCH 41/67] Automated g4 rollback of changelist 167286643 PiperOrigin-RevId: 167301509 --- tensorflow/core/example/feature_util.cc | 124 +++------- tensorflow/core/example/feature_util.h | 226 ++++++------------- tensorflow/core/example/feature_util_test.cc | 218 +----------------- 3 files changed, 111 insertions(+), 457 deletions(-) diff --git a/tensorflow/core/example/feature_util.cc b/tensorflow/core/example/feature_util.cc index f0593ede82f..6f3cc6c6c5d 100644 --- a/tensorflow/core/example/feature_util.cc +++ b/tensorflow/core/example/feature_util.cc @@ -18,129 +18,77 @@ limitations under the License. namespace tensorflow { namespace internal { -Feature& ExampleFeature(const string& name, Example* example) { - return *GetFeature(name, example); + +::tensorflow::Feature& ExampleFeature(const string& name, + ::tensorflow::Example* example) { + ::tensorflow::Features* features = example->mutable_features(); + return (*features->mutable_feature())[name]; } -} // namespace internal +} // namespace internal template <> -bool HasFeature<>(const string& key, const Features& features) { - return (features.feature().find(key) != features.feature().end()); -} - -template <> -bool HasFeature(const string& key, const Features& features) { - auto it = features.feature().find(key); - return (it != features.feature().end()) && +bool ExampleHasFeature(const string& name, + const Example& example) { + auto it = example.features().feature().find(name); + return (it != example.features().feature().end()) && (it->second.kind_case() == Feature::KindCase::kInt64List); } template <> -bool HasFeature(const string& key, const Features& features) { - auto it = features.feature().find(key); - return (it != features.feature().end()) && +bool ExampleHasFeature(const string& name, const Example& example) { + auto it = example.features().feature().find(name); + return (it != example.features().feature().end()) && (it->second.kind_case() == Feature::KindCase::kFloatList); } template <> -bool HasFeature(const string& key, const Features& features) { - auto it = features.feature().find(key); - return (it != features.feature().end()) && +bool ExampleHasFeature(const string& name, const Example& example) { + auto it = example.features().feature().find(name); + return (it != example.features().feature().end()) && (it->second.kind_case() == Feature::KindCase::kBytesList); } -bool HasFeatureList(const string& key, - const SequenceExample& sequence_example) { - auto& feature_list = sequence_example.feature_lists().feature_list(); - return (feature_list.find(key) != feature_list.end()); -} - template <> const protobuf::RepeatedField& GetFeatureValues( - const Feature& feature) { - return feature.int64_list().value(); + const string& name, const Example& example) { + return example.features().feature().at(name).int64_list().value(); } template <> protobuf::RepeatedField* GetFeatureValues( - Feature* feature) { - return feature->mutable_int64_list()->mutable_value(); + const string& name, Example* example) { + return internal::ExampleFeature(name, example) + .mutable_int64_list() + ->mutable_value(); } template <> const protobuf::RepeatedField& GetFeatureValues( - const Feature& feature) { - return feature.float_list().value(); + const string& name, const Example& example) { + return example.features().feature().at(name).float_list().value(); } template <> -protobuf::RepeatedField* GetFeatureValues(Feature* feature) { - return feature->mutable_float_list()->mutable_value(); +protobuf::RepeatedField* GetFeatureValues(const string& name, + Example* example) { + return internal::ExampleFeature(name, example) + .mutable_float_list() + ->mutable_value(); } template <> const protobuf::RepeatedPtrField& GetFeatureValues( - const Feature& feature) { - return feature.bytes_list().value(); + const string& name, const Example& example) { + return example.features().feature().at(name).bytes_list().value(); } template <> -protobuf::RepeatedPtrField* GetFeatureValues(Feature* feature) { - return feature->mutable_bytes_list()->mutable_value(); +protobuf::RepeatedPtrField* GetFeatureValues(const string& name, + Example* example) { + return internal::ExampleFeature(name, example) + .mutable_bytes_list() + ->mutable_value(); } -const protobuf::RepeatedPtrField& GetFeatureList( - const string& key, const SequenceExample& sequence_example) { - return sequence_example.feature_lists().feature_list().at(key).feature(); -} - -protobuf::RepeatedPtrField* GetFeatureList( - const string& feature_list_key, SequenceExample* sequence_example) { - return (*sequence_example->mutable_feature_lists() - ->mutable_feature_list())[feature_list_key] - .mutable_feature(); -} - -template <> -Features* GetFeatures(Features* proto) { - return proto; -} - -template <> -Features* GetFeatures(Example* proto) { - return proto->mutable_features(); -} - -template <> -const Features& GetFeatures(const Features& proto) { - return proto; -} - -template <> -const Features& GetFeatures(const Example& proto) { - return proto.features(); -} - -template <> -const protobuf::RepeatedField& GetFeatureValues( - const Feature& feature); - -template <> -protobuf::RepeatedField* GetFeatureValues( - Feature* feature); - -template <> -const protobuf::RepeatedField& GetFeatureValues( - const Feature& feature); - -template <> -protobuf::RepeatedField* GetFeatureValues(Feature* feature); - -template <> -const protobuf::RepeatedPtrField& GetFeatureValues( - const Feature& feature); - -template <> -protobuf::RepeatedPtrField* GetFeatureValues(Feature* feature); } // namespace tensorflow diff --git a/tensorflow/core/example/feature_util.h b/tensorflow/core/example/feature_util.h index 19d6d0d0d4a..4004411cb17 100644 --- a/tensorflow/core/example/feature_util.h +++ b/tensorflow/core/example/feature_util.h @@ -13,10 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// A set of lightweight wrappers which simplify access to Feature protos. +// A set of lightweight wrappers which simplify access to Example features. // // TensorFlow Example proto uses associative maps on top of oneof fields. -// SequenceExample proto uses associative map of FeatureList. // So accessing feature values is not very convenient. // // For example, to read a first value of integer feature "tag": @@ -43,59 +42,9 @@ limitations under the License. // (RepeatedPtrField for byte list). So refer to its documentation of // RepeatedField for full list of supported methods. // -// NOTE: Due to the nature of oneof proto fields setting a feature of one type -// automatically clears all values stored as another type with the same feature -// key. -// -// This library also has tools to work with SequenceExample protos. -// -// To get a value from SequenceExample.context: -// int id = GetFeatureValues("tag", se.context()).Get(0); -// To add a value to the context: -// GetFeatureValues("tag", se.mutable_context())->Add(42); -// -// To add values to feature_lists: -// AppendFeatureValues({4.0}, -// GetFeatureList("movie_ratings", &se)->Add()); -// AppendFeatureValues({5.0, 3.0}, -// GetFeatureList("movie_ratings", &se)->Add()); -// This will create a feature list keyed as "images" with two features: -// feature_lists { -// feature_list { -// key: "images" -// value { -// feature { float_list { value: [4.0] } } -// feature { float_list { value: [5.0, 3.0] } } -// } -// } } -// -// Functions exposed by this library: -// HasFeature<[FeatureType]>(key, proto) -> bool -// Returns true if a feature with the specified key, and optionally -// FeatureType, belongs to the Features or Example proto. -// HasFeatureList(key, sequence_example) -> bool -// Returns true if SequenceExample has a feature_list with the key. -// GetFeatureValues(key, proto) -> RepeatedField -// Returns values for the specified key and the FeatureType. -// Supported types for the proto: Example, Features. -// GetFeatureList(key, sequence_example) -> RepeatedPtrField -// Returns Feature protos associated with a key. -// AppendFeatureValues(begin, end, feature) -// AppendFeatureValues(container or initializer_list, feature) -// Copies values into a Feature. -// AppendFeatureValues(begin, end, key, proto) -// AppendFeatureValues(container or initializer_list, key, proto) -// Copies values into Features and Example protos with the specified key. -// -// Auxiliary functions, it is unlikely you'll need to use them directly: -// GetFeatures(proto) -> Features -// A convenience function to get Features proto. -// Supported types for the proto: Example, Features. -// GetFeature(key, proto) -> Feature* -// Returns a Feature proto for the specified key, creates a new if -// necessary. Supported types for the proto: Example, Features. -// GetFeatureValues(feature) -> RepeatedField -// Returns values of the feature for the FeatureType. +// NOTE: It is also important to mention that due to the nature of oneof proto +// fields setting a feature of one type automatically clears all values stored +// as another type with the same feature name. #ifndef TENSORFLOW_EXAMPLE_FEATURE_H_ #define TENSORFLOW_EXAMPLE_FEATURE_H_ @@ -113,11 +62,10 @@ namespace tensorflow { namespace internal { -// DEPRECATED: Use GetFeature instead. -// TODO(gorban): Update all clients in a followup CL. // Returns a reference to a feature corresponding to the name. // Note: it will create a new Feature if it is missing in the example. -Feature& ExampleFeature(const string& name, Example* example); +::tensorflow::Feature& ExampleFeature(const string& name, + ::tensorflow::Example* example); // Specializations of RepeatedFieldTrait define a type of RepeatedField // corresponding to a selected feature type. @@ -179,135 +127,89 @@ struct FeatureTrait< } // namespace internal -// Returns true if sequence_example has a feature_list with the specified key. -bool HasFeatureList(const string& key, const SequenceExample& sequence_example); - -// A family of template functions to return mutable Features proto from a -// container proto. Supported ProtoTypes: Example, Features. -template -Features* GetFeatures(ProtoType* proto); - -template -const Features& GetFeatures(const ProtoType& proto); +// Returns true if feature with the specified name belongs to the example proto. +// Doesn't check feature type. Note that specialized versions return false if +// the feature has a wrong type. +template +bool ExampleHasFeature(const string& name, const Example& example) { + return example.features().feature().find(name) != + example.features().feature().end(); +} // Base declaration of a family of template functions to return a read only -// repeated field of feature values. +// repeated field corresponding to a feature with the specified name. template const typename internal::RepeatedFieldTrait::Type& -GetFeatureValues(const Feature& feature); +GetFeatureValues(const string& name, const Example& example); -// Returns a read only repeated field corresponding to a feature with the -// specified name and FeatureType. Supported ProtoTypes: Example, Features. -template -const typename internal::RepeatedFieldTrait::Type& -GetFeatureValues(const string& key, const ProtoType& proto) { - return GetFeatureValues(GetFeatures(proto).feature().at(key)); -} - -// Returns a mutable repeated field of a feature values. +// Base declaration of a family of template functions to return a mutable +// repeated field corresponding to a feature with the specified name. template typename internal::RepeatedFieldTrait::Type* GetFeatureValues( - Feature* feature); - -// Returns a mutable repeated field corresponding to a feature with the -// specified name and FeatureType. Supported ProtoTypes: Example, Features. -template -typename internal::RepeatedFieldTrait::Type* GetFeatureValues( - const string& key, ProtoType* proto) { - ::tensorflow::Feature& feature = - (*GetFeatures(proto)->mutable_feature())[key]; - return GetFeatureValues(&feature); -} - -// Returns a Feature proto for the specified key, creates a new if necessary. -// Supported types for the proto: Example, Features. -template -Feature* GetFeature(const string& key, ProtoType* proto) { - return &(*GetFeatures(proto)->mutable_feature())[key]; -} - -// Returns a repeated field with features corresponding to a feature_list key. -const protobuf::RepeatedPtrField& GetFeatureList( - const string& key, const SequenceExample& sequence_example); - -// Returns a mutable repeated field with features corresponding to a -// feature_list key. It will create a new FeatureList if necessary. -protobuf::RepeatedPtrField* GetFeatureList( - const string& feature_list_key, SequenceExample* sequence_example); + const string& name, Example* example); +// Copies elements from the range, defined by [first, last) into a feature. template void AppendFeatureValues(IteratorType first, IteratorType last, - Feature* feature) { + const string& name, Example* example) { using FeatureType = typename internal::FeatureTrait< typename std::iterator_traits::value_type>::Type; - std::copy(first, last, - protobuf::RepeatedFieldBackInserter( - GetFeatureValues(feature))); -} - -template -void AppendFeatureValues(std::initializer_list container, - Feature* feature) { - AppendFeatureValues(container.begin(), container.end(), feature); -} - -template -void AppendFeatureValues(const ContainerType& container, Feature* feature) { - using IteratorType = typename ContainerType::const_iterator; - AppendFeatureValues(container.begin(), container.end(), - feature); -} - -// Copies elements from the range, defined by [first, last) into the feature -// obtainable from the (proto, key) combination. -template -void AppendFeatureValues(IteratorType first, IteratorType last, - const string& key, ProtoType* proto) { - AppendFeatureValues(first, last, GetFeature(key, GetFeatures(proto))); + std::copy(first, last, protobuf::RepeatedFieldBackInserter( + GetFeatureValues(name, example))); } // Copies all elements from the container into a feature. -template -void AppendFeatureValues(const ContainerType& container, const string& key, - ProtoType* proto) { +template +void AppendFeatureValues(const ContainerType& container, const string& name, + Example* example) { using IteratorType = typename ContainerType::const_iterator; - AppendFeatureValues(container.begin(), container.end(), key, - proto); + AppendFeatureValues(container.begin(), container.end(), name, + example); } -// Copies all elements from the initializer list into a Feature contained by -// Features or Example proto. -template +// Copies all elements from the initializer list into a feature. +template void AppendFeatureValues(std::initializer_list container, - const string& key, ProtoType* proto) { + const string& name, Example* example) { using IteratorType = typename std::initializer_list::const_iterator; - AppendFeatureValues(container.begin(), container.end(), key, - proto); + AppendFeatureValues(container.begin(), container.end(), name, + example); } -// Returns true if a feature with the specified key belongs to the Features. -// The template parameter pack accepts zero or one template argument - which -// is FeatureType. If the FeatureType not specified (zero template arguments) -// the function will not check the feature type. Otherwise it will return false -// if the feature has a wrong type. -template -bool HasFeature(const string& key, const Features& features); +template <> +bool ExampleHasFeature(const string& name, + const Example& example); -// Returns true if a feature with the specified key belongs to the Example. -// Doesn't check feature type if used without FeatureType, otherwise the -// specialized versions return false if the feature has a wrong type. -template -bool HasFeature(const string& key, const Example& example) { - return HasFeature(key, GetFeatures(example)); -}; +template <> +bool ExampleHasFeature(const string& name, const Example& example); -// DEPRECATED: use HasFeature instead. -// TODO(gorban): update all clients in a followup CL. -template -bool ExampleHasFeature(const string& key, const Example& example) { - return HasFeature(key, example); -} +template <> +bool ExampleHasFeature(const string& name, const Example& example); + +template <> +const protobuf::RepeatedField& GetFeatureValues( + const string& name, const Example& example); + +template <> +protobuf::RepeatedField* GetFeatureValues( + const string& name, Example* example); + +template <> +const protobuf::RepeatedField& GetFeatureValues( + const string& name, const Example& example); + +template <> +protobuf::RepeatedField* GetFeatureValues(const string& name, + Example* example); + +template <> +const protobuf::RepeatedPtrField& GetFeatureValues( + const string& name, const Example& example); + +template <> +protobuf::RepeatedPtrField* GetFeatureValues(const string& name, + Example* example); } // namespace tensorflow #endif // TENSORFLOW_EXAMPLE_FEATURE_H_ diff --git a/tensorflow/core/example/feature_util_test.cc b/tensorflow/core/example/feature_util_test.cc index 1036e28652a..eb7b90af1b2 100644 --- a/tensorflow/core/example/feature_util_test.cc +++ b/tensorflow/core/example/feature_util_test.cc @@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ + #include "tensorflow/core/example/feature_util.h" #include @@ -37,16 +38,6 @@ TEST(GetFeatureValuesInt64Test, ReadsASingleValue) { EXPECT_EQ(42, tag.Get(0)); } -TEST(GetFeatureValuesInt64Test, ReadsASingleValueFromFeature) { - Feature feature; - feature.mutable_int64_list()->add_value(42); - - auto values = GetFeatureValues(feature); - - ASSERT_EQ(1, values.size()); - EXPECT_EQ(42, values.Get(0)); -} - TEST(GetFeatureValuesInt64Test, WritesASingleValue) { Example example; @@ -57,33 +48,25 @@ TEST(GetFeatureValuesInt64Test, WritesASingleValue) { EXPECT_EQ(42, example.features().feature().at("tag").int64_list().value(0)); } -TEST(GetFeatureValuesInt64Test, WritesASingleValueToFeature) { - Feature feature; - - GetFeatureValues(&feature)->Add(42); - - ASSERT_EQ(1, feature.int64_list().value_size()); - EXPECT_EQ(42, feature.int64_list().value(0)); -} - TEST(GetFeatureValuesInt64Test, CheckUntypedFieldExistence) { Example example; - ASSERT_FALSE(HasFeature("tag", example)); + + EXPECT_FALSE(ExampleHasFeature("tag", example)); GetFeatureValues("tag", &example)->Add(0); - EXPECT_TRUE(HasFeature("tag", example)); + EXPECT_TRUE(ExampleHasFeature("tag", example)); } TEST(GetFeatureValuesInt64Test, CheckTypedFieldExistence) { Example example; GetFeatureValues("tag", &example)->Add(3.14); - ASSERT_FALSE(HasFeature("tag", example)); + ASSERT_FALSE(ExampleHasFeature("tag", example)); GetFeatureValues("tag", &example)->Add(42); - EXPECT_TRUE(HasFeature("tag", example)); + EXPECT_TRUE(ExampleHasFeature("tag", example)); auto tag_ro = GetFeatureValues("tag", example); ASSERT_EQ(1, tag_ro.size()); EXPECT_EQ(42, tag_ro.Get(0)); @@ -104,16 +87,6 @@ TEST(GetFeatureValuesInt64Test, CopyIterableToAField) { EXPECT_EQ(3, tag_ro.Get(2)); } -TEST(GetFeatureValuesFloatTest, ReadsASingleValueFromFeature) { - Feature feature; - feature.mutable_float_list()->add_value(3.14); - - auto values = GetFeatureValues(feature); - - ASSERT_EQ(1, values.size()); - EXPECT_NEAR(3.14, values.Get(0), kTolerance); -} - TEST(GetFeatureValuesFloatTest, ReadsASingleValue) { Example example; (*example.mutable_features()->mutable_feature())["tag"] @@ -126,15 +99,6 @@ TEST(GetFeatureValuesFloatTest, ReadsASingleValue) { EXPECT_NEAR(3.14, tag.Get(0), kTolerance); } -TEST(GetFeatureValuesFloatTest, WritesASingleValueToFeature) { - Feature feature; - - GetFeatureValues(&feature)->Add(3.14); - - ASSERT_EQ(1, feature.float_list().value_size()); - EXPECT_NEAR(3.14, feature.float_list().value(0), kTolerance); -} - TEST(GetFeatureValuesFloatTest, WritesASingleValue) { Example example; @@ -151,26 +115,16 @@ TEST(GetFeatureValuesFloatTest, CheckTypedFieldExistence) { Example example; GetFeatureValues("tag", &example)->Add(42); - ASSERT_FALSE(HasFeature("tag", example)); + ASSERT_FALSE(ExampleHasFeature("tag", example)); GetFeatureValues("tag", &example)->Add(3.14); - EXPECT_TRUE(HasFeature("tag", example)); + EXPECT_TRUE(ExampleHasFeature("tag", example)); auto tag_ro = GetFeatureValues("tag", example); ASSERT_EQ(1, tag_ro.size()); EXPECT_NEAR(3.14, tag_ro.Get(0), kTolerance); } -TEST(GetFeatureValuesStringTest, ReadsASingleValueFromFeature) { - Feature feature; - feature.mutable_bytes_list()->add_value("FOO"); - - auto values = GetFeatureValues(feature); - - ASSERT_EQ(1, values.size()); - EXPECT_EQ("FOO", values.Get(0)); -} - TEST(GetFeatureValuesStringTest, ReadsASingleValue) { Example example; (*example.mutable_features()->mutable_feature())["tag"] @@ -183,15 +137,6 @@ TEST(GetFeatureValuesStringTest, ReadsASingleValue) { EXPECT_EQ("FOO", tag.Get(0)); } -TEST(GetFeatureValuesStringTest, WritesASingleValueToFeature) { - Feature feature; - - *GetFeatureValues(&feature)->Add() = "FOO"; - - ASSERT_EQ(1, feature.bytes_list().value_size()); - EXPECT_EQ("FOO", feature.bytes_list().value(0)); -} - TEST(GetFeatureValuesStringTest, WritesASingleValue) { Example example; @@ -203,15 +148,15 @@ TEST(GetFeatureValuesStringTest, WritesASingleValue) { example.features().feature().at("tag").bytes_list().value(0)); } -TEST(GetFeatureValuesStringTest, CheckTypedFieldExistence) { +TEST(GetFeatureValuesBytesTest, CheckTypedFieldExistence) { Example example; GetFeatureValues("tag", &example)->Add(42); - ASSERT_FALSE(HasFeature("tag", example)); + ASSERT_FALSE(ExampleHasFeature("tag", example)); *GetFeatureValues("tag", &example)->Add() = "FOO"; - EXPECT_TRUE(HasFeature("tag", example)); + EXPECT_TRUE(ExampleHasFeature("tag", example)); auto tag_ro = GetFeatureValues("tag", example); ASSERT_EQ(1, tag_ro.size()); EXPECT_EQ("FOO", tag_ro.Get(0)); @@ -283,146 +228,5 @@ TEST(AppendFeatureValuesTest, StringVariablesUsingInitializerList) { EXPECT_EQ("BAZ", tag_ro.Get(2)); } -TEST(SequenceExampleTest, ReadsASingleValueFromContext) { - SequenceExample se; - (*se.mutable_context()->mutable_feature())["tag"] - .mutable_int64_list() - ->add_value(42); - - auto values = GetFeatureValues("tag", se.context()); - - ASSERT_EQ(1, values.size()); - EXPECT_EQ(42, values.Get(0)); -} - -TEST(SequenceExampleTest, WritesASingleValueToContext) { - SequenceExample se; - - GetFeatureValues("tag", se.mutable_context())->Add(42); - - ASSERT_EQ(1, se.context().feature().at("tag").int64_list().value_size()); - EXPECT_EQ(42, se.context().feature().at("tag").int64_list().value(0)); -} - -TEST(SequenceExampleTest, AppendFeatureValuesToContextSingleArg) { - SequenceExample se; - - AppendFeatureValues({1.1, 2.2, 3.3}, "tag", se.mutable_context()); - - auto tag_ro = GetFeatureValues("tag", se.context()); - ASSERT_EQ(3, tag_ro.size()); - EXPECT_NEAR(1.1, tag_ro.Get(0), kTolerance); - EXPECT_NEAR(2.2, tag_ro.Get(1), kTolerance); - EXPECT_NEAR(3.3, tag_ro.Get(2), kTolerance); -} - -TEST(SequenceExampleTest, CheckTypedFieldExistence) { - SequenceExample se; - - GetFeatureValues("tag", se.mutable_context())->Add(3.14); - ASSERT_FALSE(HasFeature("tag", se.context())); - - GetFeatureValues("tag", se.mutable_context())->Add(42); - - EXPECT_TRUE(HasFeature("tag", se.context())); - auto tag_ro = GetFeatureValues("tag", se.context()); - ASSERT_EQ(1, tag_ro.size()); - EXPECT_EQ(42, tag_ro.Get(0)); -} - -TEST(SequenceExampleTest, ReturnsExistingFeatureLists) { - SequenceExample se; - (*se.mutable_feature_lists()->mutable_feature_list())["tag"] - .mutable_feature() - ->Add(); - - auto feature = GetFeatureList("tag", se); - - ASSERT_EQ(1, feature.size()); -} - -TEST(SequenceExampleTest, CreatesNewFeatureLists) { - SequenceExample se; - - GetFeatureList("tag", &se)->Add(); - - EXPECT_EQ(1, se.feature_lists().feature_list().at("tag").feature_size()); -} - -TEST(SequenceExampleTest, CheckFeatureListExistence) { - SequenceExample se; - ASSERT_FALSE(HasFeatureList("tag", se)); - - GetFeatureList("tag", &se)->Add(); - - ASSERT_TRUE(HasFeatureList("tag", se)); -} - -TEST(SequenceExampleTest, AppendFeatureValuesWithInitializerList) { - SequenceExample se; - - AppendFeatureValues({1, 2, 3}, "ids", se.mutable_context()); - AppendFeatureValues({"cam1-0", "cam2-0"}, - GetFeatureList("images", &se)->Add()); - AppendFeatureValues({"cam1-1", "cam2-2"}, - GetFeatureList("images", &se)->Add()); - - EXPECT_EQ(se.DebugString(), - "context {\n" - " feature {\n" - " key: \"ids\"\n" - " value {\n" - " int64_list {\n" - " value: 1\n" - " value: 2\n" - " value: 3\n" - " }\n" - " }\n" - " }\n" - "}\n" - "feature_lists {\n" - " feature_list {\n" - " key: \"images\"\n" - " value {\n" - " feature {\n" - " bytes_list {\n" - " value: \"cam1-0\"\n" - " value: \"cam2-0\"\n" - " }\n" - " }\n" - " feature {\n" - " bytes_list {\n" - " value: \"cam1-1\"\n" - " value: \"cam2-2\"\n" - " }\n" - " }\n" - " }\n" - " }\n" - "}\n"); -} - -TEST(SequenceExampleTest, AppendFeatureValuesWithVectors) { - SequenceExample se; - - std::vector readings{1.0, 2.5, 5.0}; - AppendFeatureValues(readings, GetFeatureList("movie_ratings", &se)->Add()); - - EXPECT_EQ(se.DebugString(), - "feature_lists {\n" - " feature_list {\n" - " key: \"movie_ratings\"\n" - " value {\n" - " feature {\n" - " float_list {\n" - " value: 1\n" - " value: 2.5\n" - " value: 5\n" - " }\n" - " }\n" - " }\n" - " }\n" - "}\n"); -} - } // namespace } // namespace tensorflow From 9fe71e08a56035dd547f7f9e9f7fdc0b093d0c96 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 12:07:05 -0700 Subject: [PATCH 42/67] Adding back contrib.receptive_field test to oss. PiperOrigin-RevId: 167307311 --- tensorflow/contrib/receptive_field/BUILD | 3 --- tensorflow/contrib/receptive_field/README.md | 3 ++- .../receptive_field/python/util/receptive_field.py | 10 +++++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tensorflow/contrib/receptive_field/BUILD b/tensorflow/contrib/receptive_field/BUILD index e2d7c075104..ed2f3af08cb 100644 --- a/tensorflow/contrib/receptive_field/BUILD +++ b/tensorflow/contrib/receptive_field/BUILD @@ -47,9 +47,6 @@ py_test( name = "receptive_field_test", srcs = ["python/util/receptive_field_test.py"], srcs_version = "PY2AND3", - tags = [ - "no_oss", # see b/65254194 - ], deps = [ ":receptive_field_py", "//tensorflow/contrib/framework:framework_py", diff --git a/tensorflow/contrib/receptive_field/README.md b/tensorflow/contrib/receptive_field/README.md index f7539ec1450..b150b903b23 100644 --- a/tensorflow/contrib/receptive_field/README.md +++ b/tensorflow/contrib/receptive_field/README.md @@ -161,4 +161,5 @@ Effective padding (vertical) = 1482 ## Authors -André Araujo (andrefaraujo@) and Mark Sandler +André Araujo (github id: andrefaraujo) and Mark Sandler (github id: +marksandler) diff --git a/tensorflow/contrib/receptive_field/python/util/receptive_field.py b/tensorflow/contrib/receptive_field/python/util/receptive_field.py index 4e723829bff..db190a1a416 100644 --- a/tensorflow/contrib/receptive_field/python/util/receptive_field.py +++ b/tensorflow/contrib/receptive_field/python/util/receptive_field.py @@ -35,6 +35,10 @@ _UNCHANGED_RF_LAYER_OPS = [ "VariableV2", "Sub", "Rsqrt", "ConcatV2" ] +# Different ways in which padding modes may be spelled. +_VALID_PADDING = ["VALID", b"VALID"] +_SAME_PADDING = ["SAME", b"SAME"] + def _stride_size(node): """Computes stride size given a TF node. @@ -102,9 +106,9 @@ def _padding_size_conv_pool(node, kernel_size, stride): # depends on input size, we raise an exception. padding_attr = node.attr["padding"] logging.vlog(4, "padding_attr = %s", padding_attr) - if padding_attr.s == "VALID": + if padding_attr.s in _VALID_PADDING: padding = 0 - elif padding_attr.s == "SAME": + elif padding_attr.s in _SAME_PADDING: if kernel_size == 1: padding = 0 elif stride == 1: @@ -118,7 +122,7 @@ def _padding_size_conv_pool(node, kernel_size, stride): "padding may be different depending on the input image " "dimensionality. In this case, alignment check will be skipped.") else: - raise ValueError("Invalid padding operation") + raise ValueError("Invalid padding operation %s" % padding_attr.s) return padding From 1196fbc824b45679cba9fae8daa35cc6c02d3599 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 13:05:00 -0700 Subject: [PATCH 43/67] Use boolean literals where appropriate instead of narrowing ints PiperOrigin-RevId: 167314054 --- .../compiler/xla/service/cpu/parallel_cpu_executable.cc | 2 +- tensorflow/compiler/xla/service/elemental_ir_emitter.cc | 2 +- tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc | 2 +- tensorflow/core/framework/cancellation.cc | 4 +++- tensorflow/core/kernels/debug_ops_test.cc | 3 ++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.cc index bef4ecd480d..40fa3a67bde 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.cc +++ b/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.cc @@ -241,7 +241,7 @@ Status Executor::Run() { completion_queue_.pop_front(); break; } - } while (1); + } while (true); TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice result_slice, assignment_->GetUniqueTopLevelSlice(instruction)); void* result_buffer = diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc index b02138325ed..350dbc321fb 100644 --- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc @@ -709,7 +709,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator( } else { auto r = ir_builder_->CreateSub(q, p); auto leading_zeros = llvm_ir::EmitCallToIntrinsic( - llvm::Intrinsic::ctlz, {r, ir_builder_->getInt1(1)}, + llvm::Intrinsic::ctlz, {r, ir_builder_->getInt1(true)}, {param_ir_type}, ir_builder_); auto in_block = ir_builder_->GetInsertBlock(); diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc index d044462f9a7..5edaaba3ebe 100644 --- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc @@ -334,7 +334,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator( SetToFirstInsertPoint(loops.GetInnerLoopBodyBasicBlock(), ir_builder_); IrArray::Index input_index(index.size()); - llvm::Value* in_bounds = ir_builder_->getInt1(1); + llvm::Value* in_bounds = ir_builder_->getInt1(true); for (size_t i = 0; i < index.size(); ++i) { llvm::Value* stridden_index = ir_builder_->CreateNSWMul( index[i], ir_builder_->getInt64(window.dimensions(i).stride())); diff --git a/tensorflow/core/framework/cancellation.cc b/tensorflow/core/framework/cancellation.cc index 1cbed62939f..9da4828bbad 100644 --- a/tensorflow/core/framework/cancellation.cc +++ b/tensorflow/core/framework/cancellation.cc @@ -23,7 +23,9 @@ namespace tensorflow { const CancellationToken CancellationManager::kInvalidToken = -1; CancellationManager::CancellationManager() - : is_cancelling_(false), is_cancelled_(0), next_cancellation_token_(0) {} + : is_cancelling_(false), + is_cancelled_(false), + next_cancellation_token_(0) {} void CancellationManager::StartCancel() { gtl::FlatMap callbacks_to_run; diff --git a/tensorflow/core/kernels/debug_ops_test.cc b/tensorflow/core/kernels/debug_ops_test.cc index 89bcbc9c373..37c94865942 100644 --- a/tensorflow/core/kernels/debug_ops_test.cc +++ b/tensorflow/core/kernels/debug_ops_test.cc @@ -573,7 +573,8 @@ TEST_F(DebugNumericSummaryOpTest, UInt8Success) { TEST_F(DebugNumericSummaryOpTest, BoolSuccess) { TF_ASSERT_OK(Init(DT_BOOL)); - AddInputFromArray(TensorShape({2, 3}), {0, 0, 1, 1, 1, 0}); + AddInputFromArray(TensorShape({2, 3}), + {false, false, true, true, true, false}); TF_ASSERT_OK(RunOpKernel()); Tensor expected(allocator(), DT_DOUBLE, TensorShape({16})); From 12cf8c19575e3043ce99d363b57a313fdaabbd76 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 13:06:02 -0700 Subject: [PATCH 44/67] Fix broken link in Estimators. Move Estimators to beginning of Prog. Guide. Change title of Datasets unit. PiperOrigin-RevId: 167314186 --- tensorflow/docs_src/programmers_guide/datasets.md | 4 ++-- tensorflow/docs_src/programmers_guide/estimators.md | 2 +- tensorflow/docs_src/programmers_guide/index.md | 4 ++-- tensorflow/docs_src/programmers_guide/leftnav_files | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md index bf3cb5bf196..ba26bd5e941 100644 --- a/tensorflow/docs_src/programmers_guide/datasets.md +++ b/tensorflow/docs_src/programmers_guide/datasets.md @@ -1,4 +1,4 @@ -# Using the `Dataset` API for TensorFlow Input Pipelines +# Importing Data The `Dataset` API enables you to build complex input pipelines from simple, reusable pieces. For example, the pipeline for an image model might @@ -735,7 +735,7 @@ def dataset_input_fn(): return {"image_data": image, "date_time": parsed["date_time"]}, label - # Use `Dataset.map()` to build a pair of a feature dictionary and a label + # Use `Dataset.map()` to build a pair of a feature dictionary and a label # tensor for each example. dataset = dataset.map(parser) dataset = dataset.shuffle(buffer_size=10000) diff --git a/tensorflow/docs_src/programmers_guide/estimators.md b/tensorflow/docs_src/programmers_guide/estimators.md index a5724ea294e..755bb049c99 100644 --- a/tensorflow/docs_src/programmers_guide/estimators.md +++ b/tensorflow/docs_src/programmers_guide/estimators.md @@ -134,7 +134,7 @@ The heart of every Estimator--whether pre-made or custom--is its evaluation, and prediction. When you are using a pre-made Estimator, someone else has already implemented the model function. When relying on a custom Estimator, you must write the model function yourself. A -${$extend/estimators$companion document) +@{$extend/estimators$companion document} explains how to write the model function. diff --git a/tensorflow/docs_src/programmers_guide/index.md b/tensorflow/docs_src/programmers_guide/index.md index 22fe229422a..eef35d6dcc7 100644 --- a/tensorflow/docs_src/programmers_guide/index.md +++ b/tensorflow/docs_src/programmers_guide/index.md @@ -4,6 +4,8 @@ The documents in this unit dive into the details of writing TensorFlow code. For TensorFlow 1.3, we revised this document extensively. The units are now as follows: + * @{$programmers_guide/estimators$Estimators}, which introduces a high-level + TensorFlow API that greatly simplifies ML programming. * @{$programmers_guide/tensors$Tensors}, which explains how to create, manipulate, and access Tensors--the fundamental object in TensorFlow. * @{$programmers_guide/variables$Variables}, which details how @@ -18,8 +20,6 @@ The units are now as follows: such as Estimators or Keras, the high-level API creates and manages graphs and sessions for you, but understanding graphs and sessions can still be helpful. - * @{$programmers_guide/estimators$Estimators}, which introduces a high-level - TensorFlow API that greatly simplifies ML programming. * @{$programmers_guide/saved_model$Saving and Restoring}, which explains how to save and restore variables and models. * @{$programmers_guide/datasets$Input Pipelines}, which explains how to diff --git a/tensorflow/docs_src/programmers_guide/leftnav_files b/tensorflow/docs_src/programmers_guide/leftnav_files index 5082e7f36c8..0c42f119c95 100644 --- a/tensorflow/docs_src/programmers_guide/leftnav_files +++ b/tensorflow/docs_src/programmers_guide/leftnav_files @@ -1,8 +1,8 @@ index.md +estimators.md tensors.md variables.md graphs.md -estimators.md saved_model.md datasets.md threading_and_queues.md From f7733742d51dba09d4f222b3eb027c27c2c4d130 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Fri, 1 Sep 2017 13:16:35 -0700 Subject: [PATCH 45/67] Join an unjoined checked thread in fifo_queue_test. PiperOrigin-RevId: 167315380 --- tensorflow/python/kernel_tests/fifo_queue_test.py | 3 +++ tensorflow/python/kernel_tests/padding_fifo_queue_test.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tensorflow/python/kernel_tests/fifo_queue_test.py b/tensorflow/python/kernel_tests/fifo_queue_test.py index 85e7b635d80..748135440ec 100644 --- a/tensorflow/python/kernel_tests/fifo_queue_test.py +++ b/tensorflow/python/kernel_tests/fifo_queue_test.py @@ -1078,6 +1078,9 @@ class FIFOQueueTest(test.TestCase): self.assertEqual([50.0], dequeued_t.eval()) self.assertEqual([60.0], dequeued_t.eval()) + # Make sure the thread finishes before exiting. + thread.join() + def testBlockingEnqueueBeforeClose(self): with self.test_session() as sess: q = data_flow_ops.FIFOQueue(4, dtypes_lib.float32) diff --git a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py index 53b1897f488..d8c3f9823c3 100644 --- a/tensorflow/python/kernel_tests/padding_fifo_queue_test.py +++ b/tensorflow/python/kernel_tests/padding_fifo_queue_test.py @@ -1191,6 +1191,9 @@ class PaddingFIFOQueueTest(test.TestCase): self.assertEqual([50.0], dequeued_t.eval()) self.assertEqual([60.0], dequeued_t.eval()) + # Make sure the thread finishes before exiting. + thread.join() + def testBlockingEnqueueBeforeClose(self): with self.test_session() as sess: q = data_flow_ops.PaddingFIFOQueue(4, dtypes_lib.float32, ((),)) From 0acf5bb38a8f208c6d9f048579a076d5bc6ff0be Mon Sep 17 00:00:00 2001 From: Eugene Brevdo Date: Fri, 1 Sep 2017 13:32:02 -0700 Subject: [PATCH 46/67] Added registry for variants for op ZerosLike. * Updated tf.zeros_like python wrapper to avoid calling tf.zeros on Variants. * tf.zeros_like for variants calls the appropriate callback for the given device and type_name. PiperOrigin-RevId: 167317274 --- .../core/framework/variant_op_registry.cc | 66 ++++++++++-- .../core/framework/variant_op_registry.h | 92 ++++++++++++++++ .../framework/variant_op_registry_test.cc | 100 ++++++++++++++++++ tensorflow/core/kernels/constant_op.cc | 63 +++++++---- tensorflow/core/kernels/shape_op_test.cc | 2 +- tensorflow/core/kernels/shape_ops.h | 2 +- .../python/kernel_tests/constant_op_test.py | 41 ++++++- tensorflow/python/ops/array_ops.py | 7 +- 8 files changed, 340 insertions(+), 33 deletions(-) diff --git a/tensorflow/core/framework/variant_op_registry.cc b/tensorflow/core/framework/variant_op_registry.cc index 11756c356aa..9cc7530459e 100644 --- a/tensorflow/core/framework/variant_op_registry.cc +++ b/tensorflow/core/framework/variant_op_registry.cc @@ -88,7 +88,17 @@ bool DecodeUnaryVariant(Variant* variant) { if (decode_fn == nullptr) { return false; } - return (*decode_fn)(variant); + const string type_name = variant->TypeName(); + bool decoded = (*decode_fn)(variant); + if (!decoded) return false; + if (variant->TypeName() != type_name) { + LOG(ERROR) << "DecodeUnaryVariant: Variant type_name before decoding was: " + << type_name + << " but after decoding was: " << variant->TypeName() + << ". Treating this as a failure."; + return false; + } + return true; } // Add some basic registrations for use by others, e.g., for testing. @@ -101,15 +111,59 @@ string MaybeRemoveTFPrefix(const StringPiece& str) { } // namespace #define REGISTER_VARIANT_DECODE_TYPE(T) \ - REGISTER_UNARY_VARIANT_DECODE_FUNCTION(T, MaybeRemoveTFPrefix(TF_STR(T))); + REGISTER_UNARY_VARIANT_DECODE_FUNCTION(T, TF_STR(T)); // No encode/decode registered for std::complex<> and Eigen::half // objects yet. -TF_CALL_INTEGRAL_TYPES(REGISTER_VARIANT_DECODE_TYPE); -TF_CALL_float(REGISTER_VARIANT_DECODE_TYPE); -TF_CALL_double(REGISTER_VARIANT_DECODE_TYPE); -TF_CALL_bool(REGISTER_VARIANT_DECODE_TYPE); +REGISTER_VARIANT_DECODE_TYPE(int); +REGISTER_VARIANT_DECODE_TYPE(float); +REGISTER_VARIANT_DECODE_TYPE(bool); +REGISTER_VARIANT_DECODE_TYPE(double); #undef REGISTER_VARIANT_DECODE_TYPE +// Special casing ZerosLikeFn per device. +UnaryVariantOpRegistry::VariantZerosLikeFn* +UnaryVariantOpRegistry::GetZerosLikeFn(const string& device, + const string& type_name) { + auto found = zeros_like_fns.find(std::make_pair(device, type_name)); + if (found == zeros_like_fns.end()) return nullptr; + return &found->second; +} + +void UnaryVariantOpRegistry::RegisterZerosLikeFn( + const string& device, const string& type_name, + const VariantZerosLikeFn& zeros_like_fn) { + CHECK(!type_name.empty()) << "Need a valid name for UnaryVariantZerosLike"; + VariantZerosLikeFn* existing = GetZerosLikeFn(device, type_name); + CHECK_EQ(existing, nullptr) + << "Unary VariantZerosLikeFn for type_name: " << type_name + << " already registered for device type: " << device; + zeros_like_fns.insert( + std::pair, VariantZerosLikeFn>( + std::make_pair(device, type_name), zeros_like_fn)); +} + +namespace { + +template +Status ZerosLikeVariantPrimitiveType(OpKernelContext* ctx, const T& t, + T* t_out) { + *t_out = T(0); + return Status::OK(); +} +} // namespace + +#define REGISTER_VARIANT_ZEROS_LIKE_TYPE(T) \ + REGISTER_UNARY_VARIANT_ZEROS_LIKE_FUNCTION( \ + DEVICE_CPU, T, TF_STR(T), ZerosLikeVariantPrimitiveType); + +// No zeros_like registered for std::complex<> or Eigen::half objects yet. +REGISTER_VARIANT_ZEROS_LIKE_TYPE(int); +REGISTER_VARIANT_ZEROS_LIKE_TYPE(float); +REGISTER_VARIANT_ZEROS_LIKE_TYPE(double); +REGISTER_VARIANT_ZEROS_LIKE_TYPE(bool); + +#undef REGISTER_VARIANT_ZEROS_LIKE_TYPE + } // namespace tensorflow diff --git a/tensorflow/core/framework/variant_op_registry.h b/tensorflow/core/framework/variant_op_registry.h index 389b049fa01..37e54f82c0f 100644 --- a/tensorflow/core/framework/variant_op_registry.h +++ b/tensorflow/core/framework/variant_op_registry.h @@ -19,11 +19,13 @@ limitations under the License. #include #include +#include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/variant.h" #include "tensorflow/core/framework/variant_encode_decode.h" namespace tensorflow { +class OpKernelContext; // A global UnaryVariantOpRegistry is used to hold callback functions // for different variant types. To be used by ShapeOp, RankOp, and // SizeOp, decoding, etc. @@ -32,6 +34,8 @@ class UnaryVariantOpRegistry { public: typedef std::function VariantShapeFn; typedef std::function VariantDecodeFn; + typedef std::function + VariantZerosLikeFn; // Add a shape lookup function to the registry. void RegisterShapeFn(const string& type_name, const VariantShapeFn& shape_fn); @@ -46,11 +50,29 @@ class UnaryVariantOpRegistry { // Returns nullptr if no decode function was found for the given TypeName. VariantDecodeFn* GetDecodeFn(const string& type_name); + // Add a zeros-like function to the registry. + void RegisterZerosLikeFn(const string& device, const string& type_name, + const VariantZerosLikeFn& zeros_like_fn); + + // Returns nullptr if no zeros-like function was found for the given + // device and TypeName. + VariantZerosLikeFn* GetZerosLikeFn(const string& device, + const string& type_name); + static UnaryVariantOpRegistry* Global(); private: std::unordered_map shape_fns; std::unordered_map decode_fns; + // Map std::pair to function. + struct PairHash { + template + std::size_t operator()(const std::pair& x) const { + return std::hash()(x.first) ^ std::hash()(x.second); + } + }; + std::unordered_map, VariantZerosLikeFn, PairHash> + zeros_like_fns; }; // Gets a TensorShape from a Tensor containing a scalar Variant. @@ -72,6 +94,28 @@ Status GetUnaryVariantShape(const Tensor& variant_tensor, TensorShape* shape); // bool DecodeUnaryVariant(Variant* variant); +// Sets *z_out = zeros_like(v). The variant v must have a registered +// ZerosLike function for the given Device. Returns an Internal error +// if v does not have a registered zeros_like function for this device, or if +// ZerosLike fails. +// +// REQUIRES: +// v_out is not null. +// +template +Status CreateZerosLikeVariant(OpKernelContext* ctx, const Variant& v, + Variant* v_out) { + const string& device = DeviceName::value; + UnaryVariantOpRegistry::VariantZerosLikeFn* zeros_like_fn = + UnaryVariantOpRegistry::Global()->GetZerosLikeFn(device, v.TypeName()); + if (zeros_like_fn == nullptr) { + return errors::Internal( + "No unary variant zeros_like function found for Variant type_name: ", + v.TypeName(), " for device type: ", device); + } + return (*zeros_like_fn)(ctx, v, v_out); +} + namespace variant_op_registry_fn_registration { template @@ -120,6 +164,34 @@ class UnaryVariantDecodeRegistration { } }; +template +class UnaryVariantZerosLikeRegistration { + typedef std::function + LocalVariantZerosLikeFn; + + public: + UnaryVariantZerosLikeRegistration( + const string& device, const string& type_name, + const LocalVariantZerosLikeFn& zeros_like_fn) { + auto wrapped_fn = [type_name, zeros_like_fn](OpKernelContext* ctx, + const Variant& v, + Variant* v_out) -> Status { + CHECK_NOTNULL(v_out); + *v_out = T(); + if (v.get() == nullptr) { + return errors::Internal( + "VariantZerosLikeFn: Could not access object, type_name: ", + type_name); + } + const T& t = *v.get(); + T* t_out = v_out->get(); + return zeros_like_fn(ctx, t, t_out); + }; + UnaryVariantOpRegistry::Global()->RegisterZerosLikeFn(device, type_name, + wrapped_fn); + } +}; + }; // namespace variant_op_registry_fn_registration // Register a unary shape variant function with the signature: @@ -151,6 +223,26 @@ class UnaryVariantDecodeRegistration { T> \ register_unary_variant_op_decoder_fn_##ctr(type_name) +// Register a unary zeros_like variant function with the signature: +// Status ZerosLikeFn(OpKernelContext* ctx, const T& t, T* t_out); +// to Variants having TypeName type_name, for device string device. +#define REGISTER_UNARY_VARIANT_ZEROS_LIKE_FUNCTION(device, T, type_name, \ + zeros_like_function) \ + REGISTER_UNARY_VARIANT_ZEROS_LIKE_FUNCTION_UNIQ_HELPER( \ + __COUNTER__, device, T, type_name, zeros_like_function) + +#define REGISTER_UNARY_VARIANT_ZEROS_LIKE_FUNCTION_UNIQ_HELPER( \ + ctr, device, T, type_name, zeros_like_function) \ + REGISTER_UNARY_VARIANT_ZEROS_LIKE_FUNCTION_UNIQ(ctr, device, T, type_name, \ + zeros_like_function) + +#define REGISTER_UNARY_VARIANT_ZEROS_LIKE_FUNCTION_UNIQ( \ + ctr, device, T, type_name, zeros_like_function) \ + static variant_op_registry_fn_registration:: \ + UnaryVariantZerosLikeRegistration \ + register_unary_variant_op_decoder_fn_##ctr(device, type_name, \ + zeros_like_function) + } // end namespace tensorflow #endif // TENSORFLOW_FRAMEWORK_VARIANT_OP_REGISTRY_H_ diff --git a/tensorflow/core/framework/variant_op_registry_test.cc b/tensorflow/core/framework/variant_op_registry_test.cc index 86fef53dbe6..4e79180217a 100644 --- a/tensorflow/core/framework/variant_op_registry_test.cc +++ b/tensorflow/core/framework/variant_op_registry_test.cc @@ -15,13 +15,25 @@ limitations under the License. #include +#define EIGEN_USE_THREADS + +#if GOOGLE_CUDA +#define EIGEN_USE_GPU +#endif + #include "tensorflow/core/framework/variant_op_registry.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/test.h" namespace tensorflow { +typedef Eigen::ThreadPoolDevice CPUDevice; +typedef Eigen::GpuDevice GPUDevice; + namespace { struct VariantValue { @@ -33,7 +45,24 @@ struct VariantValue { *s = TensorShape({-0xdeadbeef}); return Status::OK(); } + static Status CPUZerosLikeFn(OpKernelContext* ctx, const VariantValue& v, + VariantValue* v_out) { + if (v.early_exit) { + return errors::InvalidArgument("early exit zeros_like!"); + } + v_out->zeros_like_set = 1; // CPU + return Status::OK(); + } + static Status GPUZerosLikeFn(OpKernelContext* ctx, const VariantValue& v, + VariantValue* v_out) { + if (v.early_exit) { + return errors::InvalidArgument("early exit zeros_like!"); + } + v_out->zeros_like_set = 2; // GPU + return Status::OK(); + } bool early_exit; + int zeros_like_set; }; REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(VariantValue, "TEST VariantValue", @@ -41,6 +70,14 @@ REGISTER_UNARY_VARIANT_SHAPE_FUNCTION(VariantValue, "TEST VariantValue", REGISTER_UNARY_VARIANT_DECODE_FUNCTION(VariantValue, "TEST VariantValue"); +REGISTER_UNARY_VARIANT_ZEROS_LIKE_FUNCTION(DEVICE_CPU, VariantValue, + "TEST VariantValue", + VariantValue::CPUZerosLikeFn); + +REGISTER_UNARY_VARIANT_ZEROS_LIKE_FUNCTION(DEVICE_GPU, VariantValue, + "TEST VariantValue", + VariantValue::GPUZerosLikeFn); + } // namespace TEST(VariantOpShapeRegistryTest, TestBasic) { @@ -101,4 +138,67 @@ TEST(VariantOpDecodeRegistryTest, TestDuplicate) { "fjfjfj already registered"); } +TEST(VariantOpZerosLikeRegistryTest, TestBasicCPU) { + EXPECT_EQ(UnaryVariantOpRegistry::Global()->GetZerosLikeFn( + DEVICE_CPU, "YOU SHALL NOT PASS"), + nullptr); + + VariantValue vv_early_exit{true /* early_exit */, 0 /* zeros_like_set */}; + Variant v = vv_early_exit; + Variant v_out = VariantValue(); + + OpKernelContext* null_context_pointer = nullptr; + Status s0 = + CreateZerosLikeVariant(null_context_pointer, v, &v_out); + EXPECT_FALSE(s0.ok()); + EXPECT_TRUE( + StringPiece(s0.error_message()).contains("early exit zeros_like")); + + VariantValue vv_ok{false /* early_exit */, 0 /* zeros_like_set */}; + v = vv_ok; + TF_EXPECT_OK( + CreateZerosLikeVariant(null_context_pointer, v, &v_out)); + VariantValue* vv_out = CHECK_NOTNULL(v_out.get()); + EXPECT_EQ(vv_out->zeros_like_set, 1); // CPU +} + +#if GOOGLE_CUDA +TEST(VariantOpZerosLikeRegistryTest, TestBasicGPU) { + EXPECT_EQ(UnaryVariantOpRegistry::Global()->GetZerosLikeFn( + DEVICE_GPU, "YOU SHALL NOT PASS"), + nullptr); + + VariantValue vv_early_exit{true /* early_exit */, 0 /* zeros_like_set */}; + Variant v = vv_early_exit; + Variant v_out = VariantValue(); + + OpKernelContext* null_context_pointer = nullptr; + Status s0 = + CreateZerosLikeVariant(null_context_pointer, v, &v_out); + EXPECT_FALSE(s0.ok()); + EXPECT_TRUE( + StringPiece(s0.error_message()).contains("early exit zeros_like")); + + VariantValue vv_ok{false /* early_exit */, 0 /* zeros_like_set */}; + v = vv_ok; + TF_EXPECT_OK( + CreateZerosLikeVariant(null_context_pointer, v, &v_out)); + VariantValue* vv_out = CHECK_NOTNULL(v_out.get()); + EXPECT_EQ(vv_out->zeros_like_set, 2); // GPU +} +#endif // GOOGLE_CUDA + +TEST(VariantOpZerosLikeRegistryTest, TestDuplicate) { + UnaryVariantOpRegistry registry; + UnaryVariantOpRegistry::VariantZerosLikeFn f; + + registry.RegisterZerosLikeFn(DEVICE_CPU, "fjfjfj", f); + EXPECT_DEATH(registry.RegisterZerosLikeFn(DEVICE_CPU, "fjfjfj", f), + "fjfjfj already registered"); + + registry.RegisterZerosLikeFn(DEVICE_GPU, "fjfjfj", f); + EXPECT_DEATH(registry.RegisterZerosLikeFn(DEVICE_GPU, "fjfjfj", f), + "fjfjfj already registered"); +} + } // namespace tensorflow diff --git a/tensorflow/core/kernels/constant_op.cc b/tensorflow/core/kernels/constant_op.cc index b4b37dd4b8e..cdc11452827 100644 --- a/tensorflow/core/kernels/constant_op.cc +++ b/tensorflow/core/kernels/constant_op.cc @@ -17,6 +17,10 @@ limitations under the License. #define EIGEN_USE_THREADS +#if GOOGLE_CUDA +#define EIGEN_USE_GPU +#endif + #include "tensorflow/core/kernels/constant_op.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" @@ -26,13 +30,14 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/kernels/fill_functor.h" #include "tensorflow/core/platform/macros.h" #ifdef TENSORFLOW_USE_SYCL #include "tensorflow/core/common_runtime/sycl/sycl_util.h" -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL namespace tensorflow { @@ -40,9 +45,8 @@ ConstantOp::ConstantOp(OpKernelConstruction* ctx) : OpKernel(ctx), tensor_(ctx->output_type(0)) { const TensorProto* proto = nullptr; OP_REQUIRES_OK(ctx, ctx->GetAttr("value", &proto)); - OP_REQUIRES_OK(ctx, - ctx->device()->MakeTensorFromProto( - *proto, AllocatorAttributes(), &tensor_)); + OP_REQUIRES_OK(ctx, ctx->device()->MakeTensorFromProto( + *proto, AllocatorAttributes(), &tensor_)); OP_REQUIRES( ctx, ctx->output_type(0) == tensor_.dtype(), errors::InvalidArgument("Type mismatch between value (", @@ -85,9 +89,9 @@ REGISTER_KERNEL(GPU, bool); #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNEL(D, TYPE) \ - REGISTER_KERNEL_BUILDER( \ - Name("Const").Device(DEVICE_##D).TypeConstraint("dtype"), \ +#define REGISTER_SYCL_KERNEL(D, TYPE) \ + REGISTER_KERNEL_BUILDER( \ + Name("Const").Device(DEVICE_##D).TypeConstraint("dtype"), \ ConstantOp); REGISTER_SYCL_KERNEL(SYCL, float); REGISTER_SYCL_KERNEL(SYCL, double); @@ -194,18 +198,18 @@ struct FillFunctor { void operator()(const SYCLDevice& d, typename TTypes::Flat out, typename TTypes::ConstScalar in) { #if !defined(EIGEN_HAS_INDEX_LIST) - Eigen::array rank1{1}; + Eigen::array rank1{1}; #else - Eigen::IndexList> rank1; + Eigen::IndexList > rank1; #endif - const int size = out.dimension(0); - Eigen::array broadcast_dims{size}; + const int size = out.dimension(0); + Eigen::array broadcast_dims{size}; - To32Bit(out).device(d) = in.reshape(rank1).broadcast(broadcast_dims); + To32Bit(out).device(d) = in.reshape(rank1).broadcast(broadcast_dims); } }; -} -#endif // TENSORFLOW_USE_SYCL +} // namespace functor +#endif // TENSORFLOW_USE_SYCL #define REGISTER_KERNEL(D, TYPE) \ REGISTER_KERNEL_BUILDER(Name("Fill") \ @@ -273,11 +277,23 @@ class ZerosLikeOp : public OpKernel { void Compute(OpKernelContext* ctx) override { const Tensor& input = ctx->input(0); - Tensor* out = nullptr; - OP_REQUIRES_OK(ctx, ctx->forward_input_or_allocate_output( - {0}, 0, input.shape(), &out)); - functor::SetZeroFunctor f; - f(ctx->eigen_device(), out->flat()); + const Device& d = ctx->eigen_device(); + if (std::is_same::value) { + OP_REQUIRES(ctx, input.dims() == 0, + errors::InvalidArgument( + "ZerosLike of non-unary Variant not supported.")); + const Variant& v = input.scalar()(); + Tensor out(cpu_allocator(), DT_VARIANT, TensorShape({})); + Variant* out_v = &(out.scalar()()); + OP_REQUIRES_OK(ctx, CreateZerosLikeVariant(ctx, v, out_v)); + ctx->set_output(0, out); + } else { + Tensor* out = nullptr; + OP_REQUIRES_OK(ctx, ctx->forward_input_or_allocate_output( + {0}, 0, input.shape(), &out)); + functor::SetZeroFunctor f; + f(d, out->flat()); + } } }; @@ -288,6 +304,7 @@ class ZerosLikeOp : public OpKernel { #define REGISTER_CPU(type) REGISTER_KERNEL(type, CPU) TF_CALL_POD_STRING_TYPES(REGISTER_CPU); +REGISTER_CPU(Variant); #undef REGISTER_CPU #ifdef TENSORFLOW_USE_SYCL @@ -315,6 +332,14 @@ REGISTER_KERNEL_BUILDER(Name("ZerosLike") .TypeConstraint("T") .HostMemory("y"), ZerosLikeOp); +// TODO(ebrevdo): Once rendezvous has been properly set up for +// Variants, we'll no longer need a HostMemory attribute for this case. +REGISTER_KERNEL_BUILDER(Name("ZerosLike") + .Device(DEVICE_GPU) + .TypeConstraint("T") + .HostMemory("x") + .HostMemory("y"), + ZerosLikeOp); #endif // GOOGLE_CUDA #undef REGISTER_KERNEL diff --git a/tensorflow/core/kernels/shape_op_test.cc b/tensorflow/core/kernels/shape_op_test.cc index a305598fe2b..96eaa4ac75b 100644 --- a/tensorflow/core/kernels/shape_op_test.cc +++ b/tensorflow/core/kernels/shape_op_test.cc @@ -101,7 +101,7 @@ TEST_F(ShapeOpTest, Simple) { Tensor variant_tensor(DT_VARIANT, TensorShape({1})); Status s = session.Run({{input, variant_tensor}}, {shape_output}, &outputs); EXPECT_FALSE(s.ok()); - ExpectHasError(s, "Shape of non-scalar Variant not supported."); + ExpectHasError(s, "Shape of non-unary Variant not supported."); } { diff --git a/tensorflow/core/kernels/shape_ops.h b/tensorflow/core/kernels/shape_ops.h index 0c39d46aeaf..ac607f4e8b8 100644 --- a/tensorflow/core/kernels/shape_ops.h +++ b/tensorflow/core/kernels/shape_ops.h @@ -35,7 +35,7 @@ inline Status GetRegularOrVariantShape(OpKernelContext* ctx, int input_index, if (ctx->input_dtype(0) == DT_VARIANT) { if (inp.dims() != 0) { return errors::InvalidArgument( - "Shape of non-scalar Variant not supported."); + "Shape of non-unary Variant not supported."); } TF_RETURN_IF_ERROR(GetUnaryVariantShape(inp, shape)); } else { diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py index df413939c76..6167cb9999b 100644 --- a/tensorflow/python/kernel_tests/constant_op_test.py +++ b/tensorflow/python/kernel_tests/constant_op_test.py @@ -32,6 +32,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import logging_ops from tensorflow.python.ops import math_ops from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -119,11 +120,11 @@ class ConstantTest(test.TestCase): variant_val=[ tensor_pb2.VariantTensorDataProto( # Match registration in variant_op_registry.cc - type_name=b"int32", + type_name=b"int", metadata=np.array(1, dtype=np.int32).tobytes()) ]) - const_op = constant_op.constant(variant_tensor).op - const_value = const_op.get_attr("value") + const = constant_op.constant(variant_tensor) + const_value = const.op.get_attr("value") # Ensure we stored the tensor proto properly. self.assertProtoEquals(variant_tensor, const_value) @@ -134,7 +135,10 @@ class ConstantTest(test.TestCase): # native numpy types cannot be passed to ops.convert_to_tensor. # TODO(ebrevdo): Add registration mechanism for # ops.convert_to_tensor and for session.run output. - const_op.run() + logging_const_op = logging_ops.Print( + const, [const], + message="Variant storing an int, decoded const value:").op + logging_const_op.run() def testStringWithNulls(self): with self.test_session(): @@ -469,6 +473,35 @@ class ZerosLikeTest(test.TestCase): self.assertEqual(y.shape, shape) self.assertAllEqual(y, np.zeros(shape, dtype=out_type)) + def testZerosLikeVariant(self): + # TODO(ebrevdo): Re-enable use_gpu=True once non-DMA Variant + # copying between CPU and GPU is supported AND we register a + # ZerosLike callback for GPU for Variant storing primitive types + # in variant_op_registry.cc. + with self.test_session(use_gpu=False): + variant_tensor = tensor_pb2.TensorProto( + dtype=dtypes_lib.variant.as_datatype_enum, + tensor_shape=tensor_shape.TensorShape([]).as_proto(), + variant_val=[ + tensor_pb2.VariantTensorDataProto( + # Match registration in variant_op_registry.cc + type_name=b"int", + metadata=np.array(1, dtype=np.int32).tobytes()) + ]) + const_variant = constant_op.constant(variant_tensor) + zeros_like = array_ops.zeros_like(const_variant) + zeros_like_op = logging_ops.Print( + zeros_like, [const_variant, zeros_like], + message="Variant storing an int, input and output of zeros_like:").op + + # Smoke test -- ensure this executes without trouble. + # Right now, non-numpy-compatible objects cannot be returned from a + # session.run call; similarly, objects that can't be converted to + # native numpy types cannot be passed to ops.convert_to_tensor. + # TODO(ebrevdo): Add registration mechanism for + # ops.convert_to_tensor and for session.run output. + zeros_like_op.run() + class OnesTest(test.TestCase): diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 2b9306e8748..33ba5df7a6e 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -1466,12 +1466,15 @@ def zeros_like(tensor, dtype=None, name=None, optimize=True): with ops.name_scope(name, "zeros_like", [tensor]) as name: tensor = ops.convert_to_tensor(tensor, name="tensor") - if tensor.shape.is_fully_defined(): + # For now, variant types must be created via zeros_like; as we need to + # pass the input variant object to the proper zeros callback. + + if tensor.shape.is_fully_defined() and tensor.dtype != dtypes.variant: # We can produce a zeros tensor independent of the value of 'tensor', # since the shape is known statically. return zeros(tensor.shape, dtype=dtype or tensor.dtype, name=name) - if dtype is not None and dtype != tensor.dtype: + if dtype is not None and dtype != tensor.dtype and dtype != dtypes.variant: return zeros( shape_internal(tensor, optimize=optimize), dtype=dtype, name=name) else: From 6d6118f14f74ec7d1ac3b4a9f0a7493624ef4caa Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Fri, 1 Sep 2017 13:41:13 -0700 Subject: [PATCH 47/67] Change global step to int64. PiperOrigin-RevId: 167318475 --- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 6748a765623..0578ae8b0f4 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -68,12 +68,11 @@ def _create_global_step(graph): return variable_scope.get_variable( ops.GraphKeys.GLOBAL_STEP, shape=[], - dtype=dtypes.int32, + dtype=dtypes.int64, initializer=init_ops.zeros_initializer(), trainable=False, use_resource=True, - collections=[ops.GraphKeys.GLOBAL_VARIABLES, - ops.GraphKeys.GLOBAL_STEP]) + collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP]) def _sync_variables_ops(): From b897ae084a9833f88ab5619bdf8940735b7d0de5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 13:41:24 -0700 Subject: [PATCH 48/67] Fixing typo. PiperOrigin-RevId: 167318502 --- tensorflow/python/ops/variable_scope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index 9093c12968f..645775239fd 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -1698,7 +1698,7 @@ def variable_scope(name_or_scope, use when doing asynchronous distributed training. Returns: - A scope that can be to captured and reused. + A scope that can be captured and reused. Raises: ValueError: when trying to reuse within a create scope, or create within From 2e7fdf03319725d3e5c62c20238f49dccb474dc5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 13:47:23 -0700 Subject: [PATCH 49/67] Fix convert_to_tensor problem for list of TensorNodes Current code fail to error out in convert_to_tensor when multiple TensorNodes are passed in. PiperOrigin-RevId: 167319274 --- tensorflow/python/eager/backprop_test.py | 14 +++++++++++++ tensorflow/python/framework/ops.py | 20 +++++++++++++------ .../python/kernel_tests/array_ops_test.py | 16 +++++++-------- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 429eeabb421..b4379055096 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -307,6 +307,20 @@ class BackpropTest(test.TestCase): [tensor_shape.TensorShape(s).as_proto() for s in shape_list], backprop.make_attr([pywrap_tensorflow.TF_ATTR_SHAPE], shape_list)) + def testMultiValueConvertToTensor(self): + x = resource_variable_ops.ResourceVariable( + initial_value=array_ops.constant([1.0]), name='x') + + def fn(): + tape.watch_variable(x) + a = math_ops.add(x.value(), 1.0) + # Make sure convert_to_tensor works correctly with list of TensorNodes. + b = array_ops.stack([a, a], axis=0) + return math_ops.reduce_mean(b) + + grad = backprop.implicit_grad(fn)()[0][1] + self.assertAllEqual([1.0], grad.numpy()) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 659bc394b92..b197e96886e 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -49,6 +49,7 @@ from tensorflow.python.framework import versions from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import compat from tensorflow.python.util import decorator_utils +from tensorflow.python.util import nest from tensorflow.python.util import tf_contextlib # Temporary global switch determining if we should enable the work-in-progress @@ -1036,12 +1037,19 @@ def internal_convert_to_tensor(value, # tracing gradients, to ensure the same behavior happens with and without # tracing. unwrapped = ag_core.getval(value) - # Fast path for EagerTensors that don't need any conversion. - if isinstance(unwrapped, EagerTensor) and context.in_eager_mode(): - # Note that we don't check that value's dtype matches the dtype - # argument. We exepct that the C runtime will do that checking - # when we execute the kernel. - return value + + if context.in_eager_mode(): + # Fast path for EagerTensors that don't need any conversion. + if isinstance(unwrapped, EagerTensor): + # Note that we don't check that value's dtype matches the dtype + # argument. We exepct that the C runtime will do that checking + # when we execute the kernel. + return value + values = nest.flatten(value) + if (len(values) > 1 and + any(isinstance(ag_core.getval(v), EagerTensor) for v in values)): + raise TypeError("Cannot convert to a eager tensor.") + if dtype is not None: dtype = dtypes.as_dtype(dtype) unwrapped_type = type(unwrapped) diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py index 392639fa179..77c5bb6d400 100644 --- a/tensorflow/python/kernel_tests/array_ops_test.py +++ b/tensorflow/python/kernel_tests/array_ops_test.py @@ -981,15 +981,15 @@ class SequenceMaskTest(test_util.TensorFlowTestCase): class ConcatSliceResourceTest(test_util.TensorFlowTestCase): + @test_util.run_in_graph_and_eager_modes() def testConcatSlice(self): - with self.test_session(): - r1 = test_ops.stub_resource_handle_op(container="a", shared_name="b") - r2 = test_ops.stub_resource_handle_op(container="a", shared_name="c") - c = array_ops.stack([r1, r2]) - s = array_ops.strided_slice(c, [1], [2]) - test_ops.resource_create_op(s).run() - with self.assertRaises(errors.AlreadyExistsError): - test_ops.resource_create_op(r2).run() + r1 = test_ops.stub_resource_handle_op(container="a", shared_name="b") + r2 = test_ops.stub_resource_handle_op(container="a", shared_name="c") + c = array_ops.stack([r1, r2]) + s = array_ops.strided_slice(c, [1], [2]) + self.evaluate(test_ops.resource_create_op(s)) + with self.assertRaises(errors.AlreadyExistsError): + self.evaluate(test_ops.resource_create_op(r2)) class IdentityTest(test_util.TensorFlowTestCase): From 390dcd29ac75ba59a3ff6ab45ab58f80d4094dc2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 13:54:05 -0700 Subject: [PATCH 50/67] Change bfloat constructor to accept a float to avoid truncation in implicit conversion from non-integer types to uint16_t. PiperOrigin-RevId: 167320150 --- tensorflow/core/framework/bfloat16_test.cc | 3 ++- tensorflow/core/framework/numeric_types.h | 9 ++++++++- tensorflow/core/kernels/cast_op.h | 9 +-------- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tensorflow/core/framework/bfloat16_test.cc b/tensorflow/core/framework/bfloat16_test.cc index 5bd95b806ff..af4e6a44116 100644 --- a/tensorflow/core/framework/bfloat16_test.cc +++ b/tensorflow/core/framework/bfloat16_test.cc @@ -23,7 +23,8 @@ namespace { TEST(Bfloat16Test, Simple) { bfloat16 a(12); - EXPECT_EQ(12, a.value); + // Floating point representation of 12: 0x41400000 + EXPECT_EQ(0x4140, a.value); } TEST(Bfloat16Test, Conversion) { diff --git a/tensorflow/core/framework/numeric_types.h b/tensorflow/core/framework/numeric_types.h index 31b88707e24..a630bee38d8 100644 --- a/tensorflow/core/framework/numeric_types.h +++ b/tensorflow/core/framework/numeric_types.h @@ -44,7 +44,14 @@ typedef Eigen::QUInt16 quint16; // see framework/bfloat16.h for description. struct bfloat16 { EIGEN_DEVICE_FUNC bfloat16() {} - EIGEN_DEVICE_FUNC explicit bfloat16(const uint16_t v) : value(v) {} + EIGEN_DEVICE_FUNC explicit bfloat16(const float v) { + const uint16_t* p = reinterpret_cast(&v); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + value = p[0]; +#else + value = p[1]; +#endif + } uint16_t value; }; diff --git a/tensorflow/core/kernels/cast_op.h b/tensorflow/core/kernels/cast_op.h index 5c24f164a41..59a991b5a8f 100644 --- a/tensorflow/core/kernels/cast_op.h +++ b/tensorflow/core/kernels/cast_op.h @@ -121,14 +121,7 @@ struct scalar_cast_op { typedef ::tensorflow::bfloat16 result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const ::tensorflow::bfloat16 operator()( const float a) const { -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - const uint16_t* p = reinterpret_cast(&a); - return ::tensorflow::bfloat16(p[0]); -#else - static_assert(::tensorflow::port::kLittleEndian, "Not a little endian system!"); - const uint16_t* p = reinterpret_cast(&a); - return ::tensorflow::bfloat16(p[1]); -#endif + return ::tensorflow::bfloat16(a); } }; From ef1286ea349d362e8a214f0e98951586791e0719 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 13:56:27 -0700 Subject: [PATCH 51/67] Internal change PiperOrigin-RevId: 167320434 --- tensorflow/core/BUILD | 10 ++++++++++ tensorflow/core/platform/default/build_config.bzl | 3 +++ 2 files changed, 13 insertions(+) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index cd083abb507..d01f7843eef 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -122,6 +122,7 @@ load( "tf_additional_gpu_tracer_cuda_deps", "tf_pyclif_proto_library", "tf_jspb_proto_library", + "tf_nano_proto_library", ) load( "//tensorflow/core:platform/default/build_config_root.bzl", @@ -212,6 +213,15 @@ tf_jspb_proto_library( deps = [":protos_all_cc"], ) +tf_nano_proto_library( + name = "protos_all_nano_proto", + field_style = "accessors", + generate_equals = 1, + generate_intdefs = 1, + visibility = ["//visibility:public"], + deps = [":protos_all_cc"], +) + exports_files([ "framework/types.proto", ]) diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 126558cac38..e1ad66c387a 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -75,6 +75,9 @@ def tf_proto_library_py(name, srcs=[], protodeps=[], deps=[], visibility=[], def tf_jspb_proto_library(**kwargs): pass +def tf_nano_proto_library(**kwargs): + pass + def tf_proto_library(name, srcs = [], has_services = None, protodeps = [], visibility = [], testonly = 0, cc_libs = [], From fe8905ed992e29467bcf69053dfce07b77f906d3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 14:42:46 -0700 Subject: [PATCH 52/67] Fix issue where some pooling op gradients on the CPU would fail when strides > ksize. RELNOTES: Enable support for strides > ksize for pooling operations. PiperOrigin-RevId: 167327007 --- tensorflow/core/kernels/ops_util.cc | 5 -- tensorflow/core/kernels/ops_util_test.cc | 38 ++++++++++-- .../kernel_tests/pooling_ops_3d_test.py | 36 ++++++++++++ .../python/kernel_tests/pooling_ops_test.py | 58 +++++++++++++++++++ 4 files changed, 126 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/kernels/ops_util.cc b/tensorflow/core/kernels/ops_util.cc index 130939263be..efacd05dd39 100644 --- a/tensorflow/core/kernels/ops_util.cc +++ b/tensorflow/core/kernels/ops_util.cc @@ -37,11 +37,6 @@ Eigen::PaddingType BrainPadding2EigenPadding(Padding padding) { Status GetBroadcastSize(const int index, const int in_size, const int ksize, const int stride, const int pad_size, int* bindex, int* bsize) { - // Cannot have strides larger than the patch size. - if (stride > ksize) { - return errors::InvalidArgument( - "stride must be less than or equal to kernel size"); - } // Cannot have index beyond the input size. if (index * stride > in_size) { return errors::InvalidArgument( diff --git a/tensorflow/core/kernels/ops_util_test.cc b/tensorflow/core/kernels/ops_util_test.cc index 42ffef6735b..9d53882deef 100644 --- a/tensorflow/core/kernels/ops_util_test.cc +++ b/tensorflow/core/kernels/ops_util_test.cc @@ -173,12 +173,6 @@ TEST_F(OpsUtilTest, Get2dOutputSizeVerbose) { VerifyGet2dOutputVerboseSizeValues(pad_struct2, error::OK); } -// Test stride > ksize fails with INVALID_ARGUMENT. -TEST_F(OpsUtilTest, GetBroadcastTest3_1_2_0) { - bcast_struct bcast = {{0, 3, 1, 2, 0}, {0, 3}}; - VerifyBoundaries(bcast, error::INVALID_ARGUMENT); -} - // Test index * stride > in_size fails with INVALID_ARGUMENT. TEST_F(OpsUtilTest, GetBroadcastTestBadIndex) { bcast_struct bcast = {{2, 3, 1, 2, 0}, {0, 3}}; @@ -281,6 +275,38 @@ TEST_F(OpsUtilTest, GetBroadcastTest3_3_3_2) { } } +// in_size = 3, ksize = 1, stride = 2, pad_size = 0 +TEST_F(OpsUtilTest, GetBroadcastTest3_1_2_0) { + bcast_struct bcast[] = { + {{0, 3, 1, 2, 0}, {0, 1}}, + {{1, 3, 1, 2, 0}, {2, 1}}, + }; + for (size_t i = 0; i < sizeof(bcast) / sizeof(bcast[0]); ++i) { + VerifyBcastValues(bcast[i]); + } +} + +// in_size = 3, ksize = 2, stride = 3, pad_size = 0 +TEST_F(OpsUtilTest, GetBroadcastTest3_2_3_0) { + bcast_struct bcast[] = { + {{0, 3, 2, 3, 0}, {0, 2}}, + }; + for (size_t i = 0; i < sizeof(bcast) / sizeof(bcast[0]); ++i) { + VerifyBcastValues(bcast[i]); + } +} + +// in_size = 3, ksize = 2, stride = 3, pad_size = 1 +TEST_F(OpsUtilTest, GetBroadcastTest3_2_3_1) { + bcast_struct bcast[] = { + {{0, 3, 2, 3, 1}, {0, 1}}, + {{1, 3, 2, 3, 1}, {2, 1}}, + }; + for (size_t i = 0; i < sizeof(bcast) / sizeof(bcast[0]); ++i) { + VerifyBcastValues(bcast[i]); + } +} + TEST_F(OpsUtilTest, SanitizeThreadSuffix) { EXPECT_EQ("_aBc123_-___", SanitizeThreadSuffix("/aBc123_- /")); } diff --git a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py index fa1553a3f6b..b01fc129538 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_3d_test.py @@ -321,6 +321,15 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="VALID") + def testMaxPoolGradValidPadding1_2_3d(self): + self._ConstructAndTestGradient( + nn_ops.max_pool3d, + input_sizes=[1, 3, 3, 3, 1], + output_sizes=[1, 2, 2, 2, 1], + window=(1, 1, 1), + strides=(2, 2, 2), + padding="VALID") + def testMaxPoolGradValidPadding2_2_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -339,6 +348,15 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="SAME") + def testMaxPoolGradSamePadding1_2_3d(self): + self._ConstructAndTestGradient( + nn_ops.max_pool3d, + input_sizes=[1, 3, 2, 4, 1], + output_sizes=[1, 2, 1, 2, 1], + window=(1, 1, 1), + strides=(2, 2, 2), + padding="SAME") + def testMaxPoolGradSamePadding2_1_3d(self): self._ConstructAndTestGradient( nn_ops.max_pool3d, @@ -375,6 +393,15 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="VALID") + def testAvgPoolGradValidPadding1_2_3d(self): + self._ConstructAndTestGradient( + nn_ops.avg_pool3d, + input_sizes=[1, 3, 3, 3, 1], + output_sizes=[1, 2, 2, 2, 1], + window=(1, 1, 1), + strides=(2, 2, 2), + padding="VALID") + def testAvgPoolGradValidPadding2_1_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, @@ -402,6 +429,15 @@ class PoolingTest(test.TestCase): strides=(1, 1, 1), padding="SAME") + def testAvgPoolGradSamePadding1_2_3d(self): + self._ConstructAndTestGradient( + nn_ops.avg_pool3d, + input_sizes=[1, 3, 2, 4, 2], + output_sizes=[1, 2, 1, 2, 2], + window=(1, 1, 1), + strides=(2, 2, 2), + padding="SAME") + def testAvgPoolGradSamePadding2_1_3d(self): self._ConstructAndTestGradient( nn_ops.avg_pool3d, diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py index da14871c872..9eb1fea8037 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_test.py @@ -998,6 +998,20 @@ class PoolingTest(test.TestCase): data_format=data_format, use_gpu=use_gpu) + def _testMaxPoolGradValidPadding1_2(self, data_format, use_gpu): + for pool_func in [gen_nn_ops._max_pool_v2, nn_ops.max_pool]: + self._ConstructAndTestGradient( + pool_func, + input_sizes=[1, 3, 3, 1], + output_sizes=[1, 2, 2, 1], + window_rows=1, + window_cols=1, + row_stride=2, + col_stride=2, + padding="VALID", + data_format=data_format, + use_gpu=use_gpu) + def _testMaxPoolGradValidPadding2_2(self, data_format, use_gpu): for pool_func in [gen_nn_ops._max_pool_v2, nn_ops.max_pool]: self._ConstructAndTestGradient( @@ -1026,6 +1040,20 @@ class PoolingTest(test.TestCase): data_format=data_format, use_gpu=use_gpu) + def _testMaxPoolGradSamePadding1_2(self, data_format, use_gpu): + for pool_func in [gen_nn_ops._max_pool_v2, nn_ops.max_pool]: + self._ConstructAndTestGradient( + pool_func, + input_sizes=[2, 2, 4, 3], + output_sizes=[2, 1, 2, 3], + window_rows=1, + window_cols=1, + row_stride=2, + col_stride=2, + padding="SAME", + data_format=data_format, + use_gpu=use_gpu) + def _testMaxPoolGradSamePadding2_1(self, data_format, use_gpu): for pool_func in [gen_nn_ops._max_pool_v2, nn_ops.max_pool]: self._ConstructAndTestGradient( @@ -1071,10 +1099,12 @@ class PoolingTest(test.TestCase): def testMaxPoolGrad(self): for (data_format, use_gpu) in GetTestConfigs(): self._testMaxPoolGradValidPadding1_1(data_format, use_gpu) + self._testMaxPoolGradValidPadding1_2(data_format, use_gpu) self._testMaxPoolGradValidPadding2_1_6(data_format, use_gpu) self._testMaxPoolGradValidPadding2_1_7(data_format, use_gpu) self._testMaxPoolGradValidPadding2_2(data_format, use_gpu) self._testMaxPoolGradSamePadding1_1(data_format, use_gpu) + self._testMaxPoolGradSamePadding1_2(data_format, use_gpu) self._testMaxPoolGradSamePadding2_1(data_format, use_gpu) self._testMaxPoolGradSamePadding2_2(data_format, use_gpu) self._testMaxPoolGradSamePadding3_1(data_format, use_gpu) @@ -1497,9 +1527,11 @@ class PoolingTest(test.TestCase): def testAvgPoolGrad(self): for (data_format, use_gpu) in GetTestConfigs(): self._testAvgPoolGradValidPadding1_1(data_format, use_gpu) + self._testAvgPoolGradValidPadding1_2(data_format, use_gpu) self._testAvgPoolGradValidPadding2_1(data_format, use_gpu) self._testAvgPoolGradValidPadding2_2(data_format, use_gpu) self._testAvgPoolGradSamePadding1_1(data_format, use_gpu) + self._testAvgPoolGradSamePadding1_2(data_format, use_gpu) self._testAvgPoolGradSamePadding2_1(data_format, use_gpu) self._testAvgPoolGradSamePadding2_2(data_format, use_gpu) self._testAvgPoolGradSamePadding3_1(data_format, use_gpu) @@ -1517,6 +1549,19 @@ class PoolingTest(test.TestCase): data_format=data_format, use_gpu=use_gpu) + def _testAvgPoolGradValidPadding1_2(self, data_format, use_gpu): + self._ConstructAndTestGradient( + nn_ops.avg_pool, + input_sizes=[2, 3, 3, 3], + output_sizes=[2, 2, 2, 3], + window_rows=1, + window_cols=1, + row_stride=2, + col_stride=2, + padding="VALID", + data_format=data_format, + use_gpu=use_gpu) + def _testAvgPoolGradValidPadding2_1(self, data_format, use_gpu): self._ConstructAndTestGradient( nn_ops.avg_pool, @@ -1556,6 +1601,19 @@ class PoolingTest(test.TestCase): data_format=data_format, use_gpu=use_gpu) + def _testAvgPoolGradSamePadding1_2(self, data_format, use_gpu): + self._ConstructAndTestGradient( + nn_ops.avg_pool, + input_sizes=[2, 2, 4, 3], + output_sizes=[2, 1, 2, 3], + window_rows=1, + window_cols=1, + row_stride=2, + col_stride=2, + padding="SAME", + data_format=data_format, + use_gpu=use_gpu) + def _testAvgPoolGradSamePadding2_1(self, data_format, use_gpu): self._ConstructAndTestGradient( nn_ops.avg_pool, From c75f0ffb80702c4937636ebb8df769d6f0897797 Mon Sep 17 00:00:00 2001 From: Mingxing Tan Date: Fri, 1 Sep 2017 14:59:36 -0700 Subject: [PATCH 53/67] Enable partial JPEG decompression. PiperOrigin-RevId: 167329034 --- tensorflow/core/lib/jpeg/jpeg_mem.cc | 229 +++++++++++++++--- tensorflow/core/lib/jpeg/jpeg_mem.h | 11 + tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc | 190 ++++++++++++++- 3 files changed, 390 insertions(+), 40 deletions(-) diff --git a/tensorflow/core/lib/jpeg/jpeg_mem.cc b/tensorflow/core/lib/jpeg/jpeg_mem.cc index 258793aa1e6..3c7e5ca696d 100644 --- a/tensorflow/core/lib/jpeg/jpeg_mem.cc +++ b/tensorflow/core/lib/jpeg/jpeg_mem.cc @@ -70,13 +70,24 @@ class FewerArgsForCompiler { int stride_; }; +// Check whether the crop window is valid, assuming crop is true. +bool IsCropWindowValid(const UncompressFlags& flags, int input_image_width, + int input_image_height) { + // Crop window is valid only if it is non zero and all the window region is + // within the original image. + return flags.crop_width > 0 && flags.crop_height > 0 && flags.crop_x >= 0 && + flags.crop_y >= 0 && + flags.crop_y + flags.crop_height <= input_image_height && + flags.crop_x + flags.crop_width <= input_image_width; +} + uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) { // unpack the argball const int datasize = argball->datasize_; const auto& flags = argball->flags_; const int ratio = flags.ratio; int components = flags.components; - int stride = flags.stride; // may be 0 + int stride = flags.stride; // may be 0 int64* const nwarn = argball->pnwarn_; // may be NULL // Can't decode if the ratio is not recognized by libjpeg @@ -159,8 +170,43 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) { return nullptr; } + JDIMENSION target_output_width = cinfo.output_width; + JDIMENSION target_output_height = cinfo.output_height; + JDIMENSION skipped_scanlines = 0; +#if !defined(WIN32) + if (flags.crop) { + // Update target output height and width based on crop window. + target_output_height = flags.crop_height; + target_output_width = flags.crop_width; + + // So far, cinfo holds the original input image information. + if (!IsCropWindowValid(flags, cinfo.output_width, cinfo.output_height)) { + LOG(ERROR) << "Invalid crop window: x=" << flags.crop_x + << ", y=" << flags.crop_y << ", w=" << target_output_width + << ", h=" << target_output_height + << " for image_width: " << cinfo.output_width + << " and image_height: " << cinfo.output_height; + jpeg_destroy_decompress(&cinfo); + return nullptr; + } + + // Update cinfo.output_width. It is tricky that cinfo.output_width must + // fall on an Minimum Coded Unit (MCU) boundary; if it doesn't, then it will + // be moved left to the nearest MCU boundary, and width will be increased + // accordingly. Therefore, the final cinfo.crop_width might differ from the + // given flags.crop_width. Please see libjpeg library for details. + JDIMENSION crop_width = flags.crop_width; + JDIMENSION crop_x = flags.crop_x; + jpeg_crop_scanline(&cinfo, &crop_x, &crop_width); + + // Update cinfo.output_scanline. + skipped_scanlines = jpeg_skip_scanlines(&cinfo, flags.crop_y); + CHECK_EQ(skipped_scanlines, flags.crop_y); + } +#endif + // check for compatible stride - const int min_stride = cinfo.output_width * components * sizeof(JSAMPLE); + const int min_stride = target_output_width * components * sizeof(JSAMPLE); if (stride == 0) { stride = min_stride; } else if (stride < min_stride) { @@ -170,47 +216,88 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) { } // Remember stride and height for use in Uncompress - argball->height_ = cinfo.output_height; + argball->height_ = target_output_height; argball->stride_ = stride; - uint8* const dstdata = argball->allocate_output_( - cinfo.output_width, cinfo.output_height, components); +#if defined(WIN32) + uint8* dstdata = nullptr; + if (flags.crop) { + dstdata = new JSAMPLE[stride * target_output_height]; + } else { + dstdata = argball->allocate_output_(target_output_width, + target_output_height, components); + } +#else + uint8* dstdata = argball->allocate_output_(target_output_width, + target_output_height, components); +#endif if (dstdata == nullptr) { jpeg_destroy_decompress(&cinfo); return nullptr; } JSAMPLE* output_line = static_cast(dstdata); - // Temporary buffer used for CMYK -> RGB conversion. + // jpeg_read_scanlines requires the buffers to be allocated based on + // cinfo.output_width, but the target image width might be different if crop + // is enabled and crop_width is not MCU aligned. In this case, we need to + // realign the scanline output to achieve the exact cropping. Notably, only + // cinfo.output_width needs to fall on MCU boundary, while cinfo.output_height + // has no such constraint. + const bool need_realign_cropped_scanline = + (target_output_width != cinfo.output_width); const bool use_cmyk = (cinfo.out_color_space == JCS_CMYK); - tempdata = use_cmyk ? new JSAMPLE[cinfo.output_width * 4] : nullptr; + + if (use_cmyk) { + // Temporary buffer used for CMYK -> RGB conversion. + tempdata = new JSAMPLE[cinfo.output_width * 4]; + } else if (need_realign_cropped_scanline) { + // Temporary buffer used for MCU-aligned scanline data. + tempdata = new JSAMPLE[cinfo.output_width * components]; + } // If there is an error reading a line, this aborts the reading. // Save the fraction of the image that has been read. - argball->height_read_ = cinfo.output_height; - while (cinfo.output_scanline < cinfo.output_height) { + argball->height_read_ = target_output_height; + + // These variables are just to avoid repeated computation in the loop. + const int max_scanlines_to_read = skipped_scanlines + target_output_height; + const int mcu_align_offset = + (cinfo.output_width - target_output_width) * (use_cmyk ? 4 : components); + while (cinfo.output_scanline < max_scanlines_to_read) { int num_lines_read = 0; - if (cinfo.out_color_space == JCS_CMYK) { + if (use_cmyk) { num_lines_read = jpeg_read_scanlines(&cinfo, &tempdata, 1); - // Convert CMYK to RGB - for (size_t i = 0; i < cinfo.output_width; ++i) { - int c = tempdata[4 * i + 0]; - int m = tempdata[4 * i + 1]; - int y = tempdata[4 * i + 2]; - int k = tempdata[4 * i + 3]; - int r, g, b; - if (cinfo.saw_Adobe_marker) { - r = (k * c) / 255; - g = (k * m) / 255; - b = (k * y) / 255; - } else { - r = (255 - k) * (255 - c) / 255; - g = (255 - k) * (255 - m) / 255; - b = (255 - k) * (255 - y) / 255; + if (num_lines_read > 0) { + // Convert CMYK to RGB if scanline read succeeded. + for (size_t i = 0; i < target_output_width; ++i) { + int offset = 4 * i; + if (need_realign_cropped_scanline) { + // Align the offset for MCU boundary. + offset += mcu_align_offset; + } + const int c = tempdata[offset + 0]; + const int m = tempdata[offset + 1]; + const int y = tempdata[offset + 2]; + const int k = tempdata[offset + 3]; + int r, g, b; + if (cinfo.saw_Adobe_marker) { + r = (k * c) / 255; + g = (k * m) / 255; + b = (k * y) / 255; + } else { + r = (255 - k) * (255 - c) / 255; + g = (255 - k) * (255 - m) / 255; + b = (255 - k) * (255 - y) / 255; + } + output_line[3 * i + 0] = r; + output_line[3 * i + 1] = g; + output_line[3 * i + 2] = b; } - output_line[3 * i + 0] = r; - output_line[3 * i + 1] = g; - output_line[3 * i + 2] = b; + } + } else if (need_realign_cropped_scanline) { + num_lines_read = jpeg_read_scanlines(&cinfo, &tempdata, 1); + if (num_lines_read > 0) { + memcpy(output_line, tempdata + mcu_align_offset, min_stride); } } else { num_lines_read = jpeg_read_scanlines(&cinfo, &output_line, 1); @@ -218,12 +305,13 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) { // Handle error cases if (num_lines_read == 0) { LOG(ERROR) << "Premature end of JPEG data. Stopped at line " - << cinfo.output_scanline << "/" << cinfo.output_height; + << cinfo.output_scanline - skipped_scanlines << "/" + << target_output_height; if (!flags.try_recover_truncated_jpeg) { - argball->height_read_ = cinfo.output_scanline; + argball->height_read_ = cinfo.output_scanline - skipped_scanlines; error = JPEGERRORS_UNEXPECTED_END_OF_DATA; } else { - for (size_t line = cinfo.output_scanline; line < cinfo.output_height; + for (size_t line = cinfo.output_scanline; line < max_scanlines_to_read; ++line) { if (line == 0) { // If even the first line is missing, fill with black color @@ -235,9 +323,9 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) { output_line += stride; } argball->height_read_ = - cinfo.output_height; // consider all lines as read + target_output_height; // consider all lines as read // prevent error-on-exit in libjpeg: - cinfo.output_scanline = cinfo.output_height; + cinfo.output_scanline = max_scanlines_to_read; } break; } @@ -248,23 +336,33 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) { delete[] tempdata; tempdata = nullptr; +#if !defined(WIN32) + if (flags.crop && cinfo.output_scanline < cinfo.output_height) { + // Skip the rest of scanlines, required by jpeg_destroy_decompress. + jpeg_skip_scanlines(&cinfo, + cinfo.output_height - flags.crop_y - flags.crop_height); + // After this, cinfo.output_height must be equal to cinfo.output_height; + // otherwise, jpeg_destroy_decompress would fail. + } +#endif + // Convert the RGB data to RGBA, with alpha set to 0xFF to indicate // opacity. // RGBRGBRGB... --> RGBARGBARGBA... if (components == 4) { // Start on the last line. JSAMPLE* scanlineptr = static_cast( - dstdata + static_cast(cinfo.output_height - 1) * stride); + dstdata + static_cast(target_output_height - 1) * stride); const JSAMPLE kOpaque = -1; // All ones appropriate for JSAMPLE. - const int right_rgb = (cinfo.output_width - 1) * 3; - const int right_rgba = (cinfo.output_width - 1) * 4; + const int right_rgb = (target_output_width - 1) * 3; + const int right_rgba = (target_output_width - 1) * 4; - for (int y = cinfo.output_height; y-- > 0;) { + for (int y = target_output_height; y-- > 0;) { // We do all the transformations in place, going backwards for each row. const JSAMPLE* rgb_pixel = scanlineptr + right_rgb; JSAMPLE* rgba_pixel = scanlineptr + right_rgba; scanlineptr -= stride; - for (int x = cinfo.output_width; x-- > 0; + for (int x = target_output_width; x-- > 0; rgba_pixel -= 4, rgb_pixel -= 3) { // We copy the 3 bytes at rgb_pixel into the 4 bytes at rgba_pixel // The "a" channel is set to be opaque. @@ -319,8 +417,61 @@ uint8* UncompressLow(const void* srcdata, FewerArgsForCompiler* argball) { LOG(ERROR) << "Unhandled case " << error; break; } - jpeg_destroy_decompress(&cinfo); +#if defined(WIN32) + // TODO(tanmingxing): delete all these code after migrating to libjpeg_turbo + // for Windows. + if (flags.crop) { + // Update target output height and width based on crop window. + target_output_height = flags.crop_height; + target_output_width = flags.crop_width; + + // cinfo holds the original input image information. + if (!IsCropWindowValid(flags, cinfo.output_width, cinfo.output_height)) { + LOG(ERROR) << "Invalid crop window: x=" << flags.crop_x + << ", y=" << flags.crop_y << ", w=" << target_output_width + << ", h=" << target_output_height + << " for image_width: " << cinfo.output_width + << " and image_height: " << cinfo.output_height; + delete[] dstdata; + jpeg_destroy_decompress(&cinfo); + return nullptr; + } + + const uint8* full_image = dstdata; + dstdata = argball->allocate_output_(target_output_width, + target_output_height, components); + if (dstdata == nullptr) { + delete[] full_image; + jpeg_destroy_decompress(&cinfo); + return nullptr; + } + + const int full_image_stride = stride; + // Update stride and hight for crop window. + const int min_stride = target_output_width * components * sizeof(JSAMPLE); + if (flags.stride == 0) { + stride = min_stride; + } + argball->height_ = target_output_height; + argball->stride_ = stride; + + if (argball->height_read_ > target_output_height) { + argball->height_read_ = target_output_height; + } + const int crop_offset = flags.crop_x * components * sizeof(JSAMPLE); + const uint8* full_image_ptr = full_image + flags.crop_y * full_image_stride; + uint8* crop_image_ptr = dstdata; + for (int i = 0; i < argball->height_read_; i++) { + memcpy(crop_image_ptr, full_image_ptr + crop_offset, min_stride); + crop_image_ptr += stride; + full_image_ptr += full_image_stride; + } + delete[] full_image; + } +#endif + + jpeg_destroy_decompress(&cinfo); return dstdata; } diff --git a/tensorflow/core/lib/jpeg/jpeg_mem.h b/tensorflow/core/lib/jpeg/jpeg_mem.h index ac34f29f221..59342d28c0f 100644 --- a/tensorflow/core/lib/jpeg/jpeg_mem.h +++ b/tensorflow/core/lib/jpeg/jpeg_mem.h @@ -61,6 +61,17 @@ struct UncompressFlags { // // Setting this has a quality/speed trade-off implication. J_DCT_METHOD dct_method = JDCT_DEFAULT; + + // Settings of crop window before decompression. + bool crop = false; + // Vertical coordinate of the top-left corner of the result in the input. + int crop_x = 0; + // Horizontal coordinate of the top-left corner of the result in the input. + int crop_y = 0; + // Width of the output image. + int crop_width = 0; + // Height of the output image. + int crop_height = 0; }; // Uncompress some raw JPEG data given by the pointer srcdata and the length diff --git a/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc b/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc index cc8646750e1..15266af1dbd 100644 --- a/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc +++ b/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc @@ -57,7 +57,7 @@ void ReadFileToStringOrDie(Env* env, const string& filename, string* output) { void TestJPEG(Env* env, const string& jpegfile) { // Read the data from the jpeg file into memory string jpeg; - ReadFileToStringOrDie(Env::Default(), jpegfile, &jpeg); + ReadFileToStringOrDie(env, jpegfile, &jpeg); const int fsize = jpeg.size(); const uint8* const temp = bit_cast(jpeg.data()); @@ -95,6 +95,194 @@ TEST(JpegMemTest, Jpeg) { TestJPEG(env, data_path + "jpeg_merge_test1_cmyk.jpg"); } +void TestCropAndDecodeJpeg(Env* env, const string& jpegfile, + const UncompressFlags& default_flags) { + // Read the data from the jpeg file into memory + string jpeg; + ReadFileToStringOrDie(env, jpegfile, &jpeg); + const int fsize = jpeg.size(); + auto temp = bit_cast(jpeg.data()); + + // Decode the whole image. + std::unique_ptr imgdata1; + int w1, h1, c1; + { + UncompressFlags flags = default_flags; + if (flags.stride == 0) { + imgdata1.reset(Uncompress(temp, fsize, flags, &w1, &h1, &c1, nullptr)); + } else { + // If stride is not zero, the default allocator would fail because it + // allocate w*h*c bytes, but the actual required bytes should be stride*h. + // Therefore, we provide a specialized allocator here. + uint8* buffer = nullptr; + imgdata1.reset(Uncompress(temp, fsize, flags, nullptr, + [&](int width, int height, int components) { + w1 = width; + h1 = height; + c1 = components; + buffer = new uint8[flags.stride * height]; + return buffer; + })); + } + ASSERT_NE(imgdata1, nullptr); + } + + auto check_crop_and_decode_func = [&](int crop_x, int crop_y, int crop_width, + int crop_height) { + std::unique_ptr imgdata2; + int w, h, c; + UncompressFlags flags = default_flags; + flags.crop = true; + flags.crop_x = crop_x; + flags.crop_y = crop_y; + flags.crop_width = crop_width; + flags.crop_height = crop_height; + if (flags.stride == 0) { + imgdata2.reset(Uncompress(temp, fsize, flags, &w, &h, &c, nullptr)); + } else { + uint8* buffer = nullptr; + imgdata2.reset(Uncompress(temp, fsize, flags, nullptr, + [&](int width, int height, int components) { + w = width; + h = height; + c = components; + buffer = new uint8[flags.stride * height]; + return buffer; + })); + } + ASSERT_NE(imgdata2, nullptr); + + ASSERT_EQ(w, crop_width); + ASSERT_EQ(h, crop_height); + ASSERT_EQ(c, c1); + + const int stride1 = (flags.stride != 0) ? flags.stride : w1 * c; + const int stride2 = (flags.stride != 0) ? flags.stride : w * c; + for (int i = 0; i < crop_height; i++) { + const uint8* p1 = &imgdata1[(i + crop_y) * stride1 + crop_x * c]; + const uint8* p2 = &imgdata2[i * stride2]; + + for (int j = 0; j < c * w; j++) { + ASSERT_EQ(p1[j], p2[j]) + << "p1 != p2 in [" << i << "][" << j / 3 << "][" << j % 3 << "]"; + } + } + }; + + // Check different crop windows. + check_crop_and_decode_func(0, 0, 5, 5); + check_crop_and_decode_func(0, 0, w1, 5); + check_crop_and_decode_func(0, 0, 5, h1); + check_crop_and_decode_func(0, 0, w1, h1); + check_crop_and_decode_func(w1 - 5, h1 - 6, 5, 6); + check_crop_and_decode_func(5, 6, 10, 15); +} + +TEST(JpegMemTest, CropAndDecodeJpeg) { + Env* env = Env::Default(); + const string data_path = kTestData; + UncompressFlags flags; + + // Test basic flags for jpeg and cmyk jpeg. + TestCropAndDecodeJpeg(env, data_path + "jpeg_merge_test1.jpg", flags); + TestCropAndDecodeJpeg(env, data_path + "jpeg_merge_test1_cmyk.jpg", flags); +} + +TEST(JpegMemTest, CropAndDecodeJpegWithRatio) { + Env* env = Env::Default(); + const string data_path = kTestData; + UncompressFlags flags; + for (int ratio : {1, 2, 4, 8}) { + flags.ratio = ratio; + TestCropAndDecodeJpeg(env, data_path + "jpeg_merge_test1.jpg", flags); + } +} + +TEST(JpegMemTest, CropAndDecodeJpegWithComponents) { + Env* env = Env::Default(); + const string data_path = kTestData; + UncompressFlags flags; + for (const int components : {0, 1, 3}) { + flags.components = components; + TestCropAndDecodeJpeg(env, data_path + "jpeg_merge_test1.jpg", flags); + } +} + +TEST(JpegMemTest, CropAndDecodeJpegWithUpScaling) { + Env* env = Env::Default(); + const string data_path = kTestData; + UncompressFlags flags; + flags.fancy_upscaling = true; + TestCropAndDecodeJpeg(env, data_path + "jpeg_merge_test1.jpg", flags); +} + +TEST(JpegMemTest, CropAndDecodeJpegWithStride) { + Env* env = Env::Default(); + const string data_path = kTestData; + + // Read the data from the jpeg file into memory + string jpeg; + ReadFileToStringOrDie(env, data_path + "jpeg_merge_test1.jpg", &jpeg); + const int fsize = jpeg.size(); + auto temp = bit_cast(jpeg.data()); + + int w, h, c; + ASSERT_TRUE(GetImageInfo(temp, fsize, &w, &h, &c)); + + // stride must be either 0 or > w*c; otherwise, uncompress would fail. + UncompressFlags flags; + flags.stride = w * c; + TestCropAndDecodeJpeg(env, data_path + "jpeg_merge_test1.jpg", flags); + flags.stride = w * c * 3; + TestCropAndDecodeJpeg(env, data_path + "jpeg_merge_test1.jpg", flags); + flags.stride = w * c + 100; + TestCropAndDecodeJpeg(env, data_path + "jpeg_merge_test1.jpg", flags); +} + +void CheckInvalidCropWindowFailed(const uint8* const temp, int fsize, int x, + int y, int w, int h) { + std::unique_ptr imgdata; + int ww, hh, cc; + UncompressFlags flags; + flags.components = 3; + flags.crop = true; + flags.crop_x = x; + flags.crop_y = y; + flags.crop_width = w; + flags.crop_height = h; + imgdata.reset(Uncompress(temp, fsize, flags, &ww, &hh, &cc, nullptr)); + CHECK(imgdata == nullptr); +} + +TEST(JpegMemTest, CropAndDecodeJpegWithInvalidCropWindow) { + Env* env = Env::Default(); + const string data_path = kTestData; + + // Read the data from the jpeg file into memory + string jpeg; + ReadFileToStringOrDie(env, data_path + "jpeg_merge_test1.jpg", &jpeg); + const int fsize = jpeg.size(); + auto temp = bit_cast(jpeg.data()); + + int w, h, c; + ASSERT_TRUE(GetImageInfo(temp, fsize, &w, &h, &c)); + + // Width and height for the crop window must be non zero. + CheckInvalidCropWindowFailed(temp, fsize, 11, 11, /*w=*/0, 11); + CheckInvalidCropWindowFailed(temp, fsize, 11, 11, 11, /*h=*/0); + + // Crop window must be non negative. + CheckInvalidCropWindowFailed(temp, fsize, /*x=*/-1, 11, 11, 11); + CheckInvalidCropWindowFailed(temp, fsize, 11, /*y=*/-1, 11, 11); + CheckInvalidCropWindowFailed(temp, fsize, 11, 11, /*w=*/-1, 11); + CheckInvalidCropWindowFailed(temp, fsize, 11, 11, 11, /*h=*/-1); + + // Invalid crop window width: x + crop_width = w + 1 > w + CheckInvalidCropWindowFailed(temp, fsize, /*x=*/w - 10, 11, 11, 11); + // Invalid crop window height: y + crop_height= h + 1 > h + CheckInvalidCropWindowFailed(temp, fsize, 11, /*y=*/h - 10, 11, 11); +} + TEST(JpegMemTest, Jpeg2) { // create known data, for size in_w x in_h const int in_w = 256; From 6af27b1e4906c60008c2202d2afb7f1ac209ae30 Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Fri, 1 Sep 2017 15:29:54 -0700 Subject: [PATCH 54/67] Revert breaking change PiperOrigin-RevId: 167332804 --- tensorflow/core/kernels/variable_ops.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/core/kernels/variable_ops.cc b/tensorflow/core/kernels/variable_ops.cc index b14e5551039..36b8ff09d73 100644 --- a/tensorflow/core/kernels/variable_ops.cc +++ b/tensorflow/core/kernels/variable_ops.cc @@ -83,7 +83,6 @@ TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL_KERNEL); IsVariableInitializedOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); -TF_CALL_bool(REGISTER_GPU_KERNELS) #undef REGISTER_GPU_KERNELS #endif // GOOGLE_CUDA From 42fcbb196052823c4393a4d5d8682ca425253f6a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 15:39:24 -0700 Subject: [PATCH 55/67] Automated g4 rollback of changelist 166396680 PiperOrigin-RevId: 167333886 --- tensorflow/contrib/cmake/tf_tests.cmake | 2 + tensorflow/core/kernels/BUILD | 9 +- tensorflow/core/kernels/l2loss_op.cc | 39 +- tensorflow/core/kernels/l2loss_op.h | 16 +- tensorflow/core/kernels/l2loss_op_gpu.cu.cc | 49 +- tensorflow/core/kernels/reduction_ops.h | 3 +- .../core/kernels/reduction_ops_common.h | 15 +- .../core/kernels/reduction_ops_gpu.cu.cc | 210 +++++- .../core/kernels/reduction_ops_gpu_kernels.h | 713 ++++++++++++++++++ tensorflow/core/kernels/reduction_ops_test.cc | 163 +++- .../core/util/permutation_input_iterator.h | 134 ++++ .../core/util/transform_output_iterator.h | 149 ++++ tensorflow/python/kernel_tests/BUILD | 20 + .../python/kernel_tests/cholesky_op_test.py | 13 +- .../python/kernel_tests/reduction_ops_test.py | 34 +- .../kernel_tests/reduction_ops_test_big.py | 179 +++++ 16 files changed, 1617 insertions(+), 131 deletions(-) create mode 100644 tensorflow/core/kernels/reduction_ops_gpu_kernels.h create mode 100644 tensorflow/core/util/permutation_input_iterator.h create mode 100644 tensorflow/core/util/transform_output_iterator.h create mode 100644 tensorflow/python/kernel_tests/reduction_ops_test_big.py diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake index 25f00de81dd..6507a9a5e07 100644 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ b/tensorflow/contrib/cmake/tf_tests.cmake @@ -289,6 +289,8 @@ if (tensorflow_BUILD_PYTHON_TESTS) # Failing with TF 1.3 (TODO) "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/estimator_test.py" "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_test.py" + # Test should only be run manually + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/reduction_ops_test_big.py" ) endif() list(REMOVE_ITEM tf_test_src_py ${tf_test_src_py_exclude}) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 6530ecf13fe..cb7e3b33165 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2581,8 +2581,9 @@ tf_kernel_library( tf_kernel_library( name = "reduction_ops", + srcs = ["reduction_ops_gpu_kernels.h"], prefix = "reduction_ops", - deps = MATH_DEPS, + deps = MATH_DEPS + if_cuda(["@cub_archive//:cub"]), ) tf_kernel_library( @@ -3064,14 +3065,16 @@ tf_kernel_library( tf_kernel_library( name = "l2loss_op", prefix = "l2loss_op", + #srcs = ["reduction_ops_gpu_kernels.h"], deps = [ + ":reduction_ops", + "//third_party/eigen3", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "//tensorflow/core:nn_grad", "//tensorflow/core:nn_ops_op_lib", - "//third_party/eigen3", - ], + ] + if_cuda(["@cub_archive//:cub"]), ) tf_cuda_cc_test( diff --git a/tensorflow/core/kernels/l2loss_op.cc b/tensorflow/core/kernels/l2loss_op.cc index 9875cd027d5..f8ed9351579 100644 --- a/tensorflow/core/kernels/l2loss_op.cc +++ b/tensorflow/core/kernels/l2loss_op.cc @@ -27,10 +27,9 @@ limitations under the License. namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; -typedef Eigen::GpuDevice GPUDevice; -template -class L2LossOp : public OpKernel { +template +class L2LossOp : public OpKernel { public: explicit L2LossOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -42,8 +41,9 @@ class L2LossOp : public OpKernel { Tensor* output = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape({}), &output)); - functor::L2Loss()(context->eigen_device(), - input.flat(), output->scalar()); + const CPUDevice& d = context->eigen_device(); + output->scalar().device(d) = + (input.flat().square() * static_cast(0.5)).sum(); } }; @@ -57,33 +57,4 @@ REGISTER_KERNEL(double); REGISTER_KERNEL(Eigen::half); #undef REGISTER_KERNEL -#if GOOGLE_CUDA -// Forward declarations of the functor specializations for GPU. -namespace functor { -#define DECLARE_GPU_SPEC(T) \ - template <> \ - void L2Loss::operator()(const GPUDevice& d, \ - typename TTypes::ConstTensor input, \ - typename TTypes::Scalar output); \ - extern template struct L2Loss; - -DECLARE_GPU_SPEC(float); -DECLARE_GPU_SPEC(double); -DECLARE_GPU_SPEC(Eigen::half); -#undef DECLARE_GPU_SPEC -} // namespace functor - -// Registration of the GPU implementations. -#define REGISTER_GPU_KERNEL(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("L2Loss").Device(DEVICE_GPU).TypeConstraint("T"), \ - L2LossOp); - -REGISTER_GPU_KERNEL(float); -REGISTER_GPU_KERNEL(double); -REGISTER_GPU_KERNEL(Eigen::half); -#undef REGISTER_GPU_KERNEL - -#endif // GOOGLE_CUDA - } // namespace tensorflow diff --git a/tensorflow/core/kernels/l2loss_op.h b/tensorflow/core/kernels/l2loss_op.h index f7204cefdd4..4953aa237cd 100644 --- a/tensorflow/core/kernels/l2loss_op.h +++ b/tensorflow/core/kernels/l2loss_op.h @@ -15,25 +15,19 @@ limitations under the License. #ifndef TENSORFLOW_KERNELS_L2LOSS_OP_H_ #define TENSORFLOW_KERNELS_L2LOSS_OP_H_ -// Functor definition for L2LossOp, must be compilable by nvcc. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" namespace tensorflow { -namespace functor { -// Functor used by L2LossOp to do the computations. template -struct L2Loss { - void operator()(const Device& d, typename TTypes::ConstTensor input, - typename TTypes::Scalar output) { - // We flatten the input tensor and reduce on dimension 0, producing - // a single number which is Mul(Sum(x^2), 0.5). - output.device(d) = (input.square() * static_cast(0.5)).sum(); - } +struct L2LossOp : public OpKernel { + explicit L2LossOp(OpKernelConstruction* context) : OpKernel(context) {} + + void Compute(OpKernelContext* context) {} }; -} // namespace functor } // namespace tensorflow #endif // TENSORFLOW_KERNELS_L2LOSS_OP_H_ diff --git a/tensorflow/core/kernels/l2loss_op_gpu.cu.cc b/tensorflow/core/kernels/l2loss_op_gpu.cu.cc index 420df370865..73b6472254c 100644 --- a/tensorflow/core/kernels/l2loss_op_gpu.cu.cc +++ b/tensorflow/core/kernels/l2loss_op_gpu.cu.cc @@ -21,12 +21,55 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/kernels/reduction_ops_common.h" +#include "tensorflow/core/kernels/reduction_ops_gpu_kernels.h" + namespace tensorflow { typedef Eigen::GpuDevice GPUDevice; -template struct functor::L2Loss; -template struct functor::L2Loss; -template struct functor::L2Loss; + +// TODO(eriche): can add specialization for half2 +template +struct squareHalf { + __host__ __device__ T operator()(const T& x) const { + return static_cast(0.5) * x * x; + } +}; + +template +class L2LossOp : public OpKernel { + public: + explicit L2LossOp(OpKernelConstruction* context) : OpKernel(context) {} + + void Compute(OpKernelContext* context) override { + // The input tensor can be of any number of dimensions, even though it's + // 2D in most typical applications. + const Tensor& input = context->input(0); + // The output is a single number. + Tensor* output = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output(0, TensorShape({}), &output)); + typedef cub::TransformInputIterator, T*> inputIterType; + inputIterType input_itr((T*)input.flat().data(), squareHalf()); + typedef const Eigen::array::Tensor::Index, 1>& ReductionAxes; + + Constants constants; + functor::ReduceImpl( + context, (T*)output->flat().data(), input_itr, 1, + input.flat().size(), 1, 1, 0, constants.kZero, cub::Sum(), T(0)); + } +}; + +// Registration of the GPU implementations. +#define REGISTER_GPU_KERNEL(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("L2Loss").Device(DEVICE_GPU).TypeConstraint("T"), \ + L2LossOp); + +REGISTER_GPU_KERNEL(float); +REGISTER_GPU_KERNEL(double); +REGISTER_GPU_KERNEL(Eigen::half); +#undef REGISTER_GPU_KERNEL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops.h b/tensorflow/core/kernels/reduction_ops.h index 5db9e6032e0..e43d2828f30 100644 --- a/tensorflow/core/kernels/reduction_ops.h +++ b/tensorflow/core/kernels/reduction_ops.h @@ -20,6 +20,7 @@ limitations under the License. #include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" namespace tensorflow { @@ -67,7 +68,7 @@ void FillIdentityEigenImpl(const Device& d, OUT_T out, const Reducer& reducer) { template struct ReduceFunctor { template - static void Reduce(const Device& d, OUT_T out, IN_T in, + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, const ReductionAxes& reduction_axes, const Reducer& reducer); diff --git a/tensorflow/core/kernels/reduction_ops_common.h b/tensorflow/core/kernels/reduction_ops_common.h index 553f8895232..71af9d88dc1 100644 --- a/tensorflow/core/kernels/reduction_ops_common.h +++ b/tensorflow/core/kernels/reduction_ops_common.h @@ -190,24 +190,24 @@ class ReductionOp : public OpKernel { Functor::FillIdentity(d, tmp_out.flat(), reducer); } else if ((helper.ndims() == 1) && helper.reduce_first_axis()) { // Reduce to a scalar. - Functor::Reduce(d, helper.out(&tmp_out), helper.in(data), + Functor::Reduce(ctx, helper.out(&tmp_out), helper.in(data), constants.kZero, reducer); } else if ((helper.ndims() == 2) && helper.reduce_first_axis()) { // Can be viewed as a reduction of a matrix along 1st dimension. - Functor::Reduce(d, helper.out(&tmp_out), helper.in(data), + Functor::Reduce(ctx, helper.out(&tmp_out), helper.in(data), constants.kZero, reducer); } else if ((helper.ndims() == 2) && !helper.reduce_first_axis()) { // Can be viewed as a reduction of a matrix along 2nd dimension. - Functor::Reduce(d, helper.out(&tmp_out), helper.in(data), + Functor::Reduce(ctx, helper.out(&tmp_out), helper.in(data), constants.kOne, reducer); } else if ((helper.ndims() == 3) && helper.reduce_first_axis()) { // Can be viewed as a reduction of a 3D tensor along 1st and 3rd // dimensions. - Functor::Reduce(d, helper.out(&tmp_out), helper.in(data), + Functor::Reduce(ctx, helper.out(&tmp_out), helper.in(data), constants.kZeroTwo, reducer); } else if ((helper.ndims() == 3) && !helper.reduce_first_axis()) { // Can be viewed as a reduction of a 3D tensor along 2nd dimension. - Functor::Reduce(d, helper.out(&tmp_out), helper.in(data), + Functor::Reduce(ctx, helper.out(&tmp_out), helper.in(data), constants.kOne, reducer); } else { // If we don't hit one of the cases above, transpose the data so that @@ -223,7 +223,7 @@ class ReductionOp : public OpKernel { const int64 unreduced = tmp_out.NumElements(); const int64 reduced = shuffled.NumElements() / unreduced; const Tensor& const_shuffled = shuffled; - Functor::Reduce(d, tmp_out.flat(), + Functor::Reduce(ctx, tmp_out.flat(), const_shuffled.shaped({unreduced, reduced}), constants.kOne, reducer); } @@ -258,9 +258,10 @@ namespace functor { template struct ReduceFunctorBase { template - static void Reduce(const Device& d, OUT_T out, IN_T in, + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, const ReductionAxes& reduction_axes, const Reducer& reducer) { + const Device& d = ctx->eigen_device(); ReduceEigenImpl(d, out, in, reduction_axes, reducer); } diff --git a/tensorflow/core/kernels/reduction_ops_gpu.cu.cc b/tensorflow/core/kernels/reduction_ops_gpu.cu.cc index ec4490db83f..8fd9165eb9f 100644 --- a/tensorflow/core/kernels/reduction_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/reduction_ops_gpu.cu.cc @@ -17,8 +17,7 @@ limitations under the License. #define EIGEN_USE_GPU -#include "tensorflow/core/framework/numeric_types.h" -#include "tensorflow/core/kernels/reduction_ops.h" +#include "tensorflow/core/kernels/reduction_ops_gpu_kernels.h" namespace tensorflow { namespace functor { @@ -33,15 +32,27 @@ typedef TTypes::Tensor::Index Index; template struct ReduceFunctor { template - static void Reduce(const GPUDevice& d, OUT_T out, IN_T in, + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, const ReductionAxes& reduction_axes, - const Reducer& reducer) { - ReduceEigenImpl(d, To32Bit(out), To32Bit(in), reduction_axes, reducer); + const Reducer& reducer); +}; + +template +struct ReduceFunctor> { + template + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, + const ReductionAxes& reduction_axes, + const Eigen::internal::SumReducer& reducer) { + ReduceImpl( + ctx, (T*)out.data(), (T*)in.data(), in.rank(), in.dimension(0), + in.rank() >= 2 ? in.dimension(1) : 1, + in.rank() >= 3 ? in.dimension(2) : 1, out.rank(), reduction_axes, + cub::Sum(), T(0)); } template static void FillIdentity(const GPUDevice& d, OUT_T out, - const Reducer& reducer) { + const Eigen::internal::SumReducer& reducer) { FillIdentityEigenImpl(d, To32Bit(out), reducer); } }; @@ -49,19 +60,30 @@ struct ReduceFunctor { template struct ReduceFunctor> { template - static void Reduce(const GPUDevice& d, OUT_T out, IN_T in, + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, const ReductionAxes& reduction_axes, const Eigen::internal::MeanReducer& reducer) { - typedef typename IN_T::Index Index; - // Eigen sum reductions are much faster on GPU than mean reductions: - // Simply trigger them by computing the sum of the weighted inputs. - Index num_coeffs_to_reduce = 1; - for (int i = 0; i < Eigen::internal::array_size::value; - ++i) { - num_coeffs_to_reduce *= in.dimension(reduction_axes[i]); - } - T scale = T(1.0 / num_coeffs_to_reduce); - out.device(d) = (in * scale).sum(reduction_axes); + int divisor = 1; + if (out.rank() == 0) + divisor = in.size(); + else if (out.rank() == 1 && in.rank() == 2 && reduction_axes[0] == 0) + divisor = in.dimension(0); + else if (out.rank() == 1 && in.rank() == 2 && reduction_axes[0] == 1) + divisor = in.dimension(1); + else if (out.rank() == 1 && in.rank() == 3 && reduction_axes[0] == 0 && + reduction_axes[1] == 2) + divisor = in.dimension(0) * in.dimension(2); + else if (out.rank() == 2 && in.rank() == 3 && reduction_axes[0] == 1) + divisor = in.dimension(1); + + DividesBy div_op(static_cast(divisor)); + TransformOutputIterator> itr((T*)out.data(), div_op); + ReduceImpl>, T*, + ReductionAxes>(ctx, itr, (T*)in.data(), in.rank(), + in.dimension(0), + in.rank() >= 2 ? in.dimension(1) : 1, + in.rank() >= 3 ? in.dimension(2) : 1, out.rank(), + reduction_axes, cub::Sum(), T(0)); } template @@ -71,15 +93,159 @@ struct ReduceFunctor> { } }; +template <> +struct ReduceFunctor> { + template + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, + const ReductionAxes& reduction_axes, + const Eigen::internal::MeanReducer& reducer) { + float divisor = 1.f; + if (out.rank() == 0) + divisor = in.size(); + else if (out.rank() == 1 && in.rank() == 2 && reduction_axes[0] == 0) + divisor = in.dimension(0); + else if (out.rank() == 1 && in.rank() == 2 && reduction_axes[0] == 1) + divisor = in.dimension(1); + else if (out.rank() == 1 && in.rank() == 3 && reduction_axes[0] == 0 && + reduction_axes[1] == 2) + divisor = in.dimension(0) * in.dimension(2); + else if (out.rank() == 2 && in.rank() == 3 && reduction_axes[0] == 1) + divisor = in.dimension(1); + DividesBy div_op(divisor); + + typedef cub::TransformInputIterator + inputIterType; + inputIterType input_itr((Eigen::half*)in.data(), HalfToFloat()); + + typedef TransformOutputIterator> + outputIterType; + outputIterType itr((Eigen::half*)out.data(), div_op); + + ReduceImpl( + ctx, itr, input_itr, in.rank(), in.dimension(0), + in.rank() >= 2 ? in.dimension(1) : 1, + in.rank() >= 3 ? in.dimension(2) : 1, out.rank(), reduction_axes, + cub::Sum(), 0.f); + } + + template + static void FillIdentity( + const GPUDevice& d, OUT_T out, + const Eigen::internal::MeanReducer& reducer) { + FillIdentityEigenImpl(d, To32Bit(out), reducer); + } +}; + +template +struct ReduceFunctor> { + template + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, + const ReductionAxes& reduction_axes, + const Eigen::internal::MaxReducer& reducer) { + ReduceImpl( + ctx, (T*)out.data(), (T*)in.data(), in.rank(), in.dimension(0), + in.rank() >= 2 ? in.dimension(1) : 1, + in.rank() >= 3 ? in.dimension(2) : 1, out.rank(), reduction_axes, + cub::Max(), std::numeric_limits::lowest()); + } + + template + static void FillIdentity(const GPUDevice& d, OUT_T out, + const Eigen::internal::MaxReducer& reducer) { + FillIdentityEigenImpl(d, To32Bit(out), reducer); + } +}; + +template +struct ReduceFunctor> { + template + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, + const ReductionAxes& reduction_axes, + const Eigen::internal::MinReducer& reducer) { + ReduceImpl( + ctx, (T*)out.data(), (T*)in.data(), in.rank(), in.dimension(0), + in.rank() >= 2 ? in.dimension(1) : 1, + in.rank() >= 3 ? in.dimension(2) : 1, out.rank(), reduction_axes, + cub::Min(), std::numeric_limits::max()); + } + + template + static void FillIdentity(const GPUDevice& d, OUT_T out, + const Eigen::internal::MinReducer& reducer) { + FillIdentityEigenImpl(d, To32Bit(out), reducer); + } +}; + +template +struct ReduceFunctor> { + template + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, + const ReductionAxes& reduction_axes, + const Eigen::internal::ProdReducer& reducer) { + ReduceImpl, T*, T*, ReductionAxes>( + ctx, (T*)out.data(), (T*)in.data(), in.rank(), in.dimension(0), + in.rank() >= 2 ? in.dimension(1) : 1, + in.rank() >= 3 ? in.dimension(2) : 1, out.rank(), reduction_axes, + Prod(), T(1)); + } + + template + static void FillIdentity(const GPUDevice& d, OUT_T out, + const Eigen::internal::ProdReducer& reducer) { + FillIdentityEigenImpl(d, To32Bit(out), reducer); + } +}; + +template <> +struct ReduceFunctor { + template + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, + const ReductionAxes& reduction_axes, + const Eigen::internal::AndReducer& reducer) { + ReduceImpl( + ctx, (bool*)out.data(), (bool*)in.data(), in.rank(), in.dimension(0), + in.rank() >= 2 ? in.dimension(1) : 1, + in.rank() >= 3 ? in.dimension(2) : 1, out.rank(), reduction_axes, And(), + true); + } + + template + static void FillIdentity(const GPUDevice& d, OUT_T out, + const Eigen::internal::AndReducer& reducer) { + FillIdentityEigenImpl(d, To32Bit(out), reducer); + } +}; + +template <> +struct ReduceFunctor { + template + static void Reduce(OpKernelContext* ctx, OUT_T out, IN_T in, + const ReductionAxes& reduction_axes, + const Eigen::internal::OrReducer& reducer) { + ReduceImpl( + ctx, (bool*)out.data(), (bool*)in.data(), in.rank(), in.dimension(0), + in.rank() >= 2 ? in.dimension(1) : 1, + in.rank() >= 3 ? in.dimension(2) : 1, out.rank(), reduction_axes, Or(), + false); + } + + template + static void FillIdentity(const GPUDevice& d, OUT_T out, + const Eigen::internal::OrReducer& reducer) { + FillIdentityEigenImpl(d, To32Bit(out), reducer); + } +}; + // T: the data type // REDUCER: the reducer functor // NUM_AXES: the number of axes to reduce // IN_DIMS: the number of dimensions of the input tensor -#define DEFINE(T, REDUCER, IN_DIMS, NUM_AXES) \ - template void ReduceFunctor::Reduce( \ - const GPUDevice& d, TTypes::Tensor out, \ - TTypes::ConstTensor in, \ - const Eigen::array& reduction_axes, \ +#define DEFINE(T, REDUCER, IN_DIMS, NUM_AXES) \ + template void ReduceFunctor::Reduce( \ + OpKernelContext* ctx, TTypes::Tensor out, \ + TTypes::ConstTensor in, \ + const Eigen::array& reduction_axes, \ const REDUCER& reducer); #define DEFINE_IDENTITY(T, REDUCER) \ diff --git a/tensorflow/core/kernels/reduction_ops_gpu_kernels.h b/tensorflow/core/kernels/reduction_ops_gpu_kernels.h new file mode 100644 index 00000000000..ce471c672c7 --- /dev/null +++ b/tensorflow/core/kernels/reduction_ops_gpu_kernels.h @@ -0,0 +1,713 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#define EIGEN_USE_GPU + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "external/cub_archive/cub/device/device_reduce.cuh" +#include "external/cub_archive/cub/device/device_segmented_reduce.cuh" +#include "external/cub_archive/cub/iterator/counting_input_iterator.cuh" +#include "external/cub_archive/cub/iterator/transform_input_iterator.cuh" +#include "external/cub_archive/cub/warp/warp_reduce.cuh" +#include "cuda/include/cuComplex.h" +#include "tensorflow/core/framework/numeric_types.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/kernels/reduction_ops.h" +#include "tensorflow/core/lib/core/bits.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_kernel_helper.h" +#include "tensorflow/core/util/permutation_input_iterator.h" +#include "tensorflow/core/util/transform_output_iterator.h" + +#include + +namespace tensorflow { +namespace functor { + +typedef Eigen::GpuDevice GPUDevice; + +template +struct Prod { + __host__ __device__ T operator()(const T& a, const T& b) const { + return a * b; + } +}; + +// needed to work around a compiler bug in nvcc - it doesn't seem to like +// the overloaded multiply op for std::complex +template <> +struct Prod> { + __host__ __device__ std::complex operator()( + const std::complex& a, const std::complex& b) const { + auto result = cuCmulf(make_cuComplex(a.real(), a.imag()), + make_cuComplex(b.real(), b.imag())); + return std::complex(result.x, result.y); + } +}; + +template <> +struct Prod> { + __host__ __device__ std::complex operator()( + const std::complex& a, const std::complex& b) const { + auto result = cuCmul(make_cuDoubleComplex(a.real(), a.imag()), + make_cuDoubleComplex(b.real(), b.imag())); + return std::complex(result.x, result.y); + } +}; + +template +struct DividesBy { + T divisor; + + __host__ __device__ explicit DividesBy(T divisor) : divisor(divisor) {} + + __host__ __device__ outT operator()(const T& x) const { return x / divisor; } +}; + +// needed to work around a compiler bug in nvcc - it doesn't seem to like +// the overloaded ops for std::complex +template <> +struct DividesBy> { + cuFloatComplex divisor; + + __host__ __device__ explicit DividesBy(std::complex divisor) + : divisor(make_cuComplex(divisor.real(), divisor.imag())) {} + + // implements + __host__ __device__ std::complex operator()( + const std::complex& x) const { + auto result = cuCdivf(make_cuComplex(x.real(), x.imag()), divisor); + return std::complex(result.x, result.y); + } +}; + +template <> +struct DividesBy> { + cuDoubleComplex divisor; + + __host__ __device__ explicit DividesBy(std::complex divisor) + : divisor(make_cuDoubleComplex(divisor.real(), divisor.imag())) {} + + // implements + __host__ __device__ std::complex operator()( + const std::complex& x) const { + auto result = cuCdiv(make_cuDoubleComplex(x.real(), x.imag()), divisor); + return std::complex(result.x, result.y); + } +}; + +template <> +struct DividesBy { + float divisor; + + __host__ __device__ explicit DividesBy(float divisor) : divisor(divisor) {} + + __host__ __device__ Eigen::half operator()(const float& x) const { + return Eigen::half(x / divisor); + } +}; + +struct HalfToFloat { + __host__ __device__ float operator()(const Eigen::half& x) const { + return Eigen::half_impl::half_to_float(x); + } +}; + +struct FloatToHalf { + __host__ __device__ Eigen::half operator()(const float& x) const { + return Eigen::half_impl::float_to_half_rtne(x); + } +}; + +struct And { + __host__ __device__ bool operator()(const bool& a, const bool& b) const { + return a && b; + } +}; + +struct Or { + __host__ __device__ bool operator()(const bool& a, const bool& b) const { + return a || b; + } +}; + +// each block does a grid strided loop and reduces its values locally +// the case of one block is used for low latency small reductions to scalars +template +__global__ void BlockReduceKernel( + T in, outT out, int num_elems, Op op, + typename std::iterator_traits::value_type initVal) { + const int bid = blockIdx.x; + const int tid = threadIdx.x; + + const int gid = bid * blockDim.x + tid; + const int stride = blockDim.x * gridDim.x; + + typedef typename std::iterator_traits::value_type value_type; + + value_type sum = initVal; + if (gid < num_elems) { + sum = in[gid]; + for (int pos = gid + stride; pos < num_elems; pos += stride) { + sum = op(sum, in[pos]); + } + } + + typedef cub::BlockReduce BlockReduce; + + __shared__ typename BlockReduce::TempStorage temp_storage; + + // only include input values in the reduction + // + // elements: ----------------- + // grid: |====|====|====|====|====| + const int num_elements_to_reduce = + max(min(num_elems - bid * blockDim.x, num_threads), 0); + + sum = BlockReduce(temp_storage) + .template Reduce(sum, op, num_elements_to_reduce); + + if (tid == 0) out[bid] = sum; +} + +// maps a warp to each row +template +__global__ void RowReduceKernel( + T in, outT out, int num_rows, int num_cols, Op op, + typename std::iterator_traits::value_type initVal) { + typedef typename std::iterator_traits::value_type value_type; + const int row = (blockIdx.x * blockDim.x + threadIdx.x) / 32; + const int lane = threadIdx.x % 32; + + if (num_cols == 1) { + int gid = threadIdx.x + blockIdx.x * blockDim.x; + if (gid < num_rows) out[gid] = in[gid]; + return; + } + + value_type sum = initVal; + int col = lane; + + if (row < num_rows && col < num_cols) { + sum = in[row * num_cols + col]; + col += 32; + for (; col < num_cols; col += 32) { + sum = op(sum, in[row * num_cols + col]); + } + } + + typedef cub::WarpReduce WarpReduce; + + __shared__ typename WarpReduce::TempStorage temp_storage; + + sum = WarpReduce(temp_storage).template Reduce(sum, op, min(num_cols, 32)); + + if (row < num_rows && lane == 0) out[row] = sum; +} + +// Works only if there are <= 16 columns +// each warps sums over multiple rows at once +template +__global__ void ColumnReduceMax16ColumnsKernel( + T in, outT out, int num_rows, int num_cols, Op op, + typename std::iterator_traits::value_type initVal) { + typedef typename std::iterator_traits::value_type value_type; + int rows_per_warp = 32 / num_cols; + + const int lane = threadIdx.x % 32; + const int lane_row = lane / num_cols; + + const int start_row_warp = + rows_per_warp * (blockIdx.y * blockDim.y + threadIdx.y); + const int start_row_lane = start_row_warp + lane_row; + int row = start_row_lane; + int col = lane % num_cols; + + value_type sum = initVal; + if (row * num_cols + col < num_rows * num_cols) + sum = in[row * num_cols + col]; + + __shared__ value_type partial_sums[32][33]; + + row += rows_per_warp * gridDim.y * blockDim.y; + for (; row < num_rows; row += rows_per_warp * gridDim.y * blockDim.y) { + int global_pos = row * num_cols + col; + if (global_pos < (num_rows * num_cols)) + sum = op(sum, in[row * num_cols + col]); + } + + const int rows_in_this_warp = min(rows_per_warp, num_rows - start_row_warp); + // not the most efficient way to do this sum + for (int i = 1; i < rows_in_this_warp; ++i) { + value_type tmp = + cub::ShuffleIndex(sum, threadIdx.x + i * num_cols, 32, 0xffffffff); + if (lane < num_cols) sum = op(sum, tmp); + } + + if (lane < num_cols) partial_sums[lane][threadIdx.y] = sum; + + __syncthreads(); + + if (threadIdx.y == 0 && threadIdx.x < num_cols) { + value_type s = partial_sums[threadIdx.x][0]; + + if (blockDim.y > 1) { + for (int row = 1; row < blockDim.y; ++row) { + s = op(s, partial_sums[threadIdx.x][row]); + } + } + + out[col * gridDim.y + blockIdx.y] = s; + } +} + +// Maps each block to a column range 32 wide +template +__global__ void ColumnReduceKernel( + T in, outT out, int num_rows, int num_cols, Op op, + typename std::iterator_traits::value_type initVal) { + typedef typename std::iterator_traits::value_type value_type; + int row = blockIdx.y * blockDim.y + threadIdx.y; + int col = blockIdx.x * 32 + threadIdx.x; + + value_type sum = initVal; + if (row < num_rows && col < num_cols) + sum = in[row * num_cols + col]; + + __shared__ value_type partial_sums[32][33]; + + row += gridDim.y * blockDim.y; + + if (col < num_cols) { + for (; row < num_rows; row += gridDim.y * blockDim.y) { + sum = op(sum, in[row * num_cols + col]); + } + } + + partial_sums[threadIdx.x][threadIdx.y] = sum; + + __syncthreads(); + + if (threadIdx.y == 0 && col < num_cols) { + value_type s = partial_sums[threadIdx.x][0]; + + // only include input values in the reduction + // elem block_rows + // - = + // - = + // # # block boundary + // - = + // - = + // # # block boundary + // - = + // = + const int numRowsThisBlock = + min(blockDim.y, num_rows - blockIdx.y * blockDim.y); + + for (int row = 1; row < numRowsThisBlock; ++row) { + s = op(s, partial_sums[threadIdx.x][row]); + } + + out[col * gridDim.y + blockIdx.y] = s; + } +} + +// does multiple warp size segmented reductions in parallel +// segments cannot cross warp boundaries (mainly used for reducing the segments +// that come from the Max16Columns column reduction kernel) +template +__global__ void CleanupSegments( + T partial_sums, outT out, int num_rows, int num_cols, int segment_size, + Op op, typename std::iterator_traits::value_type initVal) { + typedef typename std::iterator_traits::value_type value_type; + const int tid = threadIdx.x + blockIdx.x * blockDim.x; + + value_type val = initVal; + if (tid < segment_size * num_cols) + val = partial_sums[tid]; + + typedef cub::WarpReduce WarpReduce; + + __shared__ typename WarpReduce::TempStorage temp_storage; + + const bool head_flag = (threadIdx.x % segment_size) == 0; + value_type sum = + WarpReduce(temp_storage).HeadSegmentedReduce(val, head_flag, op); + + if (head_flag && tid < segment_size * num_cols) { + out[tid / segment_size] = sum; + } +} + +// assigns one thread to a column +template +__global__ void ColumnReduceSimpleKernel(T in, outT out, int num_planes, + int num_rows, int num_cols, Op op) { + typedef typename std::iterator_traits::value_type value_type; + const int gid = threadIdx.x + blockIdx.x * blockDim.x; + const int elems_per_plane = num_rows * num_cols; + + const int plane = gid / num_cols; + const int col = gid % num_cols; + + if (plane >= num_planes) return; + + if (num_rows == 1) { + out[plane * elems_per_plane + col] = in[plane * elems_per_plane + col]; + return; + } + + value_type sum = op(in[plane * elems_per_plane + col], + in[plane * elems_per_plane + num_cols + col]); + for (int row = 2; row < num_rows; ++row) { + sum = op(sum, in[plane * elems_per_plane + row * num_cols + col]); + } + + out[plane * num_cols + col] = sum; +} + +struct RowOffset { + __host__ __device__ explicit RowOffset(const int& cols) : cols_(cols) {} + + __host__ __device__ int operator()(const int& x) const { return cols_ * x; } + + int cols_; +}; + +struct GatherOp { + __host__ __device__ GatherOp(const int& extent_x, const int& extent_y, + const int& extent_z, bool kOne) + : extent_x_(extent_x), + extent_y_(extent_y), + extent_z_(extent_z), + kOne_(kOne) { + if (kOne_) + group_size_ = extent_y_; + else + group_size_ = extent_x_ * extent_z_; + } + + __host__ __device__ int operator()(const int& ind) const { + const int group = kOne_ ? ind / group_size_ : ind % group_size_; + const int offset = kOne_ ? ind % group_size_ : ind / group_size_; + + const int x = group / extent_z_; + const int z = group % extent_z_; + + return x * extent_y_ * extent_z_ + z + offset * extent_z_; + } + + int extent_x_; + int extent_y_; + int extent_z_; + bool kOne_; + int group_size_; +}; + +template +void LaunchScalarReduction(OpKernelContext* ctx, OUT_T out, IN_T in, + int in_size, Op op, T init, + const cudaStream_t& cu_stream) { + // handle situations where low latency is important better than CUB + if (in_size <= 4096) { + const int num_blocks = 1; + const int num_threads = 256; + BlockReduceKernel + <<>>(in, out, in_size, op, init); + return; + } else if (in_size <= 1 << 19) { + const int num_threads = 256; + const int num_blocks = min(32, Eigen::divup(in_size, num_threads)); + // it seems like tailoring this to the GPU + // would be more effective, but all attempts + // at making this a multiple of the number of + // multiprocessors have lead to lower perf + // in general + // TODO(eriche) investigate this more + + Tensor temp_storage; + OP_REQUIRES_OK( + ctx, + ctx->allocate_temp( + DT_INT8, TensorShape({static_cast(num_blocks * sizeof(T))}), + &temp_storage)); + + BlockReduceKernel + <<>>( + in, (T*)temp_storage.flat().data(), in_size, op, init); + + // take care that we only reduce blocks that had some valid elements in them + // TODO(eriche): CUB currently has a bug in HeadSegmentedReduce that + // requires it to be used with a full warp. Can reduce 32 -> num_blocks + // when this is fixed. + CleanupSegments<<<1, 32, 0, cu_stream>>>( + (T*)temp_storage.flat().data(), out, 1, 1, num_blocks, op, + init); + return; + } + std::size_t temp_storage_bytes = 0; + + Tensor temp_storage; + // written as a loop because it reduces clutter + // first pass allocates memory, second launches kernel(s) + for (int i = 0; i < 2; ++i) { + auto success = cub::DeviceReduce::Reduce( + i == 0 ? nullptr : temp_storage.flat().data(), + temp_storage_bytes, in, out, in_size, op, init, cu_stream); + + OP_REQUIRES( + ctx, success == 0, + errors::Internal("CUB reduce error", cudaGetErrorString(success))); + + if (i == 0) + OP_REQUIRES_OK( + ctx, + ctx->allocate_temp( + DT_INT8, TensorShape({static_cast(temp_storage_bytes)}), + &temp_storage)); + } +} + +template +void LaunchRowReduction(OpKernelContext* ctx, OUT_T out, IN_T in, int num_rows, + int num_cols, Op op, T init, + const cudaStream_t& cu_stream) { + if (num_cols < 1024) { + const int threads_per_block = 128; + const int warps_per_block = threads_per_block / 32; + int num_blocks = (num_rows + warps_per_block - 1) / warps_per_block; + + RowReduceKernel<<>>( + in, out, num_rows, num_cols, op, init); + return; + } + + // setup segment offsets with counting and transform iterator + RowOffset row_offset_op(num_cols); + cub::CountingInputIterator counting_iter(0); + cub::TransformInputIterator> + transform_iter(counting_iter, row_offset_op); + + std::size_t temp_storage_bytes = 0; + Tensor temp_storage; + for (int i = 0; i < 2; ++i) { + auto success = cub::DeviceSegmentedReduce::Reduce( + i == 0 ? nullptr : temp_storage.flat().data(), + temp_storage_bytes, in, out, num_rows, transform_iter, + transform_iter + 1, op, init, cu_stream); + + OP_REQUIRES(ctx, success == 0, + errors::Internal("CUB segmented reduce error", + cudaGetErrorString(success))); + + if (i == 0) + OP_REQUIRES_OK( + ctx, + ctx->allocate_temp( + DT_INT8, TensorShape({static_cast(temp_storage_bytes)}), + &temp_storage)); + } +} + +template +void LaunchColumnReduction_LTE16Cols(OpKernelContext* ctx, OUT_T out, IN_T in, + int extent_x, int extent_y, Op op, T init, + const cudaStream_t& cu_stream) { + int rows_per_warp = 32 / extent_y; + dim3 block_dim(32, min(Eigen::divup(extent_x, rows_per_warp), 32), 1); + dim3 grid_dim(1, + Eigen::divup(static_cast(extent_x), + rows_per_warp * block_dim.y), + 1); + + grid_dim.y = min((int)grid_dim.y, 32); + + if (grid_dim.y > 2 && grid_dim.y < 32) { + int log2 = Log2Floor(grid_dim.y); + grid_dim.y = 1 << log2; + } + + if (grid_dim.y == 1) { + ColumnReduceMax16ColumnsKernel<<>>( + in, out, extent_x, extent_y, op, init); + } else { + Tensor temp_storage; + OP_REQUIRES_OK(ctx, + ctx->allocate_temp(DT_INT8, + TensorShape({static_cast( + sizeof(T) * extent_y * grid_dim.y)}), + &temp_storage)); + ColumnReduceMax16ColumnsKernel<<>>( + in, (T*)temp_storage.flat().data(), extent_x, extent_y, op, + init); + + dim3 new_grid_dim((grid_dim.y * extent_y + 31) / 32, 1, 1); + dim3 num_threads(128, 1, 1); + CleanupSegments<<>>( + (T*)temp_storage.flat().data(), out, extent_x, extent_y, + grid_dim.y, op, init); + } +} + +template +void LaunchColumnReduction_LTE4096Cols(OpKernelContext* ctx, OUT_T out, IN_T in, + int extent_x, int extent_y, Op op, + T init, const cudaStream_t& cu_stream) { + dim3 block_dim(32, min(extent_x, 32), 1); + dim3 grid_dim((extent_y + 31) / 32, 1, 1); + + if (grid_dim.x < 16) grid_dim.y = min((extent_x + 31) / 32, 32); + + if (grid_dim.y > 2 && grid_dim.y < 32) { + int log2 = Log2Floor(grid_dim.y); + grid_dim.y = 1 << log2; + } + + if (grid_dim.y == 1) { + ColumnReduceKernel<<>>( + in, out, extent_x, extent_y, op, init); + } else { + Tensor temp_storage; + OP_REQUIRES_OK(ctx, + ctx->allocate_temp(DT_INT8, + TensorShape({static_cast( + sizeof(T) * extent_y * grid_dim.y)}), + &temp_storage)); + + ColumnReduceKernel<<>>( + in, (T*)temp_storage.flat().data(), extent_x, extent_y, op, + init); + + dim3 new_grid_dim((grid_dim.y * extent_y + 31) / 32, 1, 1); + dim3 num_threads(128, 1, 1); + CleanupSegments<<>>( + (T*)temp_storage.flat().data(), out, extent_x, extent_y, + grid_dim.y, op, init); + } +} + +template +void LaunchColumnReduction(OpKernelContext* ctx, OUT_T out, IN_T in, + int extent_x, int extent_y, Op op, T init, + const cudaStream_t& cu_stream) { + if (extent_y <= 16) { + LaunchColumnReduction_LTE16Cols(ctx, out, in, extent_x, extent_y, op, init, + cu_stream); + } else if (extent_y <= 4096) { + LaunchColumnReduction_LTE4096Cols(ctx, out, in, extent_x, extent_y, op, + init, cu_stream); + } else { + int threads_per_block = 128; + int num_blocks = Eigen::divup(extent_y, threads_per_block); + + ColumnReduceSimpleKernel<<>>( + in, out, 1, extent_x, extent_y, op); + } +} + +template +void Launch3DYReduction(OpKernelContext* ctx, OUT_T out, IN_T in, int extent_x, + int extent_y, int extent_z, Op op, T init, + const cudaStream_t& cu_stream) { + int threads_per_block = 128; + int num_blocks = + (extent_x * extent_z + threads_per_block - 1) / threads_per_block; + + // TODO(eriche): this won't be very good in the case of small x + // small z and large y. + ColumnReduceSimpleKernel<<>>( + in, out, extent_x, extent_y, extent_z, op); +} + +template +void Launch3DXZReduction(OpKernelContext* ctx, OUT_T out, IN_T in, int extent_x, + int extent_y, int extent_z, Op op, T init, + const cudaStream_t& cu_stream) { + // setup segment offsets with counting and transform iterator + RowOffset row_offset_op(extent_x * extent_z); + cub::CountingInputIterator counting_iter(0); + cub::TransformInputIterator> + transform_iter(counting_iter, row_offset_op); + + GatherOp gather_op(extent_x, extent_y, extent_z, false); + typedef cub::TransformInputIterator> + gatherIterType; + gatherIterType gather_iter(counting_iter, gather_op); + + PermutationInputIterator permute_iter(in, + gather_iter); + + std::size_t temp_storage_bytes = 0; + Tensor temp_storage; + + for (int i = 0; i < 2; ++i) { + auto success = cub::DeviceSegmentedReduce::Reduce( + i == 0 ? nullptr : temp_storage.flat().data(), + temp_storage_bytes, permute_iter, out, extent_y, transform_iter, + transform_iter + 1, op, init, cu_stream); + + OP_REQUIRES(ctx, success == 0, + errors::Internal("CUB segmented reduce error", + cudaGetErrorString(success))); + + if (i == 0) + OP_REQUIRES_OK( + ctx, + ctx->allocate_temp( + DT_INT8, TensorShape({static_cast(temp_storage_bytes)}), + &temp_storage)); + } +} + +template +void ReduceImpl(OpKernelContext* ctx, OUT_T out, IN_T in, int in_rank, + int in_dim0, int in_dim1, int in_dim2, int out_rank, + const ReductionAxes& reduction_axes, Op op, T init) { + const cudaStream_t& cu_stream = GetCudaStream(ctx); + if (out_rank == 0) { + const int in_size = in_dim0 * in_dim1 * in_dim2; + LaunchScalarReduction(ctx, out, in, in_size, op, init, cu_stream); + } else if (in_rank == 2 && out_rank == 1 && + reduction_axes[0] == 1) { // row reduction + LaunchRowReduction(ctx, out, in, in_dim0, in_dim1, op, init, cu_stream); + } else if (in_rank == 2 && out_rank == 1 && + reduction_axes[0] == 0) { // column reduction + LaunchColumnReduction(ctx, out, in, in_dim0, in_dim1, op, init, cu_stream); + } else if (in_rank == 3 && out_rank == 2 && reduction_axes[0] == 1) { + Launch3DYReduction(ctx, out, in, in_dim0, in_dim1, in_dim2, op, init, + cu_stream); + } else if (in_rank == 3 && out_rank == 1 && reduction_axes[0] == 0 && + reduction_axes[1] == 2) { + Launch3DXZReduction(ctx, out, in, in_dim0, in_dim1, in_dim2, op, init, + cu_stream); + } else { + std::stringstream ss; + ss << "Invalid reduction requested: in_rank, out_rank, axes " << in_rank + << " " << out_rank; + if (out_rank == 1) ss << " " << reduction_axes[0]; + if (out_rank == 2) ss << " " << reduction_axes[1]; + LOG(FATAL) << ss.str(); + } +} + +} // namespace functor +} // namespace tensorflow + +#endif diff --git a/tensorflow/core/kernels/reduction_ops_test.cc b/tensorflow/core/kernels/reduction_ops_test.cc index 9cdebdd4f23..9bbe993a2f9 100644 --- a/tensorflow/core/kernels/reduction_ops_test.cc +++ b/tensorflow/core/kernels/reduction_ops_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" @@ -22,14 +23,59 @@ namespace tensorflow { // Creates a Graph which "reduce"s a 3D float tensor of "num" elements // into a scalar. -static Graph* ToScalar(const string& reduce, int num) { - Graph* g = new Graph(OpRegistry::Global()); - Tensor data(DT_FLOAT, TensorShape({64, 64, num / (64 * 64)})); - data.flat().setRandom(); - Tensor axes(DT_INT32, TensorShape({3})); +template +static Graph* ToScalar(const string& reduce, int num_x, int num_y) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DataTypeToEnum::value, TensorShape({num_x, num_y})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({2})); axes.flat()(0) = 0; axes.flat()(1) = 1; - axes.flat()(2) = 2; + test::graph::Reduce(g, reduce, test::graph::Constant(g, data), + test::graph::Constant(g, axes)); + return g; +} + +static Graph* ColReduce(const string& reduce, int num_x, int num_y) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DT_FLOAT, TensorShape({num_x, num_y})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({1})); + axes.flat()(0) = 0; + test::graph::Reduce(g, reduce, test::graph::Constant(g, data), + test::graph::Constant(g, axes)); + return g; +} + +static Graph* RowReduce(const string& reduce, int num_x, int num_y) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DT_FLOAT, TensorShape({num_x, num_y})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({1})); + axes.flat()(0) = 1; + test::graph::Reduce(g, reduce, test::graph::Constant(g, data), + test::graph::Constant(g, axes)); + return g; +} + +static Graph* ThreeDYReduce(const string& reduce, int num_y, int num_z) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DT_FLOAT, TensorShape({4, num_y, num_z})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({1})); + axes.flat()(0) = 1; + test::graph::Reduce(g, reduce, test::graph::Constant(g, data), + test::graph::Constant(g, axes)); + return g; +} + +static Graph* ThreeDXZReduce(const string& reduce, int num_y, int num_z) { + auto* g = new Graph(OpRegistry::Global()); + Tensor data(DT_FLOAT, TensorShape({4, num_y, num_z})); + data.flat().setRandom(); + Tensor axes(DT_INT32, TensorShape({2})); + axes.flat()(0) = 0; + axes.flat()(1) = 2; test::graph::Reduce(g, reduce, test::graph::Constant(g, data), test::graph::Constant(g, axes)); return g; @@ -37,51 +83,100 @@ static Graph* ToScalar(const string& reduce, int num) { // Creates a bench which reduces a 3D tensor with total "num" floats // into a scalar on a "device". Runs the bench for "iters" times. +template static void ReduceToScalar(int iters, const string& device, - const string& reduce, int num) { - testing::ItemsProcessed(static_cast(iters) * num); - testing::BytesProcessed(static_cast(iters) * num * sizeof(float)); - test::Benchmark(device, ToScalar(reduce, num)).Run(iters); + const string& reduce, int num_x, int num_y) { + testing::ItemsProcessed(static_cast(iters) * num_x * num_y); + testing::BytesProcessed(static_cast(iters) * num_x * num_y * + sizeof(T)); + test::Benchmark(device, ToScalar(reduce, num_x, num_y)).Run(iters); } -static void BM_Sum3DToScalarCPU(int iters, int num) { - ReduceToScalar(iters, "cpu", "Sum", num); +static void DoRowReduce(int iters, const string& device, const string& reduce, + int num_x, int num_y) { + testing::ItemsProcessed(static_cast(iters) * num_x * num_y); + testing::BytesProcessed(static_cast(iters) * num_x * num_y * + sizeof(float)); + test::Benchmark(device, RowReduce(reduce, num_x, num_y)).Run(iters); } -BENCHMARK(BM_Sum3DToScalarCPU)->Range(1 << 13, 1 << 20); -static void BM_Max3DToScalarCPU(int iters, int num) { - ReduceToScalar(iters, "cpu", "Max", num); +static void DoColReduce(int iters, const string& device, const string& reduce, + int num_x, int num_y) { + testing::ItemsProcessed(static_cast(iters) * num_x * num_y); + testing::BytesProcessed(static_cast(iters) * num_x * num_y * + sizeof(float)); + test::Benchmark(device, ColReduce(reduce, num_x, num_y)).Run(iters); } -BENCHMARK(BM_Max3DToScalarCPU)->Range(1 << 13, 1 << 20); -static void BM_Prod3DToScalarCPU(int iters, int num) { - ReduceToScalar(iters, "cpu", "Prod", num); +static void Do3DYReduce(int iters, const string& device, const string& reduce, + int num_x, int num_y) { + testing::ItemsProcessed(static_cast(iters) * num_x * num_y); + testing::BytesProcessed(static_cast(iters) * num_x * num_y * + sizeof(float)); + test::Benchmark(device, ThreeDYReduce(reduce, num_x, num_y)).Run(iters); } -BENCHMARK(BM_Prod3DToScalarCPU)->Range(1 << 13, 1 << 20); -static void BM_Mean3DToScalarCPU(int iters, int num) { - ReduceToScalar(iters, "cpu", "Mean", num); +static void Do3DXZReduce(int iters, const string& device, const string& reduce, + int num_x, int num_y) { + testing::ItemsProcessed(static_cast(iters) * num_x * num_y); + testing::BytesProcessed(static_cast(iters) * num_x * num_y * + sizeof(float)); + test::Benchmark(device, ThreeDXZReduce(reduce, num_x, num_y)).Run(iters); } -BENCHMARK(BM_Mean3DToScalarCPU)->Range(1 << 13, 1 << 20); -static void BM_Sum3DToScalarGPU(int iters, int num) { - ReduceToScalar(iters, "gpu", "Sum", num); +static void BM_Sum2DToScalarGPU(int iters, int num_x, int num_y) { + ReduceToScalar(iters, "gpu", "Sum", num_x, num_y); } -BENCHMARK(BM_Sum3DToScalarGPU)->Range(1 << 13, 1 << 20); +BENCHMARK(BM_Sum2DToScalarGPU)->RangePair(1, 8192, 1, 8192); -static void BM_Max3DToScalarGPU(int iters, int num) { - ReduceToScalar(iters, "gpu", "Max", num); +static void BM_Sum2DToScalarGPUComplex(int iters, int num_x, int num_y) { + ReduceToScalar>(iters, "gpu", "Sum", num_x, num_y); } -BENCHMARK(BM_Max3DToScalarGPU)->Range(1 << 13, 1 << 20); +BENCHMARK(BM_Sum2DToScalarGPUComplex)->RangePair(1, 8192, 1, 8192); -static void BM_Prod3DToScalarGPU(int iters, int num) { - ReduceToScalar(iters, "gpu", "Prod", num); +static void BM_Sum2DToScalarGPUHalf(int iters, int num_x, int num_y) { + ReduceToScalar(iters, "gpu", "Sum", num_x, num_y); } -BENCHMARK(BM_Prod3DToScalarGPU)->Range(1 << 13, 1 << 20); +BENCHMARK(BM_Sum2DToScalarGPUHalf)->RangePair(1, 8192, 1, 8192); -static void BM_Mean3DToScalarGPU(int iters, int num) { - ReduceToScalar(iters, "gpu", "Mean", num); +static void BM_Sum2DRowReduceGPU(int iters, int num_x, int num_y) { + DoRowReduce(iters, "gpu", "Sum", num_x, num_y); } -BENCHMARK(BM_Mean3DToScalarGPU)->Range(1 << 13, 1 << 20); +BENCHMARK(BM_Sum2DRowReduceGPU)->RangePair(1, 8192, 1, 8192); + +static void BM_Sum2DColumnReduceGPU(int iters, int num_x, int num_y) { + DoColReduce(iters, "gpu", "Sum", num_x, num_y); +} +BENCHMARK(BM_Sum2DColumnReduceGPU)->RangePair(1, 8192, 1, 8192); + +static void BM_Sum3DYReduceGPU(int iters, int num_x, int num_y) { + Do3DYReduce(iters, "gpu", "Sum", num_x, num_y); +} +BENCHMARK(BM_Sum3DYReduceGPU)->RangePair(64, 4096, 64, 4096); + +static void BM_Sum3DXZReduceGPU(int iters, int num_x, int num_y) { + Do3DXZReduce(iters, "gpu", "Sum", num_x, num_y); +} +BENCHMARK(BM_Sum3DXZReduceGPU)->RangePair(64, 4096, 64, 4096); + +static void BM_Mean2DToScalarGPU(int iters, int num_x, int num_y) { + ReduceToScalar(iters, "gpu", "Mean", num_x, num_y); +} +BENCHMARK(BM_Mean2DToScalarGPU)->RangePair(2048, 8192, 2048, 8192); + +static void BM_Max2DToScalarGPU(int iters, int num_x, int num_y) { + ReduceToScalar(iters, "gpu", "Max", num_x, num_y); +} +BENCHMARK(BM_Max2DToScalarGPU)->RangePair(2048, 8192, 2048, 8192); + +static void BM_Min2DToScalarGPU(int iters, int num_x, int num_y) { + ReduceToScalar(iters, "gpu", "Min", num_x, num_y); +} +BENCHMARK(BM_Min2DToScalarGPU)->RangePair(2048, 8192, 2048, 8192); + +static void BM_Bool2DToScalarGPU(int iters, int num_x, int num_y) { + ReduceToScalar(iters, "gpu", "All", num_x, num_y); +} +BENCHMARK(BM_Bool2DToScalarGPU)->RangePair(2048, 8192, 2048, 8192); } // end namespace tensorflow diff --git a/tensorflow/core/util/permutation_input_iterator.h b/tensorflow/core/util/permutation_input_iterator.h new file mode 100644 index 00000000000..f6375b25157 --- /dev/null +++ b/tensorflow/core/util/permutation_input_iterator.h @@ -0,0 +1,134 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_ +#define TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_ + +#include +#include + +namespace tensorflow { + +template +class PermutationInputIterator { + public: + // Required iterator traits + typedef PermutationInputIterator self_type; ///< My own type + typedef OffsetT difference_type; ///< Type to express the result of + ///< subtracting one iterator from another + typedef ValueType + value_type; ///< The type of the element the iterator can point to + typedef ValueType* pointer; ///< The type of a pointer to an element the + ///< iterator can point to + typedef ValueType reference; ///< The type of a reference to an element the + ///< iterator can point to + + typedef std::random_access_iterator_tag + iterator_category; ///< The iterator category + + private: + InputIteratorT input_itr; + IndexIteratorT index_itr; + + public: + /// Constructor + __host__ __device__ __forceinline__ PermutationInputIterator( + InputIteratorT input_itr, ///< Input iterator to wrap + IndexIteratorT index_itr) ///< Conversion functor to wrap + : input_itr(input_itr), index_itr(index_itr) {} + + /// Postfix increment + __host__ __device__ __forceinline__ self_type operator++(int) { + self_type retval = *this; + index_itr++; + return retval; + } + + /// Prefix increment + __host__ __device__ __forceinline__ self_type operator++() { + index_itr++; + return *this; + } + + /// Indirection + __host__ __device__ __forceinline__ reference operator*() const { + return input_itr[*index_itr]; + } + + /// Addition + template + __host__ __device__ __forceinline__ self_type operator+(Distance n) const { + self_type retval(input_itr, index_itr + n); + return retval; + } + + /// Addition assignment + template + __host__ __device__ __forceinline__ self_type& operator+=(Distance n) { + index_itr += n; + return *this; + } + + /// Subtraction + template + __host__ __device__ __forceinline__ self_type operator-(Distance n) const { + self_type retval(input_itr, index_itr - n); + return retval; + } + + /// Subtraction assignment + template + __host__ __device__ __forceinline__ self_type& operator-=(Distance n) { + index_itr -= n; + return *this; + } + + /// Distance + __host__ __device__ __forceinline__ difference_type + operator-(self_type other) const { + return index_itr - other.index_itr; + } + + /// Array subscript + template + __host__ __device__ __forceinline__ reference operator[](Distance n) const { + return input_itr[index_itr[n]]; + } + + /// Structure dereference + __host__ __device__ __forceinline__ pointer operator->() { + return input_itr + *index_itr; + } + + /// Equal to + __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) { + return (index_itr == rhs.index_itr && input_itr == rhs.input_itr); + } + + /// Not equal to + __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) { + return !(*this == rhs); + } + + /// ostream operator + friend std::ostream& operator<<(std::ostream& os, const self_type& itr) { + return os; + } +}; + +} // end namespace tensorflow + +#endif // TENSORFLOW_UTIL_PERMUTATION_INPUT_ITERATOR_H_ diff --git a/tensorflow/core/util/transform_output_iterator.h b/tensorflow/core/util/transform_output_iterator.h new file mode 100644 index 00000000000..1640791ad17 --- /dev/null +++ b/tensorflow/core/util/transform_output_iterator.h @@ -0,0 +1,149 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_UTIL_TRANSFORM_OUTPUT_ITERATOR_H_ +#define TENSORFLOW_UTIL_TRANSFORM_OUTPUT_ITERATOR_H_ + +#include +#include + +namespace tensorflow { + +template +class TransformOutputIterator { + private: + // Proxy object + struct Reference { + StoreType* ptr; + ConversionOp conversion_op; + + /// Constructor + __host__ __device__ __forceinline__ Reference(StoreType* ptr, + ConversionOp conversion_op) + : ptr(ptr), conversion_op(conversion_op) {} + + /// Assignment + __host__ __device__ __forceinline__ InputType operator=(InputType val) { + *ptr = conversion_op(val); + return val; + } + }; + + public: + // Required iterator traits + typedef TransformOutputIterator self_type; ///< My own type + typedef OffsetT difference_type; ///< Type to express the result of + ///< subtracting one iterator from another + typedef void + value_type; ///< The type of the element the iterator can point to + typedef void pointer; ///< The type of a pointer to an element the iterator + ///< can point to + typedef Reference reference; ///< The type of a reference to an element the + ///< iterator can point to + + typedef std::random_access_iterator_tag + iterator_category; ///< The iterator category + + /*private:*/ + + StoreType* ptr; + ConversionOp conversion_op; + + public: + /// Constructor + template + __host__ __device__ __forceinline__ TransformOutputIterator( + QualifiedStoreType* ptr, + ConversionOp conversionOp) ///< Native pointer to wrap + : ptr(ptr), conversion_op(conversionOp) {} + + /// Postfix increment + __host__ __device__ __forceinline__ self_type operator++(int) { + self_type retval = *this; + ptr++; + return retval; + } + + /// Prefix increment + __host__ __device__ __forceinline__ self_type operator++() { + ptr++; + return *this; + } + + /// Indirection + __host__ __device__ __forceinline__ reference operator*() const { + return Reference(ptr, conversion_op); + } + + /// Addition + template + __host__ __device__ __forceinline__ self_type operator+(Distance n) const { + self_type retval(ptr + n, conversion_op); + return retval; + } + + /// Addition assignment + template + __host__ __device__ __forceinline__ self_type& operator+=(Distance n) { + ptr += n; + return *this; + } + + /// Subtraction + template + __host__ __device__ __forceinline__ self_type operator-(Distance n) const { + self_type retval(ptr - n, conversion_op); + return retval; + } + + /// Subtraction assignment + template + __host__ __device__ __forceinline__ self_type& operator-=(Distance n) { + ptr -= n; + return *this; + } + + /// Distance + __host__ __device__ __forceinline__ difference_type + operator-(self_type other) const { + return ptr - other.ptr; + } + + /// Array subscript + template + __host__ __device__ __forceinline__ reference operator[](Distance n) const { + return Reference(ptr + n, conversion_op); + } + + /// Equal to + __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) { + return (ptr == rhs.ptr); + } + + /// Not equal to + __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) { + return (ptr != rhs.ptr); + } + + /// ostream operator + friend std::ostream& operator<<(std::ostream& os, const self_type& itr) { + return os; + } +}; + +} // end namespace tensorflow + +#endif // TENSORFLOW_UTIL_TRANSFORM_OUTPUT_ITERATOR_H_ diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index ed3f02aedd6..ed89a6dc488 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -1708,6 +1708,26 @@ cuda_py_test( tags = ["no_windows_gpu"], ) +cuda_py_test( + name = "reduction_ops_test_big", + size = "medium", + srcs = ["reduction_ops_test_big.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + ], + tags = [ + "manual", + "no_gpu", + "nogpu", + "noguitar", + "notap", + ], +) + cuda_py_test( name = "relu_op_test", size = "small", diff --git a/tensorflow/python/kernel_tests/cholesky_op_test.py b/tensorflow/python/kernel_tests/cholesky_op_test.py index eb06e067a7f..de80fb30554 100644 --- a/tensorflow/python/kernel_tests/cholesky_op_test.py +++ b/tensorflow/python/kernel_tests/cholesky_op_test.py @@ -183,14 +183,11 @@ class CholeskyGradTest(test.TestCase): self.runFiniteDifferences( shapes, dtypes=(dtypes_lib.float32, dtypes_lib.float64)) - # TODO(eriche): investigate why this test fails only in opensource - # ubuntu gpu python3 - - # def testSmallMatricesComplex(self): - # np.random.seed(0) - # shapes = self.getShapes([1, 2, 10]) - # self.runFiniteDifferences( - # shapes, dtypes=(dtypes_lib.complex64, dtypes_lib.complex128)) + def testSmallMatricesComplex(self): + np.random.seed(0) + shapes = self.getShapes([1, 2, 10]) + self.runFiniteDifferences( + shapes, dtypes=(dtypes_lib.complex64, dtypes_lib.complex128)) def testOneBlockMatrices(self): np.random.seed(0) diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index 04ce99a4a63..8d6b7925e45 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -175,6 +175,24 @@ class SumReductionTest(BaseReductionTest): np_arr = self._makeIncremental((2,) * rank, dtypes.int32) self._compareAllAxes(np_arr) + def testFloat16(self): + for rank in range(1, _MAX_RANK + 1): + np_arr = self._makeIncremental((2,) * rank, dtypes.float16) + self._compareAllAxes(np_arr) + + # test that mean doesn't overflow + # only on GPU, since it has the more accurate implementation + if not test.is_gpu_available(): + return + + arr = np.ones([68000], dtype=np.float16) + + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + tf_arr = array_ops.constant(arr) + tf_mean = math_ops.reduce_mean(tf_arr, 0, False) + tf_out_mean = sess.run(tf_mean) + self.assertAllClose(tf_out_mean, 1.) + def testFloat32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float32) @@ -523,7 +541,7 @@ class MinReductionTest(test.TestCase): def testFloatReduce3D(self): # Create a 3D array of floats and reduce across all possible # dimensions - np_arr = np.arange(0, 30).reshape([2, 3, 5]).astype(np.float32) + np_arr = np.arange(1, 31).reshape([2, 3, 5]).astype(np.float32) self._compareAll(np_arr, None) self._compareAll(np_arr, []) self._compareAll(np_arr, [0]) @@ -537,7 +555,7 @@ class MinReductionTest(test.TestCase): def testDoubleReduce3D(self): # Create a 3D array of doubles and reduce across all possible # dimensions - np_arr = np.arange(0, 30).reshape([2, 3, 5]).astype(np.float64) + np_arr = np.arange(1, 31).reshape([2, 3, 5]).astype(np.float64) self._compareAll(np_arr, None) self._compareAll(np_arr, []) self._compareAll(np_arr, [0]) @@ -629,7 +647,7 @@ class MaxReductionTest(test.TestCase): def testFloatReduce3D(self): # Create a 3D array of floats and reduce across all possible # dimensions - np_arr = np.arange(0, 30).reshape([2, 3, 5]).astype(np.float32) + np_arr = np.arange(-31, -1).reshape([2, 3, 5]).astype(np.float32) self._compareAll(np_arr, None) self._compareAll(np_arr, []) self._compareAll(np_arr, [0]) @@ -643,7 +661,7 @@ class MaxReductionTest(test.TestCase): def testDoubleReduce3D(self): # Create a 3D array of doubles and reduce across all possible # dimensions - np_arr = np.arange(0, 30).reshape([2, 3, 5]).astype(np.float64) + np_arr = np.arange(-31, -1).reshape([2, 3, 5]).astype(np.float64) self._compareAll(np_arr, None) self._compareAll(np_arr, []) self._compareAll(np_arr, [0]) @@ -656,7 +674,7 @@ class MaxReductionTest(test.TestCase): def testGradient(self): s = [2, 3, 4, 2] - x = np.arange(1.0, 49.0).reshape(s).astype(np.float64) + x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64) with self.test_session(): t = ops.convert_to_tensor(x) su = math_ops.reduce_max(t, [1, 2]) @@ -666,7 +684,7 @@ class MaxReductionTest(test.TestCase): def testGradient2(self): s = [2, 3, 4, 2] - x = np.arange(1.0, 49.0).reshape(s).astype(np.float64) + x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64) with self.test_session(): t = ops.convert_to_tensor(x) su = math_ops.reduce_max(t, [1]) @@ -676,7 +694,7 @@ class MaxReductionTest(test.TestCase): def testGradient3(self): s = [2, 3, 4, 2] - x = np.arange(1.0, 49.0).reshape(s).astype(np.float64) + x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64) with self.test_session(): t = ops.convert_to_tensor(x) su = math_ops.reduce_max(t, [2]) @@ -686,7 +704,7 @@ class MaxReductionTest(test.TestCase): def testGradient4(self): s = [2, 3, 4, 2] - x = np.arange(1.0, 49.0).reshape(s).astype(np.float64) + x = np.arange(-49.0, -1.0).reshape(s).astype(np.float64) with self.test_session(): t = ops.convert_to_tensor(x) su = math_ops.reduce_max(t) diff --git a/tensorflow/python/kernel_tests/reduction_ops_test_big.py b/tensorflow/python/kernel_tests/reduction_ops_test_big.py new file mode 100644 index 00000000000..0959adb026e --- /dev/null +++ b/tensorflow/python/kernel_tests/reduction_ops_test_big.py @@ -0,0 +1,179 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functional tests for reduction ops.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import ops +from tensorflow.python.ops import math_ops +from tensorflow.python.platform import test + + +class BaseReductionTest(test.TestCase): + + def _tf_reduce(self, x, reduction_axes, keep_dims): + raise NotImplementedError() + + +class BigReductionTest(BaseReductionTest): + """Test reductions for sum and boolean all over a wide range of shapes.""" + + def _tf_reduce_max(self, x, reduction_axes, keep_dims): + return math_ops.reduce_max(x, reduction_axes, keep_dims) + + def _tf_reduce_all(self, x, reduction_axes, keep_dims): + return math_ops.reduce_all(x, reduction_axes, keep_dims) + + def _tf_reduce_mean(self, x, reduction_axes, keep_dims): + return math_ops.reduce_mean(x, reduction_axes, keep_dims) + + def _tf_reduce_sum(self, x, reduction_axes, keep_dims): + return math_ops.reduce_sum(x, reduction_axes, keep_dims) + + def testFloat32Sum(self): + # make sure we test all possible kernel invocations + # logic is the same for all ops, test just float32 for brevity + arr_ = np.ones([4097, 4097], dtype=np.float32) + for size_x in [ + 1, 2, 3, 4, 16, 17, 32, 33, 64, 65, 128, 131, 256, 263, 1024, 1025, + 4096, 4097 + ]: + for size_y in [ + 1, 2, 3, 4, 16, 17, 32, 33, 64, 65, 128, 131, 256, 263, 1024, 1025, + 4096, 4097 + ]: + arr = arr_[0:size_x, 0:size_y] + col_sum = np.ones([size_y], dtype=np.float32) * size_x + row_sum = np.ones([size_x], dtype=np.float32) * size_y + full_sum = np.ones([], dtype=np.float32) * size_x * size_y + + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + tf_row_sum = self._tf_reduce_sum(arr, 1, False) + tf_col_sum = self._tf_reduce_sum(arr, 0, False) + tf_full_sum = self._tf_reduce_sum(arr, [0, 1], False) + tf_out_row, tf_out_col, tf_out_full = sess.run( + [tf_row_sum, tf_col_sum, tf_full_sum]) + self.assertAllClose(col_sum, tf_out_col) + self.assertAllClose(row_sum, tf_out_row) + self.assertAllClose(full_sum, tf_out_full) + + arr_ = np.ones([130, 130, 130], dtype=np.float32) + for size_x in range(1, 130, 13): + for size_y in range(1, 130, 13): + for size_z in range(1, 130, 13): + arr = arr_[0:size_x, 0:size_y, 0:size_z] + sum_y = np.ones([size_x, size_z], dtype=np.float32) + sum_xz = np.ones([size_y], dtype=np.float32) + + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + tf_sum_xz = self._tf_reduce_mean(arr, [0, 2], False) + tf_sum_y = self._tf_reduce_mean(arr, 1, False) + tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y]) + self.assertAllClose(sum_y, tf_out_sum_y) + self.assertAllClose(sum_xz, tf_out_sum_xz) + + def testFloat32Max(self): + # make sure we test all possible kernel invocations + # logic is the same for all ops, test just float32 for brevity + arr_ = np.random.uniform( + low=-3, high=-1, size=[4105, 4105]).astype(np.float32) + for size_x in [ + 1, 2, 3, 4, 16, 17, 32, 33, 64, 65, 128, 131, 256, 263, 1024, 1025, + 4096, 4097 + ]: + for size_y in [ + 1, 2, 3, 4, 16, 17, 32, 33, 64, 65, 128, 131, 256, 263, 1024, 1025, + 4096, 4097 + ]: + arr = arr_[0:size_x, 0:size_y] + col_max = np.max(arr, axis=0) + row_max = np.max(arr, axis=1) + full_max = np.max(col_max) + + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + tf_row_max = self._tf_reduce_max(arr, 1, False) + tf_col_max = self._tf_reduce_max(arr, 0, False) + tf_full_max = self._tf_reduce_max(arr, [0, 1], False) + tf_out_row, tf_out_col, tf_out_full = sess.run( + [tf_row_max, tf_col_max, tf_full_max]) + self.assertAllClose(col_max, tf_out_col) + self.assertAllClose(row_max, tf_out_row) + self.assertAllClose(full_max, tf_out_full) + + arr_ = np.random.uniform( + low=-3, high=-1, size=[130, 130, 130]).astype(np.float32) + for size_x in range(1, 130, 13): + for size_y in range(1, 130, 13): + for size_z in range(1, 130, 13): + arr = arr_[0:size_x, 0:size_y, 0:size_z] + sum_y = np.max(arr, axis=1) + sum_xz = np.max(arr, axis=(0, 2)) + + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + tf_sum_xz = self._tf_reduce_max(arr, [0, 2], False) + tf_sum_y = self._tf_reduce_max(arr, 1, False) + tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y]) + self.assertAllClose(sum_y, tf_out_sum_y) + self.assertAllClose(sum_xz, tf_out_sum_xz) + + def testBooleanAll(self): + # make sure we test all possible kernel invocations + # test operation where T(0) is not the identity + arr_ = np.ones([4097, 4097], dtype=np.bool) + for size_x in [ + 1, 2, 3, 4, 16, 17, 32, 33, 64, 65, 128, 131, 256, 263, 1024, 1025, + 4096, 4097 + ]: + for size_y in [ + 1, 2, 3, 4, 16, 17, 32, 33, 64, 65, 128, 131, 256, 263, 1024, 1025, + 4096, 4097 + ]: + arr = arr_[0:size_x, 0:size_y] + col_sum = np.ones([size_y], dtype=np.bool) + row_sum = np.ones([size_x], dtype=np.bool) + full_sum = np.ones([1], dtype=np.bool).reshape([]) + + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + tf_row_sum = self._tf_reduce_all(arr, 1, False) + tf_col_sum = self._tf_reduce_all(arr, 0, False) + tf_full_sum = self._tf_reduce_all(arr, [0, 1], False) + tf_out_row, tf_out_col, tf_out_full = sess.run( + [tf_row_sum, tf_col_sum, tf_full_sum]) + self.assertAllClose(col_sum, tf_out_col) + self.assertAllClose(row_sum, tf_out_row) + self.assertAllClose(full_sum, tf_out_full) + + arr_ = np.ones([130, 130, 130], dtype=np.bool) + for size_x in range(1, 130, 13): + for size_y in range(1, 130, 13): + for size_z in range(1, 130, 13): + arr = arr_[0:size_x, 0:size_y, 0:size_z] + sum_y = np.ones([size_x, size_z], dtype=np.bool) + sum_xz = np.ones([size_y], dtype=np.bool) + + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + tf_sum_xz = self._tf_reduce_all(arr, [0, 2], False) + tf_sum_y = self._tf_reduce_all(arr, 1, False) + tf_out_sum_xz, tf_out_sum_y = sess.run([tf_sum_xz, tf_sum_y]) + self.assertAllClose(sum_y, tf_out_sum_y) + self.assertAllClose(sum_xz, tf_out_sum_xz) + + +if __name__ == "__main__": + test.main() From a4fe92514fab8d75d6acf5f593cf7e7c8cea919f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 15:56:58 -0700 Subject: [PATCH 56/67] Implement reduce_min and reduce_max gradients. Closes #12715 PiperOrigin-RevId: 167335900 --- tensorflow/cc/gradients/math_grad.cc | 66 +++++++++++++++++++++++ tensorflow/cc/gradients/math_grad_test.cc | 49 +++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/tensorflow/cc/gradients/math_grad.cc b/tensorflow/cc/gradients/math_grad.cc index 09a15fbe5f1..d90654f2e9a 100644 --- a/tensorflow/cc/gradients/math_grad.cc +++ b/tensorflow/cc/gradients/math_grad.cc @@ -687,6 +687,72 @@ Status MeanGrad(const Scope& scope, const Operation& op, } REGISTER_GRADIENT_OP("Mean", MeanGrad); +Status MinOrMaxGrad(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + // The partial derivative for any input along a "reduced" dimension + // is 1 when it is the min (or max) and 0 everywhere else. So the + // gradient calculation is identical for both operators. + // + // There's a special case for propagating gradients when there are + // multiple minima (or maxima) - we choose to divide the gradient + // equally among all matching inputs. + // + // Please note this comment + // https://github.com/tensorflow/tensorflow/issues/4886#issuecomment-256836063 + // for details. + + // Running example: + // input: [[5, 5, 5], + // [1, 2, -3]] + // reduction_indices: [1] + auto input = op.input(0); + auto reduction_indices = op.input(1); + + // [2, 3] + auto input_shape = Shape(scope, input); + + // [2, 1] + auto output_shape_kept_dims = + ReducedShapeHelper(scope, input_shape, reduction_indices); + + // for op=min (say) + // output = [5, -3] + // y = [[5], + // [-3]] + auto y = Reshape(scope, op.output(0), output_shape_kept_dims); + + // reshape([g1, g2], [2, 1]) = [[g1], + // [g2]] + auto grad = Reshape(scope, grad_inputs[0], output_shape_kept_dims); + + // indicators = equal(y, input) + // = equal([[5], [[5, 5, 5], + // [-3]], [1, 2, -3]]) + // = [[1, 1, 1], + // [0, 0, 1]] + auto indicators = Cast(scope, Equal(scope, y, input), grad_inputs[0].type()); + + // [[3], + // [1]] + auto num_selected = Reshape(scope, Sum(scope, indicators, reduction_indices), + output_shape_kept_dims); + + // [[1/3, 1/3, 1/3], + // [0, 0, 1]] + auto scale = Div(scope, indicators, num_selected); + + // [[g1/3, g1/3, g1/3], + // [0, 0, g2]] + grad_outputs->push_back(Mul(scope, scale, grad)); + + // Stop propagation along reduction_indices + grad_outputs->push_back(NoGradient()); + return scope.status(); +} +REGISTER_GRADIENT_OP("Min", MinOrMaxGrad); +REGISTER_GRADIENT_OP("Max", MinOrMaxGrad); + // MatMulGrad helper function used to compute two MatMul operations // based on input matrix transposition combinations. Status MatMulGradHelper(const Scope& scope, const bool is_batch, diff --git a/tensorflow/cc/gradients/math_grad_test.cc b/tensorflow/cc/gradients/math_grad_test.cc index 62b59b25c7a..5b1558dd820 100644 --- a/tensorflow/cc/gradients/math_grad_test.cc +++ b/tensorflow/cc/gradients/math_grad_test.cc @@ -955,6 +955,55 @@ TEST_F(NaryGradTest, Mean) { RunTest({x}, {x_shape}, {y}, {y_shape}); } +TEST_F(NaryGradTest, Min) { + TensorShape x_shape({2, 3}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); + auto y = Min(scope_, x, {-1}); + // y's shape is the result of reducing x along axes -1 (= 1) + TensorShape y_shape({2}); + Tensor x_init_value = + test::AsTensor({0.5f, 0.7f, 0.2f, 1.0f, 1.5f, -2.8f}, x_shape); + RunTest(x, x_init_value, y, y_shape); +} + +TEST_F(NaryGradTest, Max) { + TensorShape x_shape({2, 3}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); + auto y = Max(scope_, x, {-1}); + // y's shape is the result of reducing x along axes -1 (= 1) + TensorShape y_shape({2}); + Tensor x_init_value = + test::AsTensor({0.5f, 0.7f, 0.2f, 1.0f, 1.5f, -2.8f}, x_shape); + RunTest(x, x_init_value, y, y_shape); +} + +TEST_F(NaryGradTest, MinMulti) { + // Test gradient when there are multiple minima. + // Note that we cannot directly use a test Tensor with multiple + // minima, as the numeric estimator will calculate incorrect + // gradients when perturbing each entry in the Tensor (which then + // changes how many minima exist.) + // Instead, we use a single input that broadcast-multiplies a larger + // tensor with equal values, and apply reduce_min to the multiplied + // result. + TensorShape x_shape({1}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); + auto all_same = Mul(scope_, Const(scope_, {1.f, 1.f, 1.f}), x); + auto y = Min(scope_, all_same, {0}); + // y is a [3] shaped tensor reduced along dimension 0, so it is [1] shaped + TensorShape y_shape({1}); + RunTest({x}, {x_shape}, {y}, {y_shape}); +} + +TEST_F(NaryGradTest, MaxMulti) { + TensorShape x_shape({1}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); + auto all_same = Mul(scope_, Const(scope_, {1.f, 1.f, 1.f}), x); + auto y = Max(scope_, all_same, {0}); + TensorShape y_shape({1}); + RunTest({x}, {x_shape}, {y}, {y_shape}); +} + TEST_F(NaryGradTest, AddN) { TensorShape shape({3, 2, 5}); std::vector xs; From 25b301d2f80c76055f2ab4dc6ddb9cab0ab62cef Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Fri, 1 Sep 2017 16:20:38 -0700 Subject: [PATCH 57/67] Automated g4 rollback of changelist 167318475 PiperOrigin-RevId: 167338828 --- tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 0578ae8b0f4..6748a765623 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -68,11 +68,12 @@ def _create_global_step(graph): return variable_scope.get_variable( ops.GraphKeys.GLOBAL_STEP, shape=[], - dtype=dtypes.int64, + dtype=dtypes.int32, initializer=init_ops.zeros_initializer(), trainable=False, use_resource=True, - collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP]) + collections=[ops.GraphKeys.GLOBAL_VARIABLES, + ops.GraphKeys.GLOBAL_STEP]) def _sync_variables_ops(): From e09619b923f3e8ac633fd6beaa96f2298988c355 Mon Sep 17 00:00:00 2001 From: Gunhan Gulsoy Date: Fri, 1 Sep 2017 16:23:01 -0700 Subject: [PATCH 58/67] Increase the size of the flaky sample_stats_test.py PiperOrigin-RevId: 167339080 --- tensorflow/contrib/distributions/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index c78b064b4fd..c2b99d67c7f 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -341,7 +341,7 @@ cuda_py_test( cuda_py_test( name = "sample_stats_test", - size = "small", + size = "medium", srcs = ["python/kernel_tests/sample_stats_test.py"], additional_deps = [ ":distributions_py", From b541b4812289b19cd4d4ba68916ab386878f8f17 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 16:31:36 -0700 Subject: [PATCH 59/67] Update CUB to 1.7.3 PiperOrigin-RevId: 167339917 --- tensorflow/contrib/cmake/external/cub.cmake | 4 ++-- tensorflow/workspace.bzl | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/contrib/cmake/external/cub.cmake b/tensorflow/contrib/cmake/external/cub.cmake index 477572d5881..d98579d2077 100644 --- a/tensorflow/contrib/cmake/external/cub.cmake +++ b/tensorflow/contrib/cmake/external/cub.cmake @@ -14,8 +14,8 @@ # ============================================================================== include (ExternalProject) -set(cub_URL http://mirror.bazel.build/github.com/NVlabs/cub/archive/69ceda618313df8e9cac6659d607b08949455d14.tar.gz) -set(cub_HASH SHA256=87e856522c283b8ea887c3b61d7d5b252d2dd74abac4f1d756d776e721223e82) +set(cub_URL http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip) +set(cub_HASH SHA256=b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe) set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) set(cub_ARCHIVE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/cub_archive) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 2c7acd809a8..646459646da 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -683,11 +683,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "cub_archive", urls = [ - "http://mirror.bazel.build/github.com/NVlabs/cub/archive/69ceda618313df8e9cac6659d607b08949455d14.tar.gz", - "https://github.com/NVlabs/cub/archive/69ceda618313df8e9cac6659d607b08949455d14.tar.gz", + "http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.3.zip", + "https://github.com/NVlabs/cub/archive/1.7.3.zip", ], - sha256 = "87e856522c283b8ea887c3b61d7d5b252d2dd74abac4f1d756d776e721223e82", - strip_prefix = "cub-69ceda618313df8e9cac6659d607b08949455d14", + sha256 = "b7ead9e291d34ffa8074243541c1380d63be63f88de23de8ee548db573b72ebe", + strip_prefix = "cub-1.7.3", build_file = str(Label("//third_party:cub.BUILD")), ) From e83d8ab0959b6a0e50d14acbfd39c5ab79e449d5 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 1 Sep 2017 16:33:24 -0700 Subject: [PATCH 60/67] Contrib ops and kernels for summary ops which write without touching python. PiperOrigin-RevId: 167340103 --- tensorflow/BUILD | 1 + tensorflow/contrib/BUILD | 1 + tensorflow/contrib/summary/BUILD | 59 +++ tensorflow/contrib/summary/summary_ops.py | 159 +++++++ .../contrib/summary/summary_ops_test.py | 52 +++ tensorflow/core/BUILD | 2 + tensorflow/core/kernels/BUILD | 39 ++ tensorflow/core/kernels/summary_interface.cc | 432 ++++++++++++++++++ tensorflow/core/kernels/summary_interface.h | 59 +++ .../core/kernels/summary_interface_test.cc | 167 +++++++ tensorflow/core/kernels/summary_kernels.cc | 226 +++++++++ tensorflow/core/ops/summary_ops.cc | 218 +++++++++ tensorflow/python/eager/context.py | 28 -- tensorflow/python/eager/core_test.py | 7 +- 14 files changed, 1416 insertions(+), 34 deletions(-) create mode 100644 tensorflow/contrib/summary/BUILD create mode 100644 tensorflow/contrib/summary/summary_ops.py create mode 100644 tensorflow/contrib/summary/summary_ops_test.py create mode 100644 tensorflow/core/kernels/summary_interface.cc create mode 100644 tensorflow/core/kernels/summary_interface.h create mode 100644 tensorflow/core/kernels/summary_interface_test.cc create mode 100644 tensorflow/core/kernels/summary_kernels.cc create mode 100644 tensorflow/core/ops/summary_ops.cc diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 9faa0964cae..2544e26b8ec 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -336,6 +336,7 @@ filegroup( "//tensorflow/contrib/staging:all_files", "//tensorflow/contrib/stat_summarizer:all_files", "//tensorflow/contrib/stateless:all_files", + "//tensorflow/contrib/summary:all_files", "//tensorflow/contrib/tensor_forest:all_files", "//tensorflow/contrib/tensor_forest/hybrid:all_files", "//tensorflow/contrib/tensor_forest/kernels/v4:all_files", diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index d9f2b843469..84fcc0d0149 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -73,6 +73,7 @@ py_library( "//tensorflow/contrib/staging", "//tensorflow/contrib/stat_summarizer:stat_summarizer_py", "//tensorflow/contrib/stateless", + "//tensorflow/contrib/summary:summary_ops", "//tensorflow/contrib/tensor_forest:init_py", "//tensorflow/contrib/tensorboard", "//tensorflow/contrib/testing:testing_py", diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD new file mode 100644 index 00000000000..bc305022642 --- /dev/null +++ b/tensorflow/contrib/summary/BUILD @@ -0,0 +1,59 @@ +licenses(["notice"]) # Apache 2.0 + +exports_files([ + "LICENSE", +]) + +load( + "//tensorflow:tensorflow.bzl", + "py_test", + "tf_gen_op_wrapper_py", +) + +tf_gen_op_wrapper_py( + name = "gen_summary_ops", + out = "gen_summary_ops.py", + deps = ["//tensorflow/core:summary_ops_op_lib"], +) + +py_test( + name = "summary_ops_test", + srcs = ["summary_ops_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":summary_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform", + "//tensorflow/python:training", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:test", + ], +) + +py_library( + name = "summary_ops", + srcs = ["summary_ops.py"], + srcs_version = "PY2AND3", + visibility = ["//tensorflow:internal"], + deps = [ + ":gen_summary_ops", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:summary_op_util", + "//tensorflow/python:training", + "//tensorflow/python/eager:context", + ], +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py new file mode 100644 index 00000000000..05e627adf1c --- /dev/null +++ b/tensorflow/contrib/summary/summary_ops.py @@ -0,0 +1,159 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Operations to emit summaries.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.summary import gen_summary_ops +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import summary_op_util +from tensorflow.python.training import training_util + + +# Name for a collection which is expected to have at most a single boolean +# Tensor. If this tensor is True the summary ops will record summaries. +_SHOULD_RECORD_SUMMARIES_NAME = "ShouldRecordSummaries" + + +def should_record_summaries(): + """Returns boolean Tensor which is true if summaries should be recorded.""" + should_record_collection = ops.get_collection(_SHOULD_RECORD_SUMMARIES_NAME) + if not should_record_collection: + return constant_op.constant(False) + if len(should_record_collection) != 1: + raise ValueError( + "More than one tensor specified for whether summaries " + "should be recorded: %s" % should_record_collection) + return should_record_collection[0] + + +# TODO(apassos) consider how to handle local step here. +def record_summaries_every_n_global_steps(n): + """Sets the should_record_summaries Tensor to true if global_step % n == 0.""" + collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) + collection_ref[:] = [training_util.get_global_step() % n == 0] + + +def always_record_summaries(): + """Sets the should_record_summaries Tensor to always true.""" + collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) + collection_ref[:] = [constant_op.constant(True)] + + +def never_record_summaries(): + """Sets the should_record_summaries Tensor to always false.""" + collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) + collection_ref[:] = [constant_op.constant(False)] + + +def create_summary_file_writer(logdir, + max_queue=None, + flush_secs=None, + filename_suffix=None): + """Creates a summary file writer in the current context.""" + if max_queue is None: + max_queue = constant_op.constant(10) + if flush_secs is None: + flush_secs = constant_op.constant(120) + if filename_suffix is None: + filename_suffix = constant_op.constant("") + resource = gen_summary_ops.summary_writer() + gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue, + flush_secs, filename_suffix) + context.context().summary_writer_resource = resource + + +def _nothing(): + """Convenient else branch for when summaries do not record.""" + return + + +def generic(name, tensor, metadata, family=None): + """Writes a tensor summary if possible.""" + + def record(): + with summary_op_util.summary_scope( + name, family, values=[tensor]) as (tag, scope): + gen_summary_ops.write_summary(context.context().summary_writer_resource, + training_util.get_global_step(), tensor, + tag, metadata, name=scope) + return control_flow_ops.cond(should_record_summaries(), record, _nothing) + + +def scalar(name, tensor, family=None): + """Writes a scalar summary if possible.""" + + def record(): + with summary_op_util.summary_scope( + name, family, values=[tensor]) as (tag, scope): + gen_summary_ops.write_scalar_summary( + context.context().summary_writer_resource, + training_util.get_global_step(), tag, tensor, name=scope) + + return control_flow_ops.cond(should_record_summaries(), record, _nothing) + + +def histogram(name, tensor, family=None): + """Writes a histogram summary if possible.""" + + def record(): + with summary_op_util.summary_scope( + name, family, values=[tensor]) as (tag, scope): + gen_summary_ops.write_histogram_summary( + context.context().summary_writer_resource, + training_util.get_global_step(), tag, tensor, name=scope) + + return control_flow_ops.cond(should_record_summaries(), record, _nothing) + + +def image(name, tensor, bad_color=None, max_images=3, family=None): + """Writes an image summary if possible.""" + + def record(): + if bad_color is None: + bad_color_ = constant_op.constant([255, 0, 0, 255], dtype=dtypes.uint8) + with summary_op_util.summary_scope( + name, family, values=[tensor]) as (tag, scope): + gen_summary_ops.write_image_summary( + context.context().summary_writer_resource, + training_util.get_global_step(), tag, tensor, bad_color_, max_images, + name=scope) + + return control_flow_ops.cond(should_record_summaries(), record, _nothing) + + +def audio(name, tensor, sample_rate, max_outputs, family=None): + """Writes an audio summary if possible.""" + + def record(): + with summary_op_util.summary_scope( + name, family, values=[tensor]) as (tag, scope): + gen_summary_ops.write_audio_summary( + context.context().summary_writer_resource, + training_util.get_global_step(), + tag, + tensor, + sample_rate=sample_rate, + max_outputs=max_outputs, + name=scope) + + return control_flow_ops.cond(should_record_summaries(), record, _nothing) diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py new file mode 100644 index 00000000000..56c1a16f7f0 --- /dev/null +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -0,0 +1,52 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tempfile + +from tensorflow.contrib.summary import summary_ops +from tensorflow.python.eager import test +from tensorflow.python.framework import test_util +from tensorflow.python.platform import gfile +from tensorflow.python.training import training_util + + +class TargetTest(test_util.TensorFlowTestCase): + + def testShouldRecordSummary(self): + self.assertFalse(summary_ops.should_record_summaries().numpy()) + summary_ops.always_record_summaries() + self.assertTrue(summary_ops.should_record_summaries().numpy()) + + def testSummaryOps(self): + training_util.get_or_create_global_step() + logdir = tempfile.mkdtemp() + summary_ops.create_summary_file_writer(logdir, max_queue=0) + summary_ops.always_record_summaries() + summary_ops.generic('tensor', 1, '') + summary_ops.scalar('scalar', 2.0) + summary_ops.histogram('histogram', [1.0]) + summary_ops.image('image', [[[[1.0]]]]) + summary_ops.audio('audio', [[1.0]], 1.0, 1) + # The working condition of the ops is tested in the C++ test so we just + # test here that we're calling them correctly. + self.assertTrue(gfile.Exists(logdir)) + + +if __name__ == '__main__': + test.main() diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index d01f7843eef..35394eeb877 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -566,6 +566,7 @@ tf_gen_op_libs( "state_ops", "stateless_random_ops", "string_ops", + "summary_ops", "training_ops", ], ) @@ -776,6 +777,7 @@ cc_library( "//tensorflow/core/kernels:state", "//tensorflow/core/kernels:stateless_random_ops", "//tensorflow/core/kernels:string", + "//tensorflow/core/kernels:summary_kernels", "//tensorflow/core/kernels:training_ops", "//tensorflow/core/kernels:word2vec_kernels", ] + tf_additional_cloud_kernel_deps() + if_not_windows([ diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index cb7e3b33165..174ccde8b7a 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -4665,6 +4665,8 @@ filegroup( "whole_file_read_ops.*", "sample_distorted_bounding_box_op.*", "ctc_loss_op.*", + "summary_interface.*", + "summary_kernels.*", "spectrogram_convert_test_data.cc", "sql_dataset_ops.cc", # Excluded due to experimental status: @@ -5954,6 +5956,43 @@ tf_kernel_library( ], ) +cc_library( + name = "summary_interface", + srcs = ["summary_interface.cc"], + hdrs = ["summary_interface.h"], + deps = [ + "//tensorflow/compiler/xla:util", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:proto_text", + "//tensorflow/core:protos_all_cc", + ], +) + +cc_test( + name = "summary_interface_test", + srcs = ["summary_interface_test.cc"], + deps = [ + ":summary_interface", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + +tf_kernel_library( + name = "summary_kernels", + srcs = ["summary_kernels.cc"], + deps = [ + ":summary_interface", + "//tensorflow/core:framework", + "//tensorflow/core:summary_ops_op_lib", + ], +) + # ----------------------------------------------------------------------------- # Google-internal targets. These must be at the end for syncrepo. diff --git a/tensorflow/core/kernels/summary_interface.cc b/tensorflow/core/kernels/summary_interface.cc new file mode 100644 index 00000000000..19e0f702f9f --- /dev/null +++ b/tensorflow/core/kernels/summary_interface.cc @@ -0,0 +1,432 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/summary.pb.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/kernels/summary_interface.h" +#include "tensorflow/core/lib/histogram/histogram.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/png/png_io.h" +#include "tensorflow/core/lib/wav/wav_io.h" +#include "tensorflow/core/util/event.pb.h" +#include "tensorflow/core/util/events_writer.h" + +namespace tensorflow { +namespace { +template +Status TensorValueAt(Tensor t, int index, T* out) { + switch (t.dtype()) { + case DT_FLOAT: + *out = t.flat()(index); + break; + case DT_DOUBLE: + *out = t.flat()(index); + break; + case DT_HALF: + *out = T(t.flat()(index)); + break; + case DT_INT32: + *out = t.flat()(index); + break; + case DT_UINT8: + *out = t.flat()(index); + break; + case DT_INT16: + *out = t.flat()(index); + break; + case DT_INT8: + *out = t.flat()(index); + break; + case DT_BOOL: + *out = t.flat()(index); + break; + case DT_INT64: + *out = t.flat()(index); + break; + default: + return errors::Unimplemented("Scalar summary for dtype ", + DataTypeString(t.dtype()), + " is not supported."); + } + return Status::OK(); +} + +typedef Eigen::Tensor Uint8Image; + +// Add the sequence of images specified by ith_image to the summary. +// +// Factoring this loop out into a helper function lets ith_image behave +// differently in the float and uint8 cases: the float case needs a temporary +// buffer which can be shared across calls to ith_image, but the uint8 case +// does not. +Status AddImages(const string& tag, int max_images, int batch_size, int w, + int h, int depth, + const std::function& ith_image, Summary* s) { + const int N = std::min(max_images, batch_size); + for (int i = 0; i < N; ++i) { + Summary::Value* v = s->add_value(); + // The tag depends on the number of requested images (not the number + // produced.) + // + // Note that later on avisu uses "/" to figure out a consistent naming + // convention for display, so we append "/image" to guarantee that the + // image(s) won't be displayed in the global scope with no name. + if (max_images > 1) { + v->set_tag(strings::StrCat(tag, "/image/", i)); + } else { + v->set_tag(strings::StrCat(tag, "/image")); + } + + auto image = ith_image(i); + Summary::Image* si = v->mutable_image(); + si->set_height(h); + si->set_width(w); + si->set_colorspace(depth); + const int channel_bits = 8; + const int compression = -1; // Use zlib default + if (!png::WriteImageToBuffer(image.data(), w, h, w * depth, depth, + channel_bits, compression, + si->mutable_encoded_image_string(), nullptr)) { + return errors::Internal("PNG encoding failed"); + } + } + return Status::OK(); +} + +template +void NormalizeFloatImage(int hw, int depth, + typename TTypes::ConstMatrix values, + typename TTypes::ConstVec bad_color, + Uint8Image* image) { + if (!image->size()) return; // Nothing to do for empty images + + // Rescale the image to uint8 range. + // + // We are trying to generate an RGB image from a float/half tensor. We do + // not have any info about the expected range of values in the tensor + // but the generated image needs to have all RGB values within [0, 255]. + // + // We use two different algorithms to generate these values. If the + // tensor has only positive values we scale them all by 255/max(values). + // If the tensor has both negative and positive values we scale them by + // the max of their absolute values and center them around 127. + // + // This works for most cases, but does not respect the relative dynamic + // range across different instances of the tensor. + + // Compute min and max ignoring nonfinite pixels + float image_min = std::numeric_limits::infinity(); + float image_max = -image_min; + for (int i = 0; i < hw; i++) { + bool finite = true; + for (int j = 0; j < depth; j++) { + if (!Eigen::numext::isfinite(values(i, j))) { + finite = false; + break; + } + } + if (finite) { + for (int j = 0; j < depth; j++) { + float value(values(i, j)); + image_min = std::min(image_min, value); + image_max = std::max(image_max, value); + } + } + } + + // Pick an affine transform into uint8 + const float kZeroThreshold = 1e-6; + T scale, offset; + if (image_min < 0) { + float max_val = std::max(std::abs(image_min), std::abs(image_max)); + scale = T(max_val < kZeroThreshold ? 0.0f : 127.0f / max_val); + offset = T(128.0f); + } else { + scale = T(image_max < kZeroThreshold ? 0.0f : 255.0f / image_max); + offset = T(0.0f); + } + + // Transform image, turning nonfinite values to bad_color + for (int i = 0; i < hw; i++) { + bool finite = true; + for (int j = 0; j < depth; j++) { + if (!Eigen::numext::isfinite(values(i, j))) { + finite = false; + break; + } + } + if (finite) { + image->chip<0>(i) = + (values.template chip<0>(i) * scale + offset).template cast(); + } else { + image->chip<0>(i) = bad_color; + } + } +} + +template +Status NormalizeAndAddImages(const Tensor& tensor, int max_images, int h, int w, + int hw, int depth, int batch_size, + const string& base_tag, Tensor bad_color_tensor, + Summary* s) { + // For float and half images, nans and infs are replaced with bad_color. + if (bad_color_tensor.dim_size(0) < depth) { + return errors::InvalidArgument( + "expected depth <= bad_color.size, got depth = ", depth, + ", bad_color.size = ", bad_color_tensor.dim_size(0)); + } + auto bad_color_full = bad_color_tensor.vec(); + typename TTypes::ConstVec bad_color(bad_color_full.data(), depth); + + // Float images must be scaled and translated. + Uint8Image image(hw, depth); + auto ith_image = [&tensor, &image, bad_color, batch_size, hw, depth](int i) { + auto tensor_eigen = tensor.template shaped({batch_size, hw, depth}); + typename TTypes::ConstMatrix values( + &tensor_eigen(i, 0, 0), Eigen::DSizes(hw, depth)); + NormalizeFloatImage(hw, depth, values, bad_color, &image); + return image; + }; + return AddImages(base_tag, max_images, batch_size, w, h, depth, ith_image, s); +} + +} // namespace + +class SummaryWriterImpl : public SummaryWriterInterface { + public: + SummaryWriterImpl(int max_queue, int flush_millis) + : SummaryWriterInterface(), + max_queue_(max_queue), + flush_millis_(flush_millis) {} + + Status Initialize(const string& logdir, const string& filename_suffix, + Env* env) { + Status is_dir = env->IsDirectory(logdir); + if (!is_dir.ok()) { + if (is_dir.code() != tensorflow::error::NOT_FOUND) { + return is_dir; + } + TF_RETURN_IF_ERROR(env->CreateDir(logdir)); + } + mutex_lock ml(mu_); + events_writer_ = + xla::MakeUnique(io::JoinPath(logdir, "events")); + if (!events_writer_->InitWithSuffix(filename_suffix)) { + return errors::Unknown("Could not initialize events writer."); + } + last_flush_ = Env::Default()->NowMicros(); + return Status::OK(); + } + + Status Flush() override { + mutex_lock ml(mu_); + return InternalFlush(); + } + + ~SummaryWriterImpl() override { + (void)Flush(); // Ignore errors. + } + + Status WriteTensor(int64 global_step, Tensor t, const string& tag, + const string& serialized_metadata) override { + Summary s; + Summary::Value* v = s.add_value(); + t.AsProtoTensorContent(v->mutable_tensor()); + v->set_tag(tag); + v->mutable_metadata()->ParseFromString(serialized_metadata); + return Enqueue(global_step, s); + } + + Status WriteScalar(int64 global_step, Tensor t, const string& tag) override { + Summary s; + Summary::Value* v = s.add_value(); + v->set_tag(tag); + float value; + TF_RETURN_IF_ERROR(TensorValueAt(t, 0, &value)); + v->set_simple_value(value); + return Enqueue(global_step, s); + } + + Status WriteHistogram(int64 global_step, Tensor t, + const string& tag) override { + Summary s; + Summary::Value* v = s.add_value(); + v->set_tag(tag); + histogram::Histogram histo; + for (int64 i = 0; i < t.NumElements(); i++) { + double double_val; + TF_RETURN_IF_ERROR(TensorValueAt(t, i, &double_val)); + if (Eigen::numext::isnan(double_val)) { + return errors::InvalidArgument("Nan in summary histogram for: ", tag); + } else if (Eigen::numext::isinf(double_val)) { + return errors::InvalidArgument("Infinity in summary histogram for: ", + tag); + } + histo.Add(double_val); + } + + histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); + return Enqueue(global_step, s); + } + + Status WriteImage(int64 global_step, Tensor tensor, const string& tag, + int max_images, Tensor bad_color) override { + if (!(tensor.dims() == 4 && + (tensor.dim_size(3) == 1 || tensor.dim_size(3) == 3 || + tensor.dim_size(3) == 4))) { + return errors::InvalidArgument( + "Tensor must be 4-D with last dim 1, 3, or 4, not ", + tensor.shape().DebugString()); + } + if (!(tensor.dim_size(0) < (1LL << 31) && + tensor.dim_size(1) < (1LL << 31) && + tensor.dim_size(2) < (1LL << 31) && + (tensor.dim_size(1) * tensor.dim_size(2)) < (1LL << 29))) { + return errors::InvalidArgument("Tensor too large for summary ", + tensor.shape().DebugString()); + } + Summary s; + // The casts and h * w cannot overflow because of the limits above. + const int batch_size = static_cast(tensor.dim_size(0)); + const int h = static_cast(tensor.dim_size(1)); + const int w = static_cast(tensor.dim_size(2)); + const int hw = h * w; // Compact these two dims for simplicity + const int depth = static_cast(tensor.dim_size(3)); + if (tensor.dtype() == DT_UINT8) { + // For uint8 input, no normalization is necessary + auto ith_image = [&tensor, batch_size, hw, depth](int i) { + auto values = tensor.shaped({batch_size, hw, depth}); + return typename TTypes::ConstMatrix( + &values(i, 0, 0), Eigen::DSizes(hw, depth)); + }; + TF_RETURN_IF_ERROR( + AddImages(tag, max_images, batch_size, w, h, depth, ith_image, &s)); + } else if (tensor.dtype() == DT_HALF) { + TF_RETURN_IF_ERROR(NormalizeAndAddImages( + tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, &s)); + } else if (tensor.dtype() == DT_FLOAT) { + TF_RETURN_IF_ERROR(NormalizeAndAddImages( + tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, &s)); + } else { + return errors::InvalidArgument( + "Only DT_INT8, DT_HALF, and DT_FLOAT images are supported. Got ", + DataTypeString(tensor.dtype())); + } + + return Enqueue(global_step, s); + } + + Status WriteAudio(int64 global_step, Tensor tensor, const string& tag, + int max_outputs, float sample_rate) override { + if (sample_rate <= 0.0f) { + return errors::InvalidArgument("sample_rate must be > 0"); + } + const int batch_size = tensor.dim_size(0); + const int64 length_frames = tensor.dim_size(1); + const int64 num_channels = + tensor.dims() == 2 ? 1 : tensor.dim_size(tensor.dims() - 1); + Summary s; + const int N = std::min(max_outputs, batch_size); + for (int i = 0; i < N; ++i) { + Summary::Value* v = s.add_value(); + if (max_outputs > 1) { + v->set_tag(strings::StrCat(tag, "/audio/", i)); + } else { + v->set_tag(strings::StrCat(tag, "/audio")); + } + + Summary::Audio* sa = v->mutable_audio(); + sa->set_sample_rate(sample_rate); + sa->set_num_channels(num_channels); + sa->set_length_frames(length_frames); + sa->set_content_type("audio/wav"); + + auto values = + tensor.shaped({batch_size, length_frames, num_channels}); + auto channels_by_frames = typename TTypes::ConstMatrix( + &values(i, 0, 0), + Eigen::DSizes(length_frames, num_channels)); + size_t sample_rate_truncated = lrintf(sample_rate); + if (sample_rate_truncated == 0) { + sample_rate_truncated = 1; + } + TF_RETURN_IF_ERROR(wav::EncodeAudioAsS16LEWav( + channels_by_frames.data(), sample_rate_truncated, num_channels, + length_frames, sa->mutable_encoded_audio_string())); + } + + return Enqueue(global_step, s); + } + + string DebugString() override { return "SummaryWriterImpl"; } + + private: + Status Enqueue(int64 global_step, const Summary& summary) { + mutex_lock ml(mu_); + queue_.emplace_back(global_step, summary, Env::Default()->NowMicros()); + if (queue_.size() >= max_queue_ || + Env::Default()->NowMicros() - last_flush_ > 1000 * flush_millis_) { + return InternalFlush(); + } + return Status::OK(); + } + + Status InternalFlush() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + for (const EventInfo& e : queue_) { + Event event; + event.set_step(std::get<0>(e)); + *event.mutable_summary() = std::get<1>(e); + event.set_wall_time(std::get<2>(e)); + events_writer_->WriteEvent(event); + } + queue_.clear(); + if (!events_writer_->Flush()) { + return errors::InvalidArgument("Could not flush events file."); + } + last_flush_ = Env::Default()->NowMicros(); + return Status::OK(); + } + + const int max_queue_; + const int flush_millis_; + uint64 last_flush_; + using EventInfo = std::tuple; + mutex mu_; + std::vector queue_ GUARDED_BY(mu_); + // A pointer to allow deferred construction. + std::unique_ptr events_writer_ GUARDED_BY(mu_); + std::vector> registered_summaries_ + GUARDED_BY(mu_); +}; + +Status CreateSummaryWriter(int max_queue, int flush_millis, + const string& logdir, const string& filename_suffix, + Env* env, SummaryWriterInterface** result) { + SummaryWriterImpl* w = new SummaryWriterImpl(max_queue, flush_millis); + Status s = w->Initialize(logdir, filename_suffix, env); + if (!s.ok()) { + w->Unref(); + *result = nullptr; + return s; + } + *result = w; + return Status::OK(); +} + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/summary_interface.h b/tensorflow/core/kernels/summary_interface.h new file mode 100644 index 00000000000..b0861d218e3 --- /dev/null +++ b/tensorflow/core/kernels/summary_interface.h @@ -0,0 +1,59 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_ +#define TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_ + + +#include "tensorflow/core/framework/resource_mgr.h" + +namespace tensorflow { + +// Main interface for the summary writer resource. +class SummaryWriterInterface : public ResourceBase { + public: + virtual ~SummaryWriterInterface() override {} + + // Flushes all unwritten messages in the queue. + virtual Status Flush() = 0; + + // These are called in the OpKernel::Compute methods for the summary ops. + virtual Status WriteTensor(int64 global_step, Tensor t, const string& tag, + const string& serialized_metadata) = 0; + + virtual Status WriteScalar(int64 global_step, Tensor t, + const string& tag) = 0; + + virtual Status WriteHistogram(int64 global_step, Tensor t, + const string& tag) = 0; + + virtual Status WriteImage(int64 global_step, Tensor t, const string& tag, + int max_images, Tensor bad_color) = 0; + + virtual Status WriteAudio(int64 global_step, Tensor t, const string& tag, + int max_outputs_, float sample_rate) = 0; +}; + +// Creates a SummaryWriterInterface instance which writes to a file. It will +// enqueue up to max_queue summaries, and flush at least every flush_millis +// milliseconds. The summaries will be written to the directory specified by +// logdir and with the filename suffixed by filename_suffix. The caller ows a +// reference to result if the returned status is ok. +Status CreateSummaryWriter(int max_queue, int flush_millis, + const string& logdir, const string& filename_suffix, + Env* env, SummaryWriterInterface** result); + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_ diff --git a/tensorflow/core/kernels/summary_interface_test.cc b/tensorflow/core/kernels/summary_interface_test.cc new file mode 100644 index 00000000000..66bde2cb063 --- /dev/null +++ b/tensorflow/core/kernels/summary_interface_test.cc @@ -0,0 +1,167 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/core/framework/summary.pb.h" +#include "tensorflow/core/kernels/summary_interface.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/refcount.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/io/record_reader.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/util/event.pb.h" + +namespace tensorflow { +namespace { + +Status SummaryTestHelper( + const string& test_name, + std::function writer_fn, + std::function test_fn) { + SummaryWriterInterface* writer; + Env* env = Env::Default(); + TF_CHECK_OK( + CreateSummaryWriter(1, 1, testing::TmpDir(), test_name, env, &writer)); + core::ScopedUnref deleter(writer); + + TF_CHECK_OK(writer_fn(writer)); + TF_CHECK_OK(writer->Flush()); + + std::vector files; + TF_CHECK_OK(env->GetChildren(testing::TmpDir(), &files)); + bool found = false; + for (const string& f : files) { + if (StringPiece(f).contains(test_name)) { + if (found) { + return errors::Unknown("Found more than one file for ", test_name); + } + found = true; + std::unique_ptr read_file; + TF_CHECK_OK(env->NewRandomAccessFile(io::JoinPath(testing::TmpDir(), f), + &read_file)); + io::RecordReader reader(read_file.get(), io::RecordReaderOptions()); + string record; + uint64 offset = 0; + TF_CHECK_OK(reader.ReadRecord(&offset, + &record)); // The first event is irrelevant + TF_CHECK_OK(reader.ReadRecord(&offset, &record)); + Event e; + e.ParseFromString(record); + test_fn(e); + } + } + if (!found) { + return errors::Unknown("Found no file for ", test_name); + } + return Status::OK(); +} + +TEST(SummaryInterfaceTest, WriteTensor) { + TF_CHECK_OK(SummaryTestHelper("tensor_test", + [](SummaryWriterInterface* writer) { + Tensor one(DT_FLOAT, TensorShape({})); + one.scalar()() = 1.0; + TF_RETURN_IF_ERROR(writer->WriteTensor( + 2, one, "name", + SummaryMetadata().SerializeAsString())); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { + EXPECT_EQ(e.step(), 2); + CHECK_EQ(e.summary().value_size(), 1); + EXPECT_EQ(e.summary().value(0).tag(), "name"); + })); +} + +TEST(SummaryInterfaceTest, WriteScalar) { + TF_CHECK_OK(SummaryTestHelper( + "scalar_test", + [](SummaryWriterInterface* writer) { + Tensor one(DT_FLOAT, TensorShape({})); + one.scalar()() = 1.0; + TF_RETURN_IF_ERROR(writer->WriteScalar(2, one, "name")); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { + EXPECT_EQ(e.step(), 2); + CHECK_EQ(e.summary().value_size(), 1); + EXPECT_EQ(e.summary().value(0).tag(), "name"); + EXPECT_EQ(e.summary().value(0).simple_value(), 1.0); + })); +} + +TEST(SummaryInterfaceTest, WriteHistogram) { + TF_CHECK_OK(SummaryTestHelper("hist_test", + [](SummaryWriterInterface* writer) { + Tensor one(DT_FLOAT, TensorShape({})); + one.scalar()() = 1.0; + TF_RETURN_IF_ERROR( + writer->WriteHistogram(2, one, "name")); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { + EXPECT_EQ(e.step(), 2); + CHECK_EQ(e.summary().value_size(), 1); + EXPECT_EQ(e.summary().value(0).tag(), "name"); + EXPECT_TRUE(e.summary().value(0).has_histo()); + })); +} + +TEST(SummaryInterfaceTest, WriteImage) { + TF_CHECK_OK(SummaryTestHelper( + "image_test", + [](SummaryWriterInterface* writer) { + Tensor one(DT_UINT8, TensorShape({1, 1, 1, 1})); + one.scalar()() = 1; + TF_RETURN_IF_ERROR(writer->WriteImage(2, one, "name", 1, Tensor())); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { + EXPECT_EQ(e.step(), 2); + CHECK_EQ(e.summary().value_size(), 1); + EXPECT_EQ(e.summary().value(0).tag(), "name/image"); + CHECK(e.summary().value(0).has_image()); + EXPECT_EQ(e.summary().value(0).image().height(), 1); + EXPECT_EQ(e.summary().value(0).image().width(), 1); + EXPECT_EQ(e.summary().value(0).image().colorspace(), 1); + })); +} + +TEST(SummaryInterfaceTest, WriteAudio) { + TF_CHECK_OK(SummaryTestHelper( + "scalar_test", + [](SummaryWriterInterface* writer) { + Tensor one(DT_FLOAT, TensorShape({1, 1})); + one.scalar()() = 1.0; + TF_RETURN_IF_ERROR(writer->WriteAudio(2, one, "name", 1, 1)); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { + EXPECT_EQ(e.step(), 2); + CHECK_EQ(e.summary().value_size(), 1); + EXPECT_EQ(e.summary().value(0).tag(), "name/audio"); + CHECK(e.summary().value(0).has_audio()); + })); +} + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/kernels/summary_kernels.cc b/tensorflow/core/kernels/summary_kernels.cc new file mode 100644 index 00000000000..d0eca0f1e7f --- /dev/null +++ b/tensorflow/core/kernels/summary_kernels.cc @@ -0,0 +1,226 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/kernels/summary_interface.h" + +namespace tensorflow { + +REGISTER_KERNEL_BUILDER(Name("SummaryWriter").Device(DEVICE_CPU), + ResourceHandleOp); + +class CreateSummaryFileWriterOp : public OpKernel { + public: + explicit CreateSummaryFileWriterOp(OpKernelConstruction* ctx) + : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + const Tensor* tmp; + OP_REQUIRES_OK(ctx, ctx->input("logdir", &tmp)); + const string logdir = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("max_queue", &tmp)); + const int32 max_queue = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("flush_millis", &tmp)); + const int32 flush_millis = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("filename_suffix", &tmp)); + const string filename_suffix = tmp->scalar()(); + SummaryWriterInterface* s; + OP_REQUIRES_OK(ctx, CreateSummaryWriter(max_queue, flush_millis, logdir, + filename_suffix, ctx->env(), &s)); + Status status = CreateResource(ctx, HandleFromInput(ctx, 0), s); + if (!status.ok()) { + s->Unref(); + ctx->SetStatus(status); + return; + } + } +}; +REGISTER_KERNEL_BUILDER(Name("CreateSummaryFileWriter").Device(DEVICE_CPU), + CreateSummaryFileWriterOp); + +class FlushSummaryWriterOp : public OpKernel { + public: + explicit FlushSummaryWriterOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + SummaryWriterInterface* s; + OP_REQUIRES_OK(ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &s)); + core::ScopedUnref unref(s); + OP_REQUIRES_OK(ctx, s->Flush()); + } +}; +REGISTER_KERNEL_BUILDER(Name("FlushSummaryWriter").Device(DEVICE_CPU), + FlushSummaryWriterOp); + +class CloseSummaryWriterOp : public OpKernel { + public: + explicit CloseSummaryWriterOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + OP_REQUIRES_OK(ctx, DeleteResource( + ctx, HandleFromInput(ctx, 0))); + } +}; +REGISTER_KERNEL_BUILDER(Name("CloseSummaryWriter").Device(DEVICE_CPU), + CloseSummaryWriterOp); + +class WriteSummaryOp : public OpKernel { + public: + explicit WriteSummaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + SummaryWriterInterface* s; + OP_REQUIRES_OK(ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &s)); + core::ScopedUnref unref(s); + const Tensor* tmp; + OP_REQUIRES_OK(ctx, ctx->input("global_step", &tmp)); + const int64 global_step = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("tag", &tmp)); + const string& tag = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("summary_metadata", &tmp)); + const string& serialized_metadata = tmp->scalar()(); + + const Tensor* t; + OP_REQUIRES_OK(ctx, ctx->input("tensor", &t)); + + OP_REQUIRES_OK(ctx, + s->WriteTensor(global_step, *t, tag, serialized_metadata)); + } +}; +REGISTER_KERNEL_BUILDER(Name("WriteSummary").Device(DEVICE_CPU), + WriteSummaryOp); + +class WriteScalarSummaryOp : public OpKernel { + public: + explicit WriteScalarSummaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + SummaryWriterInterface* s; + OP_REQUIRES_OK(ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &s)); + core::ScopedUnref unref(s); + const Tensor* tmp; + OP_REQUIRES_OK(ctx, ctx->input("global_step", &tmp)); + const int64 global_step = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("tag", &tmp)); + const string& tag = tmp->scalar()(); + + const Tensor* t; + OP_REQUIRES_OK(ctx, ctx->input("value", &t)); + + OP_REQUIRES_OK(ctx, s->WriteScalar(global_step, *t, tag)); + } +}; +REGISTER_KERNEL_BUILDER(Name("WriteScalarSummary").Device(DEVICE_CPU), + WriteScalarSummaryOp); + +class WriteHistogramSummaryOp : public OpKernel { + public: + explicit WriteHistogramSummaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + SummaryWriterInterface* s; + OP_REQUIRES_OK(ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &s)); + core::ScopedUnref unref(s); + const Tensor* tmp; + OP_REQUIRES_OK(ctx, ctx->input("global_step", &tmp)); + const int64 global_step = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("tag", &tmp)); + const string& tag = tmp->scalar()(); + + const Tensor* t; + OP_REQUIRES_OK(ctx, ctx->input("values", &t)); + + OP_REQUIRES_OK(ctx, s->WriteHistogram(global_step, *t, tag)); + } +}; +REGISTER_KERNEL_BUILDER(Name("WriteHistogramSummary").Device(DEVICE_CPU), + WriteHistogramSummaryOp); + +class WriteImageSummaryOp : public OpKernel { + public: + explicit WriteImageSummaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + int64 max_images_tmp; + OP_REQUIRES_OK(ctx, ctx->GetAttr("max_images", &max_images_tmp)); + OP_REQUIRES(ctx, max_images_tmp < (1LL << 31), + errors::InvalidArgument("max_images must be < 2^31")); + max_images_ = static_cast(max_images_tmp); + } + + void Compute(OpKernelContext* ctx) override { + SummaryWriterInterface* s; + OP_REQUIRES_OK(ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &s)); + core::ScopedUnref unref(s); + const Tensor* tmp; + OP_REQUIRES_OK(ctx, ctx->input("global_step", &tmp)); + const int64 global_step = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("tag", &tmp)); + const string& tag = tmp->scalar()(); + const Tensor* bad_color; + OP_REQUIRES_OK(ctx, ctx->input("bad_color", &bad_color)); + OP_REQUIRES( + ctx, TensorShapeUtils::IsVector(bad_color->shape()), + errors::InvalidArgument("bad_color must be a vector, got shape ", + bad_color->shape().DebugString())); + + const Tensor* t; + OP_REQUIRES_OK(ctx, ctx->input("tensor", &t)); + + OP_REQUIRES_OK( + ctx, s->WriteImage(global_step, *t, tag, max_images_, *bad_color)); + } + + private: + int32 max_images_; +}; +REGISTER_KERNEL_BUILDER(Name("WriteImageSummary").Device(DEVICE_CPU), + WriteImageSummaryOp); + +class WriteAudioSummaryOp : public OpKernel { + public: + explicit WriteAudioSummaryOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("max_outputs", &max_outputs_)); + OP_REQUIRES(ctx, max_outputs_ > 0, + errors::InvalidArgument("max_outputs must be > 0")); + } + + void Compute(OpKernelContext* ctx) override { + SummaryWriterInterface* s; + OP_REQUIRES_OK(ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &s)); + core::ScopedUnref unref(s); + const Tensor* tmp; + OP_REQUIRES_OK(ctx, ctx->input("global_step", &tmp)); + const int64 global_step = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("tag", &tmp)); + const string& tag = tmp->scalar()(); + OP_REQUIRES_OK(ctx, ctx->input("sample_rate", &tmp)); + const float sample_rate = tmp->scalar()(); + + const Tensor* t; + OP_REQUIRES_OK(ctx, ctx->input("tensor", &t)); + + OP_REQUIRES_OK( + ctx, s->WriteAudio(global_step, *t, tag, max_outputs_, sample_rate)); + } + + private: + int max_outputs_; + bool has_sample_rate_attr_; + float sample_rate_attr_; +}; +REGISTER_KERNEL_BUILDER(Name("WriteAudioSummary").Device(DEVICE_CPU), + WriteAudioSummaryOp); + +} // namespace tensorflow diff --git a/tensorflow/core/ops/summary_ops.cc b/tensorflow/core/ops/summary_ops.cc new file mode 100644 index 00000000000..f778b487972 --- /dev/null +++ b/tensorflow/core/ops/summary_ops.cc @@ -0,0 +1,218 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); + +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference.h" + +namespace tensorflow { + +REGISTER_OP("SummaryWriter") + .Output("writer: resource") + .Attr("shared_name: string = ''") + .Attr("container: string = ''") + .SetShapeFn(shape_inference::ScalarShape) + .Doc(R"doc( +Returns a handle to be used to access a summary writer. + +The summary writer is an in-graph resource which can be used by ops to write +summaries to event files. + +writer: the summary writer resource. Scalar handle. +)doc"); + +REGISTER_OP("CreateSummaryFileWriter") + .Input("writer: resource") + .Input("logdir: string") + .Input("max_queue: int32") + .Input("flush_millis: int32") + .Input("filename_suffix: string") + .Doc(R"doc( +Creates a summary file writer accessible by the given resource handle. + +writer: A handle to the summary writer resource +logdir: Directory where the event file will be written. +max_queue: Size of the queue of pending events and summaries. +flush_millis: How often, in milliseconds, to flush the pending events and + summaries to disk. +filename_suffix: Every event file's name is suffixed with this suffix. +)doc"); + +REGISTER_OP("FlushSummaryWriter") + .Input("writer: resource") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"( +Flushes the writer's unwritten events. + +writer: A handle to the summary writer resource. +)"); + +REGISTER_OP("CloseSummaryWriter") + .Input("writer: resource") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"( +Flushes and closes the summary writer. + +Also removes it from the resource manager. To reopen, use another +CreateSummaryFileWriter op. + +writer: A handle to the summary writer resource. +)"); + +REGISTER_OP("WriteSummary") + .Input("writer: resource") + .Input("global_step: int64") + .Input("tensor: T") + .Input("tag: string") + .Input("summary_metadata: string") + .Attr("T: type") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"doc( +Outputs a `Summary` protocol buffer with a tensor. + +writer: A handle to a summary writer. +global_step: The step to write the summary for. +tensor: A tensor to serialize. +tag: The summary's tag. +summary_metadata: Serialized SummaryMetadata protocol buffer containing + plugin-related metadata for this summary. +)doc"); + +REGISTER_OP("WriteScalarSummary") + .Input("writer: resource") + .Input("global_step: int64") + .Input("tag: string") + .Input("value: T") + .Attr("T: realnumbertype") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"doc( +Writes a `Summary` protocol buffer with scalar values. + +The input `tag` and `value` must have the scalars. + +writer: A handle to a summary writer. +global_step: The step to write the summary for. +tag: Tag for the summary. +value: Value for the summary. +)doc"); + +REGISTER_OP("WriteHistogramSummary") + .Input("writer: resource") + .Input("global_step: int64") + .Input("tag: string") + .Input("values: T") + .Attr("T: realnumbertype = DT_FLOAT") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"doc( +Writes a `Summary` protocol buffer with a histogram. + +The generated +[`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) +has one summary value containing a histogram for `values`. + +This op reports an `InvalidArgument` error if any value is not finite. + +writer: A handle to a summary writer. +global_step: The step to write the summary for. +tag: Scalar. Tag to use for the `Summary.Value`. +values: Any shape. Values to use to build the histogram. +)doc"); + +REGISTER_OP("WriteImageSummary") + .Input("writer: resource") + .Input("global_step: int64") + .Input("tag: string") + .Input("tensor: T") + .Input("bad_color: uint8") + .Attr("max_images: int >= 1 = 3") + .Attr("T: {uint8, float, half} = DT_FLOAT") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"doc( +Writes a `Summary` protocol buffer with images. + +The summary has up to `max_images` summary values containing images. The +images are built from `tensor` which must be 4-D with shape `[batch_size, +height, width, channels]` and where `channels` can be: + +* 1: `tensor` is interpreted as Grayscale. +* 3: `tensor` is interpreted as RGB. +* 4: `tensor` is interpreted as RGBA. + +The images have the same number of channels as the input tensor. For float +input, the values are normalized one image at a time to fit in the range +`[0, 255]`. `uint8` values are unchanged. The op uses two different +normalization algorithms: + +* If the input values are all positive, they are rescaled so the largest one + is 255. + +* If any input value is negative, the values are shifted so input value 0.0 + is at 127. They are then rescaled so that either the smallest value is 0, + or the largest one is 255. + +The `tag` argument is a scalar `Tensor` of type `string`. It is used to +build the `tag` of the summary values: + +* If `max_images` is 1, the summary value tag is '*tag*/image'. +* If `max_images` is greater than 1, the summary value tags are + generated sequentially as '*tag*/image/0', '*tag*/image/1', etc. + +The `bad_color` argument is the color to use in the generated images for +non-finite input values. It is a `unit8` 1-D tensor of length `channels`. +Each element must be in the range `[0, 255]` (It represents the value of a +pixel in the output image). Non-finite values in the input tensor are +replaced by this tensor in the output image. The default value is the color +red. + +writer: A handle to a summary writer. +global_step: The step to write the summary for. +tag: Scalar. Used to build the `tag` attribute of the summary values. +tensor: 4-D of shape `[batch_size, height, width, channels]` where + `channels` is 1, 3, or 4. +max_images: Max number of batch elements to generate images for. +bad_color: Color to use for pixels with non-finite values. +)doc"); + +REGISTER_OP("WriteAudioSummary") + .Input("writer: resource") + .Input("global_step: int64") + .Input("tag: string") + .Input("tensor: float") + .Input("sample_rate: float") + .Attr("max_outputs: int >= 1 = 3") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"doc( +Writes a `Summary` protocol buffer with audio. + +The summary has up to `max_outputs` summary values containing audio. The +audio is built from `tensor` which must be 3-D with shape `[batch_size, +frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are +assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`. + +The `tag` argument is a scalar `Tensor` of type `string`. It is used to +build the `tag` of the summary values: + +* If `max_outputs` is 1, the summary value tag is '*tag*/audio'. +* If `max_outputs` is greater than 1, the summary value tags are + generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc. + +writer: A handle to a summary writer. +global_step: The step to write the summary for. +tag: Scalar. Used to build the `tag` attribute of the summary values. +tensor: 2-D of shape `[batch_size, frames]`. +sample_rate: The sample rate of the signal in hertz. +max_outputs: Max number of batch elements to generate audio for. +)doc"); + +} // namespace tensorflow diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 27ffdd98105..a5a93b7bbe0 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -171,16 +171,6 @@ class Context(object): """Sets summary writer resource.""" self._summary_writer_resource = resource - @property - def recording_summaries(self): - """Returns True if recording summaries is enabled in current thread..""" - return self._eager_context.recording_summaries - - @recording_summaries.setter - def recording_summaries(self, val): - """Enables recording summaries is enabled in current thread..""" - self._eager_context.recording_summaries = val - @property def device_name(self): """Returns the device name for the current thread.""" @@ -360,24 +350,6 @@ def device(name): return context().device(name) -@contextlib.contextmanager -def record_summaries(): - """Context-manager to enable recording of summaries.""" - ctx = context() - old = ctx.recording_summaries - ctx.recording_summaries = True - try: - yield - finally: - ctx.recording_summaries = old - - -def should_record_summary(): - """True if a summary should be recorded now.""" - c = context() - return c.recording_summaries and c.summary_writer_resource is not None - - def run(main=None, argv=None): """Runs the program with an optional 'main' function and 'argv' list. diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py index 7ae80aa156a..5de396f62c3 100644 --- a/tensorflow/python/eager/core_test.py +++ b/tensorflow/python/eager/core_test.py @@ -55,10 +55,6 @@ class TFETest(test_util.TensorFlowTestCase): ctx.summary_writer_resource = 'mock' self.assertEqual('mock', ctx.summary_writer_resource) - self.assertFalse(ctx.recording_summaries) - ctx.recording_summaries = True - self.assertTrue(ctx.recording_summaries) - self.assertEqual('', ctx.device_name) self.assertEqual(ctx.device_name, ctx.device_spec.to_string()) with ctx.device('GPU:0'): @@ -95,8 +91,7 @@ class TFETest(test_util.TensorFlowTestCase): return [ ctx.in_graph_mode(), ctx.in_eager_mode(), ctx.scope_name, ctx.summary_writer_resource, - ctx.recording_summaries, ctx.device_name, - ctx.num_gpus() + ctx.device_name, ctx.num_gpus() ] def get_values(ctx, values): From c54cf14ce2c8c45cb3d8dc2cdd51b068c7f09e55 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 16:59:50 -0700 Subject: [PATCH 61/67] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 167342818 --- tensorflow/go/op/wrappers.go | 316 +++++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 9583ffb38b5..dda707aea26 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -209,6 +209,95 @@ func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...Va return op.Output(0) } +// Writes a `Summary` protocol buffer with scalar values. +// +// The input `tag` and `value` must have the scalars. +// +// Arguments: +// writer: A handle to a summary writer. +// global_step: The step to write the summary for. +// tag: Tag for the summary. +// value: Value for the summary. +// +// Returns the created operation. +func WriteScalarSummary(scope *Scope, writer tf.Output, global_step tf.Output, tag tf.Output, value tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "WriteScalarSummary", + Input: []tf.Input{ + writer, global_step, tag, value, + }, + } + return scope.AddOperation(opspec) +} + +// Outputs a `Summary` protocol buffer with a tensor. +// +// Arguments: +// writer: A handle to a summary writer. +// global_step: The step to write the summary for. +// tensor: A tensor to serialize. +// tag: The summary's tag. +// summary_metadata: Serialized SummaryMetadata protocol buffer containing +// plugin-related metadata for this summary. +// +// Returns the created operation. +func WriteSummary(scope *Scope, writer tf.Output, global_step tf.Output, tensor tf.Output, tag tf.Output, summary_metadata tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "WriteSummary", + Input: []tf.Input{ + writer, global_step, tensor, tag, summary_metadata, + }, + } + return scope.AddOperation(opspec) +} + +// Flushes and closes the summary writer. +// +// Also removes it from the resource manager. To reopen, use another +// CreateSummaryFileWriter op. +// +// Arguments: +// writer: A handle to the summary writer resource. +// +// Returns the created operation. +func CloseSummaryWriter(scope *Scope, writer tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "CloseSummaryWriter", + Input: []tf.Input{ + writer, + }, + } + return scope.AddOperation(opspec) +} + +// Flushes the writer's unwritten events. +// +// Arguments: +// writer: A handle to the summary writer resource. +// +// Returns the created operation. +func FlushSummaryWriter(scope *Scope, writer tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "FlushSummaryWriter", + Input: []tf.Input{ + writer, + }, + } + return scope.AddOperation(opspec) +} + // FakeQuantWithMinMaxVarsPerChannelGradientAttr is an optional argument to FakeQuantWithMinMaxVarsPerChannelGradient. type FakeQuantWithMinMaxVarsPerChannelGradientAttr func(optionalAttr) @@ -2149,6 +2238,34 @@ func ConcatOffset(scope *Scope, concat_dim tf.Output, shape []tf.Output) (offset return offset } +// Writes a `Summary` protocol buffer with a histogram. +// +// The generated +// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) +// has one summary value containing a histogram for `values`. +// +// This op reports an `InvalidArgument` error if any value is not finite. +// +// Arguments: +// writer: A handle to a summary writer. +// global_step: The step to write the summary for. +// tag: Scalar. Tag to use for the `Summary.Value`. +// values: Any shape. Values to use to build the histogram. +// +// Returns the created operation. +func WriteHistogramSummary(scope *Scope, writer tf.Output, global_step tf.Output, tag tf.Output, values tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "WriteHistogramSummary", + Input: []tf.Input{ + writer, global_step, tag, values, + }, + } + return scope.AddOperation(opspec) +} + // Concatenates tensors along one dimension. // // Arguments: @@ -7087,6 +7204,48 @@ func ResizeNearestNeighbor(scope *Scope, images tf.Output, size tf.Output, optio return op.Output(0) } +// SummaryWriterAttr is an optional argument to SummaryWriter. +type SummaryWriterAttr func(optionalAttr) + +// SummaryWriterSharedName sets the optional shared_name attribute to value. +// If not specified, defaults to "" +func SummaryWriterSharedName(value string) SummaryWriterAttr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// SummaryWriterContainer sets the optional container attribute to value. +// If not specified, defaults to "" +func SummaryWriterContainer(value string) SummaryWriterAttr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// Returns a handle to be used to access a summary writer. +// +// The summary writer is an in-graph resource which can be used by ops to write +// summaries to event files. +// +// Returns the summary writer resource. Scalar handle. +func SummaryWriter(scope *Scope, optional ...SummaryWriterAttr) (writer tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "SummaryWriter", + + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Returns the set of files matching one or more glob patterns. // // Note that this routine only supports wildcard characters in the @@ -10570,6 +10729,61 @@ func Restore(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, dt tf. return op.Output(0) } +// WriteAudioSummaryAttr is an optional argument to WriteAudioSummary. +type WriteAudioSummaryAttr func(optionalAttr) + +// WriteAudioSummaryMaxOutputs sets the optional max_outputs attribute to value. +// +// value: Max number of batch elements to generate audio for. +// If not specified, defaults to 3 +// +// REQUIRES: value >= 1 +func WriteAudioSummaryMaxOutputs(value int64) WriteAudioSummaryAttr { + return func(m optionalAttr) { + m["max_outputs"] = value + } +} + +// Writes a `Summary` protocol buffer with audio. +// +// The summary has up to `max_outputs` summary values containing audio. The +// audio is built from `tensor` which must be 3-D with shape `[batch_size, +// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are +// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`. +// +// The `tag` argument is a scalar `Tensor` of type `string`. It is used to +// build the `tag` of the summary values: +// +// * If `max_outputs` is 1, the summary value tag is '*tag*/audio'. +// * If `max_outputs` is greater than 1, the summary value tags are +// generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc. +// +// Arguments: +// writer: A handle to a summary writer. +// global_step: The step to write the summary for. +// tag: Scalar. Used to build the `tag` attribute of the summary values. +// tensor: 2-D of shape `[batch_size, frames]`. +// sample_rate: The sample rate of the signal in hertz. +// +// Returns the created operation. +func WriteAudioSummary(scope *Scope, writer tf.Output, global_step tf.Output, tag tf.Output, tensor tf.Output, sample_rate tf.Output, optional ...WriteAudioSummaryAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "WriteAudioSummary", + Input: []tf.Input{ + writer, global_step, tag, tensor, sample_rate, + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + // FusedResizeAndPadConv2DAttr is an optional argument to FusedResizeAndPadConv2D. type FusedResizeAndPadConv2DAttr func(optionalAttr) @@ -15797,6 +16011,30 @@ func Dilation2D(scope *Scope, input tf.Output, filter tf.Output, strides []int64 return op.Output(0) } +// Creates a summary file writer accessible by the given resource handle. +// +// Arguments: +// writer: A handle to the summary writer resource +// logdir: Directory where the event file will be written. +// max_queue: Size of the queue of pending events and summaries. +// flush_millis: How often, in milliseconds, to flush the pending events and +// summaries to disk. +// filename_suffix: Every event file's name is suffixed with this suffix. +// +// Returns the created operation. +func CreateSummaryFileWriter(scope *Scope, writer tf.Output, logdir tf.Output, max_queue tf.Output, flush_millis tf.Output, filename_suffix tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "CreateSummaryFileWriter", + Input: []tf.Input{ + writer, logdir, max_queue, flush_millis, filename_suffix, + }, + } + return scope.AddOperation(opspec) +} + // EncodeBase64Attr is an optional argument to EncodeBase64. type EncodeBase64Attr func(optionalAttr) @@ -17172,6 +17410,84 @@ func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) ( return op.Output(0) } +// WriteImageSummaryAttr is an optional argument to WriteImageSummary. +type WriteImageSummaryAttr func(optionalAttr) + +// WriteImageSummaryMaxImages sets the optional max_images attribute to value. +// +// value: Max number of batch elements to generate images for. +// If not specified, defaults to 3 +// +// REQUIRES: value >= 1 +func WriteImageSummaryMaxImages(value int64) WriteImageSummaryAttr { + return func(m optionalAttr) { + m["max_images"] = value + } +} + +// Writes a `Summary` protocol buffer with images. +// +// The summary has up to `max_images` summary values containing images. The +// images are built from `tensor` which must be 4-D with shape `[batch_size, +// height, width, channels]` and where `channels` can be: +// +// * 1: `tensor` is interpreted as Grayscale. +// * 3: `tensor` is interpreted as RGB. +// * 4: `tensor` is interpreted as RGBA. +// +// The images have the same number of channels as the input tensor. For float +// input, the values are normalized one image at a time to fit in the range +// `[0, 255]`. `uint8` values are unchanged. The op uses two different +// normalization algorithms: +// +// * If the input values are all positive, they are rescaled so the largest one +// is 255. +// +// * If any input value is negative, the values are shifted so input value 0.0 +// is at 127. They are then rescaled so that either the smallest value is 0, +// or the largest one is 255. +// +// The `tag` argument is a scalar `Tensor` of type `string`. It is used to +// build the `tag` of the summary values: +// +// * If `max_images` is 1, the summary value tag is '*tag*/image'. +// * If `max_images` is greater than 1, the summary value tags are +// generated sequentially as '*tag*/image/0', '*tag*/image/1', etc. +// +// The `bad_color` argument is the color to use in the generated images for +// non-finite input values. It is a `unit8` 1-D tensor of length `channels`. +// Each element must be in the range `[0, 255]` (It represents the value of a +// pixel in the output image). Non-finite values in the input tensor are +// replaced by this tensor in the output image. The default value is the color +// red. +// +// Arguments: +// writer: A handle to a summary writer. +// global_step: The step to write the summary for. +// tag: Scalar. Used to build the `tag` attribute of the summary values. +// tensor: 4-D of shape `[batch_size, height, width, channels]` where +// `channels` is 1, 3, or 4. +// bad_color: Color to use for pixels with non-finite values. +// +// Returns the created operation. +func WriteImageSummary(scope *Scope, writer tf.Output, global_step tf.Output, tag tf.Output, tensor tf.Output, bad_color tf.Output, optional ...WriteImageSummaryAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "WriteImageSummary", + Input: []tf.Input{ + writer, global_step, tag, tensor, bad_color, + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + // Pads a tensor with zeros. // // This operation pads a `input` with zeros according to the `paddings` you From 88063cdfa12e24df56dacf3fd7ff1084a031489c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 17:10:45 -0700 Subject: [PATCH 62/67] Automated g4 rollback of changelist 167320150 PiperOrigin-RevId: 167344016 --- tensorflow/core/framework/bfloat16_test.cc | 3 +-- tensorflow/core/framework/numeric_types.h | 9 +-------- tensorflow/core/kernels/cast_op.h | 9 ++++++++- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/framework/bfloat16_test.cc b/tensorflow/core/framework/bfloat16_test.cc index af4e6a44116..5bd95b806ff 100644 --- a/tensorflow/core/framework/bfloat16_test.cc +++ b/tensorflow/core/framework/bfloat16_test.cc @@ -23,8 +23,7 @@ namespace { TEST(Bfloat16Test, Simple) { bfloat16 a(12); - // Floating point representation of 12: 0x41400000 - EXPECT_EQ(0x4140, a.value); + EXPECT_EQ(12, a.value); } TEST(Bfloat16Test, Conversion) { diff --git a/tensorflow/core/framework/numeric_types.h b/tensorflow/core/framework/numeric_types.h index a630bee38d8..31b88707e24 100644 --- a/tensorflow/core/framework/numeric_types.h +++ b/tensorflow/core/framework/numeric_types.h @@ -44,14 +44,7 @@ typedef Eigen::QUInt16 quint16; // see framework/bfloat16.h for description. struct bfloat16 { EIGEN_DEVICE_FUNC bfloat16() {} - EIGEN_DEVICE_FUNC explicit bfloat16(const float v) { - const uint16_t* p = reinterpret_cast(&v); -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - value = p[0]; -#else - value = p[1]; -#endif - } + EIGEN_DEVICE_FUNC explicit bfloat16(const uint16_t v) : value(v) {} uint16_t value; }; diff --git a/tensorflow/core/kernels/cast_op.h b/tensorflow/core/kernels/cast_op.h index 59a991b5a8f..5c24f164a41 100644 --- a/tensorflow/core/kernels/cast_op.h +++ b/tensorflow/core/kernels/cast_op.h @@ -121,7 +121,14 @@ struct scalar_cast_op { typedef ::tensorflow::bfloat16 result_type; EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const ::tensorflow::bfloat16 operator()( const float a) const { - return ::tensorflow::bfloat16(a); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + const uint16_t* p = reinterpret_cast(&a); + return ::tensorflow::bfloat16(p[0]); +#else + static_assert(::tensorflow::port::kLittleEndian, "Not a little endian system!"); + const uint16_t* p = reinterpret_cast(&a); + return ::tensorflow::bfloat16(p[1]); +#endif } }; From 1499e91856d0d7a91ad6a6128b6d45f3d81e09d0 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Fri, 1 Sep 2017 17:11:07 -0700 Subject: [PATCH 63/67] Typo PiperOrigin-RevId: 167344049 --- tensorflow/core/kernels/summary_interface.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/summary_interface.h b/tensorflow/core/kernels/summary_interface.h index b0861d218e3..ae2fbb70fe3 100644 --- a/tensorflow/core/kernels/summary_interface.h +++ b/tensorflow/core/kernels/summary_interface.h @@ -48,7 +48,7 @@ class SummaryWriterInterface : public ResourceBase { // Creates a SummaryWriterInterface instance which writes to a file. It will // enqueue up to max_queue summaries, and flush at least every flush_millis // milliseconds. The summaries will be written to the directory specified by -// logdir and with the filename suffixed by filename_suffix. The caller ows a +// logdir and with the filename suffixed by filename_suffix. The caller owns a // reference to result if the returned status is ok. Status CreateSummaryWriter(int max_queue, int flush_millis, const string& logdir, const string& filename_suffix, From 2935132bce62cbc785da68997248e449d63f5ff2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 21:48:07 -0700 Subject: [PATCH 64/67] Use the shared locks when reading the model. PiperOrigin-RevId: 167358155 --- .../boosted_trees/kernels/model_ops.cc | 4 +- .../boosted_trees/kernels/prediction_ops.cc | 4 +- .../boosted_trees/kernels/training_ops.cc | 5 +- .../python/training/functions/gbdt_batch.py | 120 +++++++++--------- 4 files changed, 70 insertions(+), 63 deletions(-) diff --git a/tensorflow/contrib/boosted_trees/kernels/model_ops.cc b/tensorflow/contrib/boosted_trees/kernels/model_ops.cc index 42112c586a5..f4ad99f779e 100644 --- a/tensorflow/contrib/boosted_trees/kernels/model_ops.cc +++ b/tensorflow/contrib/boosted_trees/kernels/model_ops.cc @@ -74,7 +74,7 @@ class TreeEnsembleStampTokenOp : public OpKernel { decision_tree_ensemble_resource; OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), &decision_tree_ensemble_resource)); - mutex_lock l(*decision_tree_ensemble_resource->get_mutex()); + tf_shared_lock l(*decision_tree_ensemble_resource->get_mutex()); core::ScopedUnref unref_me(decision_tree_ensemble_resource); Tensor* output_stamp_token_t = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(), @@ -95,7 +95,7 @@ class TreeEnsembleSerializeOp : public OpKernel { decision_tree_ensemble_resource; OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), &decision_tree_ensemble_resource)); - mutex_lock l(*decision_tree_ensemble_resource->get_mutex()); + tf_shared_lock l(*decision_tree_ensemble_resource->get_mutex()); core::ScopedUnref unref_me(decision_tree_ensemble_resource); Tensor* output_stamp_token_t = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(), diff --git a/tensorflow/contrib/boosted_trees/kernels/prediction_ops.cc b/tensorflow/contrib/boosted_trees/kernels/prediction_ops.cc index daca0495481..8ffd7f120b4 100644 --- a/tensorflow/contrib/boosted_trees/kernels/prediction_ops.cc +++ b/tensorflow/contrib/boosted_trees/kernels/prediction_ops.cc @@ -143,7 +143,7 @@ class GradientTreesPredictionOp : public OpKernel { // Release the reference to the resource once we're done using it. core::ScopedUnref unref_me(decision_tree_ensemble_resource); if (use_locking_) { - mutex_lock l(*decision_tree_ensemble_resource->get_mutex()); + tf_shared_lock l(*decision_tree_ensemble_resource->get_mutex()); DoCompute(context, decision_tree_ensemble_resource); } else { DoCompute(context, decision_tree_ensemble_resource); @@ -334,7 +334,7 @@ class GradientTreesPartitionExamplesOp : public OpKernel { // Release the reference to the resource once we're done using it. core::ScopedUnref unref_me(decision_tree_ensemble_resource); if (use_locking_) { - mutex_lock l(*decision_tree_ensemble_resource->get_mutex()); + tf_shared_lock l(*decision_tree_ensemble_resource->get_mutex()); DoCompute(context, decision_tree_ensemble_resource); } else { DoCompute(context, decision_tree_ensemble_resource); diff --git a/tensorflow/contrib/boosted_trees/kernels/training_ops.cc b/tensorflow/contrib/boosted_trees/kernels/training_ops.cc index 9e9ef1738cd..d528757cf99 100644 --- a/tensorflow/contrib/boosted_trees/kernels/training_ops.cc +++ b/tensorflow/contrib/boosted_trees/kernels/training_ops.cc @@ -656,7 +656,8 @@ class GrowTreeEnsembleOp : public OpKernel { CHECK(split->split_info.split_node().node_case() != TreeNode::NODE_NOT_SET); CHECK(tree_config->nodes(node_id).node_case() == TreeNode::kLeaf) << "Unexpected node type to split " - << tree_config->nodes(node_id).node_case(); + << tree_config->nodes(node_id).node_case() << " for node_id " << node_id + << ". Tree config: " << tree_config->DebugString(); // Add left leaf. int32 left_id = tree_config->nodes_size(); @@ -767,7 +768,7 @@ class TreeEnsembleStatsOp : public OpKernel { OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0), &decision_tree_ensemble_resource)); core::ScopedUnref unref_me(decision_tree_ensemble_resource); - mutex_lock l(*decision_tree_ensemble_resource->get_mutex()); + tf_shared_lock l(*decision_tree_ensemble_resource->get_mutex()); // Get the stamp token. const Tensor* stamp_token_t; diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py index 83c88a04426..2d28e0a9f16 100644 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py +++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py @@ -407,75 +407,81 @@ class GradientBoostedDecisionTreeModel(object): local_stamp), _refresh_local_ensemble_fn, lambda: (control_flow_ops.no_op(), ensemble_stamp)) - # Once updated, Use the the local model for prediction. + # Once updated, use the local model for prediction. with ops.control_dependencies([refresh_local_ensemble]): ensemble_stats = training_ops.tree_ensemble_stats( local_ensemble_handle, ensemble_stamp) - apply_dropout, seed = _dropout_params(mode, ensemble_stats) # We don't need dropout info - we can always restore it based on the # seed. - predictions, predictions_no_dropout, _ = ( - prediction_ops.gradient_trees_prediction( - local_ensemble_handle, - seed, - self._dense_floats, - self._sparse_float_indices, - self._sparse_float_values, - self._sparse_float_shapes, - self._sparse_int_indices, - self._sparse_int_values, - self._sparse_int_shapes, - learner_config=self._learner_config_serialized, - apply_dropout=apply_dropout, - apply_averaging=apply_averaging, - use_locking=False, - center_bias=self._center_bias, - reduce_dim=self._reduce_dim)) - partition_ids = prediction_ops.gradient_trees_partition_examples( - local_ensemble_handle, - self._dense_floats, - self._sparse_float_indices, - self._sparse_float_values, - self._sparse_float_shapes, - self._sparse_int_indices, - self._sparse_int_values, - self._sparse_int_shapes, - use_locking=False) + apply_dropout, seed = _dropout_params(mode, ensemble_stats) + # Make sure ensemble stats run. This will check that the ensemble has + # the right stamp. + with ops.control_dependencies(ensemble_stats): + predictions, predictions_no_dropout, _ = ( + prediction_ops.gradient_trees_prediction( + local_ensemble_handle, + seed, + self._dense_floats, + self._sparse_float_indices, + self._sparse_float_values, + self._sparse_float_shapes, + self._sparse_int_indices, + self._sparse_int_values, + self._sparse_int_shapes, + learner_config=self._learner_config_serialized, + apply_dropout=apply_dropout, + apply_averaging=apply_averaging, + use_locking=True, + center_bias=self._center_bias, + reduce_dim=self._reduce_dim)) + partition_ids = prediction_ops.gradient_trees_partition_examples( + local_ensemble_handle, + self._dense_floats, + self._sparse_float_indices, + self._sparse_float_values, + self._sparse_float_shapes, + self._sparse_int_indices, + self._sparse_int_values, + self._sparse_int_shapes, + use_locking=True) else: with ops.device(self._ensemble_handle.device): ensemble_stats = training_ops.tree_ensemble_stats( self._ensemble_handle, ensemble_stamp) - apply_dropout, seed = _dropout_params(mode, ensemble_stats) # We don't need dropout info - we can always restore it based on the # seed. - predictions, predictions_no_dropout, _ = ( - prediction_ops.gradient_trees_prediction( - self._ensemble_handle, - seed, - self._dense_floats, - self._sparse_float_indices, - self._sparse_float_values, - self._sparse_float_shapes, - self._sparse_int_indices, - self._sparse_int_values, - self._sparse_int_shapes, - learner_config=self._learner_config_serialized, - apply_dropout=apply_dropout, - apply_averaging=apply_averaging, - use_locking=False, - center_bias=self._center_bias, - reduce_dim=self._reduce_dim)) - partition_ids = prediction_ops.gradient_trees_partition_examples( - self._ensemble_handle, - self._dense_floats, - self._sparse_float_indices, - self._sparse_float_values, - self._sparse_float_shapes, - self._sparse_int_indices, - self._sparse_int_values, - self._sparse_int_shapes, - use_locking=False) + apply_dropout, seed = _dropout_params(mode, ensemble_stats) + # Make sure ensemble stats run. This will check that the ensemble has + # the right stamp. + with ops.control_dependencies(ensemble_stats): + predictions, predictions_no_dropout, _ = ( + prediction_ops.gradient_trees_prediction( + self._ensemble_handle, + seed, + self._dense_floats, + self._sparse_float_indices, + self._sparse_float_values, + self._sparse_float_shapes, + self._sparse_int_indices, + self._sparse_int_values, + self._sparse_int_shapes, + learner_config=self._learner_config_serialized, + apply_dropout=apply_dropout, + apply_averaging=apply_averaging, + use_locking=True, + center_bias=self._center_bias, + reduce_dim=self._reduce_dim)) + partition_ids = prediction_ops.gradient_trees_partition_examples( + self._ensemble_handle, + self._dense_floats, + self._sparse_float_indices, + self._sparse_float_values, + self._sparse_float_shapes, + self._sparse_int_indices, + self._sparse_int_values, + self._sparse_int_shapes, + use_locking=True) return _make_predictions_dict(ensemble_stamp, predictions, predictions_no_dropout, partition_ids, From 7d5cbd78a54319eeb45bca2e239ec037997dad20 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 1 Sep 2017 22:16:25 -0700 Subject: [PATCH 65/67] Update feature_util to support SequenceExample proto. PiperOrigin-RevId: 167359339 --- tensorflow/core/example/feature_util.cc | 124 +++++++--- tensorflow/core/example/feature_util.h | 226 ++++++++++++------ tensorflow/core/example/feature_util_test.cc | 228 ++++++++++++++++++- 3 files changed, 469 insertions(+), 109 deletions(-) diff --git a/tensorflow/core/example/feature_util.cc b/tensorflow/core/example/feature_util.cc index 6f3cc6c6c5d..f0593ede82f 100644 --- a/tensorflow/core/example/feature_util.cc +++ b/tensorflow/core/example/feature_util.cc @@ -18,77 +18,129 @@ limitations under the License. namespace tensorflow { namespace internal { - -::tensorflow::Feature& ExampleFeature(const string& name, - ::tensorflow::Example* example) { - ::tensorflow::Features* features = example->mutable_features(); - return (*features->mutable_feature())[name]; +Feature& ExampleFeature(const string& name, Example* example) { + return *GetFeature(name, example); } -} // namespace internal +} // namespace internal template <> -bool ExampleHasFeature(const string& name, - const Example& example) { - auto it = example.features().feature().find(name); - return (it != example.features().feature().end()) && +bool HasFeature<>(const string& key, const Features& features) { + return (features.feature().find(key) != features.feature().end()); +} + +template <> +bool HasFeature(const string& key, const Features& features) { + auto it = features.feature().find(key); + return (it != features.feature().end()) && (it->second.kind_case() == Feature::KindCase::kInt64List); } template <> -bool ExampleHasFeature(const string& name, const Example& example) { - auto it = example.features().feature().find(name); - return (it != example.features().feature().end()) && +bool HasFeature(const string& key, const Features& features) { + auto it = features.feature().find(key); + return (it != features.feature().end()) && (it->second.kind_case() == Feature::KindCase::kFloatList); } template <> -bool ExampleHasFeature(const string& name, const Example& example) { - auto it = example.features().feature().find(name); - return (it != example.features().feature().end()) && +bool HasFeature(const string& key, const Features& features) { + auto it = features.feature().find(key); + return (it != features.feature().end()) && (it->second.kind_case() == Feature::KindCase::kBytesList); } +bool HasFeatureList(const string& key, + const SequenceExample& sequence_example) { + auto& feature_list = sequence_example.feature_lists().feature_list(); + return (feature_list.find(key) != feature_list.end()); +} + template <> const protobuf::RepeatedField& GetFeatureValues( - const string& name, const Example& example) { - return example.features().feature().at(name).int64_list().value(); + const Feature& feature) { + return feature.int64_list().value(); } template <> protobuf::RepeatedField* GetFeatureValues( - const string& name, Example* example) { - return internal::ExampleFeature(name, example) - .mutable_int64_list() - ->mutable_value(); + Feature* feature) { + return feature->mutable_int64_list()->mutable_value(); } template <> const protobuf::RepeatedField& GetFeatureValues( - const string& name, const Example& example) { - return example.features().feature().at(name).float_list().value(); + const Feature& feature) { + return feature.float_list().value(); } template <> -protobuf::RepeatedField* GetFeatureValues(const string& name, - Example* example) { - return internal::ExampleFeature(name, example) - .mutable_float_list() - ->mutable_value(); +protobuf::RepeatedField* GetFeatureValues(Feature* feature) { + return feature->mutable_float_list()->mutable_value(); } template <> const protobuf::RepeatedPtrField& GetFeatureValues( - const string& name, const Example& example) { - return example.features().feature().at(name).bytes_list().value(); + const Feature& feature) { + return feature.bytes_list().value(); } template <> -protobuf::RepeatedPtrField* GetFeatureValues(const string& name, - Example* example) { - return internal::ExampleFeature(name, example) - .mutable_bytes_list() - ->mutable_value(); +protobuf::RepeatedPtrField* GetFeatureValues(Feature* feature) { + return feature->mutable_bytes_list()->mutable_value(); } +const protobuf::RepeatedPtrField& GetFeatureList( + const string& key, const SequenceExample& sequence_example) { + return sequence_example.feature_lists().feature_list().at(key).feature(); +} + +protobuf::RepeatedPtrField* GetFeatureList( + const string& feature_list_key, SequenceExample* sequence_example) { + return (*sequence_example->mutable_feature_lists() + ->mutable_feature_list())[feature_list_key] + .mutable_feature(); +} + +template <> +Features* GetFeatures(Features* proto) { + return proto; +} + +template <> +Features* GetFeatures(Example* proto) { + return proto->mutable_features(); +} + +template <> +const Features& GetFeatures(const Features& proto) { + return proto; +} + +template <> +const Features& GetFeatures(const Example& proto) { + return proto.features(); +} + +template <> +const protobuf::RepeatedField& GetFeatureValues( + const Feature& feature); + +template <> +protobuf::RepeatedField* GetFeatureValues( + Feature* feature); + +template <> +const protobuf::RepeatedField& GetFeatureValues( + const Feature& feature); + +template <> +protobuf::RepeatedField* GetFeatureValues(Feature* feature); + +template <> +const protobuf::RepeatedPtrField& GetFeatureValues( + const Feature& feature); + +template <> +protobuf::RepeatedPtrField* GetFeatureValues(Feature* feature); } // namespace tensorflow diff --git a/tensorflow/core/example/feature_util.h b/tensorflow/core/example/feature_util.h index 4004411cb17..a87c2c9a57c 100644 --- a/tensorflow/core/example/feature_util.h +++ b/tensorflow/core/example/feature_util.h @@ -13,9 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// A set of lightweight wrappers which simplify access to Example features. +// A set of lightweight wrappers which simplify access to Feature protos. // // TensorFlow Example proto uses associative maps on top of oneof fields. +// SequenceExample proto uses associative map of FeatureList. // So accessing feature values is not very convenient. // // For example, to read a first value of integer feature "tag": @@ -42,9 +43,59 @@ limitations under the License. // (RepeatedPtrField for byte list). So refer to its documentation of // RepeatedField for full list of supported methods. // -// NOTE: It is also important to mention that due to the nature of oneof proto -// fields setting a feature of one type automatically clears all values stored -// as another type with the same feature name. +// NOTE: Due to the nature of oneof proto fields setting a feature of one type +// automatically clears all values stored as another type with the same feature +// key. +// +// This library also has tools to work with SequenceExample protos. +// +// To get a value from SequenceExample.context: +// int id = GetFeatureValues("tag", se.context()).Get(0); +// To add a value to the context: +// GetFeatureValues("tag", se.mutable_context())->Add(42); +// +// To add values to feature_lists: +// AppendFeatureValues({4.0}, +// GetFeatureList("movie_ratings", &se)->Add()); +// AppendFeatureValues({5.0, 3.0}, +// GetFeatureList("movie_ratings", &se)->Add()); +// This will create a feature list keyed as "images" with two features: +// feature_lists { +// feature_list { +// key: "images" +// value { +// feature { float_list { value: [4.0] } } +// feature { float_list { value: [5.0, 3.0] } } +// } +// } } +// +// Functions exposed by this library: +// HasFeature<[FeatureType]>(key, proto) -> bool +// Returns true if a feature with the specified key, and optionally +// FeatureType, belongs to the Features or Example proto. +// HasFeatureList(key, sequence_example) -> bool +// Returns true if SequenceExample has a feature_list with the key. +// GetFeatureValues(key, proto) -> RepeatedField +// Returns values for the specified key and the FeatureType. +// Supported types for the proto: Example, Features. +// GetFeatureList(key, sequence_example) -> RepeatedPtrField +// Returns Feature protos associated with a key. +// AppendFeatureValues(begin, end, feature) +// AppendFeatureValues(container or initializer_list, feature) +// Copies values into a Feature. +// AppendFeatureValues(begin, end, key, proto) +// AppendFeatureValues(container or initializer_list, key, proto) +// Copies values into Features and Example protos with the specified key. +// +// Auxiliary functions, it is unlikely you'll need to use them directly: +// GetFeatures(proto) -> Features +// A convenience function to get Features proto. +// Supported types for the proto: Example, Features. +// GetFeature(key, proto) -> Feature* +// Returns a Feature proto for the specified key, creates a new if +// necessary. Supported types for the proto: Example, Features. +// GetFeatureValues(feature) -> RepeatedField +// Returns values of the feature for the FeatureType. #ifndef TENSORFLOW_EXAMPLE_FEATURE_H_ #define TENSORFLOW_EXAMPLE_FEATURE_H_ @@ -62,10 +113,11 @@ namespace tensorflow { namespace internal { +// DEPRECATED: Use GetFeature instead. +// TODO(gorban): Update all clients in a followup CL. // Returns a reference to a feature corresponding to the name. // Note: it will create a new Feature if it is missing in the example. -::tensorflow::Feature& ExampleFeature(const string& name, - ::tensorflow::Example* example); +Feature& ExampleFeature(const string& name, Example* example); // Specializations of RepeatedFieldTrait define a type of RepeatedField // corresponding to a selected feature type. @@ -127,89 +179,135 @@ struct FeatureTrait< } // namespace internal -// Returns true if feature with the specified name belongs to the example proto. -// Doesn't check feature type. Note that specialized versions return false if -// the feature has a wrong type. -template -bool ExampleHasFeature(const string& name, const Example& example) { - return example.features().feature().find(name) != - example.features().feature().end(); -} +// Returns true if sequence_example has a feature_list with the specified key. +bool HasFeatureList(const string& key, const SequenceExample& sequence_example); + +// A family of template functions to return mutable Features proto from a +// container proto. Supported ProtoTypes: Example, Features. +template +Features* GetFeatures(ProtoType* proto); + +template +const Features& GetFeatures(const ProtoType& proto); // Base declaration of a family of template functions to return a read only -// repeated field corresponding to a feature with the specified name. +// repeated field of feature values. template const typename internal::RepeatedFieldTrait::Type& -GetFeatureValues(const string& name, const Example& example); +GetFeatureValues(const Feature& feature); -// Base declaration of a family of template functions to return a mutable -// repeated field corresponding to a feature with the specified name. +// Returns a read only repeated field corresponding to a feature with the +// specified name and FeatureType. Supported ProtoTypes: Example, Features. +template +const typename internal::RepeatedFieldTrait::Type& +GetFeatureValues(const string& key, const ProtoType& proto) { + return GetFeatureValues(GetFeatures(proto).feature().at(key)); +} + +// Returns a mutable repeated field of a feature values. template typename internal::RepeatedFieldTrait::Type* GetFeatureValues( - const string& name, Example* example); + Feature* feature); + +// Returns a mutable repeated field corresponding to a feature with the +// specified name and FeatureType. Supported ProtoTypes: Example, Features. +template +typename internal::RepeatedFieldTrait::Type* GetFeatureValues( + const string& key, ProtoType* proto) { + ::tensorflow::Feature& feature = + (*GetFeatures(proto)->mutable_feature())[key]; + return GetFeatureValues(&feature); +} + +// Returns a Feature proto for the specified key, creates a new if necessary. +// Supported types for the proto: Example, Features. +template +Feature* GetFeature(const string& key, ProtoType* proto) { + return &(*GetFeatures(proto)->mutable_feature())[key]; +} + +// Returns a repeated field with features corresponding to a feature_list key. +const protobuf::RepeatedPtrField& GetFeatureList( + const string& key, const SequenceExample& sequence_example); + +// Returns a mutable repeated field with features corresponding to a +// feature_list key. It will create a new FeatureList if necessary. +protobuf::RepeatedPtrField* GetFeatureList( + const string& feature_list_key, SequenceExample* sequence_example); -// Copies elements from the range, defined by [first, last) into a feature. template void AppendFeatureValues(IteratorType first, IteratorType last, - const string& name, Example* example) { + Feature* feature) { using FeatureType = typename internal::FeatureTrait< typename std::iterator_traits::value_type>::Type; - std::copy(first, last, protobuf::RepeatedFieldBackInserter( - GetFeatureValues(name, example))); + std::copy(first, last, + protobuf::RepeatedFieldBackInserter( + GetFeatureValues(feature))); +} + +template +void AppendFeatureValues(std::initializer_list container, + Feature* feature) { + AppendFeatureValues(container.begin(), container.end(), feature); +} + +template +void AppendFeatureValues(const ContainerType& container, Feature* feature) { + using IteratorType = typename ContainerType::const_iterator; + AppendFeatureValues(container.begin(), container.end(), + feature); +} + +// Copies elements from the range, defined by [first, last) into the feature +// obtainable from the (proto, key) combination. +template +void AppendFeatureValues(IteratorType first, IteratorType last, + const string& key, ProtoType* proto) { + AppendFeatureValues(first, last, GetFeature(key, GetFeatures(proto))); } // Copies all elements from the container into a feature. -template -void AppendFeatureValues(const ContainerType& container, const string& name, - Example* example) { +template +void AppendFeatureValues(const ContainerType& container, const string& key, + ProtoType* proto) { using IteratorType = typename ContainerType::const_iterator; - AppendFeatureValues(container.begin(), container.end(), name, - example); + AppendFeatureValues(container.begin(), container.end(), key, + proto); } -// Copies all elements from the initializer list into a feature. -template +// Copies all elements from the initializer list into a Feature contained by +// Features or Example proto. +template void AppendFeatureValues(std::initializer_list container, - const string& name, Example* example) { + const string& key, ProtoType* proto) { using IteratorType = typename std::initializer_list::const_iterator; - AppendFeatureValues(container.begin(), container.end(), name, - example); + AppendFeatureValues(container.begin(), container.end(), key, + proto); } -template <> -bool ExampleHasFeature(const string& name, - const Example& example); +// Returns true if a feature with the specified key belongs to the Features. +// The template parameter pack accepts zero or one template argument - which +// is FeatureType. If the FeatureType not specified (zero template arguments) +// the function will not check the feature type. Otherwise it will return false +// if the feature has a wrong type. +template +bool HasFeature(const string& key, const Features& features); -template <> -bool ExampleHasFeature(const string& name, const Example& example); +// Returns true if a feature with the specified key belongs to the Example. +// Doesn't check feature type if used without FeatureType, otherwise the +// specialized versions return false if the feature has a wrong type. +template +bool HasFeature(const string& key, const Example& example) { + return HasFeature(key, GetFeatures(example)); +}; -template <> -bool ExampleHasFeature(const string& name, const Example& example); - -template <> -const protobuf::RepeatedField& GetFeatureValues( - const string& name, const Example& example); - -template <> -protobuf::RepeatedField* GetFeatureValues( - const string& name, Example* example); - -template <> -const protobuf::RepeatedField& GetFeatureValues( - const string& name, const Example& example); - -template <> -protobuf::RepeatedField* GetFeatureValues(const string& name, - Example* example); - -template <> -const protobuf::RepeatedPtrField& GetFeatureValues( - const string& name, const Example& example); - -template <> -protobuf::RepeatedPtrField* GetFeatureValues(const string& name, - Example* example); +// DEPRECATED: use HasFeature instead. +// TODO(gorban): update all clients in a followup CL. +template +bool ExampleHasFeature(const string& key, const Example& example) { + return HasFeature(key, example); +} } // namespace tensorflow #endif // TENSORFLOW_EXAMPLE_FEATURE_H_ diff --git a/tensorflow/core/example/feature_util_test.cc b/tensorflow/core/example/feature_util_test.cc index eb7b90af1b2..cd32dee306d 100644 --- a/tensorflow/core/example/feature_util_test.cc +++ b/tensorflow/core/example/feature_util_test.cc @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ - #include "tensorflow/core/example/feature_util.h" #include @@ -38,6 +37,16 @@ TEST(GetFeatureValuesInt64Test, ReadsASingleValue) { EXPECT_EQ(42, tag.Get(0)); } +TEST(GetFeatureValuesInt64Test, ReadsASingleValueFromFeature) { + Feature feature; + feature.mutable_int64_list()->add_value(42); + + auto values = GetFeatureValues(feature); + + ASSERT_EQ(1, values.size()); + EXPECT_EQ(42, values.Get(0)); +} + TEST(GetFeatureValuesInt64Test, WritesASingleValue) { Example example; @@ -48,25 +57,33 @@ TEST(GetFeatureValuesInt64Test, WritesASingleValue) { EXPECT_EQ(42, example.features().feature().at("tag").int64_list().value(0)); } +TEST(GetFeatureValuesInt64Test, WritesASingleValueToFeature) { + Feature feature; + + GetFeatureValues(&feature)->Add(42); + + ASSERT_EQ(1, feature.int64_list().value_size()); + EXPECT_EQ(42, feature.int64_list().value(0)); +} + TEST(GetFeatureValuesInt64Test, CheckUntypedFieldExistence) { Example example; - - EXPECT_FALSE(ExampleHasFeature("tag", example)); + ASSERT_FALSE(HasFeature("tag", example)); GetFeatureValues("tag", &example)->Add(0); - EXPECT_TRUE(ExampleHasFeature("tag", example)); + EXPECT_TRUE(HasFeature("tag", example)); } TEST(GetFeatureValuesInt64Test, CheckTypedFieldExistence) { Example example; GetFeatureValues("tag", &example)->Add(3.14); - ASSERT_FALSE(ExampleHasFeature("tag", example)); + ASSERT_FALSE(HasFeature("tag", example)); GetFeatureValues("tag", &example)->Add(42); - EXPECT_TRUE(ExampleHasFeature("tag", example)); + EXPECT_TRUE(HasFeature("tag", example)); auto tag_ro = GetFeatureValues("tag", example); ASSERT_EQ(1, tag_ro.size()); EXPECT_EQ(42, tag_ro.Get(0)); @@ -87,6 +104,16 @@ TEST(GetFeatureValuesInt64Test, CopyIterableToAField) { EXPECT_EQ(3, tag_ro.Get(2)); } +TEST(GetFeatureValuesFloatTest, ReadsASingleValueFromFeature) { + Feature feature; + feature.mutable_float_list()->add_value(3.14); + + auto values = GetFeatureValues(feature); + + ASSERT_EQ(1, values.size()); + EXPECT_NEAR(3.14, values.Get(0), kTolerance); +} + TEST(GetFeatureValuesFloatTest, ReadsASingleValue) { Example example; (*example.mutable_features()->mutable_feature())["tag"] @@ -99,6 +126,15 @@ TEST(GetFeatureValuesFloatTest, ReadsASingleValue) { EXPECT_NEAR(3.14, tag.Get(0), kTolerance); } +TEST(GetFeatureValuesFloatTest, WritesASingleValueToFeature) { + Feature feature; + + GetFeatureValues(&feature)->Add(3.14); + + ASSERT_EQ(1, feature.float_list().value_size()); + EXPECT_NEAR(3.14, feature.float_list().value(0), kTolerance); +} + TEST(GetFeatureValuesFloatTest, WritesASingleValue) { Example example; @@ -114,6 +150,20 @@ TEST(GetFeatureValuesFloatTest, WritesASingleValue) { TEST(GetFeatureValuesFloatTest, CheckTypedFieldExistence) { Example example; + GetFeatureValues("tag", &example)->Add(42); + ASSERT_FALSE(HasFeature("tag", example)); + + GetFeatureValues("tag", &example)->Add(3.14); + + EXPECT_TRUE(HasFeature("tag", example)); + auto tag_ro = GetFeatureValues("tag", example); + ASSERT_EQ(1, tag_ro.size()); + EXPECT_NEAR(3.14, tag_ro.Get(0), kTolerance); +} + +TEST(GetFeatureValuesFloatTest, CheckTypedFieldExistenceForDeprecatedMethod) { + Example example; + GetFeatureValues("tag", &example)->Add(42); ASSERT_FALSE(ExampleHasFeature("tag", example)); @@ -125,6 +175,16 @@ TEST(GetFeatureValuesFloatTest, CheckTypedFieldExistence) { EXPECT_NEAR(3.14, tag_ro.Get(0), kTolerance); } +TEST(GetFeatureValuesStringTest, ReadsASingleValueFromFeature) { + Feature feature; + feature.mutable_bytes_list()->add_value("FOO"); + + auto values = GetFeatureValues(feature); + + ASSERT_EQ(1, values.size()); + EXPECT_EQ("FOO", values.Get(0)); +} + TEST(GetFeatureValuesStringTest, ReadsASingleValue) { Example example; (*example.mutable_features()->mutable_feature())["tag"] @@ -137,6 +197,15 @@ TEST(GetFeatureValuesStringTest, ReadsASingleValue) { EXPECT_EQ("FOO", tag.Get(0)); } +TEST(GetFeatureValuesStringTest, WritesASingleValueToFeature) { + Feature feature; + + *GetFeatureValues(&feature)->Add() = "FOO"; + + ASSERT_EQ(1, feature.bytes_list().value_size()); + EXPECT_EQ("FOO", feature.bytes_list().value(0)); +} + TEST(GetFeatureValuesStringTest, WritesASingleValue) { Example example; @@ -148,15 +217,15 @@ TEST(GetFeatureValuesStringTest, WritesASingleValue) { example.features().feature().at("tag").bytes_list().value(0)); } -TEST(GetFeatureValuesBytesTest, CheckTypedFieldExistence) { +TEST(GetFeatureValuesStringTest, CheckTypedFieldExistence) { Example example; GetFeatureValues("tag", &example)->Add(42); - ASSERT_FALSE(ExampleHasFeature("tag", example)); + ASSERT_FALSE(HasFeature("tag", example)); *GetFeatureValues("tag", &example)->Add() = "FOO"; - EXPECT_TRUE(ExampleHasFeature("tag", example)); + EXPECT_TRUE(HasFeature("tag", example)); auto tag_ro = GetFeatureValues("tag", example); ASSERT_EQ(1, tag_ro.size()); EXPECT_EQ("FOO", tag_ro.Get(0)); @@ -228,5 +297,146 @@ TEST(AppendFeatureValuesTest, StringVariablesUsingInitializerList) { EXPECT_EQ("BAZ", tag_ro.Get(2)); } +TEST(SequenceExampleTest, ReadsASingleValueFromContext) { + SequenceExample se; + (*se.mutable_context()->mutable_feature())["tag"] + .mutable_int64_list() + ->add_value(42); + + auto values = GetFeatureValues("tag", se.context()); + + ASSERT_EQ(1, values.size()); + EXPECT_EQ(42, values.Get(0)); +} + +TEST(SequenceExampleTest, WritesASingleValueToContext) { + SequenceExample se; + + GetFeatureValues("tag", se.mutable_context())->Add(42); + + ASSERT_EQ(1, se.context().feature().at("tag").int64_list().value_size()); + EXPECT_EQ(42, se.context().feature().at("tag").int64_list().value(0)); +} + +TEST(SequenceExampleTest, AppendFeatureValuesToContextSingleArg) { + SequenceExample se; + + AppendFeatureValues({1.1, 2.2, 3.3}, "tag", se.mutable_context()); + + auto tag_ro = GetFeatureValues("tag", se.context()); + ASSERT_EQ(3, tag_ro.size()); + EXPECT_NEAR(1.1, tag_ro.Get(0), kTolerance); + EXPECT_NEAR(2.2, tag_ro.Get(1), kTolerance); + EXPECT_NEAR(3.3, tag_ro.Get(2), kTolerance); +} + +TEST(SequenceExampleTest, CheckTypedFieldExistence) { + SequenceExample se; + + GetFeatureValues("tag", se.mutable_context())->Add(3.14); + ASSERT_FALSE(HasFeature("tag", se.context())); + + GetFeatureValues("tag", se.mutable_context())->Add(42); + + EXPECT_TRUE(HasFeature("tag", se.context())); + auto tag_ro = GetFeatureValues("tag", se.context()); + ASSERT_EQ(1, tag_ro.size()); + EXPECT_EQ(42, tag_ro.Get(0)); +} + +TEST(SequenceExampleTest, ReturnsExistingFeatureLists) { + SequenceExample se; + (*se.mutable_feature_lists()->mutable_feature_list())["tag"] + .mutable_feature() + ->Add(); + + auto feature = GetFeatureList("tag", se); + + ASSERT_EQ(1, feature.size()); +} + +TEST(SequenceExampleTest, CreatesNewFeatureLists) { + SequenceExample se; + + GetFeatureList("tag", &se)->Add(); + + EXPECT_EQ(1, se.feature_lists().feature_list().at("tag").feature_size()); +} + +TEST(SequenceExampleTest, CheckFeatureListExistence) { + SequenceExample se; + ASSERT_FALSE(HasFeatureList("tag", se)); + + GetFeatureList("tag", &se)->Add(); + + ASSERT_TRUE(HasFeatureList("tag", se)); +} + +TEST(SequenceExampleTest, AppendFeatureValuesWithInitializerList) { + SequenceExample se; + + AppendFeatureValues({1, 2, 3}, "ids", se.mutable_context()); + AppendFeatureValues({"cam1-0", "cam2-0"}, + GetFeatureList("images", &se)->Add()); + AppendFeatureValues({"cam1-1", "cam2-2"}, + GetFeatureList("images", &se)->Add()); + + EXPECT_EQ(se.DebugString(), + "context {\n" + " feature {\n" + " key: \"ids\"\n" + " value {\n" + " int64_list {\n" + " value: 1\n" + " value: 2\n" + " value: 3\n" + " }\n" + " }\n" + " }\n" + "}\n" + "feature_lists {\n" + " feature_list {\n" + " key: \"images\"\n" + " value {\n" + " feature {\n" + " bytes_list {\n" + " value: \"cam1-0\"\n" + " value: \"cam2-0\"\n" + " }\n" + " }\n" + " feature {\n" + " bytes_list {\n" + " value: \"cam1-1\"\n" + " value: \"cam2-2\"\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n"); +} + +TEST(SequenceExampleTest, AppendFeatureValuesWithVectors) { + SequenceExample se; + + std::vector readings{1.0, 2.5, 5.0}; + AppendFeatureValues(readings, GetFeatureList("movie_ratings", &se)->Add()); + + EXPECT_EQ(se.DebugString(), + "feature_lists {\n" + " feature_list {\n" + " key: \"movie_ratings\"\n" + " value {\n" + " feature {\n" + " float_list {\n" + " value: 1\n" + " value: 2.5\n" + " value: 5\n" + " }\n" + " }\n" + " }\n" + " }\n" + "}\n"); +} + } // namespace } // namespace tensorflow From ddba1e0aadabe26063a28c5d1c48e2cfce44e30f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 2 Sep 2017 14:35:12 -0700 Subject: [PATCH 66/67] Replace CHECKs in v1 checkpoint loading codepath with returning errors. PiperOrigin-RevId: 167392822 --- tensorflow/core/kernels/save_restore_tensor.cc | 9 ++++++--- tensorflow/core/util/tensor_slice_reader.h | 17 +++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/kernels/save_restore_tensor.cc b/tensorflow/core/kernels/save_restore_tensor.cc index 80d49017406..6b06cf650a8 100644 --- a/tensorflow/core/kernels/save_restore_tensor.cc +++ b/tensorflow/core/kernels/save_restore_tensor.cc @@ -216,9 +216,12 @@ void RestoreTensor(OpKernelContext* context, if (output_shape.num_elements() == 0) return; -#define READER_COPY(T) \ - case DataTypeToEnum::value: \ - reader->CopySliceData(tensor_name, slice_to_load, t->flat().data()); \ +#define READER_COPY(T) \ + case DataTypeToEnum::value: \ + OP_REQUIRES(context, \ + reader->CopySliceData(tensor_name, slice_to_load, \ + t->flat().data()), \ + errors::InvalidArgument("Error copying slice data")); \ break; switch (type) { diff --git a/tensorflow/core/util/tensor_slice_reader.h b/tensorflow/core/util/tensor_slice_reader.h index eeb31295737..5932d59a159 100644 --- a/tensorflow/core/util/tensor_slice_reader.h +++ b/tensorflow/core/util/tensor_slice_reader.h @@ -165,13 +165,18 @@ bool TensorSliceReader::CopySliceData(const string& name, CHECK_GE(idx, 0) << "Failed to find the index for filename " << fname; // We read a record in the corresponding sstable const string key = EncodeTensorNameSlice(name, slice_s); - CHECK(sss_[idx]->Get(key, &value)) - << "Failed to seek to the record for tensor " << name << ", slice " - << slice_s.DebugString() << ": computed key = " << key; + if (!sss_[idx]->Get(key, &value)) { + VLOG(1) << "Failed to seek to the record for tensor " << name + << ", slice " << slice_s.DebugString() + << ": computed key = " << key; + return false; + } SavedTensorSlices sts; - CHECK(ParseProtoUnlimited(&sts, value)) - << "Failed to parse the record for tensor " << name << ", slice " - << slice_s.DebugString() << ": computed key = " << key; + if (!ParseProtoUnlimited(&sts, value)) { + VLOG(1) << "Failed to parse the record for tensor " << name << ", slice " + << slice_s.DebugString() << ": computed key = " << key; + return false; + } CopyDataFromTensorSliceToTensorSlice( tss->shape(), slice_s, slice, checkpoint::TensorProtoData(sts.data().data()), data); From d57572e996dce24abf4d9cf6ea04e7104b3d743b Mon Sep 17 00:00:00 2001 From: Martin Wicke Date: Sat, 2 Sep 2017 19:21:45 -0700 Subject: [PATCH 67/67] Merge changes from github. PiperOrigin-RevId: 167401527 --- README.md | 8 ++++++ RELEASE.md | 2 +- WORKSPACE | 8 +++--- configure.py | 10 +++---- tensorflow/cc/gradients/nn_grad.cc | 8 ++++++ tensorflow/cc/gradients/nn_grad_test.cc | 8 ++++++ .../gpu/llvm_gpu_backend/gpu_backend_lib.cc | 2 +- tensorflow/contrib/cmake/tf_tests.cmake | 2 ++ tensorflow/contrib/gdr/BUILD | 1 + tensorflow/contrib/gdr/gdr_memory_manager.h | 7 +---- tensorflow/contrib/layers/__init__.py | 1 + .../layers/python/layers/optimizers.py | 5 ++-- .../python/learn/learn_io/data_feeder.py | 24 ++++++++++------- tensorflow/contrib/makefile/Makefile | 12 ++++----- tensorflow/contrib/makefile/README.md | 3 ++- .../base_rendezvous_mgr.cc | 27 ++++++++++--------- tensorflow/core/framework/op_kernel.h | 2 +- tensorflow/core/profiler/g3doc/advise.md | 4 +-- .../core/profiler/g3doc/command_line.md | 6 ++--- tensorflow/core/profiler/g3doc/options.md | 8 +++--- .../core/profiler/g3doc/profile_memory.md | 2 +- .../g3doc/profile_model_architecture.md | 8 +++--- .../core/profiler/g3doc/profile_time.md | 12 ++++----- tensorflow/docs_src/install/install_linux.md | 6 ++--- .../docs_src/install/install_windows.md | 6 +++-- tensorflow/examples/speech_commands/README.md | 2 +- .../python/feature_column/feature_column.py | 2 +- .../feature_column/feature_column_test.py | 14 ++++++++++ .../stream_executor/device_description.h | 4 +-- tensorflow/stream_executor/kernel.h | 2 +- tensorflow/tools/ci_build/update_version.py | 5 ++-- tensorflow/tools/pip_package/BUILD | 1 + third_party/sqlite.BUILD | 4 +-- 33 files changed, 133 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 87c7b1bfa9f..5a0739a603c 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,15 @@ and discussion, and please direct specific questions to [Stack Overflow](https:/ People who are a little more adventurous can also try our nightly binaries: +**Nightly pip packages** +* We are pleased to announce that TensorFlow now offers nightly pip packages +under the [tf-nightly](https://pypi.python.org/pypi/tf-nightly) project on pypi. +Simply run `pip install tf-nightly` in a clean environment to install the nightly +tensorflow build. We currently only support CPU-only packages on Linux and Mac. +GPU packages on all platforms and Windows CPU-only packages will arrive soon! + +**Individual whl files** * Linux CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=cpu-slave)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=cpu-slave/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-python35-linux-cpu/)) * Linux GPU: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.3.0-cp27-none-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=gpu-linux/)) / [Python 3.4](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.3.0-cp34-cp34m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=gpu-linux/)) / [Python 3.5](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow_gpu-1.3.0-cp35-cp35m-linux_x86_64.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3.5,label=gpu-linux/)) * Mac CPU-only: [Python 2](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-py2-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON2,label=mac-slave/)) / [Python 3](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/lastSuccessfulBuild/artifact/pip_test/whl/tensorflow-1.3.0-py3-none-any.whl) ([build history](https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-cpu/TF_BUILD_IS_OPT=OPT,TF_BUILD_IS_PIP=PIP,TF_BUILD_PYTHON_VERSION=PYTHON3,label=mac-slave/)) diff --git a/RELEASE.md b/RELEASE.md index d120f068cae..3d497dbaa96 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -46,7 +46,7 @@ See also [TensorBoard 0.1.4](https://github.com/tensorflow/tensorboard/releases/ * Display feed values with the `print_feed` or `pf` command and clickable links in the curses UI. * Runtime profiler at the op level and the Python source line level with the `run -p` command. * Initial release of the statistical distribution library `tf.distributions`. -* GPU kernels and speed improvements for for unary `tf.where` and `tf.nn.top_k`. +* GPU kernels and speed improvements for unary `tf.where` and `tf.nn.top_k`. * Monotonic Attention wrappers added to `tf.contrib.seq2seq`. * Added `tf.contrib.signal`, a library for signal processing primitives. * Added `tf.contrib.resampler`, containing CPU and GPU ops for differentiable resampling of images. diff --git a/WORKSPACE b/WORKSPACE index 5e9b991fcca..a0fe67bf318 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,11 +2,11 @@ workspace(name = "org_tensorflow") http_archive( name = "io_bazel_rules_closure", - sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846", - strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f", + sha256 = "25f5399f18d8bf9ce435f85c6bbf671ec4820bc4396b3022cc5dc4bc66303609", + strip_prefix = "rules_closure-0.4.2", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", # 2017-06-02 - "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", + "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/0.4.2.tar.gz", # 2017-08-29 + "https://github.com/bazelbuild/rules_closure/archive/0.4.2.tar.gz", ], ) diff --git a/configure.py b/configure.py index 1a0f71ed943..186fdc9ddce 100644 --- a/configure.py +++ b/configure.py @@ -143,7 +143,7 @@ def run_shell(cmd, allow_non_zero=False): def cygpath(path): """Convert path from posix to windows.""" - return run_shell(['cygpath', '-m', path]) + return os.path.abspath(path).replace('\\', '/') def get_python_path(environ_cp, python_bin_path): @@ -196,7 +196,7 @@ def setup_python(environ_cp, bazel_version): environ_cp['PYTHON_BIN_PATH'] = '' # Convert python path to Windows style before checking lib and version - if is_cygwin(): + if is_windows() or is_cygwin(): python_bin_path = cygpath(python_bin_path) # Get PYTHON_LIB_PATH @@ -219,7 +219,7 @@ def setup_python(environ_cp, bazel_version): python_major_version = get_python_major_version(python_bin_path) # Convert python path to Windows style before writing into bazel.rc - if is_cygwin(): + if is_windows() or is_cygwin(): python_lib_path = cygpath(python_lib_path) # Set-up env variables used by python_configure.bzl @@ -600,7 +600,7 @@ def set_tf_cuda_version(environ_cp): # Find out where the CUDA toolkit is installed default_cuda_path = _DEFAULT_CUDA_PATH - if is_cygwin(): + if is_windows() or is_cygwin(): default_cuda_path = cygpath( environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN)) elif is_linux(): @@ -660,7 +660,7 @@ def set_tf_cunn_version(environ_cp): # unusable. Going through one more level of expansion to handle that. cudnn_install_path = os.path.realpath( os.path.expanduser(cudnn_install_path)) - if is_cygwin(): + if is_windows() or is_cygwin(): cudnn_install_path = cygpath(cudnn_install_path) if is_windows(): diff --git a/tensorflow/cc/gradients/nn_grad.cc b/tensorflow/cc/gradients/nn_grad.cc index 6fc73c3fa1b..ccb58e7f915 100644 --- a/tensorflow/cc/gradients/nn_grad.cc +++ b/tensorflow/cc/gradients/nn_grad.cc @@ -95,6 +95,14 @@ Status SeluGradHelper(const Scope& scope, const Operation& op, } REGISTER_GRADIENT_OP("Selu", SeluGradHelper); +Status L2LossGrad(const Scope& scope, const Operation& op, + const std::vector& grad_inputs, + std::vector* grad_outputs) { + grad_outputs->push_back(Mul(scope, op.input(0), grad_inputs[0])); + return scope.status(); +} +REGISTER_GRADIENT_OP("L2Loss", L2LossGrad); + Status BiasAddGradHelper(const Scope& scope, const Operation& op, const std::vector& grad_inputs, std::vector* grad_outputs) { diff --git a/tensorflow/cc/gradients/nn_grad_test.cc b/tensorflow/cc/gradients/nn_grad_test.cc index f9a512f29ed..affc1e1dbe6 100644 --- a/tensorflow/cc/gradients/nn_grad_test.cc +++ b/tensorflow/cc/gradients/nn_grad_test.cc @@ -122,6 +122,14 @@ TEST_F(NNGradTest, SeluGrad) { RunTest(x, x_init_value, y, shape); } +TEST_F(NNGradTest, L2LossGrad) { + TensorShape x_shape({5, 2}); + TensorShape y_shape({1}); + auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); + auto y = L2Loss(scope_, x); + RunTest(x, x_shape, y, y_shape); +} + TEST_F(NNGradTest, BiasAddGradHelper) { TensorShape shape({4, 5}); TensorShape bias_shape({5}); diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc index 2a999f52f01..2e7765c4c61 100644 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc @@ -389,7 +389,7 @@ StatusOr CompileModuleToPtx(llvm::Module* module, // Loop unrolling exposes more opportunities for SROA. Therefore, we run SROA // again after the standard optimization passes [http://b/13329423]. - // TODO(jingyue): SROA may further expose more optimization opportunities, such + // TODO(jingyue): SROA may further expose more optimization opportunities such // as more precise alias analysis and more function inlining (SROA may change // the inlining cost of a function). For now, running SROA already emits good // enough code for the evaluated benchmarks. We may want to run more diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake index 6507a9a5e07..15850bf0a4e 100644 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ b/tensorflow/contrib/cmake/tf_tests.cmake @@ -82,6 +82,7 @@ function(AddTest) set_tests_properties(${_AT_TARGET} PROPERTIES ENVIRONMENT "TEST_TMPDIR=${tempdir};TEST_SRCDIR=${testdir}" ) + set_tests_properties(${_AT_TARGET} PROPERTIES TIMEOUT "600") foreach(datafile ${_AT_DATA}) file(RELATIVE_PATH datafile_rel ${tensorflow_source_dir} ${datafile}) @@ -117,6 +118,7 @@ function(AddPythonTests) if (_AT_DEPENDS) add_dependencies(${_AT_TARGET} ${_AT_DEPENDS}) endif() + set_tests_properties(${sourcefile} PROPERTIES TIMEOUT "600") endforeach() endfunction(AddPythonTests) diff --git a/tensorflow/contrib/gdr/BUILD b/tensorflow/contrib/gdr/BUILD index 645e364d191..bebcf079ba4 100644 --- a/tensorflow/contrib/gdr/BUILD +++ b/tensorflow/contrib/gdr/BUILD @@ -62,6 +62,7 @@ tf_cuda_library( }), deps = [ ":gdr_proto_cc", + "//tensorflow/core:core_cpu_internal", "//tensorflow/core:framework", "//tensorflow/core:gpu_runtime", "//tensorflow/core:lib", diff --git a/tensorflow/contrib/gdr/gdr_memory_manager.h b/tensorflow/contrib/gdr/gdr_memory_manager.h index 7e9fe01e979..e0e2a3f624a 100644 --- a/tensorflow/contrib/gdr/gdr_memory_manager.h +++ b/tensorflow/contrib/gdr/gdr_memory_manager.h @@ -16,14 +16,9 @@ limitations under the License. #ifndef GDR_MEMORY_MANAGER_H_ #define GDR_MEMORY_MANAGER_H_ +#include "google/protobuf/any.pb.h" #include "tensorflow/core/lib/core/status.h" -namespace google { -namespace protobuf { -class Any; -} -} - namespace tensorflow { class Device; diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py index 674f5db1075..ea8d9e0c633 100644 --- a/tensorflow/contrib/layers/__init__.py +++ b/tensorflow/contrib/layers/__init__.py @@ -115,6 +115,7 @@ _allowed_symbols = ['bias_add', 'legacy_linear', 'legacy_relu', 'OPTIMIZER_CLS_NAMES', + 'OPTIMIZER_SUMMARIES', 'regression_target', 'SPARSE_FEATURE_CROSS_DEFAULT_HASH_KEY', 'summaries'] diff --git a/tensorflow/contrib/layers/python/layers/optimizers.py b/tensorflow/contrib/layers/python/layers/optimizers.py index ac217f043f1..7eb410b4c72 100644 --- a/tensorflow/contrib/layers/python/layers/optimizers.py +++ b/tensorflow/contrib/layers/python/layers/optimizers.py @@ -129,8 +129,9 @@ def optimize_loss(loss, `None` to use all trainable variables. name: The name for this operation is used to scope operations and summaries. summaries: List of internal quantities to visualize on tensorboard. If not - set only the loss and the learning rate will be reported. The - complete list is in OPTIMIZER_SUMMARIES. + set, the loss, the learning rate, and the global norm of the + gradients will be reported. The complete list of possible values + is in OPTIMIZER_SUMMARIES. colocate_gradients_with_ops: If True, try colocating gradients with the corresponding op. increment_global_step: Whether to increment `global_step`. If your model diff --git a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py b/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py index 48d79ecbbff..4c50d40aaa9 100644 --- a/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py +++ b/tensorflow/contrib/learn/python/learn/learn_io/data_feeder.py @@ -28,7 +28,6 @@ import six from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.python.framework import dtypes -from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.platform import tf_logging as logging @@ -44,7 +43,7 @@ def _get_in_out_shape(x_shape, y_shape, n_classes, batch_size=None): x_is_dict, y_is_dict = isinstance( x_shape, dict), y_shape is not None and isinstance(y_shape, dict) if y_is_dict and n_classes is not None: - assert (isinstance(n_classes, dict)) + assert isinstance(n_classes, dict) if batch_size is None: batch_size = list(x_shape.values())[0][0] if x_is_dict else x_shape[0] @@ -322,10 +321,12 @@ class DataFeeder(object): self._x = dict([(k, check_array(v, v.dtype)) for k, v in list(x.items()) ]) if x_is_dict else check_array(x, x.dtype) - self._y = None if y is None else \ - dict([(k, check_array(v, v.dtype)) for k, v in list(y.items())]) if x_is_dict else check_array(y, y.dtype) + self._y = None if y is None else ( + dict([(k, check_array(v, v.dtype)) for k, v in list(y.items())]) + if y_is_dict else check_array(y, y.dtype)) - # self.n_classes is not None means we're converting raw target indices to one-hot. + # self.n_classes is not None means we're converting raw target indices + # to one-hot. if n_classes is not None: if not y_is_dict: y_dtype = (np.int64 @@ -344,12 +345,15 @@ class DataFeeder(object): x_shape, y_shape, n_classes, batch_size) # Input dtype matches dtype of x. - self._input_dtype = dict([(k, _check_dtype(v.dtype)) for k, v in list(self._x.items())]) if x_is_dict \ - else _check_dtype(self._x.dtype) + self._input_dtype = ( + dict([(k, _check_dtype(v.dtype)) for k, v in list(self._x.items())]) + if x_is_dict else _check_dtype(self._x.dtype)) - # note: self._output_dtype = np.float32 when y is None - self._output_dtype = dict([(k, _check_dtype(v.dtype)) for k, v in list(self._y.items())]) if y_is_dict \ - else _check_dtype(self._y.dtype) if y is not None else np.float32 + # self._output_dtype == np.float32 when y is None + self._output_dtype = ( + dict([(k, _check_dtype(v.dtype)) for k, v in list(self._y.items())]) + if y_is_dict else ( + _check_dtype(self._y.dtype) if y is not None else np.float32)) # self.n_classes is None means we're passing in raw target indices if n_classes is not None and y_is_dict: diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index 98af47d7288..30897bb202d 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -316,14 +316,14 @@ ifeq ($(TARGET),IOS) IPHONESIMULATOR_SYSROOT := $(shell xcrun --sdk iphonesimulator \ --show-sdk-path) IOS_SDK_VERSION := $(shell xcrun --sdk iphoneos --show-sdk-version) - MIN_SDK_VERSION := 8.0 + MIN_SDK_VERSION := 9.0 # Override IOS_ARCH with ARMV7, ARMV7S, ARM64, or I386. IOS_ARCH := X86_64 ifeq ($(IOS_ARCH),ARMV7) CXXFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \ -arch armv7 \ -fembed-bitcode \ - -D__thread= \ + -D__thread=thread_local \ -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -mno-thumb \ @@ -347,7 +347,7 @@ ifeq ($(TARGET),IOS) CXXFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \ -arch armv7s \ -fembed-bitcode \ - -D__thread= \ + -D__thread=thread_local \ -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -mno-thumb \ @@ -371,7 +371,7 @@ ifeq ($(TARGET),IOS) CXXFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \ -arch arm64 \ -fembed-bitcode \ - -D__thread= \ + -D__thread=thread_local \ -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ @@ -395,7 +395,7 @@ ifeq ($(TARGET),IOS) -arch i386 \ -mno-sse \ -fembed-bitcode \ - -D__thread= \ + -D__thread=thread_local \ -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ @@ -418,7 +418,7 @@ ifeq ($(TARGET),IOS) CXXFLAGS += -mios-simulator-version-min=$(MIN_SDK_VERSION) \ -arch x86_64 \ -fembed-bitcode \ - -D__thread= \ + -D__thread=thread_local \ -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ diff --git a/tensorflow/contrib/makefile/README.md b/tensorflow/contrib/makefile/README.md index 835d68489eb..715eb515776 100644 --- a/tensorflow/contrib/makefile/README.md +++ b/tensorflow/contrib/makefile/README.md @@ -201,7 +201,8 @@ tensorflow/contrib/makefile/compile_ios_protobuf.sh Then, you will need to compile the nsync library for iOS: -```export HOST_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh` +```bash +export HOST_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh` export TARGET_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh -t ios` ``` diff --git a/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc b/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc index cb2fde7dba7..f91e3770498 100644 --- a/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc +++ b/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc @@ -35,14 +35,18 @@ limitations under the License. namespace tensorflow { +static void StartAbortRendevous(Rendezvous* rendez, const Status& s) { + rendez->StartAbort(s); + rendez->Unref(); +} + BaseRendezvousMgr::BaseRendezvousMgr(const WorkerEnv* worker_env) : worker_env_(worker_env) {} BaseRendezvousMgr::~BaseRendezvousMgr() { for (auto& p : table_) { - BaseRemoteRendezvous* rendez = p.second; - rendez->StartAbort(errors::Aborted("Shutdown")); - rendez->Unref(); + auto rendez = p.second; + StartAbortRendevous(rendez, errors::Aborted("Shutdown")); } } @@ -52,7 +56,7 @@ RemoteRendezvous* BaseRendezvousMgr::Find(int64 step_id) { BaseRemoteRendezvous* BaseRendezvousMgr::FindOrCreate(int64 step_id) { mutex_lock l(mu_); - Table::iterator iter = table_.find(step_id); + auto iter = table_.find(step_id); if (iter == table_.end()) { auto rr = Create(step_id, worker_env_); iter = table_.insert({step_id, rr}).first; @@ -64,7 +68,7 @@ BaseRemoteRendezvous* BaseRendezvousMgr::FindOrCreate(int64 step_id) { void BaseRendezvousMgr::RecvLocalAsync(int64 step_id, const Rendezvous::ParsedKey& parsed, Rendezvous::DoneCallback done) { - BaseRemoteRendezvous* rendez = FindOrCreate(step_id); + auto rendez = FindOrCreate(step_id); using namespace std::placeholders; Rendezvous::DoneCallback done_cb = std::bind( [rendez](Rendezvous::DoneCallback done, @@ -101,15 +105,15 @@ void BaseRendezvousMgr::Cleanup(int64 step_id) { Rendezvous* rendez = nullptr; { mutex_lock l(mu_); - Table::iterator iter = table_.find(step_id); + auto iter = table_.find(step_id); if (iter != table_.end()) { rendez = iter->second; table_.erase(iter); } } - if (!rendez) return; - rendez->StartAbort(errors::Aborted("Cleanup ", step_id)); - rendez->Unref(); + if (rendez) { + StartAbortRendevous(rendez, errors::Aborted("Cleanup ", step_id)); + } } void BaseRendezvousMgr::CleanupAll() { @@ -122,8 +126,7 @@ void BaseRendezvousMgr::CleanupAll() { table_.clear(); } for (auto rendez : rendezs) { - rendez->StartAbort(errors::Aborted("Shutdown")); - rendez->Unref(); + StartAbortRendevous(rendez, errors::Aborted("Shutdown")); } } @@ -165,7 +168,7 @@ Status BaseRemoteRendezvous::Initialize(WorkerSession* session) { session_ = session; std::swap(deferred_calls, deferred_calls_); } - for (DeferredCall& call : deferred_calls) { + for (auto& call : deferred_calls) { RecvLocalAsyncInternal(call.parsed, std::move(call.done)); } return Status::OK(); diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h index 46c21dcef01..25b35a6dd71 100644 --- a/tensorflow/core/framework/op_kernel.h +++ b/tensorflow/core/framework/op_kernel.h @@ -310,7 +310,7 @@ class OpKernelConstruction { FunctionLibraryRuntime* function_library() const { return flib_; } // The GraphDef version whose behavior we should follow. - const int graph_def_version() const { return graph_def_version_; } + int graph_def_version() const { return graph_def_version_; } // Helper routines for the OP_REQUIRES macros void CtxFailure(Status s); diff --git a/tensorflow/core/profiler/g3doc/advise.md b/tensorflow/core/profiler/g3doc/advise.md index d87b0d8603d..d0de8317f69 100644 --- a/tensorflow/core/profiler/g3doc/advise.md +++ b/tensorflow/core/profiler/g3doc/advise.md @@ -86,7 +86,7 @@ For example: * Checks RecvTensor RPC latency and bandwidth. * Checks CPU/Memory utilization of the job. -####AcceleratorUtilization Checker +#### AcceleratorUtilization Checker * Checks what percentage of time the accelerator spends on computation. #### OperationChecker @@ -100,7 +100,7 @@ For example: * Checks the most expensive graph nodes. * Checks the most expensive graph-building Python codes. -####Contribute Your Checker +#### Contribute Your Checker Follow examples of accelerator_utilization_checker.h diff --git a/tensorflow/core/profiler/g3doc/command_line.md b/tensorflow/core/profiler/g3doc/command_line.md index 857b5e64590..e2839a682f9 100644 --- a/tensorflow/core/profiler/g3doc/command_line.md +++ b/tensorflow/core/profiler/g3doc/command_line.md @@ -51,7 +51,7 @@ It defines _checkpoint_variable op type. It also provides checkpointed tensors' Note: this feature is not well maintained now. -###Start `tfprof` +### Start `tfprof` #### Build `tfprof` @@ -140,9 +140,9 @@ tfprof> -output ``` -###Examples +### Examples -####Profile Python Time +#### Profile Python Time ```shell # Requires --graph_path --op_log_path tfprof> code -max_depth 1000 -show_name_regexes .*model_analyzer.*py.* -select micros -account_type_regexes .* -order_by micros diff --git a/tensorflow/core/profiler/g3doc/options.md b/tensorflow/core/profiler/g3doc/options.md index 15712d04c25..ddee63ad42a 100644 --- a/tensorflow/core/profiler/g3doc/options.md +++ b/tensorflow/core/profiler/g3doc/options.md @@ -1,6 +1,6 @@ -##Options +## Options -###Overview +### Overview For all tfprof views, the profiles are processed with the following procedures @@ -35,7 +35,7 @@ For all tfprof views, the profiles are processed with the following procedures 4) Finally, the filtered data structure is output in a format depending on the `-output` option. -####Option Semantics In Different View +#### Option Semantics In Different View options usually have the same semantics in different views. However, some can vary. For example `-max_depth` in scope view means the depth of name scope tree. In op view, it means the length of operation list. @@ -68,7 +68,7 @@ output_bytes: The memory output by the operation. It's not necessarily requested by the current operation. For example, it can be a tensor forwarded from input to output, with in-place mutation. -###Docs +### Docs `-max_depth`: Show nodes that are at most this number of hops from starting node in the data structure. diff --git a/tensorflow/core/profiler/g3doc/profile_memory.md b/tensorflow/core/profiler/g3doc/profile_memory.md index a00683d0626..6eda5abdd97 100644 --- a/tensorflow/core/profiler/g3doc/profile_memory.md +++ b/tensorflow/core/profiler/g3doc/profile_memory.md @@ -1,4 +1,4 @@ -##Profile Memory +## Profile Memory It is generally a good idea to visualize the memory usage in timeline. It allows you to see the memory consumption of each GPU over time. diff --git a/tensorflow/core/profiler/g3doc/profile_model_architecture.md b/tensorflow/core/profiler/g3doc/profile_model_architecture.md index a42b2e918da..61bb66bd21b 100644 --- a/tensorflow/core/profiler/g3doc/profile_model_architecture.md +++ b/tensorflow/core/profiler/g3doc/profile_model_architecture.md @@ -1,9 +1,9 @@ -##Profile Model Architecture +## Profile Model Architecture * [Profile Model Parameters](#profile-model-parameters) * [Profile Model Float Operations](#profile-model-float-operations) -###Profile Model Parameters +### Profile Model Parameters Notes: `VariableV2` operation type might contain variables created by TensorFlow @@ -39,9 +39,9 @@ param_stats = tf.profiler.profile( sys.stdout.write('total_params: %d\n' % param_stats.total_parameters) ``` -###Profile Model Float Operations +### Profile Model Float Operations -####Caveats +#### Caveats For an operation to have float operation statistics: diff --git a/tensorflow/core/profiler/g3doc/profile_time.md b/tensorflow/core/profiler/g3doc/profile_time.md index e11a75553b2..4aafc697a9b 100644 --- a/tensorflow/core/profiler/g3doc/profile_time.md +++ b/tensorflow/core/profiler/g3doc/profile_time.md @@ -1,4 +1,4 @@ -##Profile Time +## Profile Time * [Times in TensorFlow and tfprof](#times-in-tensorflow-and-tfprof) * [Profile by Python Code](#profile-by-python-code) @@ -7,7 +7,7 @@ * [Profile by Name Scope](#profile-by-name-scope) -###Times in TensorFlow and tfprof +### Times in TensorFlow and tfprof When we run a model, Tensorflow schedules and runs the nodes (operations) in the graph. An operation can be placed on an accelerator or on CPU. @@ -37,7 +37,7 @@ When an operation is placed on CPU, it will completely run on CPU. Hence, should be 0. -###Profile by Python Code +### Profile by Python Code ```python # In code view, the time of each line of Python code is the aggregated # times of all operations created by that line. @@ -112,7 +112,7 @@ Set ```-output timeline:outfile=``` to generate timeline instead of st -###Profile by Operation Type +### Profile by Operation Type ```python # In op view, you can view the aggregated time of each operation type. tfprof> op -select micros,occurrence -order_by micros @@ -138,7 +138,7 @@ MatMul 618.97ms (63.56%, 16.51%), |/job:worker/replica:0/ ``` -###Profile by Graph +### Profile by Graph Usually, use graph view to generate a timeline to visualize the result. @@ -163,7 +163,7 @@ Open a Chrome browser, enter URL chrome://tracing and load the timeline file. ****************************************************** ``` -###Profile by Name Scope +### Profile by Name Scope Usually scope view allows you to pin point the problematic places if you have properly named your operations with tf.name_scope or tf.variable_scope. diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 43e09906f73..d5e481520c4 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -151,10 +151,10 @@ Take the following steps to install TensorFlow with Virtualenv: (tensorflow)$ pip install --upgrade tensorflow-gpu # for Python 2.7 and GPU (tensorflow)$ pip3 install --upgrade tensorflow-gpu # for Python 3.n and GPU - If the preceding command succeeds, skip Step 5. If the preceding - command fails, perform Step 5. + If the preceding command succeeds, skip Step 6. If the preceding + command fails, perform Step 6. - 5. (Optional) If Step 4 failed (typically because you invoked a pip version + 6. (Optional) If Step 5 failed (typically because you invoked a pip version lower than 8.1), install TensorFlow in the active virtualenv environment by issuing a command of the following format: diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md index be6a490ff9b..3025c9971ab 100644 --- a/tensorflow/docs_src/install/install_windows.md +++ b/tensorflow/docs_src/install/install_windows.md @@ -71,12 +71,14 @@ Use that package at your own risk. ## Installing with native pip -If the following version of Python is not installed on your machine, +If one of the following versions of Python is not installed on your machine, install it now: * [Python 3.5.x 64-bit from python.org](https://www.python.org/downloads/release/python-352/) + * [Python 3.6.x 64-bit from python.org](https://www.python.org/downloads/release/python-362/) -Note that Python 3.5.x comes with the pip3 package manager, which is the +-TensorFlow supports Python 3.5.x and 3.6.x on Windows. +Note that Python 3 comes with the pip3 package manager, which is the program you'll use to install TensorFlow. To install TensorFlow, start a terminal. Then issue the appropriate diff --git a/tensorflow/examples/speech_commands/README.md b/tensorflow/examples/speech_commands/README.md index 3b782101292..63be04ee582 100644 --- a/tensorflow/examples/speech_commands/README.md +++ b/tensorflow/examples/speech_commands/README.md @@ -1,4 +1,4 @@ # Speech Commands Example This is a basic speech recognition example. For more information, see the -tutorial at http://tensorflow.org/tutorials/audio_recognition. +tutorial at https://www.tensorflow.org/versions/master/tutorials/audio_recognition. diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index 44ab1a622e8..a8434d0c991 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -2473,7 +2473,7 @@ class _IndicatorColumn(_DenseColumn, weighted_column = sparse_ops.sparse_merge( sp_ids=id_tensor, sp_values=weight_tensor, - vocab_size=self._variable_shape[-1]) + vocab_size=int(self._variable_shape[-1])) return sparse_ops.sparse_tensor_to_dense(weighted_column) dense_id_tensor = sparse_ops.sparse_tensor_to_dense( diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index b14ec73ba26..30577763910 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -3206,6 +3206,20 @@ class IndicatorColumnTest(test.TestCase): with _initialized_session(): self.assertAllEqual([[0, 0, 1], [1, 0, 0]], indicator_tensor.eval()) + def test_transform_with_weighted_column(self): + # Github issue 12557 + ids = fc.categorical_column_with_vocabulary_list( + key='ids', vocabulary_list=('a', 'b', 'c')) + weights = fc.weighted_categorical_column(ids, 'weights') + indicator = fc.indicator_column(weights) + features = { + 'ids': constant_op.constant(['c', 'b', 'a'], shape=(1, 3)), + 'weights': constant_op.constant([2., 4., 6.], shape=(1, 3)) + } + indicator_tensor = _transform_features(features, [indicator])[indicator] + with _initialized_session(): + self.assertAllEqual([[6., 4., 2.]], indicator_tensor.eval()) + def test_linear_model(self): animal = fc.indicator_column( fc.categorical_column_with_identity('animal', num_buckets=4)) diff --git a/tensorflow/stream_executor/device_description.h b/tensorflow/stream_executor/device_description.h index 8e3155475c3..f2b35bcb434 100644 --- a/tensorflow/stream_executor/device_description.h +++ b/tensorflow/stream_executor/device_description.h @@ -82,7 +82,7 @@ class DeviceDescription { // Returns the limit on the number of simultaneously resident blocks // on a multiprocessor. - const uint64 blocks_per_core_limit() const { return blocks_per_core_limit_; } + uint64 blocks_per_core_limit() const { return blocks_per_core_limit_; } // Returns the limit on the total number of threads that can be launched in a // single block; i.e. the limit on x * y * z dimensions of a ThreadDim. @@ -141,7 +141,7 @@ class DeviceDescription { uint64 device_memory_size() const { return device_memory_size_; } // Returns the device's core clock rate in GHz. - const float clock_rate_ghz() const { return clock_rate_ghz_; } + float clock_rate_ghz() const { return clock_rate_ghz_; } // Returns whether ECC is enabled. bool ecc_enabled() const { return ecc_enabled_; } diff --git a/tensorflow/stream_executor/kernel.h b/tensorflow/stream_executor/kernel.h index d9d40d77bd9..8ef091f929c 100644 --- a/tensorflow/stream_executor/kernel.h +++ b/tensorflow/stream_executor/kernel.h @@ -302,7 +302,7 @@ class KernelArgIterator { // // Returns a default-constructed KernelArg if there is no next argument. KernelArg next() { - KernelArg result; + KernelArg result = {}; if (!has_next()) { return result; } else if ((shmem_indices_iter_ != shmem_indices_end_) && diff --git a/tensorflow/tools/ci_build/update_version.py b/tensorflow/tools/ci_build/update_version.py index e525e113974..4405678a6b8 100755 --- a/tensorflow/tools/ci_build/update_version.py +++ b/tensorflow/tools/ci_build/update_version.py @@ -276,8 +276,9 @@ def check_for_lingering_string(lingering_string): """Check for given lingering strings.""" formatted_string = lingering_string.replace(".", r"\.") try: - linger_strs = subprocess.check_output( - ['grep', '-rnoH', formatted_string, TF_SRC_DIR]).split("\n") + linger_str_output = subprocess.check_output( + ["grep", "-rnoH", formatted_string, TF_SRC_DIR]) + linger_strs = linger_str_output.decode("utf8").split("\n") except subprocess.CalledProcessError: linger_strs = [] diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index bb6334942f3..d62316964f8 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -84,6 +84,7 @@ py_binary( "//tensorflow/python/saved_model", "//tensorflow/python:spectral_ops_test_util", "//tensorflow/python/tools:tools_pip", + "//tensorflow/python/eager:eager_pip", # These targets don't build on Windows yet. Exclude them for now. # "//tensorflow/contrib/ndlstm", # "//tensorflow/contrib/slim", diff --git a/third_party/sqlite.BUILD b/third_party/sqlite.BUILD index f593b71542a..9840d7b1514 100644 --- a/third_party/sqlite.BUILD +++ b/third_party/sqlite.BUILD @@ -2,9 +2,9 @@ # Sqlite3 library. Provides utilities for interacting # with sqlite3 databases. -licenses(["notice"]) # BSD/MIT-like license +licenses(["unencumbered"]) # Public Domain -exports_files(["LICENSE"]) +# exports_files(["LICENSE"]) cc_library( name = "sqlite",